feat: adding fileref support for mp_dirlist, as well as a directory column on the output dataset

all.sas

@@ -380,7 +380,7 @@ options noquotelenmax;
   @brief Returns the engine type of a SAS library
   @details Usage:
 
-      %put %mf_getEngine(SASHELP);
+      %put %mf_getengine(SASHELP);
 
   returns:
   > V9
@@ -398,9 +398,10 @@ options noquotelenmax;
 
   @version 9.2
   @author Allan Bowe
-**/
 
-%macro mf_getEngine(libref
+**/ /** \cond */
+
+%macro mf_getengine(libref
 )/*/STORE SOURCE*/;
   %local dsid engnum rc engine;
 
@@ -419,7 +420,8 @@ options noquotelenmax;
  &engine
 
 %mend;
-/**
+
+/** \endcond *//**
   @file
   @brief Returns the size of a file in bytes.
   @details Provide full path/filename.extension to the file, eg:
@@ -1316,13 +1318,23 @@ Usage:
 **/
 
 %macro mf_trimstr(basestr,trimstr);
-%local trimlen trimval;
+%local baselen trimlen trimval;
+
+/* return if basestr is shorter than trimstr (or 0) */
+%let baselen=%length(%superq(basestr));
 %let trimlen=%length(%superq(trimstr));
+%if &baselen < &trimlen or &baselen=0 %then %return;
+
+/* obtain the characters from the end of basestr */
 %let trimval=%qsubstr(%superq(basestr)
   ,%length(%superq(basestr))-&trimlen+1
   ,&trimlen);
 
-%if %superq(trimval)=%superq(trimstr) %then %do;
+/* compare and if matching, chop it off! */
+%if %superq(basestr)=%superq(trimstr) %then %do;
+  %return;
+%end;
+%else %if %superq(trimval)=%superq(trimstr) %then %do;
   %qsubstr(%superq(basestr),1,%length(%superq(basestr))-&trimlen)
 %end;
 %else %do;
@@ -1475,6 +1487,12 @@ Usage:
     the particulars of an environment.  For instance, can stream custom
     results back to the client in an STP Web App context, or completely stop
     in the case of a batch run.
+  
+  Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored Process
+  environments.  This macro takes a unique approach - we set the SAS syscc to 0,
+  run `stpsrvset('program error', 0)` (if SAS 9) and then - we open a macro 
+  but don't close it!  This provides a graceful abort for SAS web services in all 
+  web enabled environments.
 
   @param mac= to contain the name of the calling macro
   @param msg= message to be returned
@@ -1535,8 +1553,15 @@ Usage:
     %end;
 
     %if %symexist(SYS_JES_JOB_URI) %then %do;
-      /* refer web service output to file service in one hit */
+      /* setup webout */
-      filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json";
+      OPTIONS NOBOMFILE;
+      %if "X&SYS_JES_JOB_URI.X"="XX" %then %do;
+          filename _webout temp lrecl=999999 mod;
+      %end;
+      %else %do;
+        filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
+          name="_webout.json" lrecl=999999 mod;
+      %end;
     %end;
 
     /* send response in SASjs JSON format */
@@ -1932,17 +1957,17 @@ run;
 %mend;/**
   @file
   @brief Returns all files and subdirectories within a specified parent
-  @details When used with getattrs=NO, is not OS specific (uses dopen / dread). 
+  @details When used with getattrs=NO, is not OS specific (uses dopen / dread).
 
   If getattrs=YES then the doptname / foptname functions are used to scan all
-  properties - any characters that are not valid in a SAS name (v7) are simply 
+  properties - any characters that are not valid in a SAS name (v7) are simply
   stripped, and the table is transposed so theat each property is a column
-  and there is one file per row.  An attempt is made to get all properties 
+  and there is one file per row.  An attempt is made to get all properties
   whether a file or folder, but some files/folders cannot be accessed, and so
   not all properties can / will be populated.
 
   Credit for the rename approach:
-  https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003 
+  https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003
 
 
   usage:
@@ -1951,21 +1976,25 @@ run;
 
       %mp_dirlist(outds=cwdfileprops, getattrs=YES)
 
-  @warning In a Unix environment, the existence of a named pipe will cause this 
+      %mp_dirlist(fref=MYFREF)
+
+  @warning In a Unix environment, the existence of a named pipe will cause this
   macro to hang.  Therefore this tool should be used with caution in a SAS 9 web
   application, as it can use up all available multibridge sessions if requests
   are resubmitted.
-  If anyone finds a way to positively identify a named pipe using SAS (without 
+  If anyone finds a way to positively identify a named pipe using SAS (without
   X CMD) do please raise an issue!
 
 
   @param path= for which to return contents
+  @param fref= Provide a DISK engine fileref as an alternative to PATH
   @param outds= the output dataset to create
-  @param getattrs= YES/NO (default=NO).  Uses doptname and foptname to return 
+  @param getattrs= YES/NO (default=NO).  Uses doptname and foptname to return
-  all attributes for each file / folder.  
+  all attributes for each file / folder.
 
 
   @returns outds contains the following variables:
+   - directory (containing folder)
    - file_or_folder (file / folder)
    - filepath (path/to/file.name)
    - filename (just the file name)
@@ -1978,18 +2007,26 @@ run;
 **/
 
 %macro mp_dirlist(path=%sysfunc(pathname(work))
+    , fref=0
     , outds=work.mp_dirlist
     , getattrs=NO
 )/*/STORE SOURCE*/;
 %let getattrs=%upcase(&getattrs)XX;
 
-data &outds (compress=no keep=file_or_folder filepath filename ext msg);
+data &outds (compress=no keep=file_or_folder filepath filename ext msg directory);
-  length filepath $500 fref fref2 $8 file_or_folder $6 filename $80 ext $20 msg $200;
+  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80 ext $20 msg $200;
-  rc = filename(fref, "&path");
+  %if &fref=0 %then %do;
+    rc = filename(fref, "&path");
+  %end;
+  %else %do;
+    fref="&fref";
+    rc=0;
+  %end;
   if rc = 0 then do;
      did = dopen(fref);
+     directory=dinfo(did,'Directory');
      if did=0 then do;
-        putlog "NOTE: This directory is empty - &path";
+        putlog "NOTE: This directory is empty - " directory;
         msg=sysmsg();
         put _all_;
         stop;
@@ -2004,7 +2041,7 @@ data &outds (compress=no keep=file_or_folder filepath filename ext msg);
   dnum = dnum(did);
   do i = 1 to dnum;
     filename = dread(did, i);
-    rc = filename(fref2, "&path/"!!filename);
+    rc = filename(fref2, cats(directory,'/',filename));
     midd=dopen(fref2);
     dmsg=sysmsg();
     if did > 0 then file_or_folder='folder';
@@ -2013,12 +2050,12 @@ data &outds (compress=no keep=file_or_folder filepath filename ext msg);
     fmsg=sysmsg();
     if midf > 0 then file_or_folder='file';
     rc=fclose(midf);
-    
+
-    if index(fmsg,'File is in use') or index(dmsg,'is not a directory') 
+    if index(fmsg,'File is in use') or index(dmsg,'is not a directory')
       then file_or_folder='file';
     else if index(fmsg, 'Insufficient authorization') then file_or_folder='file';
     else if file_or_folder='' then file_or_folder='locked';
-      
+
     if file_or_folder='file' then do;
       ext = prxchange('s/.*\.{1,1}(.*)/$1/', 1, filename);
       if filename = ext then ext = ' ';
@@ -2051,7 +2088,7 @@ run;
       else do i=1 to foptnum(fid);
         infoname=foptname(fid,i);
         infoval=finfo(fid,infoname);
-        sasname=compress(infoname, '_', 'adik');	
+        sasname=compress(infoname, '_', 'adik');
         if anydigit(sasname)=1 then sasname=substr(sasname,anyalpha(sasname));
         if upcase(sasname) ne 'FILENAME' then output;
       end;
@@ -2068,7 +2105,7 @@ run;
       else do i=1 to doptnum(fid);
         infoname=doptname(fid,i);
         infoval=dinfo(fid,infoname);
-        sasname=compress(infoname, '_', 'adik');	
+        sasname=compress(infoname, '_', 'adik');
         if anydigit(sasname)=1 then sasname=substr(sasname,anyalpha(sasname));
         if upcase(sasname) ne 'FILENAME' then output;
       end;
@@ -2274,8 +2311,7 @@ proc sql
   order by ranuni(42)
 %end;
   ;
-
+reset outobs=max;
-
 create table datalines1 as
    select name,type,length,varnum,format,label from dictionary.columns
    where libname="%upcase(%scan(&base_ds,1))"
@@ -2505,7 +2541,6 @@ create table &outds as
    datetime2 format or regular decimal type
   @version 9.3
   @author Allan Bowe
-  @source https://github.com/sasjs/core
 **/
 
 %macro mp_getddl(libref,ds,fref=getddl,flavour=SAS,showlog=NO,schema=
@@ -2634,7 +2669,7 @@ run;
       file &fref mod;
       put ');';
     run;
-  
+
     /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
     data _null_;
       *length ds $128;
@@ -2814,7 +2849,7 @@ run;
 
   %end;
 %end;
-%if &showlog=YES %then %do;
+%if %upcase(&showlog)=YES %then %do;
   options ps=max;
   data _null_;
     infile &fref;
@@ -2899,9 +2934,9 @@ create table &outds (rename=(
   @brief Guess the primary key of a table
   @details Tries to guess the primary key of a table based on the following logic:
 
-  * Columns with nulls are ignored
+      * Columns with nulls are ignored
-  * Return only column combinations that provide unique results
+      * Return only column combinations that provide unique results
-  * Start from one column, then move out to include composite keys of 2 to 6 columns
+      * Start from one column, then move out to include composite keys of 2 to 6 columns
 
   The library of the target should be assigned before using this macro.
 
@@ -2947,7 +2982,7 @@ create table &outds (rename=(
   /* get null count and row count */
   %let tmpvar=%mf_getuniquename();
   proc sql noprint;
-  create table _data_ as select 
+  create table _data_ as select
     count(*) as &tmpvar
   %do i=1 %to &vcnt;
     %let var=%scan(&vars,&i);
@@ -2981,10 +3016,10 @@ create table &outds (rename=(
     %put &sysmacroname: &baseds has no combination of unique records! Exiting.;
     %return;
   %end;
-  
+
   /* now check cardinality */
   proc sql noprint;
-  create table _data_ as select 
+  create table _data_ as select
   %do i=1 %to &ppkcnt;
     %let var=%scan(&posspks,&i);
     count(distinct &var) as &var
@@ -3108,7 +3143,7 @@ create table &outds (rename=(
       %end;
     %end;
   %end;
- 
+
   %if &ppkcnt=4 %then %do;
     %put &sysmacroname: No more PK guess possible;
     %return;
@@ -3124,7 +3159,7 @@ create table &outds (rename=(
         %let lev3=%scan(&posspks,&k);
         %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
           %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then 
+          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
           %do m=5 %to &ppkcnt;
             %let lev5=%scan(&posspks,&m);
             %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then %do;
@@ -3146,7 +3181,7 @@ create table &outds (rename=(
       %end;
     %end;
   %end;
- 
+
   %if &ppkcnt=5 %then %do;
     %put &sysmacroname: No more PK guess possible;
     %return;
@@ -3162,17 +3197,17 @@ create table &outds (rename=(
         %let lev3=%scan(&posspks,&k);
         %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
           %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then 
+          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
           %do m=5 %to &ppkcnt;
             %let lev5=%scan(&posspks,&m);
-            %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then 
+            %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then
             %do n=6 %to &ppkcnt;
               %let lev6=%scan(&posspks,&n);
-              %if &lev1 ne &lev6 & &lev2 ne &lev6 & &lev3 ne &lev6 
+              %if &lev1 ne &lev6 & &lev2 ne &lev6 & &lev3 ne &lev6
-              & &lev4 ne &lev6 & &lev5 ne &lev6 %then 
+              & &lev4 ne &lev6 & &lev5 ne &lev6 %then
               %do;
                 /* check for four level uniqueness */
-                proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5 &lev6) 
+                proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5 &lev6)
                   out=&tmpds noduprec;
                   by _all_;
                 run;
@@ -3191,7 +3226,7 @@ create table &outds (rename=(
       %end;
     %end;
   %end;
- 
+
   %if &ppkcnt=6 %then %do;
     %put &sysmacroname: No more PK guess possible;
     %return;
@@ -3697,7 +3732,7 @@ proc sort; by descending sumcols memname libname; run;
   @brief Searches all data in a library
   @details
   Scans an entire library and creates a copy of any table
-    containing a specific string or numeric value.  Only 
+    containing a specific string OR numeric value.  Only 
     matching records are written out.
     If both a string and numval are provided, the string
     will take precedence.
@@ -3738,9 +3773,13 @@ proc sort; by descending sumcols memname libname; run;
   ,filter_text=%str(1=1)
 )/*/STORE SOURCE*/;
 
-%local table_list table table_num table colnum col start_tm vars type coltype;
+%local table_list table table_num table colnum col start_tm check_tm vars type coltype;
 %put process began at %sysfunc(datetime(),datetime19.);
 
+%if &syscc ge 4 %then %do;
+  %put %str(WAR)NING: SYSCC=&syscc on macro entry;
+  %return;
+%end;
 
 %if &string = %then %let type=N;
 %else %let type=C;
@@ -3772,6 +3811,7 @@ proc sql
     %put NO COLUMNS IN &lib..&table!  This will be skipped.;
   %end;
   %else %do;
+    %let check_tm=%sysfunc(datetime());
     /* build sql statement */
     create table mpsearch.&table as select * from &lib..&table
       where %unquote(&filter_text) and 
@@ -3782,14 +3822,19 @@ proc sql
       %let coltype=%mf_getvartype(&lib..&table,&col);
       %if &type=C and &coltype=C %then %do;
         /* if a char column, see if it contains the string */
-        or (&col ? "&string")
+        or ("&col"n ? "&string")
       %end;
       %else %if &type=N and &coltype=N %then %do;
         /* if numeric match exactly */
-        or (&col = &numval)
+        or ("&col"n = &numval)
       %end;
     %end;
     );
+    %put Search query for &table took %sysevalf(%sysfunc(datetime())-&check_tm) seconds;
+    %if &sqlrc ne 0 %then %do;
+      %put %str(WAR)NING: SQLRC=&sqlrc when processing &table;
+      %return;
+    %end;
     %if %mf_nobs(mpsearch.&table)=0 %then %do;
       drop table mpsearch.&table;
     %end;
@@ -7082,6 +7127,7 @@ data _null_;
         when ('&#x0a;') rec='0A'x;
         when ('&#x0d;') rec='0D'x;
         when ('&#36;' ) rec='$'  ;
+        when ('&#x09;') rec='09'x;
         otherwise putlog "WARNING: missing value for " entity=;
       end;
       rc =fput(fileid, substr(rec,1,1));
@@ -7996,6 +8042,7 @@ data _null_;
         when ('&#x0a;') rec='0A'x;
         when ('&#x0d;') rec='0D'x;
         when ('&#36;' ) rec='$'  ;
+        when ('&#x09;') rec='09'x;
         otherwise putlog "%str(WARN)ING: missing value for " entity=;
       end;
       rc =fput(fileid, substr(rec,1,1));
@@ -8630,6 +8677,7 @@ run;
           when ('&#x0a;') rec='0A'x;
           when ('&#x0d;') rec='0D'x;
           when ('&#36;' ) rec='$'  ;
+          when ('&#x09;') rec='09'x;
           otherwise putlog "WARNING: missing value for " entity=;
         end;
         rc =fput(fileid, substr(rec,1,1));
@@ -10594,7 +10642,7 @@ run;
 libname &libref1 JSON fileref=&fname1;
 data _null_;
   set &libref1..links;
-  if rel='members' then call symputx('mref',quote(trim(href)),'l');
+  if rel='members' then call symputx('mref',quote("&base_uri"!!trim(href)),'l');
 run;
 
 /* get the children */
@@ -10624,7 +10672,7 @@ run;
   %put NOTE:;%put NOTE- &sysmacroname: &path/&name NOT FOUND;%put NOTE- ;
   %return;
 %end;
-proc http method="DELETE" url="&uri" &oauth_bearer;
+proc http method="DELETE" url="&base_uri&uri" &oauth_bearer;
   headers 
 %if &grant_type=authorization_code %then %do;
       "Authorization"="Bearer &&&access_token_var"
@@ -10883,9 +10931,9 @@ libname &libref1 JSON fileref=&fname1;
 data _null_;
   set &libref1..links;
   if rel='deleteRecursively' then
-    call symputx('href',quote(trim(href)),'l');
+    call symputx('href',quote("&base_uri"!!trim(href)),'l');
   else if rel='members' then
-    call symputx('mref',quote(cats(href,'?recursive=true')),'l');
+    call symputx('mref',quote(cats("&base_uri",href,'?recursive=true')),'l');
 run;
 
 /* before we can delete the folder, we need to delete the children */
@@ -10906,7 +10954,7 @@ data _null_;
   set &libref1a..items_links;
   if href=:'/folders/folders' then return;
   if rel='deleteResource' then
-    call execute('proc http method="DELETE" url='!!quote(trim(href))
+    call execute('proc http method="DELETE" url='!!quote("&base_uri"!!trim(href))
     !!'; headers "Authorization"="Bearer &&&access_token_var" '
     !!' "Accept"="*/*";run; /**/');
 run;
@@ -10991,8 +11039,8 @@ libname &libref1 clear;
 )
 
 %mend;/**
-  @file mv_getgroups.sas
+  @file mv_getclients.sas
-  @brief Creates a dataset with a list of viya groups
+  @brief Get a list of Viya Clients
   @details First, be sure you have an access token (which requires an app token).
 
   Using the macros here:
@@ -11012,7 +11060,7 @@ libname &libref1 clear;
 
   Now we can run the macro!
 
-      %mv_getgroups()
+      %mv_getclients()
 
   @param access_token_var= The global macro variable to contain the access token
   @param grant_type= valid values are "password" or "authorization_code" (unquoted).
@@ -11157,7 +11205,7 @@ options noquotelenmax;
 %if "&root"="/" %then %do;
   /* if root just list root folders */
   proc http method='GET' out=&fname1 &oauth_bearer
-      url='%sysfunc(getoption(servicesbaseurl))/folders/rootFolders';
+      url="&base_uri/folders/rootFolders";
   %if &grant_type=authorization_code %then %do;
       headers "Authorization"="Bearer &&&access_token_var";
   %end;
@@ -11180,7 +11228,7 @@ options noquotelenmax;
   /* now get the followon link to list members */
   data _null_;
     set &libref1..links;
-    if rel='members' then call symputx('href',quote(trim(href)),'l');
+    if rel='members' then call symputx('href',quote("&base_uri"!!trim(href)),'l');
   run;
   %local fname2 libref2;
   %let fname2=%mf_getuniquefileref();
@@ -11443,24 +11491,15 @@ libname &libref1 clear;
 %mend;/**
   @file mv_getusergroups.sas
   @brief Creates a dataset with a list of groups for a particular user
-  @details First, be sure you have an access token (which requires an app token).
+  @details If using outside of Viya SPRE, then an access token is needed.
 
-  Using the macros here:
+  Compile the macros here:
 
       filename mc url
         "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
 
-  An administrator needs to set you up with an access code:
+  Then run the macro!
-
-      %mv_registerclient(outds=client)
-
-  Navigate to the url from the log (opting in to the groups) and paste the
-  access code below:
-
-      %mv_tokenauth(inds=client,code=wKDZYTEPK6)
-
-  Now we can run the macro!
 
       %mv_getusergroups(&sysuserid,outds=users)
 
@@ -11497,7 +11536,7 @@ libname &libref1 clear;
     %let &access_token_var=;
 %end;
 %put &sysmacroname: grant_type=&grant_type;
-%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password 
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
   ,mac=&sysmacroname
@@ -11515,7 +11554,7 @@ options noquotelenmax;
 
 proc http method='GET' out=&fname1 &oauth_bearer
   url="&base_uri/identities/users/&user/memberships?limit=10000";
-  headers 
+  headers
 %if &grant_type=authorization_code %then %do;
          "Authorization"="Bearer &&&access_token_var"
 %end;
@@ -11846,7 +11885,7 @@ run;
 %end;
 
 /* prepare url */
-%if &grant_type=authorization_code %then %do;
+%if %index(%superq(grant_type),authorization_code) %then %do;
   data _null_;
     if symexist('_baseurl') then do;
       url=symget('_baseurl');
@@ -11865,7 +11904,7 @@ run;
 %put CLIENT_SECRET=&client_secret;
 %put GRANT_TYPE=&grant_type;
 %put;
-%if &grant_type=authorization_code %then %do;
+%if %index(%superq(grant_type),authorization_code) %then %do;
   /* cannot use base_uri here as it includes the protocol which may be incorrect externally */
   %put NOTE: The developer must also register below and select 'openid' to get the grant code:;
   %put NOTE- ;
@@ -11932,6 +11971,7 @@ libname &libref clear;
   @param pass= If grant_type=password then provide the password here
   @param access_token_var= The global macro variable to contain the access token
   @param refresh_token_var= The global macro variable to contain the refresh token
+  @param base_uri= The Viya API server location
 
   @version VIYA V.03.04
   @author Allan Bowe
@@ -11956,6 +11996,7 @@ libname &libref clear;
     ,pass=
     ,access_token_var=ACCESS_TOKEN
     ,refresh_token_var=REFRESH_TOKEN
+    ,base_uri=#NOTSET#
   );
 %global &access_token_var &refresh_token_var;
 
@@ -12002,8 +12043,7 @@ run;
 /**
  * Request access token
  */
-%local base_uri; /* location of rest apis */
+%if &base_uri=#NOTSET# %then %let base_uri=%mf_getplatform(VIYARESTAPI);
-%let base_uri=%mf_getplatform(VIYARESTAPI);
 
 %let fref2=%mf_getuniquefileref();
 proc http method='POST' in=&grantstring out=&fref2

base/mf_trimstr.sas

@@ -23,13 +23,23 @@
 **/
 
 %macro mf_trimstr(basestr,trimstr);
-%local trimlen trimval;
+%local baselen trimlen trimval;
+
+/* return if basestr is shorter than trimstr (or 0) */
+%let baselen=%length(%superq(basestr));
 %let trimlen=%length(%superq(trimstr));
+%if &baselen < &trimlen or &baselen=0 %then %return;
+
+/* obtain the characters from the end of basestr */
 %let trimval=%qsubstr(%superq(basestr)
   ,%length(%superq(basestr))-&trimlen+1
   ,&trimlen);
 
-%if %superq(trimval)=%superq(trimstr) %then %do;
+/* compare and if matching, chop it off! */
+%if %superq(basestr)=%superq(trimstr) %then %do;
+  %return;
+%end;
+%else %if %superq(trimval)=%superq(trimstr) %then %do;
   %qsubstr(%superq(basestr),1,%length(%superq(basestr))-&trimlen)
 %end;
 %else %do;

base/mp_abort.sas

@@ -5,6 +5,12 @@
     the particulars of an environment.  For instance, can stream custom
     results back to the client in an STP Web App context, or completely stop
     in the case of a batch run.
+  
+  Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored Process
+  environments.  This macro takes a unique approach - we set the SAS syscc to 0,
+  run `stpsrvset('program error', 0)` (if SAS 9) and then - we open a macro 
+  but don't close it!  This provides a graceful abort for SAS web services in all 
+  web enabled environments.
 
   @param mac= to contain the name of the calling macro
   @param msg= message to be returned
@@ -65,8 +71,15 @@
     %end;
 
     %if %symexist(SYS_JES_JOB_URI) %then %do;
-      /* refer web service output to file service in one hit */
+      /* setup webout */
-      filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json";
+      OPTIONS NOBOMFILE;
+      %if "X&SYS_JES_JOB_URI.X"="XX" %then %do;
+          filename _webout temp lrecl=999999 mod;
+      %end;
+      %else %do;
+        filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
+          name="_webout.json" lrecl=999999 mod;
+      %end;
     %end;
 
     /* send response in SASjs JSON format */

base/mp_dirlist.sas

@@ -1,17 +1,17 @@
 /**
   @file
   @brief Returns all files and subdirectories within a specified parent
-  @details When used with getattrs=NO, is not OS specific (uses dopen / dread). 
+  @details When used with getattrs=NO, is not OS specific (uses dopen / dread).
 
   If getattrs=YES then the doptname / foptname functions are used to scan all
-  properties - any characters that are not valid in a SAS name (v7) are simply 
+  properties - any characters that are not valid in a SAS name (v7) are simply
   stripped, and the table is transposed so theat each property is a column
-  and there is one file per row.  An attempt is made to get all properties 
+  and there is one file per row.  An attempt is made to get all properties
   whether a file or folder, but some files/folders cannot be accessed, and so
   not all properties can / will be populated.
 
   Credit for the rename approach:
-  https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003 
+  https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003
 
 
   usage:
@@ -20,21 +20,25 @@
 
       %mp_dirlist(outds=cwdfileprops, getattrs=YES)
 
-  @warning In a Unix environment, the existence of a named pipe will cause this 
+      %mp_dirlist(fref=MYFREF)
+
+  @warning In a Unix environment, the existence of a named pipe will cause this
   macro to hang.  Therefore this tool should be used with caution in a SAS 9 web
   application, as it can use up all available multibridge sessions if requests
   are resubmitted.
-  If anyone finds a way to positively identify a named pipe using SAS (without 
+  If anyone finds a way to positively identify a named pipe using SAS (without
   X CMD) do please raise an issue!
 
 
   @param path= for which to return contents
+  @param fref= Provide a DISK engine fileref as an alternative to PATH
   @param outds= the output dataset to create
-  @param getattrs= YES/NO (default=NO).  Uses doptname and foptname to return 
+  @param getattrs= YES/NO (default=NO).  Uses doptname and foptname to return
-  all attributes for each file / folder.  
+  all attributes for each file / folder.
 
 
   @returns outds contains the following variables:
+   - directory (containing folder)
    - file_or_folder (file / folder)
    - filepath (path/to/file.name)
    - filename (just the file name)
@@ -47,18 +51,26 @@
 **/
 
 %macro mp_dirlist(path=%sysfunc(pathname(work))
+    , fref=0
     , outds=work.mp_dirlist
     , getattrs=NO
 )/*/STORE SOURCE*/;
 %let getattrs=%upcase(&getattrs)XX;
 
-data &outds (compress=no keep=file_or_folder filepath filename ext msg);
+data &outds (compress=no keep=file_or_folder filepath filename ext msg directory);
-  length filepath $500 fref fref2 $8 file_or_folder $6 filename $80 ext $20 msg $200;
+  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80 ext $20 msg $200;
-  rc = filename(fref, "&path");
+  %if &fref=0 %then %do;
+    rc = filename(fref, "&path");
+  %end;
+  %else %do;
+    fref="&fref";
+    rc=0;
+  %end;
   if rc = 0 then do;
      did = dopen(fref);
+     directory=dinfo(did,'Directory');
      if did=0 then do;
-        putlog "NOTE: This directory is empty - &path";
+        putlog "NOTE: This directory is empty - " directory;
         msg=sysmsg();
         put _all_;
         stop;
@@ -73,7 +85,7 @@ data &outds (compress=no keep=file_or_folder filepath filename ext msg);
   dnum = dnum(did);
   do i = 1 to dnum;
     filename = dread(did, i);
-    rc = filename(fref2, "&path/"!!filename);
+    rc = filename(fref2, cats(directory,'/',filename));
     midd=dopen(fref2);
     dmsg=sysmsg();
     if did > 0 then file_or_folder='folder';
@@ -82,12 +94,12 @@ data &outds (compress=no keep=file_or_folder filepath filename ext msg);
     fmsg=sysmsg();
     if midf > 0 then file_or_folder='file';
     rc=fclose(midf);
-    
+
-    if index(fmsg,'File is in use') or index(dmsg,'is not a directory') 
+    if index(fmsg,'File is in use') or index(dmsg,'is not a directory')
       then file_or_folder='file';
     else if index(fmsg, 'Insufficient authorization') then file_or_folder='file';
     else if file_or_folder='' then file_or_folder='locked';
-      
+
     if file_or_folder='file' then do;
       ext = prxchange('s/.*\.{1,1}(.*)/$1/', 1, filename);
       if filename = ext then ext = ' ';
@@ -120,7 +132,7 @@ run;
       else do i=1 to foptnum(fid);
         infoname=foptname(fid,i);
         infoval=finfo(fid,infoname);
-        sasname=compress(infoname, '_', 'adik');	
+        sasname=compress(infoname, '_', 'adik');
         if anydigit(sasname)=1 then sasname=substr(sasname,anyalpha(sasname));
         if upcase(sasname) ne 'FILENAME' then output;
       end;
@@ -137,7 +149,7 @@ run;
       else do i=1 to doptnum(fid);
         infoname=doptname(fid,i);
         infoval=dinfo(fid,infoname);
-        sasname=compress(infoname, '_', 'adik');	
+        sasname=compress(infoname, '_', 'adik');
         if anydigit(sasname)=1 then sasname=substr(sasname,anyalpha(sasname));
         if upcase(sasname) ne 'FILENAME' then output;
       end;

base/mp_ds2cards.sas

@@ -103,8 +103,7 @@ proc sql
   order by ranuni(42)
 %end;
   ;
-
+reset outobs=max;
-
 create table datalines1 as
    select name,type,length,varnum,format,label from dictionary.columns
    where libname="%upcase(%scan(&base_ds,1))"

base/mp_getddl.sas

@@ -159,7 +159,7 @@ run;
       file &fref mod;
       put ');';
     run;
-  
+
     /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
     data _null_;
       *length ds $128;
@@ -339,7 +339,7 @@ run;
 
   %end;
 %end;
-%if &showlog=YES %then %do;
+%if %upcase(&showlog)=YES %then %do;
   options ps=max;
   data _null_;
     infile &fref;

base/mp_guesspk.sas

@@ -3,9 +3,9 @@
   @brief Guess the primary key of a table
   @details Tries to guess the primary key of a table based on the following logic:
 
-  * Columns with nulls are ignored
+      * Columns with nulls are ignored
-  * Return only column combinations that provide unique results
+      * Return only column combinations that provide unique results
-  * Start from one column, then move out to include composite keys of 2 to 6 columns
+      * Start from one column, then move out to include composite keys of 2 to 6 columns
 
   The library of the target should be assigned before using this macro.
 
@@ -51,7 +51,7 @@
   /* get null count and row count */
   %let tmpvar=%mf_getuniquename();
   proc sql noprint;
-  create table _data_ as select 
+  create table _data_ as select
     count(*) as &tmpvar
   %do i=1 %to &vcnt;
     %let var=%scan(&vars,&i);
@@ -85,10 +85,10 @@
     %put &sysmacroname: &baseds has no combination of unique records! Exiting.;
     %return;
   %end;
-  
+
   /* now check cardinality */
   proc sql noprint;
-  create table _data_ as select 
+  create table _data_ as select
   %do i=1 %to &ppkcnt;
     %let var=%scan(&posspks,&i);
     count(distinct &var) as &var
@@ -212,7 +212,7 @@
       %end;
     %end;
   %end;
- 
+
   %if &ppkcnt=4 %then %do;
     %put &sysmacroname: No more PK guess possible;
     %return;
@@ -228,7 +228,7 @@
         %let lev3=%scan(&posspks,&k);
         %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
           %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then 
+          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
           %do m=5 %to &ppkcnt;
             %let lev5=%scan(&posspks,&m);
             %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then %do;
@@ -250,7 +250,7 @@
       %end;
     %end;
   %end;
- 
+
   %if &ppkcnt=5 %then %do;
     %put &sysmacroname: No more PK guess possible;
     %return;
@@ -266,17 +266,17 @@
         %let lev3=%scan(&posspks,&k);
         %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
           %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then 
+          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
           %do m=5 %to &ppkcnt;
             %let lev5=%scan(&posspks,&m);
-            %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then 
+            %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then
             %do n=6 %to &ppkcnt;
               %let lev6=%scan(&posspks,&n);
-              %if &lev1 ne &lev6 & &lev2 ne &lev6 & &lev3 ne &lev6 
+              %if &lev1 ne &lev6 & &lev2 ne &lev6 & &lev3 ne &lev6
-              & &lev4 ne &lev6 & &lev5 ne &lev6 %then 
+              & &lev4 ne &lev6 & &lev5 ne &lev6 %then
               %do;
                 /* check for four level uniqueness */
-                proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5 &lev6) 
+                proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5 &lev6)
                   out=&tmpds noduprec;
                   by _all_;
                 run;
@@ -295,7 +295,7 @@
       %end;
     %end;
   %end;
- 
+
   %if &ppkcnt=6 %then %do;
     %put &sysmacroname: No more PK guess possible;
     %return;

base/mp_searchdata.sas

@@ -3,7 +3,7 @@
   @brief Searches all data in a library
   @details
   Scans an entire library and creates a copy of any table
-    containing a specific string or numeric value.  Only 
+    containing a specific string OR numeric value.  Only 
     matching records are written out.
     If both a string and numval are provided, the string
     will take precedence.
@@ -44,9 +44,13 @@
   ,filter_text=%str(1=1)
 )/*/STORE SOURCE*/;
 
-%local table_list table table_num table colnum col start_tm vars type coltype;
+%local table_list table table_num table colnum col start_tm check_tm vars type coltype;
 %put process began at %sysfunc(datetime(),datetime19.);
 
+%if &syscc ge 4 %then %do;
+  %put %str(WAR)NING: SYSCC=&syscc on macro entry;
+  %return;
+%end;
 
 %if &string = %then %let type=N;
 %else %let type=C;
@@ -78,6 +82,7 @@ proc sql
     %put NO COLUMNS IN &lib..&table!  This will be skipped.;
   %end;
   %else %do;
+    %let check_tm=%sysfunc(datetime());
     /* build sql statement */
     create table mpsearch.&table as select * from &lib..&table
       where %unquote(&filter_text) and 
@@ -88,14 +93,19 @@ proc sql
       %let coltype=%mf_getvartype(&lib..&table,&col);
       %if &type=C and &coltype=C %then %do;
         /* if a char column, see if it contains the string */
-        or (&col ? "&string")
+        or ("&col"n ? "&string")
       %end;
       %else %if &type=N and &coltype=N %then %do;
         /* if numeric match exactly */
-        or (&col = &numval)
+        or ("&col"n = &numval)
       %end;
     %end;
     );
+    %put Search query for &table took %sysevalf(%sysfunc(datetime())-&check_tm) seconds;
+    %if &sqlrc ne 0 %then %do;
+      %put %str(WAR)NING: SQLRC=&sqlrc when processing &table;
+      %return;
+    %end;
     %if %mf_nobs(mpsearch.&table)=0 %then %do;
       drop table mpsearch.&table;
     %end;

doxy/DoxygenLayout.xml

@@ -16,7 +16,7 @@
       <tab type="classmembers" visible="no" title="" intro=""/>
     </tab>
 
-    <tab type="filelist" visible="yes" title="" intro="List of Files Used in the Macro Core Library"/>
+    <tab type="filelist" visible="yes" title="" intro="List of Files Used in the Macro-Core Library"/>
 
     <tab type="examples" visible="yes" title="" intro=""/>
   </navindex>

doxy/build.sh

@@ -36,5 +36,9 @@ echo 'core.sasjs.io' > CNAME
 git add *
 git commit -m "build.sh build on $(date +%F:%H:%M:%S)"
 git push
+npx sitemap-generator-cli https://core.sasjs.io
+git add *
+git commit -m "adding sitemap"
+git push
 
 echo "check it out:  https://sasjs.github.io/core.github.io/files.html"

meta/mm_getdocument.sas

@@ -124,6 +124,7 @@ data _null_;
         when ('
') rec='0A'x;
         when ('
') rec='0D'x;
         when ('$' ) rec='$'  ;
+        when ('	') rec='09'x;
         otherwise putlog "WARNING: missing value for " entity=;
       end;
       rc =fput(fileid, substr(rec,1,1));

meta/mm_getstpcode.sas

@@ -123,6 +123,7 @@ data _null_;
         when ('
') rec='0A'x;
         when ('
') rec='0D'x;
         when ('$' ) rec='$'  ;
+        when ('	') rec='09'x;
         otherwise putlog "%str(WARN)ING: missing value for " entity=;
       end;
       rc =fput(fileid, substr(rec,1,1));

meta/mm_getwebappsrvprops.sas

@@ -99,6 +99,7 @@ run;
           when ('
') rec='0A'x;
           when ('
') rec='0D'x;
           when ('$' ) rec='$'  ;
+          when ('	') rec='09'x;
           otherwise putlog "WARNING: missing value for " entity=;
         end;
         rc =fput(fileid, substr(rec,1,1));

viya/mv_deletefoldermember.sas

@@ -98,7 +98,7 @@ run;
 libname &libref1 JSON fileref=&fname1;
 data _null_;
   set &libref1..links;
-  if rel='members' then call symputx('mref',quote(trim(href)),'l');
+  if rel='members' then call symputx('mref',quote("&base_uri"!!trim(href)),'l');
 run;
 
 /* get the children */
@@ -128,7 +128,7 @@ run;
   %put NOTE:;%put NOTE- &sysmacroname: &path/&name NOT FOUND;%put NOTE- ;
   %return;
 %end;
-proc http method="DELETE" url="&uri" &oauth_bearer;
+proc http method="DELETE" url="&base_uri&uri" &oauth_bearer;
   headers 
 %if &grant_type=authorization_code %then %do;
       "Authorization"="Bearer &&&access_token_var"

viya/mv_deleteviyafolder.sas

@@ -88,9 +88,9 @@ libname &libref1 JSON fileref=&fname1;
 data _null_;
   set &libref1..links;
   if rel='deleteRecursively' then
-    call symputx('href',quote(trim(href)),'l');
+    call symputx('href',quote("&base_uri"!!trim(href)),'l');
   else if rel='members' then
-    call symputx('mref',quote(cats(href,'?recursive=true')),'l');
+    call symputx('mref',quote(cats("&base_uri",href,'?recursive=true')),'l');
 run;
 
 /* before we can delete the folder, we need to delete the children */
@@ -111,7 +111,7 @@ data _null_;
   set &libref1a..items_links;
   if href=:'/folders/folders' then return;
   if rel='deleteResource' then
-    call execute('proc http method="DELETE" url='!!quote(trim(href))
+    call execute('proc http method="DELETE" url='!!quote("&base_uri"!!trim(href))
     !!'; headers "Authorization"="Bearer &&&access_token_var" '
     !!' "Accept"="*/*";run; /**/');
 run;

viya/mv_getclients.sas

@@ -1,6 +1,6 @@
 /**
-  @file mv_getgroups.sas
+  @file mv_getclients.sas
-  @brief Creates a dataset with a list of viya groups
+  @brief Get a list of Viya Clients
   @details First, be sure you have an access token (which requires an app token).
 
   Using the macros here:
@@ -20,7 +20,7 @@
 
   Now we can run the macro!
 
-      %mv_getgroups()
+      %mv_getclients()
 
   @param access_token_var= The global macro variable to contain the access token
   @param grant_type= valid values are "password" or "authorization_code" (unquoted).

viya/mv_getfoldermembers.sas

@@ -64,7 +64,7 @@ options noquotelenmax;
 %if "&root"="/" %then %do;
   /* if root just list root folders */
   proc http method='GET' out=&fname1 &oauth_bearer
-      url='%sysfunc(getoption(servicesbaseurl))/folders/rootFolders';
+      url="&base_uri/folders/rootFolders";
   %if &grant_type=authorization_code %then %do;
       headers "Authorization"="Bearer &&&access_token_var";
   %end;
@@ -87,7 +87,7 @@ options noquotelenmax;
   /* now get the followon link to list members */
   data _null_;
     set &libref1..links;
-    if rel='members' then call symputx('href',quote(trim(href)),'l');
+    if rel='members' then call symputx('href',quote("&base_uri"!!trim(href)),'l');
   run;
   %local fname2 libref2;
   %let fname2=%mf_getuniquefileref();

viya/mv_getusergroups.sas

@@ -1,24 +1,15 @@
 /**
   @file mv_getusergroups.sas
   @brief Creates a dataset with a list of groups for a particular user
-  @details First, be sure you have an access token (which requires an app token).
+  @details If using outside of Viya SPRE, then an access token is needed.
 
-  Using the macros here:
+  Compile the macros here:
 
       filename mc url
         "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
 
-  An administrator needs to set you up with an access code:
+  Then run the macro!
-
-      %mv_registerclient(outds=client)
-
-  Navigate to the url from the log (opting in to the groups) and paste the
-  access code below:
-
-      %mv_tokenauth(inds=client,code=wKDZYTEPK6)
-
-  Now we can run the macro!
 
       %mv_getusergroups(&sysuserid,outds=users)
 
@@ -55,7 +46,7 @@
     %let &access_token_var=;
 %end;
 %put &sysmacroname: grant_type=&grant_type;
-%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password 
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
   ,mac=&sysmacroname
@@ -73,7 +64,7 @@ options noquotelenmax;
 
 proc http method='GET' out=&fname1 &oauth_bearer
   url="&base_uri/identities/users/&user/memberships?limit=10000";
-  headers 
+  headers
 %if &grant_type=authorization_code %then %do;
          "Authorization"="Bearer &&&access_token_var"
 %end;

viya/mv_registerclient.sas

@@ -187,7 +187,7 @@ run;
 %end;
 
 /* prepare url */
-%if &grant_type=authorization_code %then %do;
+%if %index(%superq(grant_type),authorization_code) %then %do;
   data _null_;
     if symexist('_baseurl') then do;
       url=symget('_baseurl');
@@ -206,7 +206,7 @@ run;
 %put CLIENT_SECRET=&client_secret;
 %put GRANT_TYPE=&grant_type;
 %put;
-%if &grant_type=authorization_code %then %do;
+%if %index(%superq(grant_type),authorization_code) %then %do;
   /* cannot use base_uri here as it includes the protocol which may be incorrect externally */
   %put NOTE: The developer must also register below and select 'openid' to get the grant code:;
   %put NOTE- ;

viya/mv_tokenauth.sas

@@ -38,6 +38,7 @@
   @param pass= If grant_type=password then provide the password here
   @param access_token_var= The global macro variable to contain the access token
   @param refresh_token_var= The global macro variable to contain the refresh token
+  @param base_uri= The Viya API server location
 
   @version VIYA V.03.04
   @author Allan Bowe
@@ -62,6 +63,7 @@
     ,pass=
     ,access_token_var=ACCESS_TOKEN
     ,refresh_token_var=REFRESH_TOKEN
+    ,base_uri=#NOTSET#
   );
 %global &access_token_var &refresh_token_var;
 
@@ -108,8 +110,7 @@ run;
 /**
  * Request access token
  */
-%local base_uri; /* location of rest apis */
+%if &base_uri=#NOTSET# %then %let base_uri=%mf_getplatform(VIYARESTAPI);
-%let base_uri=%mf_getplatform(VIYARESTAPI);
 
 %let fref2=%mf_getuniquefileref();
 proc http method='POST' in=&grantstring out=&fref2