fix: removing unnecessary cond flags in mf_abort, adding abort logic in mp_csv2ds

fix: base_uri

.editorconfig

@@ -0,0 +1,12 @@
+# http://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+end_of_line = lf
+insert_final_newline = false
+trim_trailing_whitespace = true
+
+

.gitignore

@@ -1 +1,2 @@
 node_modules
+.DS_Store

base/mf_getengine.sas

@@ -3,7 +3,7 @@
   @brief Returns the engine type of a SAS library
   @details Usage:
 
-      %put %mf_getEngine(SASHELP);
+      %put %mf_getengine(SASHELP);
 
   returns:
   > V9
@@ -21,9 +21,10 @@
 
   @version 9.2
   @author Allan Bowe
+
 **/
 
-%macro mf_getEngine(libref
+%macro mf_getengine(libref
 )/*/STORE SOURCE*/;
   %local dsid engnum rc engine;
 
@@ -41,4 +42,4 @@
 
  &engine
 
-%mend;
+%mend;

base/mf_getplatform.sas

@@ -12,6 +12,7 @@
 
   <h4> Dependencies </h4>
   @li mf_mval.sas
+  @li mf_trimstr.sas
 
   @version 9.4 / 3.4
   @author Allan Bowe
@@ -57,6 +58,6 @@
   %else 0;
 %end;
 %else %if &switch=VIYARESTAPI %then %do;
-  %sysfunc(getoption(servicesbaseurl))
+  %mf_trimstr(%sysfunc(getoption(servicesbaseurl)),/)
 %end;
 %mend;

base/mf_trimstr.sas

@@ -0,0 +1,49 @@
+/**
+  @file mf_trimstr.sas
+  @brief Removes character(s) from the end, if they exist
+  @details If the designated characters exist at the end of the string, they 
+  are removed
+
+        %put %mf_trimstr(/blah/,/); * /blah;
+        %put %mf_trimstr(/blah/,h); * /blah/;
+        %put %mf_trimstr(/blah/,h/);* /bla;
+
+  <h4> Dependencies </h4>
+
+
+  @param basestr The string to be modified
+  @param trimstr The string to be removed from the end of `basestr`, if it exists
+
+  @return output returns result with the value of `trimstr` removed from the end
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_trimstr(basestr,trimstr);
+%local baselen trimlen trimval;
+
+/* return if basestr is shorter than trimstr (or 0) */
+%let baselen=%length(%superq(basestr));
+%let trimlen=%length(%superq(trimstr));
+%if &baselen < &trimlen or &baselen=0 %then %return;
+
+/* obtain the characters from the end of basestr */
+%let trimval=%qsubstr(%superq(basestr)
+  ,%length(%superq(basestr))-&trimlen+1
+  ,&trimlen);
+
+/* compare and if matching, chop it off! */
+%if %superq(basestr)=%superq(trimstr) %then %do;
+  %return;
+%end;
+%else %if %superq(trimval)=%superq(trimstr) %then %do;
+  %qsubstr(%superq(basestr),1,%length(%superq(basestr))-&trimlen)
+%end;
+%else %do;
+  &basestr
+%end;
+
+%mend;

base/mp_abort.sas

@@ -5,6 +5,12 @@
     the particulars of an environment.  For instance, can stream custom
     results back to the client in an STP Web App context, or completely stop
     in the case of a batch run.
+  
+  Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored Process
+  environments.  This macro takes a unique approach - we set the SAS syscc to 0,
+  run `stpsrvset('program error', 0)` (if SAS 9) and then - we open a macro 
+  but don't close it!  This provides a graceful abort for SAS web services in all 
+  web enabled environments.
 
   @param mac= to contain the name of the calling macro
   @param msg= message to be returned
@@ -65,8 +71,15 @@
     %end;
 
     %if %symexist(SYS_JES_JOB_URI) %then %do;
-      /* refer web service output to file service in one hit */
-      filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json";
+      /* setup webout */
+      OPTIONS NOBOMFILE;
+      %if "X&SYS_JES_JOB_URI.X"="XX" %then %do;
+          filename _webout temp lrecl=999999 mod;
+      %end;
+      %else %do;
+        filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
+          name="_webout.json" lrecl=999999 mod;
+      %end;
     %end;
 
     /* send response in SASjs JSON format */

base/mp_csv2ds.sas

@@ -0,0 +1,144 @@
+/**
+  @file mp_csv2ds.sas
+  @brief Efficient import of arbitrary CSV using a dataset as template
+  @details Used to import relevant columns from a large CSV using
+  a dataset to provide the types and lengths.  Assumes that a header
+  row is provided, and datarows start on line 2.  Extra columns in
+  both the CSV and base dataset are ignored.
+
+  Usage:
+
+      filename mycsv temp;
+      data _null_;
+        file mycsv;
+        put 'name,age,nickname';
+        put 'John,48,Jonny';
+        put 'Jennifer,23,Jen';
+      run;
+
+      %mp_csv2ds(inref=mycsv,outds=myds,baseds=sashelp.class)
+
+
+  @param inref= fileref to the CSV
+  @param outds= output ds (lib.ds format)
+  @param view= Set to YES or NO to determine whether the output should be
+    a view or not.  Default is NO (not a view).
+  @param baseds= Template dataset on which to create the input statement.
+    Is used to determine types, lengths, and any informats.
+
+  @version 9.2
+  @author Allan Bowe
+
+  <h4> Dependencies </h4>
+  @li mp_abort.sas
+  @li mf_existds.sas
+
+**/
+
+%macro mp_csv2ds(inref=0,outds=0,baseds=0,view=NO);
+
+%mp_abort(iftrue=( &inref=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the INREF variable must be provided)
+)
+%mp_abort(iftrue=( %superq(outds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the OUTDS variable must be provided)
+)
+%mp_abort(iftrue=( &baseds=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS variable must be provided)
+)
+%mp_abort(iftrue=( &baseds=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS variable must be provided)
+)
+%mp_abort(iftrue=( %mf_existds(&baseds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS dataset (&baseds) needs to be assigned, and to exist)
+)
+
+/* count rows */
+%local hasheader; %let hasheader=0;
+data _null_;
+  if _N_ > 1 then do;
+     call symputx('hasheader',1,'l');
+     stop;
+  end;
+  infile &inref;
+  input;
+run;
+%mp_abort(iftrue=( &hasheader=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(No header row in &inref)
+)
+
+/* get the variables in the CSV */
+data _data_;
+  infile &inref;
+  input;
+  length name $32;
+  do i=1 to countc(_infile_,',')+1;
+    name=upcase(scan(_infile_,i,','));
+    output;
+  end;
+  stop;
+run;
+%local csv_vars;%let csv_vars=&syslast;
+
+/* get the variables in the dataset */
+proc contents noprint data=&baseds
+  out=_data_ (keep=name type length format: informat);
+run;
+%local base_vars; %let base_vars=&syslast;
+
+proc sql undo_policy=none;
+create table &csv_vars as
+  select a.*
+    ,b.type
+    ,b.length
+    ,b.format
+    ,b.formatd
+    ,b.formatl
+    ,b.informat
+  from &csv_vars a
+  left join &base_vars b
+  on a.name=upcase(b.name)
+  order by i;
+
+/* prepare the input statement */
+%local instat dropvars;
+data _null_;
+  set &syslast end=last;
+  length in dropvars $32767;
+  retain in dropvars;
+  if missing(type) then do;
+    informat='$1.';
+    dropvars=catx(' ',dropvars,name);
+  end;
+  else if missing(informat) then do;
+    if type=1 then informat='best.';
+    else informat=cats('$',length,'.');
+  end;
+  else informat=cats(informat,'.');
+  in=catx(' ',in,name,':',informat);
+  if last then do;
+    call symputx('instat',in,'l');
+    call symputx('dropvars',dropvars,'l');
+  end;
+run;
+
+/* import the CSV */
+data &outds
+  %if %upcase(&view)=YES %then %do;
+   /view=&outds
+  %end;
+  ;
+  infile &inref dsd firstobs=2;
+  input &instat;
+  %if %length(&dropvars)>0 %then %do;
+    drop &dropvars;
+  %end;
+run;
+
+%mend;

base/mp_dirlist.sas

@@ -1,17 +1,17 @@
 /**
   @file
   @brief Returns all files and subdirectories within a specified parent
-  @details When used with getattrs=NO, is not OS specific (uses dopen / dread). 
+  @details When used with getattrs=NO, is not OS specific (uses dopen / dread).
 
   If getattrs=YES then the doptname / foptname functions are used to scan all
-  properties - any characters that are not valid in a SAS name (v7) are simply 
+  properties - any characters that are not valid in a SAS name (v7) are simply
   stripped, and the table is transposed so theat each property is a column
-  and there is one file per row.  An attempt is made to get all properties 
+  and there is one file per row.  An attempt is made to get all properties
   whether a file or folder, but some files/folders cannot be accessed, and so
   not all properties can / will be populated.
 
   Credit for the rename approach:
-  https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003 
+  https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003
 
 
   usage:
@@ -20,21 +20,25 @@
 
       %mp_dirlist(outds=cwdfileprops, getattrs=YES)
 
-  @warning In a Unix environment, the existence of a named pipe will cause this 
+      %mp_dirlist(fref=MYFREF)
+
+  @warning In a Unix environment, the existence of a named pipe will cause this
   macro to hang.  Therefore this tool should be used with caution in a SAS 9 web
   application, as it can use up all available multibridge sessions if requests
   are resubmitted.
-  If anyone finds a way to positively identify a named pipe using SAS (without 
+  If anyone finds a way to positively identify a named pipe using SAS (without
   X CMD) do please raise an issue!
 
 
   @param path= for which to return contents
+  @param fref= Provide a DISK engine fileref as an alternative to PATH
   @param outds= the output dataset to create
-  @param getattrs= YES/NO (default=NO).  Uses doptname and foptname to return 
-  all attributes for each file / folder.  
+  @param getattrs= YES/NO (default=NO).  Uses doptname and foptname to return
+  all attributes for each file / folder.
 
 
   @returns outds contains the following variables:
+   - directory (containing folder)
    - file_or_folder (file / folder)
    - filepath (path/to/file.name)
    - filename (just the file name)
@@ -47,18 +51,26 @@
 **/
 
 %macro mp_dirlist(path=%sysfunc(pathname(work))
+    , fref=0
     , outds=work.mp_dirlist
     , getattrs=NO
 )/*/STORE SOURCE*/;
 %let getattrs=%upcase(&getattrs)XX;
 
-data &outds (compress=no keep=file_or_folder filepath filename ext msg);
-  length filepath $500 fref fref2 $8 file_or_folder $6 filename $80 ext $20 msg $200;
-  rc = filename(fref, "&path");
+data &outds (compress=no keep=file_or_folder filepath filename ext msg directory);
+  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80 ext $20 msg $200;
+  %if &fref=0 %then %do;
+    rc = filename(fref, "&path");
+  %end;
+  %else %do;
+    fref="&fref";
+    rc=0;
+  %end;
   if rc = 0 then do;
      did = dopen(fref);
+     directory=dinfo(did,'Directory');
      if did=0 then do;
-        putlog "NOTE: This directory is empty - &path";
+        putlog "NOTE: This directory is empty - " directory;
         msg=sysmsg();
         put _all_;
         stop;
@@ -73,7 +85,8 @@ data &outds (compress=no keep=file_or_folder filepath filename ext msg);
   dnum = dnum(did);
   do i = 1 to dnum;
     filename = dread(did, i);
-    rc = filename(fref2, "&path/"!!filename);
+    filepath=cats(directory,'/',filename);
+    rc = filename(fref2,filepath);
     midd=dopen(fref2);
     dmsg=sysmsg();
     if did > 0 then file_or_folder='folder';
@@ -82,12 +95,12 @@ data &outds (compress=no keep=file_or_folder filepath filename ext msg);
     fmsg=sysmsg();
     if midf > 0 then file_or_folder='file';
     rc=fclose(midf);
-    
-    if index(fmsg,'File is in use') or index(dmsg,'is not a directory') 
+
+    if index(fmsg,'File is in use') or index(dmsg,'is not a directory')
       then file_or_folder='file';
     else if index(fmsg, 'Insufficient authorization') then file_or_folder='file';
     else if file_or_folder='' then file_or_folder='locked';
-      
+
     if file_or_folder='file' then do;
       ext = prxchange('s/.*\.{1,1}(.*)/$1/', 1, filename);
       if filename = ext then ext = ' ';
@@ -96,7 +109,6 @@ data &outds (compress=no keep=file_or_folder filepath filename ext msg);
       ext='';
       file_or_folder='folder';
     end;
-    filepath="&path/"!!filename;
     output;
   end;
   rc = dclose(did);
@@ -120,7 +132,7 @@ run;
       else do i=1 to foptnum(fid);
         infoname=foptname(fid,i);
         infoval=finfo(fid,infoname);
-        sasname=compress(infoname, '_', 'adik');	
+        sasname=compress(infoname, '_', 'adik');
         if anydigit(sasname)=1 then sasname=substr(sasname,anyalpha(sasname));
         if upcase(sasname) ne 'FILENAME' then output;
       end;
@@ -137,7 +149,7 @@ run;
       else do i=1 to doptnum(fid);
         infoname=doptname(fid,i);
         infoval=dinfo(fid,infoname);
-        sasname=compress(infoname, '_', 'adik');	
+        sasname=compress(infoname, '_', 'adik');
         if anydigit(sasname)=1 then sasname=substr(sasname,anyalpha(sasname));
         if upcase(sasname) ne 'FILENAME' then output;
       end;

base/mp_ds2cards.sas

@@ -103,8 +103,7 @@ proc sql
   order by ranuni(42)
 %end;
   ;
-
-
+reset outobs=max;
 create table datalines1 as
    select name,type,length,varnum,format,label from dictionary.columns
    where libname="%upcase(%scan(&base_ds,1))"

base/mp_getddl.sas

@@ -13,11 +13,13 @@
         label x='blah';
       run;
       proc sql; describe table &syslast;
-
       %mp_getddl(work,test,flavour=tsql,showlog=YES)
 
+  <h4> Dependencies </h4>
+  @li mp_getconstraints.sas
+
   @param lib libref of the library to create DDL for.  Should be assigned.
-  @param ds dataset to create ddl for
+  @param ds dataset to create ddl for (optional)
   @param fref= the fileref to which to write the DDL.  If not preassigned, will
     be assigned to TEMP.
   @param flavour= The type of DDL to create (default=SAS). Supported=TSQL
@@ -26,11 +28,8 @@
     ,else libref)
   @param applydttm= for non SAS DDL, choose if columns are created with native
    datetime2 format or regular decimal type
-
   @version 9.3
   @author Allan Bowe
-  @source https://github.com/sasjs/core
-
 **/
 
 %macro mp_getddl(libref,ds,fref=getddl,flavour=SAS,showlog=NO,schema=
@@ -54,6 +53,21 @@ create table _data_ as
   ;
 %local tabinfo; %let tabinfo=&syslast;
 
+create table _data_ as
+  select * from dictionary.columns
+  where upcase(libname)="%upcase(&libref)"
+  %if %length(&ds)>0 %then %do;
+    and upcase(memname)="%upcase(&ds)"
+  %end;
+  ;
+%local colinfo; %let colinfo=&syslast;
+
+%local dsnlist;
+  select distinct upcase(memname) into: dsnlist
+  separated by ' '
+  from &syslast
+;
+
 create table _data_ as
   select * from dictionary.indexes
   where upcase(libname)="%upcase(&libref)"
@@ -64,18 +78,43 @@ create table _data_ as
   ;
 %local idxinfo; %let idxinfo=&syslast;
 
-create table _data_ as
-  select * from dictionary.columns
-  where upcase(libname)="%upcase(&libref)"
-  %if %length(&ds)>0 %then %do;
-    and upcase(memname)="%upcase(&ds)"
-  %end;
-  ;
-%local colinfo; %let colinfo=&syslast;
-%local dsnlist;
-select distinct upcase(memname) into: dsnlist
-  separated by ' '
-  from &syslast;
+/* Extract all Primary Key and Unique data constraints */
+%mp_getconstraints(lib=%upcase(&libref),ds=%upcase(&ds),outds=_data_)
+%local colconst; %let colconst=&syslast;
+
+%macro addConst();
+  %global constraints_used;
+  data _null_;
+    length ctype $11 constraint_name_orig $256 constraints_used $5000;
+    set &colconst (where=(table_name="&curds" and constraint_type in ('PRIMARY','UNIQUE'))) end=last;
+    file &fref mod;
+    by constraint_type constraint_name;
+    retain constraints_used;
+    constraint_name_orig=constraint_name;
+    if upcase(strip(constraint_type)) = 'PRIMARY' then ctype='PRIMARY KEY';
+    else ctype=strip(constraint_type);
+    %if &flavour=TSQL %then %do;
+      column_name=catt('[',column_name,']');
+      constraint_name=catt('[',constraint_name,']');
+    %end;
+    %else %if &flavour=PGSQL %then %do;
+      column_name=catt('"',column_name,'"');
+      constraint_name=catt('"',constraint_name,'"');
+    %end;
+    if first.constraint_name then do;
+      constraints_used = catx(' ', constraints_used, constraint_name_orig);
+      put "   ,CONSTRAINT " constraint_name ctype "(" ;
+      put '     ' column_name;
+    end;
+  else put '     ,' column_name;
+  if last.constraint_name then do;
+    put "   )";
+    call symput('constraints_used',strip(constraints_used));
+  end;
+  run;
+  %put &=constraints_used;
+%mend;
+
 data _null_;
   file &fref;
   put "/* DDL generated by &sysuserid on %sysfunc(datetime(),datetime19.) */";
@@ -84,14 +123,15 @@ run;
 %local x curds;
 %if &flavour=SAS %then %do;
   data _null_;
-    file &fref;
+    file &fref mod;
+    put "/* SAS Flavour DDL for %upcase(&libref).&curds */";
     put "proc sql;";
   run;
   %do x=1 %to %sysfunc(countw(&dsnlist));
     %let curds=%scan(&dsnlist,&x);
     data _null_;
       file &fref mod;
-      length nm lab $1024;
+      length nm lab $1024 typ $20;
       set &colinfo (where=(upcase(memname)="&curds")) end=last;
 
       if _n_=1 then do;
@@ -105,27 +145,40 @@ run;
       end;
       else put "   ,"@@;
       if length(format)>1 then fmt=" format="!!cats(format);
-      len=" length="!!cats(length);
-      lab=" label="!!quote(trim(label));
+      if length(label)>1 then lab=" label="!!quote(trim(label));
       if notnull='yes' then notnul=' not null';
-      put name type len fmt notnul lab;
-      if last then put ');';
+      if type='char' then typ=cats('char(',length,')');
+      else if length ne 8 then typ='num length='!!left(length);
+      else typ='num';
+      put name typ fmt notnul lab;
     run;
 
+    /* Extra step for data constraints */
+    %addConst()
+
     data _null_;
-      length ds $128;
-      set &idxinfo (where=(memname="&curds")) end=last;
+      file &fref mod;
+      put ');';
+    run;
+
+    /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
+    data _null_;
+      *length ds $128;
+      set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
       file &fref mod;
       by idxusage indxname;
-      if unique='yes' then uniq=' unique';
-      ds=cats(libname,'.',memname);
+/*       ds=cats(libname,'.',memname); */
       if first.indxname then do;
-        put 'create ' uniq ' index ' indxname;
-        put '  on ' ds '(' name @@;
+          put 'CREATE UNIQUE INDEX ' indxname "ON &libref..&curds (" ;
+          put '  ' name ;
+      end;
+      else put '  ,' name ;
+      *else put '    ,' name ;
+      if last.indxname then do;
+        put ');';
       end;
-      else put ',' name @@;
-      if last.indxname then put ');';
     run;
+
 /*
     ods output IntegrityConstraints=ic;
     proc contents data=testali out2=info;
@@ -146,7 +199,7 @@ run;
     %let curds=%scan(&dsnlist,&x);
     data _null_;
       file &fref mod;
-      put "/* DDL for &schema..&curds */";
+      put "/* TSQL Flavour DDL for &schema..&curds */";
     data _null_;
       file &fref mod;
       set &colinfo (where=(upcase(memname)="&curds")) end=last;
@@ -169,31 +222,30 @@ run;
       else if length le 8000 then fmt='[varchar]('!!cats(length)!!')';
       else fmt=cats('[varchar](max)');
       if notnull='yes' then notnul=' NOT NULL';
-      put name fmt notnul;
+      put "[" name +(-1) "]" fmt notnul;
     run;
+
+    /* Extra step for data constraints */
+    %addConst()
+
+    /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
     data _null_;
-      length ds $128;
-      set &idxinfo (where=(memname="&curds"));
+      *length ds $128;
+      set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
       file &fref mod;
       by idxusage indxname;
-      if unique='yes' then uniq=' unique';
-      ds=cats(libname,'.',memname);
+      *ds=cats(libname,'.',memname);
       if first.indxname then do;
-        if unique='yes' and nomiss='yes' then do;
-          put '  ,constraint [' indxname '] PRIMARY KEY';
-        end;
-        else if unique='yes' then do;
-          /* add nonclustered in case of multiple unique indexes */
-          put '  ,index [' indxname '] UNIQUE NONCLUSTERED';
-        end;
-        put '  (';
-        put '    [' name ']';
+        /* add nonclustered in case of multiple unique indexes */
+        put '   ,index [' indxname +(-1) '] UNIQUE NONCLUSTERED (';
+        put '     [' name +(-1) ']';
       end;
-      else put '    ,[' name ']';
+      else put '     ,[' name +(-1) ']';
       if last.indxname then do;
-        put '  )';
+        put '   )';
       end;
     run;
+
     data _null_;
       file &fref mod;
       put ')';
@@ -225,7 +277,9 @@ run;
     from dictionary.libnames
     where libname="&libref" and engine='POSTGRES';
   %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
-
+  data _null_;
+    file &fref mod;
+    put "CREATE SCHEMA &schema;";
   %do x=1 %to %sysfunc(countw(&dsnlist));
     %let curds=%scan(&dsnlist,&x);
     data _null_;
@@ -253,37 +307,40 @@ run;
       else if type='num' then fmt=' DOUBLE PRECISION';
       else fmt='VARCHAR('!!cats(length)!!')';
       if notnull='yes' then notnul=' NOT NULL';
-      put name fmt notnul;
-    run;
-    data _null_;
-      length ds $128;
-      set &idxinfo (where=(memname="&curds"));
-      file &fref mod;
-      by idxusage indxname;
-      if unique='yes' then uniq=' unique';
-      ds=cats(libname,'.',memname);
-      if first.indxname then do;
-        if unique='yes' and nomiss='yes' then do;
-          put '  ,PRIMARY KEY ';
-        end;
-        else if unique='yes' then do;
-          /* add nonclustered in case of multiple unique indexes */
-          put '  ,UNIQUE ';
-        end;
-        put '  (' name ;
-      end;
-      else put '    ,' name ;
-      if last.indxname then do;
-        put '  )';
-      end;
+      /* quote column names in case they represent reserved words */
+      name2=quote(trim(name));
+      put name2 fmt notnul;
     run;
+
+    /* Extra step for data constraints */
+    %addConst()
+
     data _null_;
       file &fref mod;
       put ');';
     run;
+
+    /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
+    data _null_;
+      *length ds $128;
+      set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
+      file &fref mod;
+      by idxusage indxname;
+/*       ds=cats(libname,'.',memname); */
+      if first.indxname then do;
+          put 'CREATE UNIQUE INDEX "' indxname +(-1) '" ' "ON &schema..&curds (" ;
+          put '  "' name +(-1) '"' ;
+      end;
+      else put '  ,"' name +(-1) '"';
+      *else put '    ,' name ;
+      if last.indxname then do;
+        put ');';
+      end;
+    run;
+
   %end;
 %end;
-%if &showlog=YES %then %do;
+%if %upcase(&showlog)=YES %then %do;
   options ps=max;
   data _null_;
     infile &fref;
@@ -292,4 +349,4 @@ run;
   run;
 %end;
 
-%mend;
+%mend;

base/mp_guesspk.sas

@@ -3,9 +3,9 @@
   @brief Guess the primary key of a table
   @details Tries to guess the primary key of a table based on the following logic:
 
-  * Columns with nulls are ignored
-  * Return only column combinations that provide unique results
-  * Start from one column, then move out to include composite keys of 2 to 6 columns
+      * Columns with nulls are ignored
+      * Return only column combinations that provide unique results
+      * Start from one column, then move out to include composite keys of 2 to 6 columns
 
   The library of the target should be assigned before using this macro.
 
@@ -51,7 +51,7 @@
   /* get null count and row count */
   %let tmpvar=%mf_getuniquename();
   proc sql noprint;
-  create table _data_ as select 
+  create table _data_ as select
     count(*) as &tmpvar
   %do i=1 %to &vcnt;
     %let var=%scan(&vars,&i);
@@ -85,10 +85,10 @@
     %put &sysmacroname: &baseds has no combination of unique records! Exiting.;
     %return;
   %end;
-  
+
   /* now check cardinality */
   proc sql noprint;
-  create table _data_ as select 
+  create table _data_ as select
   %do i=1 %to &ppkcnt;
     %let var=%scan(&posspks,&i);
     count(distinct &var) as &var
@@ -212,7 +212,7 @@
       %end;
     %end;
   %end;
- 
+
   %if &ppkcnt=4 %then %do;
     %put &sysmacroname: No more PK guess possible;
     %return;
@@ -228,7 +228,7 @@
         %let lev3=%scan(&posspks,&k);
         %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
           %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then 
+          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
           %do m=5 %to &ppkcnt;
             %let lev5=%scan(&posspks,&m);
             %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then %do;
@@ -250,7 +250,7 @@
       %end;
     %end;
   %end;
- 
+
   %if &ppkcnt=5 %then %do;
     %put &sysmacroname: No more PK guess possible;
     %return;
@@ -266,17 +266,17 @@
         %let lev3=%scan(&posspks,&k);
         %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
           %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then 
+          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
           %do m=5 %to &ppkcnt;
             %let lev5=%scan(&posspks,&m);
-            %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then 
+            %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then
             %do n=6 %to &ppkcnt;
               %let lev6=%scan(&posspks,&n);
-              %if &lev1 ne &lev6 & &lev2 ne &lev6 & &lev3 ne &lev6 
-              & &lev4 ne &lev6 & &lev5 ne &lev6 %then 
+              %if &lev1 ne &lev6 & &lev2 ne &lev6 & &lev3 ne &lev6
+              & &lev4 ne &lev6 & &lev5 ne &lev6 %then
               %do;
                 /* check for four level uniqueness */
-                proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5 &lev6) 
+                proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5 &lev6)
                   out=&tmpds noduprec;
                   by _all_;
                 run;
@@ -295,7 +295,7 @@
       %end;
     %end;
   %end;
- 
+
   %if &ppkcnt=6 %then %do;
     %put &sysmacroname: No more PK guess possible;
     %return;

base/mp_searchdata.sas

@@ -3,7 +3,7 @@
   @brief Searches all data in a library
   @details
   Scans an entire library and creates a copy of any table
-    containing a specific string or numeric value.  Only 
+    containing a specific string OR numeric value.  Only 
     matching records are written out.
     If both a string and numval are provided, the string
     will take precedence.
@@ -44,9 +44,13 @@
   ,filter_text=%str(1=1)
 )/*/STORE SOURCE*/;
 
-%local table_list table table_num table colnum col start_tm vars type coltype;
+%local table_list table table_num table colnum col start_tm check_tm vars type coltype;
 %put process began at %sysfunc(datetime(),datetime19.);
 
+%if &syscc ge 4 %then %do;
+  %put %str(WAR)NING: SYSCC=&syscc on macro entry;
+  %return;
+%end;
 
 %if &string = %then %let type=N;
 %else %let type=C;
@@ -78,6 +82,7 @@ proc sql
     %put NO COLUMNS IN &lib..&table!  This will be skipped.;
   %end;
   %else %do;
+    %let check_tm=%sysfunc(datetime());
     /* build sql statement */
     create table mpsearch.&table as select * from &lib..&table
       where %unquote(&filter_text) and 
@@ -88,14 +93,19 @@ proc sql
       %let coltype=%mf_getvartype(&lib..&table,&col);
       %if &type=C and &coltype=C %then %do;
         /* if a char column, see if it contains the string */
-        or (&col ? "&string")
+        or ("&col"n ? "&string")
       %end;
       %else %if &type=N and &coltype=N %then %do;
         /* if numeric match exactly */
-        or (&col = &numval)
+        or ("&col"n = &numval)
       %end;
     %end;
     );
+    %put Search query for &table took %sysevalf(%sysfunc(datetime())-&check_tm) seconds;
+    %if &sqlrc ne 0 %then %do;
+      %put %str(WAR)NING: SQLRC=&sqlrc when processing &table;
+      %return;
+    %end;
     %if %mf_nobs(mpsearch.&table)=0 %then %do;
       drop table mpsearch.&table;
     %end;

base/mp_streamfile.sas

@@ -47,6 +47,7 @@
   %end;
 %end;
 %else %if &contentype=EXCEL %then %do;
+  /* suitable for XLS format */
   %if &platform=SASMETA %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/vnd.ms-excel');
@@ -59,6 +60,19 @@
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
+%else %if &contentype=XLSX %then %do;
+  %if &platform=SASMETA %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type','application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
+      rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
+    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
+      contenttype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' 
+      contentdisp="attachment; filename=&outname";
+  %end;
+%end;
 %else %if &contentype=TEXT %then %do;
   %if &platform=SASMETA %then %do;
     data _null_;

doxy/DoxygenLayout.xml

@@ -16,7 +16,7 @@
       <tab type="classmembers" visible="no" title="" intro=""/>
     </tab>
 
-    <tab type="filelist" visible="yes" title="" intro="List of Files Used in the Macro Core Library"/>
+    <tab type="filelist" visible="yes" title="" intro="List of Files Used in the Macro-Core Library"/>
 
     <tab type="examples" visible="yes" title="" intro=""/>
   </navindex>

doxy/build.sh

@@ -36,5 +36,9 @@ echo 'core.sasjs.io' > CNAME
 git add *
 git commit -m "build.sh build on $(date +%F:%H:%M:%S)"
 git push
+npx sitemap-generator-cli https://core.sasjs.io
+git add *
+git commit -m "adding sitemap"
+git push
 
 echo "check it out:  https://sasjs.github.io/core.github.io/files.html"

meta/mm_adduser2group.sas

@@ -69,7 +69,7 @@ run;
   %return;
 %end;
 
-%if %length(&syscc) ge 4 %then %do;
+%if &syscc ge 4 %then %do;
   %put WARNING:  SYSCC=&syscc, exiting &sysmacroname;
   %return;
 %end;

meta/mm_getdocument.sas

@@ -124,6 +124,7 @@ data _null_;
         when ('
') rec='0A'x;
         when ('
') rec='0D'x;
         when ('$' ) rec='$'  ;
+        when ('	') rec='09'x;
         otherwise putlog "WARNING: missing value for " entity=;
       end;
       rc =fput(fileid, substr(rec,1,1));

meta/mm_getstpcode.sas

@@ -123,6 +123,7 @@ data _null_;
         when ('
') rec='0A'x;
         when ('
') rec='0D'x;
         when ('$' ) rec='$'  ;
+        when ('	') rec='09'x;
         otherwise putlog "%str(WARN)ING: missing value for " entity=;
       end;
       rc =fput(fileid, substr(rec,1,1));

meta/mm_gettableid.sas

@@ -0,0 +1,67 @@
+/**
+  @file mm_gettableid.sas
+  @brief Get the metadata id for a particular table
+  @details Provide a libref and table name to return the corresponding metadata id
+  in an output datataset.
+
+  Usage:
+
+      - get a table id
+      %mm_gettableid(libref=METALIB,ds=SOMETABLE,outds=iwant)
+
+  @param libref= The libref to search
+  @param ds= The input dataset to check
+  @param outds= the dataset to create that contains the `tableuri`
+  @param mDebug= set to 1 to show debug messages in the log
+
+  @returns outds  dataset containing `tableuri` and `tablename`
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mm_gettableid(
+     libref=
+    ,ds=
+    ,outds=work.mm_gettableid
+    ,mDebug=0
+)/*/STORE SOURCE*/;
+
+%local mD;
+%if &mDebug=1 %then %let mD=;
+%else %let mD=%str(*);
+%&mD.put Executing &sysmacroname..sas;
+%&mD.put _local_;
+
+data &outds;
+  length uri usingpkguri id type tableuri tablename tmpuri $256;
+  call missing(of _all_);
+  keep tableuri tablename;
+  n=1;
+  rc=0;
+  if metadata_getnobj("omsobj:SASLibrary?@Libref='&libref'",n,uri)<1 then do;
+    put "Library &libref not found";
+    stop;
+  end;
+  &mD.putlog "uri is " uri;
+  if metadata_getnasn(uri, "UsingPackages", 1, usingpkguri)>0 then do;
+    rc=metadata_resolve(usingpkguri,type,id);
+    &mD.putlog "Type is " type;
+  end;
+
+  if type='DatabaseSchema' then tmpuri=usingpkguri;
+  else tmpuri=uri;
+  
+  t=1;
+  do while(metadata_getnasn(tmpuri, "Tables", t, tableuri)>0);
+    t+1;
+    rc= metadata_getattr(tableuri, "Name", tablename);
+    &mD.putlog "Table is " tablename;
+    if upcase(tablename)="%upcase(&ds)" then do;
+      output;
+    end;
+  end;
+run;
+
+%mend;

meta/mm_getwebappsrvprops.sas

@@ -99,6 +99,7 @@ run;
           when ('
') rec='0A'x;
           when ('
') rec='0D'x;
           when ('$' ) rec='$'  ;
+          when ('	') rec='09'x;
           otherwise putlog "WARNING: missing value for " entity=;
         end;
         rc =fput(fileid, substr(rec,1,1));

viya/mv_createfolder.sas

@@ -96,7 +96,7 @@ options noquotelenmax;
     data _null_;
       set &libref1..links;
       if rel='createChild' then
-        call symputx('href',quote(trim(href)),'l');
+        call symputx('href',quote("&base_uri"!!trim(href)),'l');
     run;
   %end;
   %else %if &SYS_PROCHTTP_STATUS_CODE=404 %then %do;

viya/mv_createwebservice.sas

@@ -143,7 +143,7 @@ libname &libref1 JSON fileref=&fname1;
 
 data _null_;
   set &libref1..links;
-  if rel='members' then call symputx('membercheck',quote(trim(href)),'l');
+  if rel='members' then call symputx('membercheck',quote("&base_uri"!!trim(href)),'l');
   else if rel='self' then call symputx('parentFolderUri',href,'l');
 run;
 data _null_;
@@ -609,7 +609,7 @@ proc http method='POST'
     in=&fname3
     out=&fname4
     &oauth_bearer
-    url="/jobDefinitions/definitions?parentFolderUri=&parentFolderUri";
+    url="&base_uri/jobDefinitions/definitions?parentFolderUri=&parentFolderUri";
     headers 'Content-Type'='application/vnd.sas.job.definition+json'
   %if &grant_type=authorization_code %then %do;
             "Authorization"="Bearer &&&access_token_var"

viya/mv_deletefoldermember.sas

@@ -98,7 +98,7 @@ run;
 libname &libref1 JSON fileref=&fname1;
 data _null_;
   set &libref1..links;
-  if rel='members' then call symputx('mref',quote(trim(href)),'l');
+  if rel='members' then call symputx('mref',quote("&base_uri"!!trim(href)),'l');
 run;
 
 /* get the children */
@@ -128,7 +128,7 @@ run;
   %put NOTE:;%put NOTE- &sysmacroname: &path/&name NOT FOUND;%put NOTE- ;
   %return;
 %end;
-proc http method="DELETE" url="&uri" &oauth_bearer;
+proc http method="DELETE" url="&base_uri&uri" &oauth_bearer;
   headers 
 %if &grant_type=authorization_code %then %do;
       "Authorization"="Bearer &&&access_token_var"

viya/mv_deletejes.sas

@@ -95,7 +95,7 @@ run;
 libname &libref1 JSON fileref=&fname1;
 data _null_;
   set &libref1..links;
-  if rel='members' then call symputx('mref',quote(trim(href)),'l');
+  if rel='members' then call symputx('mref',quote("&base_uri"!!trim(href)),'l');
 run;
 
 /* get the children */
@@ -117,7 +117,7 @@ libname &libref1a JSON fileref=&fname1a;
 data _null_;
   set &libref1a..items;
   if contenttype='jobDefinition' and upcase(name)="%upcase(&name)" then do;
-    call symputx('uri',uri,'l');
+    call symputx('uri',cats("&base_uri",uri),'l');
     call symputx('found',1,'l');
   end;
 run;

viya/mv_deleteviyafolder.sas

@@ -88,9 +88,9 @@ libname &libref1 JSON fileref=&fname1;
 data _null_;
   set &libref1..links;
   if rel='deleteRecursively' then
-    call symputx('href',quote(trim(href)),'l');
+    call symputx('href',quote("&base_uri"!!trim(href)),'l');
   else if rel='members' then
-    call symputx('mref',quote(cats(href,'?recursive=true')),'l');
+    call symputx('mref',quote(cats("&base_uri",href,'?recursive=true')),'l');
 run;
 
 /* before we can delete the folder, we need to delete the children */
@@ -111,7 +111,7 @@ data _null_;
   set &libref1a..items_links;
   if href=:'/folders/folders' then return;
   if rel='deleteResource' then
-    call execute('proc http method="DELETE" url='!!quote(trim(href))
+    call execute('proc http method="DELETE" url='!!quote("&base_uri"!!trim(href))
     !!'; headers "Authorization"="Bearer &&&access_token_var" '
     !!' "Accept"="*/*";run; /**/');
 run;

viya/mv_getclients.sas

@@ -1,6 +1,6 @@
 /**
-  @file mv_getgroups.sas
-  @brief Creates a dataset with a list of viya groups
+  @file mv_getclients.sas
+  @brief Get a list of Viya Clients
   @details First, be sure you have an access token (which requires an app token).
 
   Using the macros here:
@@ -20,7 +20,7 @@
 
   Now we can run the macro!
 
-      %mv_getgroups()
+      %mv_getclients()
 
   @param access_token_var= The global macro variable to contain the access token
   @param grant_type= valid values are "password" or "authorization_code" (unquoted).

viya/mv_getfoldermembers.sas

@@ -64,7 +64,7 @@ options noquotelenmax;
 %if "&root"="/" %then %do;
   /* if root just list root folders */
   proc http method='GET' out=&fname1 &oauth_bearer
-      url='%sysfunc(getoption(servicesbaseurl))/folders/rootFolders';
+      url="&base_uri/folders/rootFolders";
   %if &grant_type=authorization_code %then %do;
       headers "Authorization"="Bearer &&&access_token_var";
   %end;
@@ -87,7 +87,7 @@ options noquotelenmax;
   /* now get the followon link to list members */
   data _null_;
     set &libref1..links;
-    if rel='members' then call symputx('href',quote(trim(href)),'l');
+    if rel='members' then call symputx('href',quote("&base_uri"!!trim(href)),'l');
   run;
   %local fname2 libref2;
   %let fname2=%mf_getuniquefileref();

viya/mv_getusergroups.sas

@@ -1,24 +1,15 @@
 /**
   @file mv_getusergroups.sas
   @brief Creates a dataset with a list of groups for a particular user
-  @details First, be sure you have an access token (which requires an app token).
+  @details If using outside of Viya SPRE, then an access token is needed.
 
-  Using the macros here:
+  Compile the macros here:
 
       filename mc url
         "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
 
-  An administrator needs to set you up with an access code:
-
-      %mv_registerclient(outds=client)
-
-  Navigate to the url from the log (opting in to the groups) and paste the
-  access code below:
-
-      %mv_tokenauth(inds=client,code=wKDZYTEPK6)
-
-  Now we can run the macro!
+  Then run the macro!
 
       %mv_getusergroups(&sysuserid,outds=users)
 
@@ -55,7 +46,7 @@
     %let &access_token_var=;
 %end;
 %put &sysmacroname: grant_type=&grant_type;
-%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password 
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
   ,mac=&sysmacroname
@@ -73,7 +64,7 @@ options noquotelenmax;
 
 proc http method='GET' out=&fname1 &oauth_bearer
   url="&base_uri/identities/users/&user/memberships?limit=10000";
-  headers 
+  headers
 %if &grant_type=authorization_code %then %do;
          "Authorization"="Bearer &&&access_token_var"
 %end;

viya/mv_registerclient.sas

@@ -187,7 +187,7 @@ run;
 %end;
 
 /* prepare url */
-%if &grant_type=authorization_code %then %do;
+%if %index(%superq(grant_type),authorization_code) %then %do;
   data _null_;
     if symexist('_baseurl') then do;
       url=symget('_baseurl');
@@ -206,7 +206,7 @@ run;
 %put CLIENT_SECRET=&client_secret;
 %put GRANT_TYPE=&grant_type;
 %put;
-%if &grant_type=authorization_code %then %do;
+%if %index(%superq(grant_type),authorization_code) %then %do;
   /* cannot use base_uri here as it includes the protocol which may be incorrect externally */
   %put NOTE: The developer must also register below and select 'openid' to get the grant code:;
   %put NOTE- ;

viya/mv_tokenauth.sas

@@ -38,6 +38,7 @@
   @param pass= If grant_type=password then provide the password here
   @param access_token_var= The global macro variable to contain the access token
   @param refresh_token_var= The global macro variable to contain the refresh token
+  @param base_uri= The Viya API server location
 
   @version VIYA V.03.04
   @author Allan Bowe
@@ -62,6 +63,7 @@
     ,pass=
     ,access_token_var=ACCESS_TOKEN
     ,refresh_token_var=REFRESH_TOKEN
+    ,base_uri=#NOTSET#
   );
 %global &access_token_var &refresh_token_var;
 
@@ -108,8 +110,7 @@ run;
 /**
  * Request access token
  */
-%local base_uri; /* location of rest apis */
-%let base_uri=%mf_getplatform(VIYARESTAPI);
+%if &base_uri=#NOTSET# %then %let base_uri=%mf_getplatform(VIYARESTAPI);
 
 %let fref2=%mf_getuniquefileref();
 proc http method='POST' in=&grantstring out=&fref2