fix: adding base_uri in all calls to cover instances where viya calls are not on localhost

fix: base_uri

.editorconfig

@@ -0,0 +1,12 @@
+# http://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+end_of_line = lf
+insert_final_newline = false
+trim_trailing_whitespace = true
+
+

.gitignore

@@ -1 +1,2 @@
 node_modules
+.DS_Store

all.sas

@@ -510,6 +510,7 @@ options noquotelenmax;
 
   <h4> Dependencies </h4>
   @li mf_mval.sas
+  @li mf_trimstr.sas
 
   @version 9.4 / 3.4
   @author Allan Bowe
@@ -555,7 +556,7 @@ options noquotelenmax;
   %else 0;
 %end;
 %else %if &switch=VIYARESTAPI %then %do;
-  %sysfunc(getoption(servicesbaseurl))
+  %mf_trimstr(%sysfunc(getoption(servicesbaseurl)),/)
 %end;
 %mend;/**
   @file
@@ -1290,6 +1291,44 @@ Usage:
 %macro mf_nobs(libds
 )/*/STORE SOURCE*/;
   %mf_getattrn(&libds,NLOBS)
+%mend;/**
+  @file mf_trimstr.sas
+  @brief Removes character(s) from the end, if they exist
+  @details If the designated characters exist at the end of the string, they 
+  are removed
+
+        %put %mf_trimstr(/blah/,/); * /blah;
+        %put %mf_trimstr(/blah/,h); * /blah/;
+        %put %mf_trimstr(/blah/,h/);* /bla;
+
+  <h4> Dependencies </h4>
+
+
+  @param basestr The string to be modified
+  @param trimstr The string to be removed from the end of `basestr`, if it exists
+
+  @return output returns result with the value of `trimstr` removed from the end
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_trimstr(basestr,trimstr);
+%local trimlen trimval;
+%let trimlen=%length(%superq(trimstr));
+%let trimval=%qsubstr(%superq(basestr)
+  ,%length(%superq(basestr))-&trimlen+1
+  ,&trimlen);
+
+%if %superq(trimval)=%superq(trimstr) %then %do;
+  %qsubstr(%superq(basestr),1,%length(%superq(basestr))-&trimlen)
+%end;
+%else %do;
+  &basestr
+%end;
+
 %mend;/**
   @file
   @brief Creates a Unique ID based on system time in a friendly format
@@ -1436,6 +1475,12 @@ Usage:
     the particulars of an environment.  For instance, can stream custom
     results back to the client in an STP Web App context, or completely stop
     in the case of a batch run.
+  
+  Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored Process
+  environments.  This macro takes a unique approach - we set the SAS syscc to 0,
+  run `stpsrvset('program error', 0)` (if SAS 9) and then - we open a macro 
+  but don't close it!  This provides a graceful abort for SAS web services in all 
+  web enabled environments.
 
   @param mac= to contain the name of the calling macro
   @param msg= message to be returned
@@ -2449,11 +2494,13 @@ create table &outds as
         label x='blah';
       run;
       proc sql; describe table &syslast;
-
       %mp_getddl(work,test,flavour=tsql,showlog=YES)
 
+  <h4> Dependencies </h4>
+  @li mp_getconstraints.sas
+
   @param lib libref of the library to create DDL for.  Should be assigned.
-  @param ds dataset to create ddl for
+  @param ds dataset to create ddl for (optional)
   @param fref= the fileref to which to write the DDL.  If not preassigned, will
     be assigned to TEMP.
   @param flavour= The type of DDL to create (default=SAS). Supported=TSQL
@@ -2462,11 +2509,9 @@ create table &outds as
     ,else libref)
   @param applydttm= for non SAS DDL, choose if columns are created with native
    datetime2 format or regular decimal type
-
   @version 9.3
   @author Allan Bowe
   @source https://github.com/sasjs/core
-
 **/
 
 %macro mp_getddl(libref,ds,fref=getddl,flavour=SAS,showlog=NO,schema=
@@ -2490,6 +2535,21 @@ create table _data_ as
   ;
 %local tabinfo; %let tabinfo=&syslast;
 
+create table _data_ as
+  select * from dictionary.columns
+  where upcase(libname)="%upcase(&libref)"
+  %if %length(&ds)>0 %then %do;
+    and upcase(memname)="%upcase(&ds)"
+  %end;
+  ;
+%local colinfo; %let colinfo=&syslast;
+
+%local dsnlist;
+  select distinct upcase(memname) into: dsnlist
+  separated by ' '
+  from &syslast
+;
+
 create table _data_ as
   select * from dictionary.indexes
   where upcase(libname)="%upcase(&libref)"
@@ -2500,18 +2560,43 @@ create table _data_ as
   ;
 %local idxinfo; %let idxinfo=&syslast;
 
-create table _data_ as
-  select * from dictionary.columns
-  where upcase(libname)="%upcase(&libref)"
-  %if %length(&ds)>0 %then %do;
-    and upcase(memname)="%upcase(&ds)"
-  %end;
-  ;
-%local colinfo; %let colinfo=&syslast;
-%local dsnlist;
-select distinct upcase(memname) into: dsnlist
-  separated by ' '
-  from &syslast;
+/* Extract all Primary Key and Unique data constraints */
+%mp_getconstraints(lib=%upcase(&libref),ds=%upcase(&ds),outds=_data_)
+%local colconst; %let colconst=&syslast;
+
+%macro addConst();
+  %global constraints_used;
+  data _null_;
+    length ctype $11 constraint_name_orig $256 constraints_used $5000;
+    set &colconst (where=(table_name="&curds" and constraint_type in ('PRIMARY','UNIQUE'))) end=last;
+    file &fref mod;
+    by constraint_type constraint_name;
+    retain constraints_used;
+    constraint_name_orig=constraint_name;
+    if upcase(strip(constraint_type)) = 'PRIMARY' then ctype='PRIMARY KEY';
+    else ctype=strip(constraint_type);
+    %if &flavour=TSQL %then %do;
+      column_name=catt('[',column_name,']');
+      constraint_name=catt('[',constraint_name,']');
+    %end;
+    %else %if &flavour=PGSQL %then %do;
+      column_name=catt('"',column_name,'"');
+      constraint_name=catt('"',constraint_name,'"');
+    %end;
+    if first.constraint_name then do;
+      constraints_used = catx(' ', constraints_used, constraint_name_orig);
+      put "   ,CONSTRAINT " constraint_name ctype "(" ;
+      put '     ' column_name;
+    end;
+  else put '     ,' column_name;
+  if last.constraint_name then do;
+    put "   )";
+    call symput('constraints_used',strip(constraints_used));
+  end;
+  run;
+  %put &=constraints_used;
+%mend;
+
 data _null_;
   file &fref;
   put "/* DDL generated by &sysuserid on %sysfunc(datetime(),datetime19.) */";
@@ -2520,7 +2605,8 @@ run;
 %local x curds;
 %if &flavour=SAS %then %do;
   data _null_;
-    file &fref;
+    file &fref mod;
+    put "/* SAS Flavour DDL for %upcase(&libref).&curds */";
     put "proc sql;";
   run;
   %do x=1 %to %sysfunc(countw(&dsnlist));
@@ -2545,23 +2631,34 @@ run;
       lab=" label="!!quote(trim(label));
       if notnull='yes' then notnul=' not null';
       put name type len fmt notnul lab;
-      if last then put ');';
     run;
 
+    /* Extra step for data constraints */
+    %addConst()
+
     data _null_;
-      length ds $128;
-      set &idxinfo (where=(memname="&curds")) end=last;
+      file &fref mod;
+      put ');';
+    run;
+  
+    /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
+    data _null_;
+      *length ds $128;
+      set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
       file &fref mod;
       by idxusage indxname;
-      if unique='yes' then uniq=' unique';
-      ds=cats(libname,'.',memname);
+/*       ds=cats(libname,'.',memname); */
       if first.indxname then do;
-        put 'create ' uniq ' index ' indxname;
-        put '  on ' ds '(' name @@;
+          put 'CREATE UNIQUE INDEX ' indxname "ON &libref..&curds (" ;
+          put '  ' name ;
+      end;
+      else put '  ,' name ;
+      *else put '    ,' name ;
+      if last.indxname then do;
+        put ');';
       end;
-      else put ',' name @@;
-      if last.indxname then put ');';
     run;
+
 /*
     ods output IntegrityConstraints=ic;
     proc contents data=testali out2=info;
@@ -2582,7 +2679,7 @@ run;
     %let curds=%scan(&dsnlist,&x);
     data _null_;
       file &fref mod;
-      put "/* DDL for &schema..&curds */";
+      put "/* TSQL Flavour DDL for &schema..&curds */";
     data _null_;
       file &fref mod;
       set &colinfo (where=(upcase(memname)="&curds")) end=last;
@@ -2605,31 +2702,30 @@ run;
       else if length le 8000 then fmt='[varchar]('!!cats(length)!!')';
       else fmt=cats('[varchar](max)');
       if notnull='yes' then notnul=' NOT NULL';
-      put name fmt notnul;
+      put "[" name +(-1) "]" fmt notnul;
     run;
+
+    /* Extra step for data constraints */
+    %addConst()
+
+    /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
     data _null_;
-      length ds $128;
-      set &idxinfo (where=(memname="&curds"));
+      *length ds $128;
+      set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
       file &fref mod;
       by idxusage indxname;
-      if unique='yes' then uniq=' unique';
-      ds=cats(libname,'.',memname);
+      *ds=cats(libname,'.',memname);
       if first.indxname then do;
-        if unique='yes' and nomiss='yes' then do;
-          put '  ,constraint [' indxname '] PRIMARY KEY';
-        end;
-        else if unique='yes' then do;
-          /* add nonclustered in case of multiple unique indexes */
-          put '  ,index [' indxname '] UNIQUE NONCLUSTERED';
-        end;
-        put '  (';
-        put '    [' name ']';
+        /* add nonclustered in case of multiple unique indexes */
+        put '   ,index [' indxname +(-1) '] UNIQUE NONCLUSTERED (';
+        put '     [' name +(-1) ']';
       end;
-      else put '    ,[' name ']';
+      else put '     ,[' name +(-1) ']';
       if last.indxname then do;
-        put '  )';
+        put '   )';
       end;
     run;
+
     data _null_;
       file &fref mod;
       put ')';
@@ -2653,7 +2749,77 @@ run;
     run;
   %end;
 %end;
+%else %if &flavour=PGSQL %then %do;
+  /* if schema does not exist, set to be same as libref */
+  %local schemaactual;
+  proc sql noprint;
+  select sysvalue into: schemaactual
+    from dictionary.libnames
+    where libname="&libref" and engine='POSTGRES';
+  %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
+  data _null_;
+    file &fref mod;
+    put "CREATE SCHEMA &schema;";
+  %do x=1 %to %sysfunc(countw(&dsnlist));
+    %let curds=%scan(&dsnlist,&x);
+    data _null_;
+      file &fref mod;
+      put "/* Postgres Flavour DDL for &schema..&curds */";
+    data _null_;
+      file &fref mod;
+      set &colinfo (where=(upcase(memname)="&curds")) end=last;
+      length fmt $32;
+      if _n_=1 then do;
+        if memtype='DATA' then do;
+          put "CREATE TABLE &schema..&curds (";
+        end;
+        else do;
+          put "CREATE VIEW &schema..&curds (";
+        end;
+        put "    "@@;
+      end;
+      else put "   ,"@@;
+      format=upcase(format);
+      if 1=0 then; /* dummy if */
+      %if &applydttm=YES %then %do;
+        else if format=:'DATETIME' then fmt=' TIMESTAMP ';
+      %end;
+      else if type='num' then fmt=' DOUBLE PRECISION';
+      else fmt='VARCHAR('!!cats(length)!!')';
+      if notnull='yes' then notnul=' NOT NULL';
+      /* quote column names in case they represent reserved words */
+      name2=quote(trim(name));
+      put name2 fmt notnul;
+    run;
 
+    /* Extra step for data constraints */
+    %addConst()
+
+    data _null_;
+      file &fref mod;
+      put ');';
+    run;
+
+    /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
+    data _null_;
+      *length ds $128;
+      set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
+      file &fref mod;
+      by idxusage indxname;
+/*       ds=cats(libname,'.',memname); */
+      if first.indxname then do;
+          put 'CREATE UNIQUE INDEX "' indxname +(-1) '" ' "ON &schema..&curds (" ;
+          put '  "' name +(-1) '"' ;
+      end;
+      else put '  ,"' name +(-1) '"';
+      *else put '    ,' name ;
+      if last.indxname then do;
+        put ');';
+      end;
+    run;
+
+  %end;
+%end;
 %if &showlog=YES %then %do;
   options ps=max;
   data _null_;
@@ -2663,8 +2829,7 @@ run;
   run;
 %end;
 
-%mend;
-/**
+%mend;/**
   @file mp_getmaxvarlengths.sas
   @brief Scans a dataset to find the max length of the variable values
   @details  
@@ -3080,7 +3245,9 @@ create table &outds (rename=(
   * PROCJSON (default)
   * DATASTEP 
 
-  @param dbg= Typically used with an _debug (numeric) option
+  @param dbg= DEPRECATED - was used to conditionally add PRETTY to
+    proc json but this can cause line truncation in large files.
+    
 
   @version 9.2
   @author Allan Bowe
@@ -3106,9 +3273,8 @@ create table &outds (rename=(
     proc sql;drop table &tempds;
     data &tempds /view=&tempds;set &ds; 
     %if &fmt=N %then format _numeric_ best32.;;
-    proc json out=&jref
+    proc json out=&jref pretty
         %if &action=ARR %then nokeys ;
-        %if &dbg ge 131  %then pretty ;
         ;export &tempds / nosastags fmtnumeric;
     run;
     proc sql;drop view &tempds;
@@ -3811,6 +3977,7 @@ proc sql
   %end;
 %end;
 %else %if &contentype=EXCEL %then %do;
+  /* suitable for XLS format */
   %if &platform=SASMETA %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/vnd.ms-excel');
@@ -3823,6 +3990,19 @@ proc sql
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
+%else %if &contentype=XLSX %then %do;
+  %if &platform=SASMETA %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type','application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
+      rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
+    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
+      contenttype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' 
+      contentdisp="attachment; filename=&outname";
+  %end;
+%end;
 %else %if &contentype=TEXT %then %do;
   %if &platform=SASMETA %then %do;
     data _null_;
@@ -4099,19 +4279,20 @@ ods package publish archive properties
 ods package close;
 
 %mend;/**
-  @file
+  @file mm_adduser2group.sas
   @brief Adds a user to a group
   @details Adds a user to a metadata group.  The macro first checks whether the
     user is in that group, and if not, the user is added.
 
   Usage:
 
-    %mm_adduser2group(user=sasdemo
-      ,group=someGroup)
+      %mm_adduser2group(user=sasdemo
+        ,group=someGroup)
 
 
   @param user= the user name (not displayname)
   @param group= the group to which to add the user
+  @param mdebug= set to 1 to show debug info in log
 
   @warning the macro does not check inherited group memberships - it looks at
     direct members only
@@ -4168,7 +4349,7 @@ run;
   %return;
 %end;
 
-%if %length(&syscc) ge 4 %then %do;
+%if &syscc ge 4 %then %do;
   %put WARNING:  SYSCC=&syscc, exiting &sysmacroname;
   %return;
 %end;
@@ -6053,9 +6234,8 @@ data _null_;
   put '    proc sql;drop table &tempds; ';
   put '    data &tempds /view=&tempds;set &ds; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
-  put '    proc json out=&jref ';
+  put '    proc json out=&jref pretty ';
   put '        %if &action=ARR %then nokeys ; ';
-  put '        %if &dbg ge 131  %then pretty ; ';
   put '        ;export &tempds / nosastags fmtnumeric; ';
   put '    run; ';
   put '    proc sql;drop view &tempds; ';
@@ -6150,7 +6330,8 @@ data _null_;
   put '%end; ';
   put '%mend; ';
   put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y); ';
-  put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug; ';
+  put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
+  put '  sasjs_tables; ';
   put '%local i tempds; ';
   put ' ';
   put '%if &action=FETCH %then %do; ';
@@ -6177,6 +6358,7 @@ data _null_;
   put '        if _n_<20 then putlog _infile_; ';
   put '      %end; ';
   put '    run; ';
+  put '    %let sasjs_tables=&sasjs_tables &&_webin_name&i; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
@@ -6906,6 +7088,7 @@ data _null_;
         when ('&#x0a;') rec='0A'x;
         when ('&#x0d;') rec='0D'x;
         when ('&#36;' ) rec='$'  ;
+        when ('&#x09;') rec='09'x;
         otherwise putlog "WARNING: missing value for " entity=;
       end;
       rc =fput(fileid, substr(rec,1,1));
@@ -7820,6 +8003,7 @@ data _null_;
         when ('&#x0a;') rec='0A'x;
         when ('&#x0d;') rec='0D'x;
         when ('&#36;' ) rec='$'  ;
+        when ('&#x09;') rec='09'x;
         otherwise putlog "%str(WARN)ING: missing value for " entity=;
       end;
       rc =fput(fileid, substr(rec,1,1));
@@ -7960,6 +8144,72 @@ run;
 
 %mend;
 /**
+  @file mm_gettableid.sas
+  @brief Get the metadata id for a particular table
+  @details Provide a libref and table name to return the corresponding metadata id
+  in an output datataset.
+
+  Usage:
+
+      - get a table id
+      %mm_gettableid(libref=METALIB,ds=SOMETABLE,outds=iwant)
+
+  @param libref= The libref to search
+  @param ds= The input dataset to check
+  @param outds= the dataset to create that contains the `tableuri`
+  @param mDebug= set to 1 to show debug messages in the log
+
+  @returns outds  dataset containing `tableuri` and `tablename`
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mm_gettableid(
+     libref=
+    ,ds=
+    ,outds=work.mm_gettableid
+    ,mDebug=0
+)/*/STORE SOURCE*/;
+
+%local mD;
+%if &mDebug=1 %then %let mD=;
+%else %let mD=%str(*);
+%&mD.put Executing &sysmacroname..sas;
+%&mD.put _local_;
+
+data &outds;
+  length uri usingpkguri id type tableuri tablename tmpuri $256;
+  call missing(of _all_);
+  keep tableuri tablename;
+  n=1;
+  rc=0;
+  if metadata_getnobj("omsobj:SASLibrary?@Libref='&libref'",n,uri)<1 then do;
+    put "Library &libref not found";
+    stop;
+  end;
+  &mD.putlog "uri is " uri;
+  if metadata_getnasn(uri, "UsingPackages", 1, usingpkguri)>0 then do;
+    rc=metadata_resolve(usingpkguri,type,id);
+    &mD.putlog "Type is " type;
+  end;
+
+  if type='DatabaseSchema' then tmpuri=usingpkguri;
+  else tmpuri=uri;
+  
+  t=1;
+  do while(metadata_getnasn(tmpuri, "Tables", t, tableuri)>0);
+    t+1;
+    rc= metadata_getattr(tableuri, "Name", tablename);
+    &mD.putlog "Table is " tablename;
+    if upcase(tablename)="%upcase(&ds)" then do;
+      output;
+    end;
+  end;
+run;
+
+%mend;/**
   @file
   @brief Creates a dataset with all metadata tables for a particular library
   @details Will only show the tables to which a user has the requisite
@@ -8388,6 +8638,7 @@ run;
           when ('&#x0a;') rec='0A'x;
           when ('&#x0d;') rec='0D'x;
           when ('&#36;' ) rec='$'  ;
+          when ('&#x09;') rec='09'x;
           otherwise putlog "WARNING: missing value for " entity=;
         end;
         rc =fput(fileid, substr(rec,1,1));
@@ -9188,7 +9439,8 @@ run;
 
 **/
 %macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y);
-%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug;
+%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug 
+  sasjs_tables;
 %local i tempds;
 
 %if &action=FETCH %then %do;
@@ -9215,6 +9467,7 @@ run;
         if _n_<20 then putlog _infile_;
       %end;
     run;
+    %let sasjs_tables=&sasjs_tables &&_webin_name&i;
   %end;
 %end;
 
@@ -9539,7 +9792,7 @@ options noquotelenmax;
     data _null_;
       set &libref1..links;
       if rel='createChild' then
-        call symputx('href',quote(trim(href)),'l');
+        call symputx('href',quote("&base_uri"!!trim(href)),'l');
     run;
   %end;
   %else %if &SYS_PROCHTTP_STATUS_CODE=404 %then %do;
@@ -9738,7 +9991,7 @@ libname &libref1 JSON fileref=&fname1;
 
 data _null_;
   set &libref1..links;
-  if rel='members' then call symputx('membercheck',quote(trim(href)),'l');
+  if rel='members' then call symputx('membercheck',quote("&base_uri"!!trim(href)),'l');
   else if rel='self' then call symputx('parentFolderUri',href,'l');
 run;
 data _null_;
@@ -9836,9 +10089,8 @@ data _null_;
   put '    proc sql;drop table &tempds; ';
   put '    data &tempds /view=&tempds;set &ds; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
-  put '    proc json out=&jref ';
+  put '    proc json out=&jref pretty ';
   put '        %if &action=ARR %then nokeys ; ';
-  put '        %if &dbg ge 131  %then pretty ; ';
   put '        ;export &tempds / nosastags fmtnumeric; ';
   put '    run; ';
   put '    proc sql;drop view &tempds; ';
@@ -9933,7 +10185,8 @@ data _null_;
   put '%end; ';
   put '%mend; ';
   put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y); ';
-  put '%global _webin_file_count _webin_fileuri _debug _omittextlog ; ';
+  put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
+  put '  sasjs_tables SYS_JES_JOB_URI; ';
   put '%if %index("&_debug",log) %then %let _debug=131; ';
   put ' ';
   put '%local i tempds; ';
@@ -9947,11 +10200,11 @@ data _null_;
   put '  %if not %symexist(_webin_fileuri1) %then %do; ';
   put '    %let _webin_file_count=%eval(&_webin_file_count+0); ';
   put '    %let _webin_fileuri1=&_webin_fileuri; ';
+  put '    %let _webin_name1=&_webin_name; ';
   put '  %end; ';
   put ' ';
-  put '  %if %symexist(sasjs_tables) %then %do; ';
-  put '    /* small volumes of non-special data are sent as params for responsiveness */ ';
-  put '    /* to do - deal with escaped values  */ ';
+  put '  /* if the sasjs_tables param is passed, we expect param based upload */ ';
+  put '  %if %length(&sasjs_tables.XX)>2 %then %do; ';
   put '    filename _sasjs "%sysfunc(pathname(work))/sasjs.lua"; ';
   put '    data _null_; ';
   put '      file _sasjs; ';
@@ -10030,13 +10283,19 @@ data _null_;
   put '      infile indata firstobs=2 dsd termstr=crlf ; ';
   put '      input &input_statement; ';
   put '    run; ';
+  put '    %let sasjs_tables=&sasjs_tables &&_webin_name&i; ';
   put '  %end; ';
   put '%end; ';
   put '%else %if &action=OPEN %then %do; ';
   put '  /* setup webout */ ';
   put '  OPTIONS NOBOMFILE; ';
-  put '  filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" ';
-  put '    name="_webout.json" lrecl=999999 mod; ';
+  put '  %if "X&SYS_JES_JOB_URI.X"="XX" %then %do; ';
+  put '     filename _webout temp lrecl=999999 mod; ';
+  put '  %end; ';
+  put '  %else %do; ';
+  put '    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" ';
+  put '      name="_webout.json" lrecl=999999 mod; ';
+  put '  %end; ';
   put ' ';
   put '  /* setup temp ref */ ';
   put '  %if %upcase(&fref) ne _WEBOUT %then %do; ';
@@ -10066,13 +10325,13 @@ data _null_;
   put '      i+1; ';
   put '      call symputx(''wt''!!left(i),name); ';
   put '      call symputx(''wtcnt'',i); ';
-  put '    data _null_; file &fref; put ",""WORK"":{"; ';
+  put '    data _null_; file &fref mod; put ",""WORK"":{"; ';
   put '    %do i=1 %to &wtcnt; ';
   put '      %let wt=&&wt&i; ';
   put '      proc contents noprint data=&wt ';
   put '        out=_data_ (keep=name type length format:); ';
   put '      run;%let tempds=%scan(&syslast,2,.); ';
-  put '      data _null_; file &fref; ';
+  put '      data _null_; file &fref mod; ';
   put '        dsid=open("WORK.&wt",''is''); ';
   put '        nlobs=attrn(dsid,''NLOBS''); ';
   put '        nvars=attrn(dsid,''NVARS''); ';
@@ -10083,9 +10342,9 @@ data _null_;
   put '        put '',"nvars":'' nvars; ';
   put '      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP) ';
   put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP) ';
-  put '      data _null_; file &fref;put "}"; ';
+  put '      data _null_; file &fref mod;put "}"; ';
   put '    %end; ';
-  put '    data _null_; file &fref;put "}";run; ';
+  put '    data _null_; file &fref mod;put "}";run; ';
   put '  %end; ';
   put ' ';
   put '  /* close off json */ ';
@@ -10198,7 +10457,7 @@ proc http method='POST'
     in=&fname3
     out=&fname4
     &oauth_bearer
-    url="/jobDefinitions/definitions?parentFolderUri=&parentFolderUri";
+    url="&base_uri/jobDefinitions/definitions?parentFolderUri=&parentFolderUri";
     headers 'Content-Type'='application/vnd.sas.job.definition+json'
   %if &grant_type=authorization_code %then %do;
             "Authorization"="Bearer &&&access_token_var"
@@ -10344,7 +10603,7 @@ run;
 libname &libref1 JSON fileref=&fname1;
 data _null_;
   set &libref1..links;
-  if rel='members' then call symputx('mref',quote(trim(href)),'l');
+  if rel='members' then call symputx('mref',quote("&base_uri"!!trim(href)),'l');
 run;
 
 /* get the children */
@@ -10374,7 +10633,7 @@ run;
   %put NOTE:;%put NOTE- &sysmacroname: &path/&name NOT FOUND;%put NOTE- ;
   %return;
 %end;
-proc http method="DELETE" url="&uri" &oauth_bearer;
+proc http method="DELETE" url="&base_uri&uri" &oauth_bearer;
   headers 
 %if &grant_type=authorization_code %then %do;
       "Authorization"="Bearer &&&access_token_var"
@@ -10492,7 +10751,7 @@ run;
 libname &libref1 JSON fileref=&fname1;
 data _null_;
   set &libref1..links;
-  if rel='members' then call symputx('mref',quote(trim(href)),'l');
+  if rel='members' then call symputx('mref',quote("&base_uri"!!trim(href)),'l');
 run;
 
 /* get the children */
@@ -10514,7 +10773,7 @@ libname &libref1a JSON fileref=&fname1a;
 data _null_;
   set &libref1a..items;
   if contenttype='jobDefinition' and upcase(name)="%upcase(&name)" then do;
-    call symputx('uri',uri,'l');
+    call symputx('uri',cats("&base_uri",uri),'l');
     call symputx('found',1,'l');
   end;
 run;
@@ -10633,9 +10892,9 @@ libname &libref1 JSON fileref=&fname1;
 data _null_;
   set &libref1..links;
   if rel='deleteRecursively' then
-    call symputx('href',quote(trim(href)),'l');
+    call symputx('href',quote("&base_uri"!!trim(href)),'l');
   else if rel='members' then
-    call symputx('mref',quote(cats(href,'?recursive=true')),'l');
+    call symputx('mref',quote(cats("&base_uri",href,'?recursive=true')),'l');
 run;
 
 /* before we can delete the folder, we need to delete the children */
@@ -10656,7 +10915,7 @@ data _null_;
   set &libref1a..items_links;
   if href=:'/folders/folders' then return;
   if rel='deleteResource' then
-    call execute('proc http method="DELETE" url='!!quote(trim(href))
+    call execute('proc http method="DELETE" url='!!quote("&base_uri"!!trim(href))
     !!'; headers "Authorization"="Bearer &&&access_token_var" '
     !!' "Accept"="*/*";run; /**/');
 run;
@@ -10907,7 +11166,7 @@ options noquotelenmax;
 %if "&root"="/" %then %do;
   /* if root just list root folders */
   proc http method='GET' out=&fname1 &oauth_bearer
-      url='%sysfunc(getoption(servicesbaseurl))/folders/rootFolders';
+      url="&base_uri/folders/rootFolders";
   %if &grant_type=authorization_code %then %do;
       headers "Authorization"="Bearer &&&access_token_var";
   %end;
@@ -10930,7 +11189,7 @@ options noquotelenmax;
   /* now get the followon link to list members */
   data _null_;
     set &libref1..links;
-    if rel='members' then call symputx('href',quote(trim(href)),'l');
+    if rel='members' then call symputx('href',quote("&base_uri"!!trim(href)),'l');
   run;
   %local fname2 libref2;
   %let fname2=%mf_getuniquefileref();
@@ -11954,7 +12213,8 @@ filename &fref1 clear;
 
 **/
 %macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y);
-%global _webin_file_count _webin_fileuri _debug _omittextlog ;
+%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name
+  sasjs_tables SYS_JES_JOB_URI;
 %if %index("&_debug",log) %then %let _debug=131; 
 
 %local i tempds;
@@ -11968,11 +12228,11 @@ filename &fref1 clear;
   %if not %symexist(_webin_fileuri1) %then %do;
     %let _webin_file_count=%eval(&_webin_file_count+0);
     %let _webin_fileuri1=&_webin_fileuri;
+    %let _webin_name1=&_webin_name;
   %end;
 
-  %if %symexist(sasjs_tables) %then %do;
-    /* small volumes of non-special data are sent as params for responsiveness */
-    /* to do - deal with escaped values  */
+  /* if the sasjs_tables param is passed, we expect param based upload */
+  %if %length(&sasjs_tables.XX)>2 %then %do;
     filename _sasjs "%sysfunc(pathname(work))/sasjs.lua";
     data _null_;
       file _sasjs;
@@ -12051,13 +12311,19 @@ filename &fref1 clear;
       infile indata firstobs=2 dsd termstr=crlf ;
       input &input_statement;
     run;
+    %let sasjs_tables=&sasjs_tables &&_webin_name&i;
   %end;
 %end;
 %else %if &action=OPEN %then %do;
   /* setup webout */
   OPTIONS NOBOMFILE;
-  filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
-    name="_webout.json" lrecl=999999 mod;
+  %if "X&SYS_JES_JOB_URI.X"="XX" %then %do;
+     filename _webout temp lrecl=999999 mod;
+  %end;
+  %else %do;
+    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" 
+      name="_webout.json" lrecl=999999 mod; 
+  %end;
 
   /* setup temp ref */
   %if %upcase(&fref) ne _WEBOUT %then %do;
@@ -12087,13 +12353,13 @@ filename &fref1 clear;
       i+1;
       call symputx('wt'!!left(i),name);
       call symputx('wtcnt',i);
-    data _null_; file &fref; put ",""WORK"":{";
+    data _null_; file &fref mod; put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
       proc contents noprint data=&wt
         out=_data_ (keep=name type length format:);
       run;%let tempds=%scan(&syslast,2,.);
-      data _null_; file &fref;
+      data _null_; file &fref mod;
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
         nvars=attrn(dsid,'NVARS');
@@ -12104,9 +12370,9 @@ filename &fref1 clear;
         put ',"nvars":' nvars;
       %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
       %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
-      data _null_; file &fref;put "}";
+      data _null_; file &fref mod;put "}";
     %end;
-    data _null_; file &fref;put "}";run;
+    data _null_; file &fref mod;put "}";run;
   %end;
 
   /* close off json */

base/mf_getplatform.sas

@@ -12,6 +12,7 @@
 
   <h4> Dependencies </h4>
   @li mf_mval.sas
+  @li mf_trimstr.sas
 
   @version 9.4 / 3.4
   @author Allan Bowe
@@ -57,6 +58,6 @@
   %else 0;
 %end;
 %else %if &switch=VIYARESTAPI %then %do;
-  %sysfunc(getoption(servicesbaseurl))
+  %mf_trimstr(%sysfunc(getoption(servicesbaseurl)),/)
 %end;
 %mend;

base/mf_trimstr.sas

@@ -0,0 +1,39 @@
+/**
+  @file mf_trimstr.sas
+  @brief Removes character(s) from the end, if they exist
+  @details If the designated characters exist at the end of the string, they 
+  are removed
+
+        %put %mf_trimstr(/blah/,/); * /blah;
+        %put %mf_trimstr(/blah/,h); * /blah/;
+        %put %mf_trimstr(/blah/,h/);* /bla;
+
+  <h4> Dependencies </h4>
+
+
+  @param basestr The string to be modified
+  @param trimstr The string to be removed from the end of `basestr`, if it exists
+
+  @return output returns result with the value of `trimstr` removed from the end
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_trimstr(basestr,trimstr);
+%local trimlen trimval;
+%let trimlen=%length(%superq(trimstr));
+%let trimval=%qsubstr(%superq(basestr)
+  ,%length(%superq(basestr))-&trimlen+1
+  ,&trimlen);
+
+%if %superq(trimval)=%superq(trimstr) %then %do;
+  %qsubstr(%superq(basestr),1,%length(%superq(basestr))-&trimlen)
+%end;
+%else %do;
+  &basestr
+%end;
+
+%mend;

base/mp_abort.sas

@@ -5,6 +5,12 @@
     the particulars of an environment.  For instance, can stream custom
     results back to the client in an STP Web App context, or completely stop
     in the case of a batch run.
+  
+  Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored Process
+  environments.  This macro takes a unique approach - we set the SAS syscc to 0,
+  run `stpsrvset('program error', 0)` (if SAS 9) and then - we open a macro 
+  but don't close it!  This provides a graceful abort for SAS web services in all 
+  web enabled environments.
 
   @param mac= to contain the name of the calling macro
   @param msg= message to be returned

base/mp_getddl.sas

@@ -13,11 +13,13 @@
         label x='blah';
       run;
       proc sql; describe table &syslast;
-
       %mp_getddl(work,test,flavour=tsql,showlog=YES)
 
+  <h4> Dependencies </h4>
+  @li mp_getconstraints.sas
+
   @param lib libref of the library to create DDL for.  Should be assigned.
-  @param ds dataset to create ddl for
+  @param ds dataset to create ddl for (optional)
   @param fref= the fileref to which to write the DDL.  If not preassigned, will
     be assigned to TEMP.
   @param flavour= The type of DDL to create (default=SAS). Supported=TSQL
@@ -26,11 +28,9 @@
     ,else libref)
   @param applydttm= for non SAS DDL, choose if columns are created with native
    datetime2 format or regular decimal type
-
   @version 9.3
   @author Allan Bowe
   @source https://github.com/sasjs/core
-
 **/
 
 %macro mp_getddl(libref,ds,fref=getddl,flavour=SAS,showlog=NO,schema=
@@ -54,6 +54,21 @@ create table _data_ as
   ;
 %local tabinfo; %let tabinfo=&syslast;
 
+create table _data_ as
+  select * from dictionary.columns
+  where upcase(libname)="%upcase(&libref)"
+  %if %length(&ds)>0 %then %do;
+    and upcase(memname)="%upcase(&ds)"
+  %end;
+  ;
+%local colinfo; %let colinfo=&syslast;
+
+%local dsnlist;
+  select distinct upcase(memname) into: dsnlist
+  separated by ' '
+  from &syslast
+;
+
 create table _data_ as
   select * from dictionary.indexes
   where upcase(libname)="%upcase(&libref)"
@@ -64,18 +79,43 @@ create table _data_ as
   ;
 %local idxinfo; %let idxinfo=&syslast;
 
-create table _data_ as
-  select * from dictionary.columns
-  where upcase(libname)="%upcase(&libref)"
-  %if %length(&ds)>0 %then %do;
-    and upcase(memname)="%upcase(&ds)"
-  %end;
-  ;
-%local colinfo; %let colinfo=&syslast;
-%local dsnlist;
-select distinct upcase(memname) into: dsnlist
-  separated by ' '
-  from &syslast;
+/* Extract all Primary Key and Unique data constraints */
+%mp_getconstraints(lib=%upcase(&libref),ds=%upcase(&ds),outds=_data_)
+%local colconst; %let colconst=&syslast;
+
+%macro addConst();
+  %global constraints_used;
+  data _null_;
+    length ctype $11 constraint_name_orig $256 constraints_used $5000;
+    set &colconst (where=(table_name="&curds" and constraint_type in ('PRIMARY','UNIQUE'))) end=last;
+    file &fref mod;
+    by constraint_type constraint_name;
+    retain constraints_used;
+    constraint_name_orig=constraint_name;
+    if upcase(strip(constraint_type)) = 'PRIMARY' then ctype='PRIMARY KEY';
+    else ctype=strip(constraint_type);
+    %if &flavour=TSQL %then %do;
+      column_name=catt('[',column_name,']');
+      constraint_name=catt('[',constraint_name,']');
+    %end;
+    %else %if &flavour=PGSQL %then %do;
+      column_name=catt('"',column_name,'"');
+      constraint_name=catt('"',constraint_name,'"');
+    %end;
+    if first.constraint_name then do;
+      constraints_used = catx(' ', constraints_used, constraint_name_orig);
+      put "   ,CONSTRAINT " constraint_name ctype "(" ;
+      put '     ' column_name;
+    end;
+  else put '     ,' column_name;
+  if last.constraint_name then do;
+    put "   )";
+    call symput('constraints_used',strip(constraints_used));
+  end;
+  run;
+  %put &=constraints_used;
+%mend;
+
 data _null_;
   file &fref;
   put "/* DDL generated by &sysuserid on %sysfunc(datetime(),datetime19.) */";
@@ -84,7 +124,8 @@ run;
 %local x curds;
 %if &flavour=SAS %then %do;
   data _null_;
-    file &fref;
+    file &fref mod;
+    put "/* SAS Flavour DDL for %upcase(&libref).&curds */";
     put "proc sql;";
   run;
   %do x=1 %to %sysfunc(countw(&dsnlist));
@@ -109,23 +150,34 @@ run;
       lab=" label="!!quote(trim(label));
       if notnull='yes' then notnul=' not null';
       put name type len fmt notnul lab;
-      if last then put ');';
     run;
 
+    /* Extra step for data constraints */
+    %addConst()
+
     data _null_;
-      length ds $128;
-      set &idxinfo (where=(memname="&curds")) end=last;
+      file &fref mod;
+      put ');';
+    run;
+  
+    /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
+    data _null_;
+      *length ds $128;
+      set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
       file &fref mod;
       by idxusage indxname;
-      if unique='yes' then uniq=' unique';
-      ds=cats(libname,'.',memname);
+/*       ds=cats(libname,'.',memname); */
       if first.indxname then do;
-        put 'create ' uniq ' index ' indxname;
-        put '  on ' ds '(' name @@;
+          put 'CREATE UNIQUE INDEX ' indxname "ON &libref..&curds (" ;
+          put '  ' name ;
+      end;
+      else put '  ,' name ;
+      *else put '    ,' name ;
+      if last.indxname then do;
+        put ');';
       end;
-      else put ',' name @@;
-      if last.indxname then put ');';
     run;
+
 /*
     ods output IntegrityConstraints=ic;
     proc contents data=testali out2=info;
@@ -146,7 +198,7 @@ run;
     %let curds=%scan(&dsnlist,&x);
     data _null_;
       file &fref mod;
-      put "/* DDL for &schema..&curds */";
+      put "/* TSQL Flavour DDL for &schema..&curds */";
     data _null_;
       file &fref mod;
       set &colinfo (where=(upcase(memname)="&curds")) end=last;
@@ -169,31 +221,30 @@ run;
       else if length le 8000 then fmt='[varchar]('!!cats(length)!!')';
       else fmt=cats('[varchar](max)');
       if notnull='yes' then notnul=' NOT NULL';
-      put name fmt notnul;
+      put "[" name +(-1) "]" fmt notnul;
     run;
+
+    /* Extra step for data constraints */
+    %addConst()
+
+    /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
     data _null_;
-      length ds $128;
-      set &idxinfo (where=(memname="&curds"));
+      *length ds $128;
+      set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
       file &fref mod;
       by idxusage indxname;
-      if unique='yes' then uniq=' unique';
-      ds=cats(libname,'.',memname);
+      *ds=cats(libname,'.',memname);
       if first.indxname then do;
-        if unique='yes' and nomiss='yes' then do;
-          put '  ,constraint [' indxname '] PRIMARY KEY';
-        end;
-        else if unique='yes' then do;
-          /* add nonclustered in case of multiple unique indexes */
-          put '  ,index [' indxname '] UNIQUE NONCLUSTERED';
-        end;
-        put '  (';
-        put '    [' name ']';
+        /* add nonclustered in case of multiple unique indexes */
+        put '   ,index [' indxname +(-1) '] UNIQUE NONCLUSTERED (';
+        put '     [' name +(-1) ']';
       end;
-      else put '    ,[' name ']';
+      else put '     ,[' name +(-1) ']';
       if last.indxname then do;
-        put '  )';
+        put '   )';
       end;
     run;
+
     data _null_;
       file &fref mod;
       put ')';
@@ -217,7 +268,77 @@ run;
     run;
   %end;
 %end;
+%else %if &flavour=PGSQL %then %do;
+  /* if schema does not exist, set to be same as libref */
+  %local schemaactual;
+  proc sql noprint;
+  select sysvalue into: schemaactual
+    from dictionary.libnames
+    where libname="&libref" and engine='POSTGRES';
+  %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
+  data _null_;
+    file &fref mod;
+    put "CREATE SCHEMA &schema;";
+  %do x=1 %to %sysfunc(countw(&dsnlist));
+    %let curds=%scan(&dsnlist,&x);
+    data _null_;
+      file &fref mod;
+      put "/* Postgres Flavour DDL for &schema..&curds */";
+    data _null_;
+      file &fref mod;
+      set &colinfo (where=(upcase(memname)="&curds")) end=last;
+      length fmt $32;
+      if _n_=1 then do;
+        if memtype='DATA' then do;
+          put "CREATE TABLE &schema..&curds (";
+        end;
+        else do;
+          put "CREATE VIEW &schema..&curds (";
+        end;
+        put "    "@@;
+      end;
+      else put "   ,"@@;
+      format=upcase(format);
+      if 1=0 then; /* dummy if */
+      %if &applydttm=YES %then %do;
+        else if format=:'DATETIME' then fmt=' TIMESTAMP ';
+      %end;
+      else if type='num' then fmt=' DOUBLE PRECISION';
+      else fmt='VARCHAR('!!cats(length)!!')';
+      if notnull='yes' then notnul=' NOT NULL';
+      /* quote column names in case they represent reserved words */
+      name2=quote(trim(name));
+      put name2 fmt notnul;
+    run;
 
+    /* Extra step for data constraints */
+    %addConst()
+
+    data _null_;
+      file &fref mod;
+      put ');';
+    run;
+
+    /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
+    data _null_;
+      *length ds $128;
+      set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
+      file &fref mod;
+      by idxusage indxname;
+/*       ds=cats(libname,'.',memname); */
+      if first.indxname then do;
+          put 'CREATE UNIQUE INDEX "' indxname +(-1) '" ' "ON &schema..&curds (" ;
+          put '  "' name +(-1) '"' ;
+      end;
+      else put '  ,"' name +(-1) '"';
+      *else put '    ,' name ;
+      if last.indxname then do;
+        put ');';
+      end;
+    run;
+
+  %end;
+%end;
 %if &showlog=YES %then %do;
   options ps=max;
   data _null_;
@@ -227,4 +348,4 @@ run;
   run;
 %end;
 
-%mend;
+%mend;

base/mp_jsonout.sas

@@ -40,7 +40,9 @@
   * PROCJSON (default)
   * DATASTEP 
 
-  @param dbg= Typically used with an _debug (numeric) option
+  @param dbg= DEPRECATED - was used to conditionally add PRETTY to
+    proc json but this can cause line truncation in large files.
+    
 
   @version 9.2
   @author Allan Bowe
@@ -66,9 +68,8 @@
     proc sql;drop table &tempds;
     data &tempds /view=&tempds;set &ds; 
     %if &fmt=N %then format _numeric_ best32.;;
-    proc json out=&jref
+    proc json out=&jref pretty
         %if &action=ARR %then nokeys ;
-        %if &dbg ge 131  %then pretty ;
         ;export &tempds / nosastags fmtnumeric;
     run;
     proc sql;drop view &tempds;

base/mp_streamfile.sas

@@ -47,6 +47,7 @@
   %end;
 %end;
 %else %if &contentype=EXCEL %then %do;
+  /* suitable for XLS format */
   %if &platform=SASMETA %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/vnd.ms-excel');
@@ -59,6 +60,19 @@
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
+%else %if &contentype=XLSX %then %do;
+  %if &platform=SASMETA %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type','application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
+      rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
+    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
+      contenttype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' 
+      contentdisp="attachment; filename=&outname";
+  %end;
+%end;
 %else %if &contentype=TEXT %then %do;
   %if &platform=SASMETA %then %do;
     data _null_;

doxy/DoxygenLayout.xml

@@ -16,7 +16,7 @@
       <tab type="classmembers" visible="no" title="" intro=""/>
     </tab>
 
-    <tab type="filelist" visible="yes" title="" intro="List of Files Used in the Macro Core Library"/>
+    <tab type="filelist" visible="yes" title="" intro="List of Files Used in the Macro-Core Library"/>
 
     <tab type="examples" visible="yes" title="" intro=""/>
   </navindex>

doxy/build.sh

@@ -30,7 +30,7 @@ doxygen Doxyfile
 # refresh github pages site
 git clone git@github.com:sasjs/core.github.io.git
 cd core.github.io
-git rm -r *
+rm -r *
 mv $BUILD_FOLDER/out/doxy/* .
 echo 'core.sasjs.io' > CNAME
 git add *

meta/mm_adduser2group.sas

@@ -1,17 +1,18 @@
 /**
-  @file
+  @file mm_adduser2group.sas
   @brief Adds a user to a group
   @details Adds a user to a metadata group.  The macro first checks whether the
     user is in that group, and if not, the user is added.
 
   Usage:
 
-    %mm_adduser2group(user=sasdemo
-      ,group=someGroup)
+      %mm_adduser2group(user=sasdemo
+        ,group=someGroup)
 
 
   @param user= the user name (not displayname)
   @param group= the group to which to add the user
+  @param mdebug= set to 1 to show debug info in log
 
   @warning the macro does not check inherited group memberships - it looks at
     direct members only
@@ -68,7 +69,7 @@ run;
   %return;
 %end;
 
-%if %length(&syscc) ge 4 %then %do;
+%if &syscc ge 4 %then %do;
   %put WARNING:  SYSCC=&syscc, exiting &sysmacroname;
   %return;
 %end;

meta/mm_createwebservice.sas

@@ -104,9 +104,8 @@ data _null_;
   put '    proc sql;drop table &tempds; ';
   put '    data &tempds /view=&tempds;set &ds; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
-  put '    proc json out=&jref ';
+  put '    proc json out=&jref pretty ';
   put '        %if &action=ARR %then nokeys ; ';
-  put '        %if &dbg ge 131  %then pretty ; ';
   put '        ;export &tempds / nosastags fmtnumeric; ';
   put '    run; ';
   put '    proc sql;drop view &tempds; ';
@@ -201,7 +200,8 @@ data _null_;
   put '%end; ';
   put '%mend; ';
   put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y); ';
-  put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug; ';
+  put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
+  put '  sasjs_tables; ';
   put '%local i tempds; ';
   put ' ';
   put '%if &action=FETCH %then %do; ';
@@ -228,6 +228,7 @@ data _null_;
   put '        if _n_<20 then putlog _infile_; ';
   put '      %end; ';
   put '    run; ';
+  put '    %let sasjs_tables=&sasjs_tables &&_webin_name&i; ';
   put '  %end; ';
   put '%end; ';
   put ' ';

meta/mm_getdocument.sas

@@ -124,6 +124,7 @@ data _null_;
         when ('
') rec='0A'x;
         when ('
') rec='0D'x;
         when ('$' ) rec='$'  ;
+        when ('	') rec='09'x;
         otherwise putlog "WARNING: missing value for " entity=;
       end;
       rc =fput(fileid, substr(rec,1,1));

meta/mm_getstpcode.sas

@@ -123,6 +123,7 @@ data _null_;
         when ('
') rec='0A'x;
         when ('
') rec='0D'x;
         when ('$' ) rec='$'  ;
+        when ('	') rec='09'x;
         otherwise putlog "%str(WARN)ING: missing value for " entity=;
       end;
       rc =fput(fileid, substr(rec,1,1));

meta/mm_gettableid.sas

@@ -0,0 +1,67 @@
+/**
+  @file mm_gettableid.sas
+  @brief Get the metadata id for a particular table
+  @details Provide a libref and table name to return the corresponding metadata id
+  in an output datataset.
+
+  Usage:
+
+      - get a table id
+      %mm_gettableid(libref=METALIB,ds=SOMETABLE,outds=iwant)
+
+  @param libref= The libref to search
+  @param ds= The input dataset to check
+  @param outds= the dataset to create that contains the `tableuri`
+  @param mDebug= set to 1 to show debug messages in the log
+
+  @returns outds  dataset containing `tableuri` and `tablename`
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mm_gettableid(
+     libref=
+    ,ds=
+    ,outds=work.mm_gettableid
+    ,mDebug=0
+)/*/STORE SOURCE*/;
+
+%local mD;
+%if &mDebug=1 %then %let mD=;
+%else %let mD=%str(*);
+%&mD.put Executing &sysmacroname..sas;
+%&mD.put _local_;
+
+data &outds;
+  length uri usingpkguri id type tableuri tablename tmpuri $256;
+  call missing(of _all_);
+  keep tableuri tablename;
+  n=1;
+  rc=0;
+  if metadata_getnobj("omsobj:SASLibrary?@Libref='&libref'",n,uri)<1 then do;
+    put "Library &libref not found";
+    stop;
+  end;
+  &mD.putlog "uri is " uri;
+  if metadata_getnasn(uri, "UsingPackages", 1, usingpkguri)>0 then do;
+    rc=metadata_resolve(usingpkguri,type,id);
+    &mD.putlog "Type is " type;
+  end;
+
+  if type='DatabaseSchema' then tmpuri=usingpkguri;
+  else tmpuri=uri;
+  
+  t=1;
+  do while(metadata_getnasn(tmpuri, "Tables", t, tableuri)>0);
+    t+1;
+    rc= metadata_getattr(tableuri, "Name", tablename);
+    &mD.putlog "Table is " tablename;
+    if upcase(tablename)="%upcase(&ds)" then do;
+      output;
+    end;
+  end;
+run;
+
+%mend;

meta/mm_getwebappsrvprops.sas

@@ -99,6 +99,7 @@ run;
           when ('
') rec='0A'x;
           when ('
') rec='0D'x;
           when ('$' ) rec='$'  ;
+          when ('	') rec='09'x;
           otherwise putlog "WARNING: missing value for " entity=;
         end;
         rc =fput(fileid, substr(rec,1,1));

meta/mm_webout.sas

@@ -33,7 +33,8 @@
 
 **/
 %macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y);
-%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug;
+%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug 
+  sasjs_tables;
 %local i tempds;
 
 %if &action=FETCH %then %do;
@@ -60,6 +61,7 @@
         if _n_<20 then putlog _infile_;
       %end;
     run;
+    %let sasjs_tables=&sasjs_tables &&_webin_name&i;
   %end;
 %end;
 

viya/mv_createfolder.sas

@@ -96,7 +96,7 @@ options noquotelenmax;
     data _null_;
       set &libref1..links;
       if rel='createChild' then
-        call symputx('href',quote(trim(href)),'l');
+        call symputx('href',quote("&base_uri"!!trim(href)),'l');
     run;
   %end;
   %else %if &SYS_PROCHTTP_STATUS_CODE=404 %then %do;

viya/mv_createwebservice.sas

@@ -143,7 +143,7 @@ libname &libref1 JSON fileref=&fname1;
 
 data _null_;
   set &libref1..links;
-  if rel='members' then call symputx('membercheck',quote(trim(href)),'l');
+  if rel='members' then call symputx('membercheck',quote("&base_uri"!!trim(href)),'l');
   else if rel='self' then call symputx('parentFolderUri',href,'l');
 run;
 data _null_;
@@ -241,9 +241,8 @@ data _null_;
   put '    proc sql;drop table &tempds; ';
   put '    data &tempds /view=&tempds;set &ds; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
-  put '    proc json out=&jref ';
+  put '    proc json out=&jref pretty ';
   put '        %if &action=ARR %then nokeys ; ';
-  put '        %if &dbg ge 131  %then pretty ; ';
   put '        ;export &tempds / nosastags fmtnumeric; ';
   put '    run; ';
   put '    proc sql;drop view &tempds; ';
@@ -338,7 +337,8 @@ data _null_;
   put '%end; ';
   put '%mend; ';
   put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y); ';
-  put '%global _webin_file_count _webin_fileuri _debug _omittextlog ; ';
+  put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
+  put '  sasjs_tables SYS_JES_JOB_URI; ';
   put '%if %index("&_debug",log) %then %let _debug=131; ';
   put ' ';
   put '%local i tempds; ';
@@ -352,11 +352,11 @@ data _null_;
   put '  %if not %symexist(_webin_fileuri1) %then %do; ';
   put '    %let _webin_file_count=%eval(&_webin_file_count+0); ';
   put '    %let _webin_fileuri1=&_webin_fileuri; ';
+  put '    %let _webin_name1=&_webin_name; ';
   put '  %end; ';
   put ' ';
-  put '  %if %symexist(sasjs_tables) %then %do; ';
-  put '    /* small volumes of non-special data are sent as params for responsiveness */ ';
-  put '    /* to do - deal with escaped values  */ ';
+  put '  /* if the sasjs_tables param is passed, we expect param based upload */ ';
+  put '  %if %length(&sasjs_tables.XX)>2 %then %do; ';
   put '    filename _sasjs "%sysfunc(pathname(work))/sasjs.lua"; ';
   put '    data _null_; ';
   put '      file _sasjs; ';
@@ -435,13 +435,19 @@ data _null_;
   put '      infile indata firstobs=2 dsd termstr=crlf ; ';
   put '      input &input_statement; ';
   put '    run; ';
+  put '    %let sasjs_tables=&sasjs_tables &&_webin_name&i; ';
   put '  %end; ';
   put '%end; ';
   put '%else %if &action=OPEN %then %do; ';
   put '  /* setup webout */ ';
   put '  OPTIONS NOBOMFILE; ';
-  put '  filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" ';
-  put '    name="_webout.json" lrecl=999999 mod; ';
+  put '  %if "X&SYS_JES_JOB_URI.X"="XX" %then %do; ';
+  put '     filename _webout temp lrecl=999999 mod; ';
+  put '  %end; ';
+  put '  %else %do; ';
+  put '    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" ';
+  put '      name="_webout.json" lrecl=999999 mod; ';
+  put '  %end; ';
   put ' ';
   put '  /* setup temp ref */ ';
   put '  %if %upcase(&fref) ne _WEBOUT %then %do; ';
@@ -471,13 +477,13 @@ data _null_;
   put '      i+1; ';
   put '      call symputx(''wt''!!left(i),name); ';
   put '      call symputx(''wtcnt'',i); ';
-  put '    data _null_; file &fref; put ",""WORK"":{"; ';
+  put '    data _null_; file &fref mod; put ",""WORK"":{"; ';
   put '    %do i=1 %to &wtcnt; ';
   put '      %let wt=&&wt&i; ';
   put '      proc contents noprint data=&wt ';
   put '        out=_data_ (keep=name type length format:); ';
   put '      run;%let tempds=%scan(&syslast,2,.); ';
-  put '      data _null_; file &fref; ';
+  put '      data _null_; file &fref mod; ';
   put '        dsid=open("WORK.&wt",''is''); ';
   put '        nlobs=attrn(dsid,''NLOBS''); ';
   put '        nvars=attrn(dsid,''NVARS''); ';
@@ -488,9 +494,9 @@ data _null_;
   put '        put '',"nvars":'' nvars; ';
   put '      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP) ';
   put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP) ';
-  put '      data _null_; file &fref;put "}"; ';
+  put '      data _null_; file &fref mod;put "}"; ';
   put '    %end; ';
-  put '    data _null_; file &fref;put "}";run; ';
+  put '    data _null_; file &fref mod;put "}";run; ';
   put '  %end; ';
   put ' ';
   put '  /* close off json */ ';
@@ -603,7 +609,7 @@ proc http method='POST'
     in=&fname3
     out=&fname4
     &oauth_bearer
-    url="/jobDefinitions/definitions?parentFolderUri=&parentFolderUri";
+    url="&base_uri/jobDefinitions/definitions?parentFolderUri=&parentFolderUri";
     headers 'Content-Type'='application/vnd.sas.job.definition+json'
   %if &grant_type=authorization_code %then %do;
             "Authorization"="Bearer &&&access_token_var"

viya/mv_deletefoldermember.sas

@@ -98,7 +98,7 @@ run;
 libname &libref1 JSON fileref=&fname1;
 data _null_;
   set &libref1..links;
-  if rel='members' then call symputx('mref',quote(trim(href)),'l');
+  if rel='members' then call symputx('mref',quote("&base_uri"!!trim(href)),'l');
 run;
 
 /* get the children */
@@ -128,7 +128,7 @@ run;
   %put NOTE:;%put NOTE- &sysmacroname: &path/&name NOT FOUND;%put NOTE- ;
   %return;
 %end;
-proc http method="DELETE" url="&uri" &oauth_bearer;
+proc http method="DELETE" url="&base_uri&uri" &oauth_bearer;
   headers 
 %if &grant_type=authorization_code %then %do;
       "Authorization"="Bearer &&&access_token_var"

viya/mv_deletejes.sas

@@ -95,7 +95,7 @@ run;
 libname &libref1 JSON fileref=&fname1;
 data _null_;
   set &libref1..links;
-  if rel='members' then call symputx('mref',quote(trim(href)),'l');
+  if rel='members' then call symputx('mref',quote("&base_uri"!!trim(href)),'l');
 run;
 
 /* get the children */
@@ -117,7 +117,7 @@ libname &libref1a JSON fileref=&fname1a;
 data _null_;
   set &libref1a..items;
   if contenttype='jobDefinition' and upcase(name)="%upcase(&name)" then do;
-    call symputx('uri',uri,'l');
+    call symputx('uri',cats("&base_uri",uri),'l');
     call symputx('found',1,'l');
   end;
 run;

viya/mv_deleteviyafolder.sas

@@ -88,9 +88,9 @@ libname &libref1 JSON fileref=&fname1;
 data _null_;
   set &libref1..links;
   if rel='deleteRecursively' then
-    call symputx('href',quote(trim(href)),'l');
+    call symputx('href',quote("&base_uri"!!trim(href)),'l');
   else if rel='members' then
-    call symputx('mref',quote(cats(href,'?recursive=true')),'l');
+    call symputx('mref',quote(cats("&base_uri",href,'?recursive=true')),'l');
 run;
 
 /* before we can delete the folder, we need to delete the children */
@@ -111,7 +111,7 @@ data _null_;
   set &libref1a..items_links;
   if href=:'/folders/folders' then return;
   if rel='deleteResource' then
-    call execute('proc http method="DELETE" url='!!quote(trim(href))
+    call execute('proc http method="DELETE" url='!!quote("&base_uri"!!trim(href))
     !!'; headers "Authorization"="Bearer &&&access_token_var" '
     !!' "Accept"="*/*";run; /**/');
 run;

viya/mv_getfoldermembers.sas

@@ -64,7 +64,7 @@ options noquotelenmax;
 %if "&root"="/" %then %do;
   /* if root just list root folders */
   proc http method='GET' out=&fname1 &oauth_bearer
-      url='%sysfunc(getoption(servicesbaseurl))/folders/rootFolders';
+      url="&base_uri/folders/rootFolders";
   %if &grant_type=authorization_code %then %do;
       headers "Authorization"="Bearer &&&access_token_var";
   %end;
@@ -87,7 +87,7 @@ options noquotelenmax;
   /* now get the followon link to list members */
   data _null_;
     set &libref1..links;
-    if rel='members' then call symputx('href',quote(trim(href)),'l');
+    if rel='members' then call symputx('href',quote("&base_uri"!!trim(href)),'l');
   run;
   %local fname2 libref2;
   %let fname2=%mf_getuniquefileref();

viya/mv_webout.sas

@@ -36,7 +36,8 @@
 
 **/
 %macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y);
-%global _webin_file_count _webin_fileuri _debug _omittextlog ;
+%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name
+  sasjs_tables SYS_JES_JOB_URI;
 %if %index("&_debug",log) %then %let _debug=131; 
 
 %local i tempds;
@@ -50,11 +51,11 @@
   %if not %symexist(_webin_fileuri1) %then %do;
     %let _webin_file_count=%eval(&_webin_file_count+0);
     %let _webin_fileuri1=&_webin_fileuri;
+    %let _webin_name1=&_webin_name;
   %end;
 
-  %if %symexist(sasjs_tables) %then %do;
-    /* small volumes of non-special data are sent as params for responsiveness */
-    /* to do - deal with escaped values  */
+  /* if the sasjs_tables param is passed, we expect param based upload */
+  %if %length(&sasjs_tables.XX)>2 %then %do;
     filename _sasjs "%sysfunc(pathname(work))/sasjs.lua";
     data _null_;
       file _sasjs;
@@ -133,13 +134,19 @@
       infile indata firstobs=2 dsd termstr=crlf ;
       input &input_statement;
     run;
+    %let sasjs_tables=&sasjs_tables &&_webin_name&i;
   %end;
 %end;
 %else %if &action=OPEN %then %do;
   /* setup webout */
   OPTIONS NOBOMFILE;
-  filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
-    name="_webout.json" lrecl=999999 mod;
+  %if "X&SYS_JES_JOB_URI.X"="XX" %then %do;
+     filename _webout temp lrecl=999999 mod;
+  %end;
+  %else %do;
+    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" 
+      name="_webout.json" lrecl=999999 mod; 
+  %end;
 
   /* setup temp ref */
   %if %upcase(&fref) ne _WEBOUT %then %do;
@@ -169,13 +176,13 @@
       i+1;
       call symputx('wt'!!left(i),name);
       call symputx('wtcnt',i);
-    data _null_; file &fref; put ",""WORK"":{";
+    data _null_; file &fref mod; put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
       proc contents noprint data=&wt
         out=_data_ (keep=name type length format:);
       run;%let tempds=%scan(&syslast,2,.);
-      data _null_; file &fref;
+      data _null_; file &fref mod;
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
         nvars=attrn(dsid,'NVARS');
@@ -186,9 +193,9 @@
         put ',"nvars":' nvars;
       %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
       %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
-      data _null_; file &fref;put "}";
+      data _null_; file &fref mod;put "}";
     %end;
-    data _null_; file &fref;put "}";run;
+    data _null_; file &fref mod;put "}";run;
   %end;
 
   /* close off json */