Merge pull request #220 from sasjs/perf

fix: using findc instead of regex for faster parsing

.git-hooks/pre-commit

@@ -1,5 +1,12 @@
 #!/bin/bash
-sasjs lint
+
+# Ensure lint is passing
+LINT=`sasjs lint`
+if [[ "$LINT" != "✔ All matched files use @sasjs/lint code style!" ]]; then
+  echo "$LINT"
+  echo "To commit in spite of these warnings, use the -n parameter."
+  exit 1
+fi
 
 # Avoid commits to the master branch
 BRANCH=`git rev-parse --abbrev-ref HEAD`

.gitpod.yml

@@ -1,7 +1,7 @@
 tasks:
-  - init: |
-      nvm install --lts
-      npm i -g @sasjs/cli
+  - init: npm install -g npm
+  - command: npm i
+  - command: npm i -g @sasjs/cli
 
 image:
   file: .gitpod.dockerfile
@@ -24,4 +24,4 @@ github:
     # add a "Review in Gitpod" button to pull requests (defaults to false)
     addBadge: false
     # add a label once the prebuild is ready to pull requests (defaults to false)
-    addLabel: prebuilt-in-gitpod
+    addLabel: prebuilt-in-gitpod

README.md

@@ -217,6 +217,15 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 ![](https://starchart.cc/sasjs/core.svg)
 
+## Other SAS Repositories
+
+The following repositories are also worth checking out:
+
+* [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
+* [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)
+* [KatjaGlassConsulting/SMILE-SmartSASMacros](https://github.com/KatjaGlassConsulting/SMILE-SmartSASMacros)
+* [scottbass/sas](https://github.com/scottbass/SAS)
+* [yabwon/sas_packages](https://github.com/yabwon/SAS_PACKAGES)
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->

base/mf_deletefile.sas

@@ -0,0 +1,31 @@
+/**
+  @file
+  @brief Deletes a physical file, if it exists
+  @details Usage:
+
+      %mf_writefile(&sasjswork/myfile.txt,l1=some content)
+
+      %mf_deletefile(&sasjswork/myfile.txt)
+
+      %mf_deletefile(&sasjswork/myfile.txt)
+
+
+  @param filepath Full path to the target file
+
+  @returns The return code from the fdelete() invocation
+
+  <h4> Related Macros </h4>
+  @li mf_deletefile.test.sas
+  @li mf_writefile.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_deletefile(file
+)/*/STORE SOURCE*/;
+  %local rc fref;
+  %let rc= %sysfunc(filename(fref,&file));
+  %if %sysfunc(fdelete(&fref)) ne 0 %then %put %sysfunc(sysmsg());
+  %let rc= %sysfunc(filename(fref));
+%mend mf_deletefile;

base/mf_existfeature.sas

@@ -9,19 +9,17 @@
 
         %put %mf_existfeature(PROCLUA);
 
-  @param feature the feature to detect.  Leave blank to list all in log.
+  @param [in] feature The feature to detect.
 
   @return output returns 1 or 0 (or -1 if not found)
 
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
 
-
   @version 8
   @author Allan Bowe
 **/
 /** @cond */
-
 %macro mf_existfeature(feature
 )/*/STORE SOURCE*/;
   %let feature=%upcase(&feature);
@@ -29,7 +27,11 @@
   %let platform=%mf_getplatform();
 
   %if &feature= %then %do;
-    %put Supported features:  PROCLUA;
+    %put No feature was requested for detection;
+  %end;
+  %else %if &feature=COLCONSTRAINTS %then %do;
+    %if %substr(&sysver,1,1)=4 %then 0;
+    %else 1;
   %end;
   %else %if &feature=PROCLUA %then %do;
     /* https://blogs.sas.com/content/sasdummy/2015/08/03/using-lua-within-your-sas-programs */
@@ -43,5 +45,4 @@
     %put &sysmacroname: &feature not found;
   %end;
 %mend mf_existfeature;
-
-/** @endcond */
+/** @endcond */

base/mf_getuser.sas

@@ -33,7 +33,8 @@
   %else %if %symexist(&metavar) %then %do;
     %if %length(&&&metavar)=0 %then %let user=&sysuserid;
     /* sometimes SAS will add @domain extension - remove for consistency */
-    %else %let user=%scan(&&&metavar,1,@);
+    /* but be sure to quote in case of usernames with commas */
+    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@));
   %end;
   %else %let user=&sysuserid;
 

base/mf_verifymacvars.sas

@@ -59,7 +59,7 @@
 
   %goto exit_success;
   %exit_err:
-    %put %str(ERR)OR: &abortmsg;
+    %put &abortmsg;
     %mf_abort(iftrue=(&mabort ne SOFT),
       mac=mf_verifymacvars,
       msg=%str(&abortmsg)

base/mp_abort.sas

@@ -85,7 +85,7 @@
   %end;
 
   /* Stored Process Server web app context */
-  %if %symexist(_metaperson)
+  %if %symexist(_METAFOLDER)
     or "&SYSPROCESSNAME "="Compute Server "
     or &mode=INCLUDE
   %then %do;
@@ -161,12 +161,14 @@
     /* send response in SASjs JSON format */
     data _null_;
       file _webout mod lrecl=32000 encoding='utf-8';
-      length msg $32767 ;
+      length msg syswarningtext syserrortext $32767 ;
       sasdatetime=datetime();
       msg=symget('msg');
     %if &logline>0 %then %do;
       msg=cats(msg,'\n\nLog Extract:\n',symget('logmsg'));
     %end;
+      /* escape the escapes */
+      msg=tranwrd(msg,'\','\\');
       /* escape the quotes */
       msg=tranwrd(msg,'"','\"');
       /* ditch the CRLFs as chrome complains */
@@ -260,4 +262,4 @@
   %end;
 %mend mp_abort;
 
-/** @endcond */
+/** @endcond */

base/mp_assertscope.sas

@@ -106,7 +106,7 @@
 
   proc compare
     base=&scopeds(where=(upcase(name) not in (%mf_getquotedstr(&ilist))))
-    compare=&ds;
+    compare=&ds noprint;
   run;
 
   %if &sysinfo=0 %then %do;

base/mp_dirlist.sas

@@ -94,6 +94,13 @@ data &out_ds(compress=no
   %end;
   if rc = 0 then do;
     did = dopen(fref);
+    if did=0 then do;
+      putlog "NOTE: This directory is empty, or does not exist - &path";
+      msg=sysmsg();
+      put msg;
+      put _all_;
+      stop;
+    end;
     /* attribute is OS-dependent - could be "Directory" or "Directory Name" */
     numopts=doptnum(did);
     do i=1 to numopts;
@@ -101,12 +108,6 @@ data &out_ds(compress=no
       if foption=:'Directory' then i=numopts;
     end;
     directory=dinfo(did,foption);
-    if did=0 then do;
-      putlog "NOTE: This directory is empty - " directory;
-      msg=sysmsg();
-      put _all_;
-      stop;
-    end;
     rc = filename(fref);
   end;
   else do;
@@ -151,6 +152,7 @@ run;
   data &out_ds;
     set &out_ds;
     length infoname infoval $60 fref $8;
+    if _n_=1 then call missing(fref);
     rc=filename(fref,filepath);
     drop rc infoname fid i close fref;
     if file_or_folder='file' then do;
@@ -242,4 +244,4 @@ run;
 proc sql;
 drop table &out_ds;
 
-%mend mp_dirlist;
+%mend mp_dirlist;

base/mp_ds2cards.sas

@@ -48,6 +48,7 @@
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
 %macro mp_ds2cards(base_ds=, tgt_ds=
@@ -219,7 +220,8 @@ data _null_;
     put '  @file';
     put "  @brief Datalines for %upcase(%scan(&base_ds,2)) dataset";
     put "  @details Generated by %nrstr(%%)mp_ds2cards()";
-    put "  Available on github.com/sasjs/core";
+    put "    Source: https://github.com/sasjs/core";
+    put '  @cond ';
     put '**/';
     put "data &tgt_ds &indexes;";
     put "attrib ";
@@ -264,6 +266,7 @@ data _null_;
   if __lastobs then do;
     put ';;;;';
     put 'run;';
+    put '/** @endcond **/';
     stop;
   end;
 run;
@@ -283,4 +286,5 @@ quit;
 %put NOTE-;%put NOTE-;
 %put NOTE- %sysfunc(dequote(&cards_file.));
 %put NOTE-;%put NOTE-;
-%mend mp_ds2cards;
+%mend mp_ds2cards;
+/** @endcond **/

base/mp_getpk.sas

@@ -55,7 +55,8 @@
 )/*/STORE SOURCE*/;
 
 
-%local engine schema ds1 ds2 ds3 dsn tabs1 tabs2 sum pk4sure pkdefault finalpks;
+%local engine schema ds1 ds2 ds3 dsn tabs1 tabs2 sum pk4sure pkdefault finalpks
+  pkfromindex;
 
 %let lib=%upcase(&lib);
 %let ds=%upcase(&ds);
@@ -70,6 +71,7 @@
 %let sum=%mf_getuniquename(prefix=getpk_sum);
 %let pk4sure=%mf_getuniquename(prefix=getpk_pk4sure);
 %let pkdefault=%mf_getuniquename(prefix=getpk_pkdefault);
+%let pkfromindex=%mf_getuniquename(prefix=getpk_pkfromindex);
 %let finalpks=%mf_getuniquename(prefix=getpk_finalpks);
 
 %local dbg;
@@ -180,9 +182,23 @@ create table &ds1 as
       and a.constraint_name=b.constraint_name
     order by 1,2,3,4;
 
+  /* extract cols from the relevant unique INDEXES */
+  create table &pkfromindex as
+    select libname as libref
+      ,memname as table_name
+      ,indxname as constraint_name
+      ,indxpos as constraint_order
+      ,name
+    from dictionary.indexes
+    where nomiss='yes' and unique='yes' and upcase(libname)="&lib"
+  %if &ds ne 0 %then %do;
+      and upcase(memname)="&ds"
+  %end;
+    order by 1,2,3,4;
+
   /* create one table */
   data &finalpks;
-    set &pkdefault &pk4sure ;
+    set &pkdefault &pk4sure &pkfromindex;
     pk_ind=1;
     /* if there are multiple unique constraints, take the first */
     by libref table_name constraint_name;
@@ -256,4 +272,4 @@ create table &outds as
   iftrue=(&mdebug=0)
 )
 
-%mend mp_getpk;
+%mend mp_getpk;

base/mp_gsubfile.sas

@@ -48,6 +48,11 @@
   outfile=0
 )/*/STORE SOURCE*/;
 
+  %if "%substr(&sysver,1,4)"="V.04" %then %do;
+    %put %str(ERR)OR: Viya 4 does not support the IO library in lua;
+    %return;
+  %end;
+
   %ml_gsubfile()
 
 %mend mp_gsubfile;

base/mp_hashdataset.sas

@@ -1,8 +1,8 @@
 /**
   @file
   @brief Returns a unique hash for a dataset
-  @details Ignores metadata attributes, used only to hash values. Compared
-  datasets must be in the same order.
+  @details Ignores metadata attributes, used only to hash values. If used to
+  compare datasets, they must have their columns and rows in the same order.
 
       %mp_hashdataset(sashelp.class,outds=myhash)
 
@@ -17,13 +17,16 @@
   @li mf_getattrn.sas
   @li mf_getuniquename.sas
   @li mf_getvarlist.sas
-  @li mf_getvartype.sas
+  @li mp_md5.sas
+
+  <h4> Related Files </h4>
+  @li mp_hashdataset.test.sas
 
   @param [in] libds dataset to hash
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
   @param [in] iftrue= A condition under which the macro should be executed.
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
-  will contain one column (hashkey) with one observation (a hex32.
+  will contain one column (hashkey) with one observation (a $hex32.
   representation of the input hash)
   |hashkey:$32.|
   |---|
@@ -35,48 +38,48 @@
 
 %macro mp_hashdataset(
   libds,
-  outds=,
+  outds=work._data_,
   salt=,
   iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
-  %if not(%eval(%unquote(&iftrue))) %then %return;
+%local keyvar /* roll up the md5 */
+  prevkeyvar /* retain prev record md5 */
+  lastvar /* last var in input ds */
+  cvars nvars;
 
-  %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
-    %put %str(WARN)ING: Dataset &libds is empty, or is not a dataset;
-  %end;
-  %else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
-    %put %str(ERR)OR: Dataset &libds is not a dataset;
-  %end;
-  %else %do;
-    %local keyvar /* roll up the md5 */
-      prevkeyvar /* retain prev record md5 */
-      lastvar /* last var in input ds */
-      varlist var i;
-    /* avoid naming conflict for hash key vars */
-    %let keyvar=%mf_getuniquename();
-    %let prevkeyvar=%mf_getuniquename();
-    %let lastvar=%mf_getuniquename();
-    %let varlist=%mf_getvarlist(&libds);
-    data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
-      length &prevkeyvar &keyvar $32;
-      retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
-      set &libds end=&lastvar;
-      /* hash should include previous row */
-      &keyvar=put(md5(&prevkeyvar
-      /* loop every column, hashing every individual value */
-    %do i=1 %to %sysfunc(countw(&varlist));
-      %let var=%scan(&varlist,&i,%str( ));
-      %if %mf_getvartype(&libds,&var)=C %then %do;
-          !!put(md5(trim(&var)),$hex32.)
-      %end;
-      %else %do;
-          !!put(md5(trim(put(&var*1,binary64.))),$hex32.)
-      %end;
-    %end;
-      ),$hex32.);
-      &prevkeyvar=&keyvar;
-      if &lastvar then output;
-    run;
-  %end;
-%mend mp_hashdataset;
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+/* avoid naming conflict for hash key vars */
+%let keyvar=%mf_getuniquename();
+%let prevkeyvar=%mf_getuniquename();
+%let lastvar=%mf_getuniquename();
+
+%if %mf_getattrn(&libds,NLOBS)=0 %then %do;
+  data &outds;
+    length hashkey $32;
+    retain hashkey "%sysfunc(md5(%str(&salt)),$hex32.)";
+    output;
+    stop;
+  run;
+  %put &sysmacroname: Dataset &libds is empty, or is not a dataset;
+  %put &sysmacroname: hashkey of &outds is based on salt (&salt) only;
+%end;
+%else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
+  %put %str(ERR)OR: Dataset &libds is not a dataset;
+%end;
+%else %do;
+  data &outds(rename=(&keyvar=hashkey) keep=&keyvar)/nonote2err;
+    length &prevkeyvar &keyvar $32;
+    retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
+    set &libds end=&lastvar;
+    /* hash should include previous row */
+    &keyvar=%mp_md5(
+      cvars=%mf_getvarlist(&libds,typefilter=C) &prevkeyvar,
+      nvars=%mf_getvarlist(&libds,typefilter=N)
+    );
+    &prevkeyvar=&keyvar;
+    if &lastvar then output;
+  run;
+%end;
+%mend mp_hashdataset;

base/mp_init.sas

@@ -73,4 +73,4 @@ options
 %end;
 ;
 
-%mend mp_init;
+%mend mp_init;

base/mp_jsonout.sas

@@ -199,12 +199,23 @@
       format _numeric_ bart.;
     %do i=1 %to &numcols;
       %if &&typelong&i=char or &fmt=Y %then %do;
-        &&name&i='"'!!trim(prxchange('s/"/\"/',-1,
-                    prxchange('s/'!!'0A'x!!'/\n/',-1,
-                    prxchange('s/'!!'0D'x!!'/\r/',-1,
-                    prxchange('s/'!!'09'x!!'/\t/',-1,
-                    prxchange('s/\\/\\\\/',-1,&&name&i)
-        )))))!!'"';
+        if findc(&&name&i,'"\'!!'0A0D09000E0F01021011'x) then do;
+          &&name&i='"'!!trim(
+            prxchange('s/"/\\"/',-1,        /* double quote */
+            prxchange('s/\x0A/\n/',-1,      /* new line */
+            prxchange('s/\x0D/\r/',-1,      /* carriage return */
+            prxchange('s/\x09/\\t/',-1,     /* tab */
+            prxchange('s/\x00/\\u0000/',-1, /* NUL */
+            prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+            prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+            prxchange('s/\x01/\\u0001/',-1, /* SOH */
+            prxchange('s/\x02/\\u0002/',-1, /* STX */
+            prxchange('s/\x10/\\u0010/',-1, /* DLE */
+            prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+            prxchange('s/\\/\\\\/',-1,&&name&i)
+          ))))))))))))!!'"';
+        end;
+        else &&name&i=quote(cats(&&name&i));
       %end;
     %end;
     run;

base/mp_loadformat.sas

@@ -40,13 +40,13 @@
   @li mp_abort.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
+  @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
   @li mddl_dc_difftable.sas
   @li mddl_dc_locktable.sas
   @li mp_loadformat.test.sas
   @li mp_lockanytable.sas
-  @li mp_storediffs.sas
   @li mp_stackdiffs.sas
 
 
@@ -134,7 +134,7 @@ run;
   * First, extract only relevant formats from the catalog
   */
 proc sql noprint;
-select distinct fmtname into: fmtlist separated by ' ' from &libds;
+select distinct upcase(fmtname) into: fmtlist separated by ' ' from &libds;
 
 %mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
 
@@ -144,8 +144,11 @@ select distinct fmtname into: fmtlist separated by ' ' from &libds;
   */
 %mddl_sas_cntlout(libds=&template)
 data &inlibds;
+  length &delete_col $3;
   if 0 then set &template;
   set &libds;
+  if &delete_col='' then &delete_col='No';
+  fmtname=upcase(fmtname);
   if missing(type) then do;
     if substr(fmtname,1,1)='$' then type='C';
     else type='N';
@@ -156,7 +159,6 @@ data &inlibds;
   end;
 run;
 
-
 /**
   * Identify new records
   */
@@ -263,7 +265,7 @@ options ibufsize=&ibufsize;
     %end;
 
     %mp_storediffs(&libcat-FC
-      ,&inlibds
+      ,&base_fmts
       ,FMTNAME START
       ,delds=&outds_del
       ,modds=&outds_mod
@@ -272,6 +274,9 @@ options ibufsize=&ibufsize;
       ,mdebug=&mdebug
     )
 
+    proc append base=&auditlibds data=&storediffs;
+    run;
+
     %if &locklibds ne 0 %then %do;
       %mp_lockanytable(UNLOCK
         ,lib=%scan(&auditlibds,1,.)
@@ -295,4 +300,4 @@ options ibufsize=&ibufsize;
   %put &sysmacroname exit vars:;
   %put _local_;
 %end;
-%mend mp_loadformat;
+%mend mp_loadformat;

base/mp_md5.sas

@@ -54,5 +54,5 @@ put(md5(
     &sep put(md5(trim(put(ifn(missing(&var),&var,&var*1),binary64.))),$hex32.)
     %let sep=!!;
   %end;
-),hex32.)
+),$hex32.)
 %mend mp_md5;

base/mp_replace.sas

@@ -70,7 +70,7 @@ filename &inref &infile lrecl=1 recfm=n;
 
 data &ds1;
   infile &inref;
-  input sourcechar $ 1. @@;
+  input sourcechar $char1. @@;
   format sourcechar hex2.;
 run;
 

base/mp_streamfile.sas

@@ -105,12 +105,20 @@ run;
   %end;
 %end;
 %else %if &contentype=HTML %then %do;
-  %if &platform=SASVIYA %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type','text/html');
+      rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
     filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
-      contenttype="text/html";
+      contenttype="text/html"
+      contentdisp="attachment; filename=&outname";
   %end;
   %else %if &platform=SASJS %then %do;
     %mfs_httpheader(Content-type,text/html)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
   %end;
 %end;
 %else %if &contentype=TEXT %then %do;

fcmp/mcf_stpsrv_header.sas

@@ -1,110 +0,0 @@
-/**
-  @file
-  @brief Provides a replacement for the stpsrv_header function
-  @details The stpsrv_header is normally a built-in function, used to set the
-  headers for SAS 9 Stored Processes as documented here:
-  https://go.documentation.sas.com/doc/en/itechcdc/9.4/stpug/srvhead.htm
-
-  The purpose of this custom function is to provide a replacement when running
-  similar code as a web service against
-  [sasjs/server](https://github.com/sasjs/server).  It operates by creating a
-  text file with the headers.  The location of this text file is determined by
-  a macro variable (`sasjs_stpsrv_header_loc`) which needs to be injected into
-  each service by the calling process, eg:
-
-      %let sasjs_stpsrv_header_loc = C:/temp/some_uuid/stpsrv_header.txt;
-
-  Note - the function works by appending headers to the file.  If multiple same-
-  named headers are provided, they will all be appended - the calling process
-  needs to pick up the last one.  This will mean removing the attribute if the
-  final record has an empty value.
-
-  The function takes the following (positional) parameters:
-
-  | PARAMETER | DESCRIPTION |
-  |------------|-------------|
-  | name $  | name of the header attribute to create|
-  | value  $  | value of the header attribute|
-
-  It returns 0 if successful, or -1 if an error occured.
-
-  Usage:
-
-      %let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/stpsrv_header.txt;
-
-      %mcf_stpsrv_header(wrap=YES, insert_cmplib=YES)
-
-      data _null_;
-        rc=stpsrv_header('Content-type','application/text');
-        rc=stpsrv_header('Content-disposition',"attachment; filename=file.txt");
-      run;
-
-      data _null_;
-        infile "&sasjs_stpsrv_header_loc";
-        input;
-        putlog _infile_;
-      run;
-
-
-  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
-  @param [out] lib= (work) The output library in which to create the catalog.
-  @param [out] cat= (sasjs) The output catalog in which to create the package.
-  @param [out] pkg= (utils) The output package in which to create the function.
-    Uses a 3 part format:  libref.catalog.package
-  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
-    values inserted only if needed.
-
-  <h4> SAS Macros </h4>
-  @li mcf_init.sas
-
-  <h4> Related Programs </h4>
-  @li mcf_stpsrv_header.test.sas
-  @li mp_init.sas
-
-**/
-
-%macro mcf_stpsrv_header(wrap=NO
-  ,insert_cmplib=DEPRECATED
-  ,lib=WORK
-  ,cat=SASJS
-  ,pkg=UTILS
-)/*/STORE SOURCE*/;
-%local i var cmpval found;
-%if %mcf_init(stpsrv_header)=1 %then %return;
-
-%if &wrap=YES  %then %do;
-  proc fcmp outlib=&lib..&cat..&pkg;
-%end;
-
-function stpsrv_header(name $, value $);
-  length loc $128 val $512;
-  loc=symget('sasjs_stpsrv_header_loc');
-  val=trim(name)!!': '!!value;
-  length fref $8;
-  rc=filename(fref,loc);
-  if (rc ne 0) then return( -1 );
-  fid = fopen(fref,'a');
-  if (fid = 0) then return( -1 );
-  rc=fput(fid, val);
-  rc=fwrite(fid);
-  rc=fclose(fid);
-  rc=filename(fref);
-  return(0);
-endsub;
-
-%if &wrap=YES %then %do;
-  quit;
-%end;
-
-/* insert the CMPLIB if not already there */
-%let cmpval=%sysfunc(getoption(cmplib));
-%let found=0;
-%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
-  %let var=%scan(&cmpval,&i,%str( %(%)));
-  %if &var=&lib..&cat %then %let found=1;
-%end;
-%if &found=0 %then %do;
-  options insert=(CMPLIB=(&lib..&cat));
-%end;
-
-%mend mcf_stpsrv_header;

meta/mm_assignlib.sas

@@ -13,8 +13,8 @@
   <h4> SAS Macros </h4>
   @li mp_abort.sas
 
-  @param libref the libref (not name) of the metadata library
-  @param mAbort= If not assigned, HARD will call %mp_abort(), SOFT will
+  @param [in] libref The libref (not name) of the metadata library
+  @param [in] mAbort= If not assigned, HARD will call %mp_abort(), SOFT will
     silently return
 
   @returns libname statement
@@ -28,11 +28,11 @@
     libref
     ,mAbort=HARD
 )/*/STORE SOURCE*/;
-
+%local mp_abort msg;
+%let mp_abort=0;
 %if %sysfunc(libref(&libref)) %then %do;
-  %local mp_abort msg; %let mp_abort=0;
   data _null_;
-    length liburi LibName $200;
+    length liburi LibName msg $200;
     call missing(of _all_);
     nobj=metadata_getnobj("omsobj:SASLibrary?@Libref='&libref'",1,liburi);
     if nobj=1 then do;
@@ -46,7 +46,24 @@
           * not always helpful though.  One example, previously received:
           * NOTE: Libref XX refers to the same library metadata as libref XX.
           */
-        call symputx('msg',sysmsg(),'l');
+        msg=sysmsg();
+        if msg=:'ERROR: Libref SAVE is not assigned.' then do;
+          msg=catx(" ",
+            "Could not assign %upcase(&libref).",
+            "Please check metadata permissions!  Libname:",libname,
+            "Liburi:",liburi
+          );
+        end;
+        else if msg="ERROR: User does not have appropriate authorization "!!
+          "level for library SAVE."
+        then do;
+          msg=catx(" ",
+            "ERROR: User does not have appropriate authorization level",
+            "for library %upcase(&libref), libname:",libname,
+            "Liburi:",liburi
+          );
+        end;
+        call symputx('msg',msg,'l');
         if "&mabort"='HARD' then call symputx('mp_abort',1,'l');
       end;
       else do;
@@ -65,20 +82,16 @@
     end;
   run;
 
-  %if &mp_abort=1 %then %do;
-    %mp_abort(iftrue= (&mp_abort=1)
-      ,mac=mm_assignlib.sas
-      ,msg=&msg
-    )
-    %return;
-  %end;
-  %else %if %length(&msg)>2 %then %do;
-    %put NOTE: &msg;
-    %return;
-  %end;
+  %put NOTE: &msg;
 
 %end;
 %else %do;
   %put NOTE: Library &libref is already assigned;
 %end;
+
+%mp_abort(iftrue= (&mp_abort=1)
+  ,mac=mm_assignlib.sas
+  ,msg=%superq(msg)
+)
+
 %mend mm_assignlib;

meta/mm_createwebservice.sas

@@ -232,12 +232,23 @@ data _null_;
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
-  put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
-  put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
-  put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
-  put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '        )))))!!''"''; ';
+  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; ';
+  put '          &&name&i=''"''!!trim( ';
+  put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '            prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '            prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '            prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '            prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '            prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '          ))))))))))))!!''"''; ';
+  put '        end; ';
+  put '        else &&name&i=quote(cats(&&name&i)); ';
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
@@ -421,14 +432,16 @@ data _null_;
   put '    put '',"_METAPERSON": '' _METAPERSON; ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
+  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
+  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';
   put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
@@ -453,7 +466,8 @@ data _null_;
   put '  %else %if %symexist(&metavar) %then %do; ';
   put '    %if %length(&&&metavar)=0 %then %let user=&sysuserid; ';
   put '    /* sometimes SAS will add @domain extension - remove for consistency */ ';
-  put '    %else %let user=%scan(&&&metavar,1,@); ';
+  put '    /* but be sure to quote in case of usernames with commas */ ';
+  put '    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@)); ';
   put '  %end; ';
   put '  %else %let user=&sysuserid; ';
   put ' ';

meta/mm_deletelibrary.sas

@@ -78,6 +78,7 @@ filename &fname2 clear;
 %local isgone;
 data _null_;
   length type uri $256;
+  call missing (of _all_);
   rc=metadata_resolve("omsobj:SASLibrary?@Id='&liburi'",type,uri);
   call symputx('isgone',type,'l');
 run;

meta/mm_spkexport.sas

@@ -49,20 +49,25 @@
 
 
   <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_isblank.sas
   @li mf_loc.sas
   @li mm_tree.sas
-  @li mf_getuniquefileref.sas
-  @li mf_isblank.sas
   @li mp_abort.sas
 
-  @param metaloc= the metadata folder to export
-  @param secureref= fileref containing the username / password (should point to
-    a file in a secure location).  Leave blank to substitute $bash type vars.
-  @param outref= fileref to which to write the command
-  @param cmdoutloc= the directory to which the command will write the SPK
-    (default=WORK)
-  @param cmdoutname= the name of the spk / log files to create (will be
-    identical just with .spk or .log extension)
+
+  @param [in] metaloc= the metadata folder to export
+  @param [in] secureref= fileref containing the username / password (should
+    point to a file in a secure location). Leave blank to substitute $bash vars.
+  @param [in] excludevars= (0) A space seperated list of macro variable names,
+    each of which contains a value that should be used to filter the output
+    objects.
+  @param [out] outref= fileref to which to write the command
+  @param [out] cmdoutloc= (%sysfunc(pathname(work))) The directory to which the
+    command will write the SPK
+  @param [out] cmdoutname= (mmxport) The name of the spk / log files to create
+    (will be identical just with .spk or .log extension)
 
   @version 9.4
   @author Allan Bowe
@@ -71,6 +76,7 @@
 
 %macro mm_spkexport(metaloc=
   ,secureref=
+  ,excludevars=0
   ,outref=
   ,cmdoutloc=%sysfunc(pathname(work))
   ,cmdoutname=mmxport
@@ -82,7 +88,7 @@
 %end;
 
 /* set creds */
-%local mmxuser mmxpath;
+%local mmxuser mmxpath i var;
 %let mmxuser=$1;
 %let mmxpass=$2;
 %if %mf_isblank(&secureref)=0 %then %do;
@@ -90,35 +96,51 @@
 %end;
 
 /* setup metadata connection options */
-%local host port platform_object_path connx_string;
+%local host port platform_object_path ds;
 %let host=%sysfunc(getoption(metaserver));
 %let port=%sysfunc(getoption(metaport));
 %let platform_object_path=%mf_loc(POF);
+%let ds=%mf_getuniquename(prefix=spkexportable);
 
-%let connx_string=%str(-host &host -port &port -user &mmxuser %trim(
-  )-password &mmxpass);
-
-%mm_tree(root=%str(&metaloc) ,types=EXPORTABLE ,outds=exportable)
+%mm_tree(root=%str(&metaloc),types=EXPORTABLE ,outds=&ds)
 
 %if %mf_isblank(&outref)=1 %then %let outref=%mf_getuniquefileref();
 
 data _null_;
-  set exportable end=last;
+  set &ds end=last;
   file &outref lrecl=32767;
   length str $32767;
   if _n_=1 then do;
+    put "# Script generated by &sysuserid on %sysfunc(datetime(),datetime19.)";
     put "cd ""&platform_object_path"" \";
-    put "; ./ExportPackage &connx_string -disableX11 \";
-    put " -package ""&cmdoutloc/&cmdoutname..spk"" \";
+    put "; ./ExportPackage -host &host -port &port -user &mmxuser \";
+    put "  -disableX11 -password &mmxpass \";
+    put "  -package ""&cmdoutloc/&cmdoutname..spk"" \";
   end;
+/* exclude particular patterns from the exported SPK */
+%if "&excludevars" ne "0" %then %do;
+  %do i=1 %to %sysfunc(countw(&excludevars));
+    %let var=%scan(&excludevars,&i);
+    if _n_=1 then do;
+      length excludestr&i $1000;
+      retain excludestr&i;
+      excludestr&i=symget("&var");
+      putlog excludestr&i=;
+      putlog path=;
+    end;
+    if index(path,cats(excludestr&i))=0 and index(name,cats(excludestr&i))=0;
+  %end;
+  /* ignore top level folder else all subcontent will be exported regardless */
+  if _n_>1;
+%end;
   str=' -objects '!!cats('"',path,'/',name,"(",publictype,')" \');
   put str;
   if last then put " -log ""&cmdoutloc/&cmdoutname..log"" 2>&1 ";
 run;
 
 %mp_abort(iftrue= (&syscc ne 0)
-  ,mac=&sysmacroname
+  ,mac=mm_spkexport
   ,msg=%str(syscc=&syscc)
 )
 
-%mend mm_spkexport;
+%mend mm_spkexport;

meta/mm_webout.sas

@@ -157,14 +157,16 @@
     put ',"_METAPERSON": ' _METAPERSON;
     put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";

package.json

@@ -26,14 +26,13 @@
   "homepage": "https://core.sasjs.io",
   "main": "index.js",
   "scripts": {
-    "build": "sasjs cbd -t viya",
+    "build": "sasjs cbd -t server",
     "docs": "sasjs doc -t docsonly && ./sasjs/utils/build.sh",
-    "test": "sasjs test -t viya",
+    "test": "sasjs test -t server",
     "lint": "sasjs lint",
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "3.6.0",
-    "@sasjs/core": "4.4.7"
+    "@sasjs/cli": "3.13.0"
   }
 }

sasjs/sasjsconfig.json

@@ -4,11 +4,8 @@
     "base",
     "ddl",
     "fcmp",
-    "meta",
-    "metax",
-    "server",
-    "viya",
     "lua",
+    "server",
     "tests/crossplatform",
     "tests/ddl"
   ],
@@ -28,25 +25,20 @@
       "mcTestAppLoc": "/Public/temp/macrocore"
     }
   },
-  "defaultTarget": "viya",
+  "defaultTarget": "server",
   "targets": [
     {
       "name": "viya",
-      "serverUrl": "https://sas.analytium.co.uk",
+      "serverUrl": "",
       "serverType": "SASVIYA",
       "httpsAgentOptions": {
         "allowInsecureRequests": false
       },
-      "appLoc": "/Public/temp/macrocore",
+      "appLoc": "/Public/app/macrocore",
       "macroFolders": [
+        "viya",
         "tests/viyaonly"
       ],
-      "programFolders": [],
-      "binaryFolders": [],
-      "deployConfig": {
-        "deployServicePack": true,
-        "deployScripts": []
-      },
       "contextName": "SAS Job Execution compute context"
     },
     {
@@ -58,6 +50,8 @@
       },
       "appLoc": "/Shared Data/temp/macrocore",
       "macroFolders": [
+        "meta",
+        "metax",
         "tests/sas9only"
       ],
       "programFolders": [],
@@ -78,6 +72,7 @@
       },
       "appLoc": "/sasjs/core",
       "macroFolders": [
+        "server",
         "tests/serveronly"
       ],
       "programFolders": [],
@@ -92,9 +87,27 @@
       "serverType": "SAS9",
       "appLoc": "dummy",
       "macroFolders": [
+        "meta",
+        "metax",
+        "server",
+        "viya",
         "tests/sas9only",
         "tests/viyaonly"
       ]
+    },
+    {
+      "name": "viya4",
+      "serverUrl": "https://azureuse011059.my-trials.sas.com",
+      "serverType": "SASVIYA",
+      "appLoc": "/Public/temp/macrocore",
+      "macroFolders": [
+        "viya",
+        "tests/viyaonly"
+      ],
+      "deployConfig": {
+        "deployServicePack": true
+      },
+      "contextName": "SAS Job Execution compute context"
     }
   ]
-}
+}

server/ms_createfile.sas

@@ -29,11 +29,16 @@
     ,mdebug=0
   );
 
-%local fname0 fname1 boundary fname statcd msg;
+%local fname0 fname1 fname2 boundary fname statcd msg optval;
 %let fname0=%mf_getuniquefileref();
 %let fname1=%mf_getuniquefileref();
+%let fname2=%mf_getuniquefileref();
 %let boundary=%mf_getuniquename();
 
+/* avoid sending bom marker to API */
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
 data _null_;
   file &fname0 termstr=crlf;
   infile &inref end=eof;
@@ -55,17 +60,25 @@ data _null_;
   end;
 run;
 
+data _null_;
+  file &fname1;
+  put "Content-Type: multipart/form-data; boundary=&boundary";
+run;
+
 %if &mdebug=1 %then %do;
   data _null_;
     infile &fname0;
     input;
     put _infile_;
+  data _null_;
+    infile &fname1;
+    input;
+    put _infile_;
   run;
 %end;
 
-proc http method='POST' in=&fname0 out=&fname1
+proc http method='POST' in=&fname0 headerin=&fname1 out=&fname2
   url="&_sasjs_apiserverurl/SASjsApi/drive/file";
-  headers "Content-Type"="multipart/form-data; boundary=&boundary";
 %if &mdebug=1 %then %do;
   debug level=1;
 %end;
@@ -73,7 +86,7 @@ run;
 
 %let statcd=0;
 data _null_;
-  infile &fname1;
+  infile &fname2;
   input;
   putlog _infile_;
   if _infile_='{"status":"success"}' then call symputx('statcd',1,'l');
@@ -86,4 +99,7 @@ run;
   ,msg=%superq(msg)
 )
 
+/* reset options */
+options &optval;
+
 %mend ms_createfile;

server/ms_deletefile.sas

@@ -0,0 +1,35 @@
+/**
+  @file
+  @brief Deletes a file from SASjs Drive
+  @details Deletes a file from SASjs Drive, if it exists.
+
+  Example:
+
+      filename stpcode temp;
+      data _null_;
+        file stpcode;
+        put '%put hello world;';
+      run;
+      %ms_createfile(/some/stored/program.sas, inref=stpcode)
+
+      %ms_deletefile(/some/stored/program.sas)
+
+  @param [in] driveloc The full path to the file in SASjs Drive
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+
+**/
+
+%macro ms_deletefile(driveloc
+    ,mdebug=0
+  );
+
+proc http method='DELETE'
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file?_filePath=&driveloc";
+%if &mdebug=1 %then %do;
+  debug level=2;
+%end;
+run;
+
+
+%mend ms_deletefile;

server/ms_getfile.sas

@@ -0,0 +1,41 @@
+/**
+  @file
+  @brief Gets a file from SASjs Drive
+  @details Fetches a file on SASjs Drive and stores it in the output fileref.
+
+  Example:
+
+      %ms_getfile(/some/stored/file.ext, outref=myfile)
+
+  @param [in] driveloc The full path to the file in SASjs Drive
+  @param [out] outref= (msgetfil) The fileref to contain the file.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+
+**/
+
+%macro ms_getfile(driveloc
+    ,outref=msgetfil
+    ,mdebug=0
+  );
+
+/* use the recfm in a separate fileref to avoid issues with subsequent reads */
+%local binaryfref floc;
+%let binaryfref=%mf_getuniquefileref();
+%let floc=%sysfunc(pathname(work))/%mf_getuniquename().txt;
+filename &outref "&floc";
+filename &binaryfref "&floc" recfm=n;
+
+proc http method='GET' out=&binaryfref
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file?_filePath=&driveloc";
+%if &mdebug=1 %then %do;
+  debug level=2;
+%end;
+run;
+
+filename &binaryfref clear;
+
+%mend ms_getfile;

server/ms_webout.sas

@@ -1,6 +1,6 @@
 /**
   @file
-  @brief Send data to/from @sasjs/server
+  @brief Send data to/from sasjs/server
   @details This macro should be added to the start of each web service,
   **immediately** followed by a call to:
 
@@ -114,6 +114,7 @@
     data _null_;
       set &tempds;
       if not (upcase(name) =:"DATA"); /* ignore temp datasets */
+      if not (upcase(name)=:"_DATA_");
       i+1;
       call symputx(cats('wt',i),name,'l');
       call symputx('wtcnt',i,'l');
@@ -146,7 +147,8 @@
     put ",""_DEBUG"" : ""&_debug"" ";
     put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
     SYSHOSTINFOLONG=quote(trim(symget('SYSHOSTINFOLONG')));
     put ',"SYSHOSTINFOLONG" : ' SYSHOSTINFOLONG;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
@@ -161,7 +163,8 @@
     put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length autoexec $512;
     autoexec=quote(urlencode(trim(getoption('autoexec'))));

tests/crossplatform/mcf_stpsrv_header.test.sas

@@ -1,39 +0,0 @@
-/**
-  @file
-  @brief Testing mcf_stpsrv_header macro
-
-  <h4> SAS Macros </h4>
-  @li mcf_stpsrv_header.sas
-  @li mp_assert.sas
-
-**/
-
-%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/stpsrv_header.txt;
-
-%mcf_stpsrv_header(wrap=YES, insert_cmplib=YES)
-
-data _null_;
-  rc=stpsrv_header('Content-type','application/text');
-  rc=stpsrv_header('Content-disposition',"attachment; filename=file.txt");
-run;
-
-%let test1=FAIL;
-%let test2=FAIL;
-
-data _null_;
-  infile "&sasjs_stpsrv_header_loc";
-  input;
-  if _n_=1 and _infile_='Content-type: application/text'
-  then call symputx('test1','PASS');
-  else if _n_=2 & _infile_='Content-disposition: attachment; filename=file.txt'
-  then call symputx('test2','PASS');
-run;
-
-%mp_assert(
-  iftrue=(%str(&test1)=%str(PASS)),
-  desc=Check first header line
-)
-%mp_assert(
-  iftrue=(%str(&test2)=%str(PASS)),
-  desc=Check second header line
-)

tests/crossplatform/mf_deletefile.test.sas

@@ -0,0 +1,29 @@
+/**
+  @file
+  @brief Testing mf_deletefile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_deletefile.sas
+  @li mf_writefile.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%let test1file=&sasjswork/myfile1.txt;
+
+%mf_writefile(&test1file,l1=some content)
+
+%mp_assert(
+  iftrue=(%sysfunc(fileexist(&test1file))=1),
+  desc=Check &test1file exists
+)
+
+%mp_assertscope(SNAPSHOT)
+%mf_deletefile(&test1file)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(%sysfunc(fileexist(&test1file))=0),
+  desc=Check &test1file no longer exists
+)

tests/crossplatform/mp_dirlist.test.sas

@@ -47,4 +47,18 @@ run;
   iftrue=(%mf_nobs(work.mytable3)=2),
   desc=Top level returned,
   outds=work.test_results
-)
+)
+
+%mp_dirlist(path=&root/b, outds=work.myTable4)
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable4)=0),
+  desc=Empty table for empty directory,
+  outds=work.test_results
+)
+
+%mp_dirlist(path=&root/notexisting, outds=work.myTable5)
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable5)=0),
+  desc=Empty table for non-existing directory,
+  outds=work.test_results
+)

tests/crossplatform/mp_getpk.test.sas

@@ -88,4 +88,50 @@ run;
 /* constraint capture at library level is functional - uses first 2 tests */
 %mp_getpk(work,outds=test4)
 
-%mp_assertdsobs(work.test4,test=ATLEAST 2)
+%mp_assertdsobs(work.test4,test=ATLEAST 2)
+
+/* unique & not null INDEX captured */
+proc sql;
+create table work.example5(
+  TX_FROM float format=datetime19.,
+  DD_TYPE char(16),
+  DD_SOURCE char(2048),
+  DD_SHORTDESC char(256)
+);
+proc datasets lib=work noprint;
+  modify example5;
+  index create tx_from /nomiss unique;
+quit;
+%mp_getpk(work,ds=example5,outds=test5)
+data _null_;
+  set work.test5;
+  call symputx('test5',pk_fields);
+run;
+%mp_assert(
+  iftrue=("&test5"="TX_FROM"),
+  desc=mp_getpk captures single column not null unique index,
+  outds=work.test_results
+)
+
+/* unique & not null COMPOSITE INDEX captured */
+proc sql;
+create table work.example6(
+  TX_FROM float format=datetime19.,
+  DD_TYPE char(16),
+  DD_SOURCE char(2048),
+  DD_SHORTDESC char(256)
+);
+proc datasets lib=work noprint;
+  modify example6;
+  index create pk_6=(tx_from dd_type) /nomiss unique;
+quit;
+%mp_getpk(work,ds=example6,outds=test6)
+data _null_;
+  set work.test6;
+  call symputx('test6',pk_fields);
+run;
+%mp_assert(
+  iftrue=("&test6"="TX_FROM DD_TYPE"),
+  desc=mp_getpk captures multiple column not null unique index,
+  outds=work.test_results
+)

tests/crossplatform/mp_gsubfile.test.sas

@@ -8,6 +8,13 @@
 
 **/
 
+
+%macro gsubtest();
+%if "%substr(&sysver,1,4)"="V.04" %then %do;
+  %put %str(ERR)OR: Viya 4 does not support the IO library in lua;
+  %return;
+%end;
+
 /**
   * test 1 - simple replace
   */
@@ -63,4 +70,8 @@ run;
   iftrue=("&strcheck2b"="&str2"),
   desc=Check that multi line replacement was successful (line3),
   outds=work.test_results
-)
+)
+
+%mend gsubtest;
+
+%gsubtest()

tests/crossplatform/mp_hashdataset.test.sas

@@ -0,0 +1,61 @@
+/**
+  @file
+  @brief Testing mp_hashdataset.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_hashdataset.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+/* test 1 - regular DS */
+data work.test;
+  set sashelp.vextfl;
+  missval=.;
+  misscval='';
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mp_hashdataset(test)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Regular test works,
+  outds=work.test_results
+)
+
+%mp_hashdataset(test,outds=work.test2)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=hash with output runs without errors,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.test2)=1),
+  desc=output has 1 row,
+  outds=work.test_results
+)
+
+
+data work.test3a;
+  set work.test;
+  stop;
+run;
+%mp_hashdataset(test3a,outds=work.test3b)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=hash with zero-row input runs without errors,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.test3b)=1),
+  desc=test 3 output has 1 row,
+  outds=work.test_results
+)

tests/crossplatform/mp_jsonout.test.3.sas

@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Testing mp_jsonout.sas macro with non-standard chars
+
+  <h4> SAS Macros </h4>
+  @li mp_jsonout.sas
+  @li mp_assert.sas
+
+**/
+
+filename webref temp;
+
+data demo;
+  do x='"','0A'x,'0D'x,'09'x,'00'x,'0E'x,'0F'x,'01'x,'02'x,'10'x,'11'x,'\';
+    output;
+  end;
+run;
+%mp_jsonout(OPEN,jref=webref)
+%mp_jsonout(OBJ,demo,jref=webref)
+%mp_jsonout(CLOSE,jref=webref)
+
+data _null_;
+  infile webref;
+  input;
+  putlog _infile_;
+run;
+
+libname web JSON fileref=webref;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking for error condition with special chars export,
+  outds=work.test_results
+)
+
+/*
+data _null_;
+  set work.demo (in=start) web.demo (in=end);
+  put (_all_)(=);
+run;
+proc sql;
+describe table work.demo;
+describe table web.demo;
+*/
+
+proc compare base=work.demo compare=web.demo(keep=x);
+quit;
+
+%mp_assert(
+  iftrue=(&sysinfo=0),
+  desc=Returned json is identical to input table for all special chars,
+  outds=work.test_results
+)

tests/crossplatform/mp_loadformat.test.sas

@@ -3,6 +3,7 @@
   @brief Testing mp_loadformat.sas macro
 
   <h4> SAS Macros </h4>
+  @li mddl_dc_difftable.sas
   @li mp_loadformat.sas
   @li mp_assert.sas
   @li mp_assertscope.sas
@@ -12,6 +13,8 @@
 /* prep format catalog */
 libname perm (work);
 
+%mddl_dc_difftable(libds=perm.audit)
+
 data work.loadfmts;
   length fmtname $32;
   eexcl='Y';
@@ -49,7 +52,7 @@ run;
 %mp_loadformat(perm.testcat
   ,work.stagedata
   ,loadtarget=YES
-  ,auditlibds=0
+  ,auditlibds=perm.audit
   ,locklibds=0
   ,delete_col=deleteme
   ,outds_add=add_test1
@@ -73,4 +76,18 @@ run;
   iftrue=(%mf_nobs(mod_test1)=100),
   desc=Test 1 - mod obs,
   outds=work.test_results
-)
+)
+%mp_assert(
+  iftrue=(%mf_nobs(perm.audit)=7329),
+  desc=Test 1 - audit table updated,
+  outds=work.test_results
+)
+data work.difftest;
+  set perm.audit;
+  where is_diff=1;
+run;
+%mp_assert(
+  iftrue=(%mf_nobs(work.difftest)>0),
+  desc=Test 1 - diffs were found,
+  outds=work.test_results
+)

tests/crossplatform/mp_replace.test.sas

@@ -63,3 +63,33 @@ run;
   desc=Checking second replace 3rd row,
   outds=work.test_results
 )
+
+
+%let test3="&sasjswork/file3.txt";
+%let str=%str(replace.string.with.dots   );
+%let rep=%str( more.dots);
+data _null_;
+  file &test3;
+  put 'blahblah';
+  put "blahblah&str.blah&str. replace &str.X";
+  put "blahbreplacewith&str.spacesahblah";
+run;
+%mp_replace(&test3, findvar=str, replacevar=rep)
+
+data _null_;
+  infile &test3;
+  input;
+  if _n_=2 then call symputx('test3resulta',_infile_);
+  if _n_=3 then call symputx('test3resultb',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&test3resulta" = "blahblah&rep.blah&rep. replace &rep.X"),
+  desc=Checking third replace 2nd row (dots),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test3resultb" = "blahbreplacewith&rep.spacesahblah"),
+  desc=Checking third replace 3rd row (dots),
+  outds=work.test_results
+)

tests/sas9only/mm_spkexport.test.sas

@@ -0,0 +1,41 @@
+/**
+  @file
+  @brief Testing mm_webout macro
+
+  <h4> SAS Macros </h4>
+  @li mm_spkexport.sas
+  @li mp_assert.sas
+  @li mp_as
+
+**/
+
+
+%* create sample text file as input to the macro;
+filename tmp temp;
+data _null_;
+  file tmp;
+  put '%let mmxuser="sasdemo";';
+  put '%let mmxpass="Mars321";';
+run;
+
+filename myref "%sysfunc(pathname(work))/mmxexport.sh"
+  permission='A::u::rwx,A::g::r-x,A::o::---';
+%mp_assertscope(SNAPSHOT)
+%mm_spkexport(metaloc=%str(/Shared Data)
+    ,outref=myref
+    ,secureref=tmp
+    ,cmdoutloc=%str(/tmp)
+)
+%mp_assertscope(COMPARE)
+
+data _null_;
+  infile tmp;
+  input;
+  putlog _infile_;
+  call symputx('nobs',_n_);
+run;
+
+%mp_assert(
+  iftrue=(&nobs>2),
+  desc=Check if content was created
+)

tests/serveronly/ms_deletefile.test.sas

@@ -0,0 +1,61 @@
+/**
+  @file
+  @brief Testing ms_deletefile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li ms_createfile.sas
+  @li ms_deletefile.sas
+  @li ms_getfile.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+
+/* first make a remote file */
+filename stpcode temp;
+%let fname=%mf_getuniquename();
+data _null_;
+  file stpcode;
+  put "data &fname;run;";
+run;
+%ms_createfile(/sasjs/tests/&fname..sas
+  ,inref=stpcode
+  ,mdebug=1
+)
+
+%ms_getfile(/sasjs/tests/&fname..sas,outref=testref)
+
+%let test1=0;
+data _null_;
+  infile testref;
+  input;
+  call symputx('test1',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&test1"="data &fname;run;"),
+  desc=Make sure the file was created,
+  outds=work.test_results
+)
+
+%mp_assertscope(SNAPSHOT)
+%ms_deletefile(/sasjs/tests/&fname..sas,mdebug=1)
+%mp_assertscope(COMPARE)
+
+%ms_getfile(/sasjs/tests/&fname..sas,outref=testref2)
+
+%let test2=0;
+data _null_;
+  infile testref2;
+  input;
+  call symputx('test2',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&test2"="%str(Err)or: File does not exist."),
+  desc=Make sure the file was deleted,
+  outds=work.test_results
+)
+
+

tests/serveronly/ms_getfile.test.sas

@@ -0,0 +1,45 @@
+/**
+  @file
+  @brief Testing ms_getfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li ms_createfile.sas
+  @li ms_getfile.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+
+/* first make a remote file */
+filename stpcode temp;
+%let fname=%mf_getuniquename();
+data _null_;
+  file stpcode;
+  put "data &fname;run;";
+run;
+%ms_createfile(/sasjs/tests/&fname..sas
+  ,inref=stpcode
+  ,mdebug=1
+)
+
+%mp_assertscope(SNAPSHOT)
+%ms_getfile(/sasjs/tests/&fname..sas,outref=testref)
+%mp_assertscope(COMPARE)
+
+%let test1=0;
+data _null_;
+  infile testref;
+  input;
+  call symputx('test1',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&test1"="data &fname;run;"),
+  desc=Checking file was created with the same content,
+  outds=work.test_results
+)
+
+
+
+

tests/viyaonly/mv_deleteviyafolder.test.sas

@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief Testing mv_deleteviyafolder macro function
+
+  <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mfv_existfolder.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mv_createfolder.sas
+  @li mv_deleteviyafolder.sas
+
+**/
+
+options mprint sgen;
+
+%let folder=%mf_uid();
+%let tgtfolder=&mcTestAppLoc/temp/&folder;
+
+/* create a folder */
+%mv_createfolder(path=&tgtfolder)
+
+
+%mp_assert(
+  iftrue=(%mfv_existfolder(&tgtfolder)=1),
+  desc=Check if created folder exists
+)
+
+%mp_assertscope(SNAPSHOT)
+%mv_deleteviyafolder(path=&tgtfolder)
+/* ignore proc json vars */
+%mp_assertscope(COMPARE
+  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADP2LEN MCLIB0_JADVLEN MCLIB2_JADP1LEN
+    MCLIB2_JADVLEN
+)
+
+%mp_assert(
+  iftrue=(%mfv_existfolder(&tgtfolder)=0),
+  desc=Check if deleted folder is gone
+)
+
+/* delete folder with content */
+%mv_createfolder(path=&tgtfolder/content/and/stuff)
+%mp_assert(
+  iftrue=(%mfv_existfolder(&tgtfolder/content/and/stuff)=1),
+  desc=Check if folder with content exists
+)
+%mv_deleteviyafolder(path=&tgtfolder)
+%mp_assert(
+  iftrue=(%mfv_existfolder(&tgtfolder)=0),
+  desc=Check if deleted folder with subfolders is gone
+)

tests/viyaonly/mv_getjobcode.test.sas

@@ -4,6 +4,7 @@
 
   <h4> SAS Macros </h4>
   @li mp_assert.sas
+  @li mp_assertscope.sas
   @li mv_createjob.sas
   @li mv_getjobcode.sas
 
@@ -27,11 +28,17 @@ run;
 )
 
 /* now get the code back */
+%mp_assertscope(SNAPSHOT)
 %mv_getjobcode(
   path=&mcTestAppLoc/services/temp,
   name=some_job,
   outref=mycode
 )
+/* exclude automatic proc json macro variables from scope check */
+%mp_assertscope(COMPARE,
+  ignorelist=MCLIB2_JADP1LEN MCLIB2_JADP2LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
+    MCLIB2_JADP3LEN
+)
 
 %let diditexist=NO;
 data work.test1;
@@ -46,4 +53,4 @@ run;
 %mp_assert(
   iftrue=(&diditexist=NO),
   desc=Check if the code that was sent was successfully retrieved
-)
+)

tests/viyaonly/mv_getjoblog.test.sas

@@ -4,6 +4,7 @@
 
   <h4> SAS Macros </h4>
   @li mp_assert.sas
+  @li mp_assertscope.sas
   @li mv_createjob.sas
   @li mv_jobexecute.sas
   @li mv_jobwaitfor.sas
@@ -49,8 +50,12 @@ data _null_;
 run;
 
 %* Finally, fetch the log;
-%mv_getjoblog(uri=%str(&uri),outref=mylog)
-
+%mp_assertscope(SNAPSHOT)
+%mv_getjoblog(uri=%str(&uri),outref=mylog,mdebug=1)
+/* ignore auto proc json vars */
+%mp_assertscope(COMPARE
+  ,ignorelist=MCLIB2_JADP2LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
+)
 
 data _null_;
   infile mylog end=eof;
@@ -67,4 +72,4 @@ run;
 %mp_assert(
   iftrue=(%str(&found)=1),
   desc=Check if the log was still fetched even though endsas was submitted
-)
+)

tests/viyaonly/mv_webout.test.sas

@@ -1,15 +1,33 @@
 /**
   @file
-  @brief Testing mm_webout macro
+  @brief Testing mv_webout macro
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
   @li mv_webout.sas
   @li mp_assert.sas
+  @li mp_assertdsobs.sas
 
 **/
 
 
+/* testing FETCHing (WEB approach) */
+
+data _null_;
+  call symputx('sasjs1data','area:$char4.'!!'0d0a'x!!'Adak');
+  call symputx('sasjs_tables','areas');
+run;
+%put &=sasjs1data;
+
+%mv_webout(FETCH)
+
+%mp_assertdsobs(work.areas,
+  desc=Test input table has 1 row,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+
 %let fref=%mf_getuniquefileref();
 %global _metaperson;
 data some datasets;
@@ -38,4 +56,4 @@ run;
 %mp_assert(
   iftrue=(%str(&checkval)=%str(&sysvlong)),
   desc=Check if the sysvlong value was created
-)
+)

viya/mv_createwebservice.sas

@@ -376,12 +376,23 @@ data _null_;
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
-  put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
-  put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
-  put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
-  put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '        )))))!!''"''; ';
+  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; ';
+  put '          &&name&i=''"''!!trim( ';
+  put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '            prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '            prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '            prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '            prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '            prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '          ))))))))))))!!''"''; ';
+  put '        end; ';
+  put '        else &&name&i=quote(cats(&&name&i)); ';
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
@@ -454,7 +465,7 @@ data _null_;
   put '  sasjs_tables SYS_JES_JOB_URI; ';
   put '%if %index("&_debug",log) %then %let _debug=131; ';
   put ' ';
-  put '%local i tempds; ';
+  put '%local i tempds table; ';
   put '%let action=%upcase(&action); ';
   put ' ';
   put '%if &action=FETCH %then %do; ';
@@ -469,60 +480,35 @@ data _null_;
   put '  %end; ';
   put ' ';
   put '  /* if the sasjs_tables param is passed, we expect param based upload */ ';
-  put '  %if %length(&sasjs_tables.XX)>2 %then %do; ';
-  put '    filename _sasjs "%sysfunc(pathname(work))/sasjs.lua"; ';
-  put '    data _null_; ';
-  put '      file _sasjs; ';
-  put '      put ''s=sas.symget("sasjs_tables")''; ';
-  put '      put ''if(s:sub(1,7) == "%nrstr(")''; ';
-  put '      put ''then''; ';
-  put '      put '' tablist=s:sub(8,s:len()-1)''; ';
-  put '      put ''else''; ';
-  put '      put '' tablist=s''; ';
-  put '      put ''end''; ';
-  put '      put ''for i = 1,sas.countw(tablist) ''; ';
-  put '      put ''do ''; ';
-  put '      put ''  tab=sas.scan(tablist,i)''; ';
-  put '      put ''  sasdata=""''; ';
-  put '      put ''  if (sas.symexist("sasjs"..i.."data0")==0)''; ';
-  put '      put ''  then''; ';
-  put '      /* TODO - condense this logic */ ';
-  put '      put ''    s=sas.symget("sasjs"..i.."data")''; ';
-  put '      put ''    if(s:sub(1,7) == "%nrstr(")''; ';
-  put '      put ''    then''; ';
-  put '      put ''      sasdata=s:sub(8,s:len()-1)''; ';
-  put '      put ''    else''; ';
-  put '      put ''      sasdata=s''; ';
-  put '      put ''    end''; ';
-  put '      put ''  else''; ';
-  put '      put ''    for d = 1, sas.symget("sasjs"..i.."data0")''; ';
-  put '      put ''    do''; ';
-  put '      put ''      s=sas.symget("sasjs"..i.."data"..d)''; ';
-  put '      put ''      if(s:sub(1,7) == "%nrstr(")''; ';
-  put '      put ''      then''; ';
-  put '      put ''        sasdata=sasdata..s:sub(8,s:len()-1)''; ';
-  put '      put ''      else''; ';
-  put '      put ''        sasdata=sasdata..s''; ';
-  put '      put ''      end''; ';
-  put '      put ''    end''; ';
-  put '      put ''  end''; ';
-  put '      put ''  file = io.open(sas.pathname("work").."/"..tab..".csv", "a")''; ';
-  put '      put ''  io.output(file)''; ';
-  put '      put ''  io.write(sasdata)''; ';
-  put '      put ''  io.close(file)''; ';
-  put '      put ''end''; ';
-  put '    run; ';
-  put '    %inc _sasjs; ';
+  put '  %if %length(&sasjs_tables.X)>1 %then %do; ';
   put ' ';
-  put '    /* now read in the data */ ';
+  put '    /* convert data from macro variables to datasets */ ';
   put '    %do i=1 %to %sysfunc(countw(&sasjs_tables)); ';
-  put '      %local table; %let table=%scan(&sasjs_tables,&i); ';
+  put '      %let table=%scan(&sasjs_tables,&i,%str( )); ';
+  put '      %if %symexist(sasjs&i.data0)=0 %then %let sasjs&i.data0=1; ';
+  put '      data _null_; ';
+  put '        file "%sysfunc(pathname(work))/&table..csv" recfm=n; ';
+  put '        retain nrflg 0; ';
+  put '        length line $32767; ';
+  put '        do i=1 to &&sasjs&i.data0; ';
+  put '          if &&sasjs&i.data0=1 then line=symget("sasjs&i.data"); ';
+  put '          else line=symget(cats("sasjs&i.data",i)); ';
+  put '          if i=1 and substr(line,1,7)=''%nrstr('' then do; ';
+  put '            nrflg=1; ';
+  put '            line=substr(line,8); ';
+  put '          end; ';
+  put '          if i=&&sasjs&i.data0 and nrflg=1 then do; ';
+  put '            line=substr(line,1,length(line)-1); ';
+  put '          end; ';
+  put '          put line +(-1) @; ';
+  put '        end; ';
+  put '      run; ';
   put '      data _null_; ';
   put '        infile "%sysfunc(pathname(work))/&table..csv" termstr=crlf ; ';
   put '        input; ';
   put '        if _n_=1 then call symputx(''input_statement'',_infile_); ';
   put '        list; ';
-  put '      data &table; ';
+  put '      data work.&table; ';
   put '        infile "%sysfunc(pathname(work))/&table..csv" firstobs=2 dsd ';
   put '          termstr=crlf; ';
   put '        input &input_statement; ';
@@ -623,13 +609,15 @@ data _null_;
   put '    put ",""_DEBUG"" : ""&_debug"" "; ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
+  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
+  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';
   put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
@@ -655,7 +643,8 @@ data _null_;
   put '  %else %if %symexist(&metavar) %then %do; ';
   put '    %if %length(&&&metavar)=0 %then %let user=&sysuserid; ';
   put '    /* sometimes SAS will add @domain extension - remove for consistency */ ';
-  put '    %else %let user=%scan(&&&metavar,1,@); ';
+  put '    /* but be sure to quote in case of usernames with commas */ ';
+  put '    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@)); ';
   put '  %end; ';
   put '  %else %let user=&sysuserid; ';
   put ' ';

viya/mv_deleteviyafolder.sas

@@ -8,10 +8,16 @@
       %mv_deleteviyafolder(path=/Public/test)
 
 
-  @param path= The full path of the folder to be deleted
-  @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values are "password" or "authorization_code" (unquoted).
-    The default is authorization_code.
+  @param [in] path= The full path of the folder to be deleted
+  @param [in] access_token_var= (ACCESS_TOKEN) The global macro variable to
+    contain the access token
+  @param [in] grant_type= (sas_services) Valid values are:
+    @li password
+    @li authorization_code
+    @li detect - will check if access_token exists, if not will use sas_services
+      if a SASStudioV session else authorization_code.  Default option.
+    @li sas_services - will use oauth_bearer=sas_services.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
 
 
   @version VIYA V.03.04
@@ -19,6 +25,7 @@
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas
+  @li mf_existds.sas
   @li mf_getplatform.sas
   @li mf_getuniquefileref.sas
   @li mf_getuniquelibref.sas
@@ -29,6 +36,7 @@
 %macro mv_deleteviyafolder(path=
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
+    ,mdebug=0
   );
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
@@ -105,14 +113,17 @@ run;
 %let libref1a=%mf_getuniquelibref();
 libname &libref1a JSON fileref=&fname1a;
 
-data _null_;
-  set &libref1a..items_links;
-  if href=:'/folders/folders' then return;
-  if rel='deleteResource' then
-    call execute('proc http method="DELETE" url='!!quote("&base_uri"!!trim(href))
-    !!'; headers "Authorization"="Bearer &&&access_token_var" '
-    !!' "Accept"="*/*";run; /**/');
-run;
+%if %mf_existds(&libref1a..items_links) %then %do;
+  data _null_;
+    set &libref1a..items_links;
+    if href=:'/folders/folders' then return;
+    if rel='deleteResource' then
+      call execute('proc http method="DELETE" url='
+      !!quote("&base_uri"!!trim(href))
+      !!'; headers "Authorization"="Bearer &&&access_token_var" '
+      !!' "Accept"="*/*";run; /**/');
+  run;
+%end;
 
 %put &sysmacroname: perform the delete operation ;
 %local fname2;
@@ -133,9 +144,11 @@ run;
 %end;
 %else %put &sysmacroname: &path successfully deleted;
 
-/* clear refs */
-filename &fname1 clear;
-filename &fname2 clear;
-libname &libref1 clear;
+%if &mdebug=0 %then %do;
+  /* clear refs */
+  filename &fname1 clear;
+  filename &fname2 clear;
+  libname &libref1 clear;
+%end;
 
-%mend mv_deleteviyafolder;
+%mend mv_deleteviyafolder;

viya/mv_getgroups.sas

@@ -32,7 +32,7 @@
     ,grant_type=sas_services
     ,outds=work.viyagroups
   );
-%local oauth_bearer;
+%local oauth_bearer base_uri fname1 libref1;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
   %else %let grant_type=sas_services;
@@ -50,11 +50,10 @@
 )
 
 options noquotelenmax;
-%local base_uri; /* location of rest apis */
+/* location of rest apis */
 %let base_uri=%mf_getplatform(VIYARESTAPI);
 
 /* fetching folder details for provided path */
-%local fname1;
 %let fname1=%mf_getuniquefileref();
 %let libref1=%mf_getuniquelibref();
 
@@ -78,9 +77,8 @@ data &outds;
 run;
 
 
-
 /* clear refs */
 filename &fname1 clear;
 libname &libref1 clear;
 
-%mend mv_getgroups;
+%mend mv_getgroups;

viya/mv_getjobcode.sas

@@ -33,7 +33,6 @@
   @li mf_getplatform.sas
   @li mf_getuniquefileref.sas
   @li mv_getfoldermembers.sas
-  @li ml_json.sas
 
 **/
 
@@ -44,9 +43,9 @@
     ,grant_type=sas_services
     ,mdebug=0
   );
-%local dbg;
+%local dbg bufsize varcnt fname1 fname2 errmsg;
 %if &mdebug=1 %then %do;
-  %put &sysmacroname entry vars:;
+  %put &sysmacroname local entry vars:;
   %put _local_;
 %end;
 %else %let dbg=*;
@@ -104,7 +103,6 @@ run;
 )
 
 /* prepare request*/
-%local  fname1;
 %let fname1=%mf_getuniquefileref();
 proc http method='GET' out=&fname1 &oauth_bearer
   url="&base_uri&joburi";
@@ -114,37 +112,95 @@ proc http method='GET' out=&fname1 &oauth_bearer
   %end;
   ;
 run;
-%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
-%do;
-  data _null_;infile &fname1;input;putlog _infile_;run;
-  %mp_abort(mac=&sysmacroname
-    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
-  )
-%end;
-%local  fname2 fname3 fpath1 fpath2 fpath3;
-%let fname2=%mf_getuniquefileref();
-%let fname3=%mf_getuniquefileref();
-%let fpath1=%sysfunc(pathname(&fname1));
-%let fpath2=%sysfunc(pathname(&fname2));
-%let fpath3=%sysfunc(pathname(&fname3));
 
-/* compile the lua JSON module */
-%ml_json()
-/* read using LUA - this allows the code to be of any length */
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fname1;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  ,mac=&sysmacroname
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+%let fname2=%mf_getuniquefileref();
+filename &fname2 temp ;
+
+/* cannot use lua IO package as not available in Viya 4 */
+/* so use data step to read the JSON until the string `"code":"` is found */
 data _null_;
-  file "&fpath3..lua";
-  put '
-    infile = io.open (sas.symget("fpath1"), "r")
-    outfile = io.open (sas.symget("fpath2"), "w")
-    io.input(infile)
-    local resp=json.decode(io.read())
-    local job=resp["code"]
-    outfile:write(job)
-    io.close(infile)
-    io.close(outfile)
-  ';
+  file &fname2 recfm=n;
+  infile &fname1 lrecl=1 recfm=n;
+  input sourcechar $char1. @@;
+  format sourcechar hex2.;
+  retain startwrite 0;
+  if startwrite=0 and sourcechar='"' then do;
+    reentry:
+    input sourcechar $ 1. @@;
+    if sourcechar='c' then do;
+      reentry2:
+      input sourcechar $ 1. @@;
+      if sourcechar='o' then do;
+        input sourcechar $ 1. @@;
+        if sourcechar='d' then do;
+          input sourcechar $ 1. @@;
+          if sourcechar='e' then do;
+            input sourcechar $ 1. @@;
+            if sourcechar='"' then do;
+              input sourcechar $ 1. @@;
+              if sourcechar=':' then do;
+                input sourcechar $ 1. @@;
+                if sourcechar='"' then do;
+                  putlog 'code found';
+                  startwrite=1;
+                  input sourcechar $ 1. @@;
+                end;
+              end;
+              else if sourcechar='c' then goto reentry2;
+            end;
+          end;
+          else if sourcechar='"' then goto reentry;
+        end;
+        else if sourcechar='"' then goto reentry;
+      end;
+      else if sourcechar='"' then goto reentry;
+    end;
+    else if sourcechar='"' then goto reentry;
+  end;
+  /* once the `"code":"` string is found, write until unescaped `"` is found */
+  if startwrite=1 then do;
+    if sourcechar='\' then do;
+      input sourcechar $ 1. @@;
+      if sourcechar in ('"','\') then put sourcechar char1.;
+      else if sourcechar='n' then put '0A'x;
+      else if sourcechar='r' then put '0D'x;
+      else if sourcechar='t' then put '09'x;
+      else if sourcechar='u' then do;
+        length uni $4;
+        input uni $ 4. @@;
+        sourcechar=unicode('\u'!!uni);
+        put sourcechar char1.;
+      end;
+      else do;
+        call symputx('errmsg',"Uncaught escape char: "!!sourcechar,'l');
+        call symputx('syscc',99);
+        stop;
+      end;
+    end;
+    else if sourcechar='"' then stop;
+    else put sourcechar char1.;
+  end;
 run;
-%inc "&fpath3..lua";
+
+%mp_abort(iftrue=("&syscc"="99")
+  ,mac=mv_getjobcode
+  ,msg=%str(&errmsg)
+)
+
 /* export to desired destination */
 %if "&outref"="0" %then %do;
   data _null_;
@@ -169,7 +225,6 @@ run;
   /* clear refs */
   filename &fname1 clear;
   filename &fname2 clear;
-  filename &fname3 clear;
 %end;
 
 %mend mv_getjobcode;

viya/mv_getjoblog.sas

@@ -86,7 +86,8 @@
   @li mp_abort.sas
   @li mf_getplatform.sas
   @li mf_existfileref.sas
-  @li ml_json.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
 
 **/
 
@@ -95,7 +96,7 @@
     ,grant_type=sas_services
     ,mdebug=0
   );
-%local dbg;
+%local dbg libref1 libref2 loglocation fname1 fname2;
 %if &mdebug=1 %then %do;
   %put &sysmacroname entry vars:;
   %put _local_;
@@ -154,8 +155,8 @@ options noquotelenmax;
 %let base_uri=%mf_getplatform(VIYARESTAPI);
 
 /* prepare request*/
-%local  fname1;
 %let fname1=%mf_getuniquefileref();
+%let fname2=%mf_getuniquefileref();
 proc http method='GET' out=&fname1 &oauth_bearer
   url="&base_uri&uri";
   headers
@@ -175,37 +176,19 @@ run;
     ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
   )
 %end;
-%local  fname2 fname3 fpath1 fpath2 fpath3;
-%let fname2=%mf_getuniquefileref();
-%let fname3=%mf_getuniquefileref();
-%let fpath1=%sysfunc(pathname(&fname1));
-%let fpath2=%sysfunc(pathname(&fname2));
-%let fpath3=%sysfunc(pathname(&fname3));
 
-/* compile the lua JSON module */
-%ml_json()
-/* read using LUA - this allows the code to be of any length */
+%let libref1=%mf_getuniquelibref();
+libname &libref1 JSON fileref=&fname1;
 data _null_;
-  file "&fpath3..lua";
-  put '
-    infile = io.open (sas.symget("fpath1"), "r")
-    outfile = io.open (sas.symget("fpath2"), "w")
-    io.input(infile)
-    local resp=json.decode(io.read())
-    local logloc=resp["logLocation"]
-    outfile:write(logloc)
-    io.close(infile)
-    io.close(outfile)
-  ';
+  set &libref1..root;
+  call symputx('loglocation',loglocation,'l');
 run;
-%inc "&fpath3..lua";
-/* get log path*/
+
+/* validate log path*/
 %let errflg=1;
-%let errmsg=No entry in &fname2 fileref;
+%let errmsg=No loglocation entry in &fname1 fileref;
 data _null_;
-  infile &fname2;
-  input;
-  uri=cats(_infile_);
+  uri=symget('loglocation');
   if length(uri)<12 then do;
     call symputx('errflg',1);
     call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
@@ -232,7 +215,7 @@ run;
 
 /* we have a log uri - now fetch the log */
 %&dbg.put &sysmacroname: querying &base_uri&logloc/content;
-proc http method='GET' out=&fname1 &oauth_bearer
+proc http method='GET' out=&fname2 &oauth_bearer
   url="&base_uri&logloc/content?limit=10000";
   headers
   %if &grant_type=authorization_code %then %do;
@@ -243,14 +226,14 @@ run;
 
 %if &mdebug=1 %then %do;
   %put &sysmacroname: fetching log content from &base_uri&logloc/content;
-  data _null_;infile &fname1;input;putlog _infile_;run;
+  data _null_;infile &fname2;input;putlog _infile_;run;
 %end;
 
 %if &SYS_PROCHTTP_STATUS_CODE=400 %then %do;
   /* fetch log from parent session */
   %let logloc=%substr(&logloc,1,%index(&logloc,%str(/jobs/))-1);
   %&dbg.put &sysmacroname: Now querying &base_uri&logloc/log/content;
-  proc http method='GET' out=&fname1 &oauth_bearer
+  proc http method='GET' out=&fname2 &oauth_bearer
     url="&base_uri&logloc/log/content?limit=10000";
     headers
     %if &grant_type=authorization_code %then %do;
@@ -260,47 +243,32 @@ run;
   run;
   %if &mdebug=1 %then %do;
     %put &sysmacroname: fetching log content from &base_uri&logloc/log/content;
-    data _null_;infile &fname1;input;putlog _infile_;run;
+    data _null_;infile &fname2;input;putlog _infile_;run;
   %end;
 %end;
 
 %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201
 %then %do;
   %if &mdebug ne 1 %then %do; /* have already output above */
-    data _null_;infile &fname1;input;putlog _infile_;run;
+    data _null_;infile &fname2;input;putlog _infile_;run;
   %end;
   %mp_abort(mac=&sysmacroname
     ,msg=%str(logfetch: &SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
   )
 %end;
-data _null_;
-  file "&fpath3..lua";
-  put '
-    infile = io.open (sas.symget("fpath1"), "r")
-    outfile = io.open (sas.symget("fpath2"), "w")
-    io.input(infile)
-    local resp=json.decode(io.read())
-    for i, v in pairs(resp["items"]) do
-      outfile:write(v.line,"\n")
-    end
-    io.close(infile)
-    io.close(outfile)
-  ';
-run;
-%inc "&fpath3..lua";
 
-/* write log out to the specified fileref */
+%let libref2=%mf_getuniquelibref();
+libname &libref2 JSON fileref=&fname2;
 data _null_;
-  infile &fname2 end=last;
   file &outref mod;
   if _n_=1 then do;
     put "/** SASJS Viya Job Log Extract start: &uri **/";
   end;
-  input;
-  put _infile_;
+  set &libref2..items end=last;
   %if &mdebug=1 %then %do;
-    putlog _infile_;
+    putlog line;
   %end;
+  put line;
   if last then do;
     put "/** SASJS Viya Job Log Extract end: &uri **/";
   end;
@@ -309,7 +277,8 @@ run;
 %if &mdebug=0 %then %do;
   filename &fname1 clear;
   filename &fname2 clear;
-  filename &fname3 clear;
+  libname &libref1 clear;
+  libname &libref2 clear;
 %end;
 %else %do;
   %put &sysmacroname exit vars:;

viya/mv_webout.sas

@@ -49,7 +49,7 @@
   sasjs_tables SYS_JES_JOB_URI;
 %if %index("&_debug",log) %then %let _debug=131;
 
-%local i tempds;
+%local i tempds table;
 %let action=%upcase(&action);
 
 %if &action=FETCH %then %do;
@@ -64,60 +64,35 @@
   %end;
 
   /* if the sasjs_tables param is passed, we expect param based upload */
-  %if %length(&sasjs_tables.XX)>2 %then %do;
-    filename _sasjs "%sysfunc(pathname(work))/sasjs.lua";
-    data _null_;
-      file _sasjs;
-      put 's=sas.symget("sasjs_tables")';
-      put 'if(s:sub(1,7) == "%nrstr(")';
-      put 'then';
-      put ' tablist=s:sub(8,s:len()-1)';
-      put 'else';
-      put ' tablist=s';
-      put 'end';
-      put 'for i = 1,sas.countw(tablist) ';
-      put 'do ';
-      put '  tab=sas.scan(tablist,i)';
-      put '  sasdata=""';
-      put '  if (sas.symexist("sasjs"..i.."data0")==0)';
-      put '  then';
-      /* TODO - condense this logic */
-      put '    s=sas.symget("sasjs"..i.."data")';
-      put '    if(s:sub(1,7) == "%nrstr(")';
-      put '    then';
-      put '      sasdata=s:sub(8,s:len()-1)';
-      put '    else';
-      put '      sasdata=s';
-      put '    end';
-      put '  else';
-      put '    for d = 1, sas.symget("sasjs"..i.."data0")';
-      put '    do';
-      put '      s=sas.symget("sasjs"..i.."data"..d)';
-      put '      if(s:sub(1,7) == "%nrstr(")';
-      put '      then';
-      put '        sasdata=sasdata..s:sub(8,s:len()-1)';
-      put '      else';
-      put '        sasdata=sasdata..s';
-      put '      end';
-      put '    end';
-      put '  end';
-      put '  file = io.open(sas.pathname("work").."/"..tab..".csv", "a")';
-      put '  io.output(file)';
-      put '  io.write(sasdata)';
-      put '  io.close(file)';
-      put 'end';
-    run;
-    %inc _sasjs;
+  %if %length(&sasjs_tables.X)>1 %then %do;
 
-    /* now read in the data */
+    /* convert data from macro variables to datasets */
     %do i=1 %to %sysfunc(countw(&sasjs_tables));
-      %local table; %let table=%scan(&sasjs_tables,&i);
+      %let table=%scan(&sasjs_tables,&i,%str( ));
+      %if %symexist(sasjs&i.data0)=0 %then %let sasjs&i.data0=1;
+      data _null_;
+        file "%sysfunc(pathname(work))/&table..csv" recfm=n;
+        retain nrflg 0;
+        length line $32767;
+        do i=1 to &&sasjs&i.data0;
+          if &&sasjs&i.data0=1 then line=symget("sasjs&i.data");
+          else line=symget(cats("sasjs&i.data",i));
+          if i=1 and substr(line,1,7)='%nrstr(' then do;
+            nrflg=1;
+            line=substr(line,8);
+          end;
+          if i=&&sasjs&i.data0 and nrflg=1 then do;
+            line=substr(line,1,length(line)-1);
+          end;
+          put line +(-1) @;
+        end;
+      run;
       data _null_;
         infile "%sysfunc(pathname(work))/&table..csv" termstr=crlf ;
         input;
         if _n_=1 then call symputx('input_statement',_infile_);
         list;
-      data &table;
+      data work.&table;
         infile "%sysfunc(pathname(work))/&table..csv" firstobs=2 dsd
           termstr=crlf;
         input &input_statement;
@@ -218,13 +193,15 @@
     put ",""_DEBUG"" : ""&_debug"" ";
     put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";