fix: mp_gsubfile() now works with multiline files (and we have a multiline test to go with it)

fix: prevening truncation of _debug in mp_abort.sas and more reliable way to fetch syswarningtext and syserrortext
chore: fixing example in header for mp_gsubfile()
2025-12-11 14:34:35 +00:00 · 2021-09-16 18:30:17 +01:00 · 2021-09-16 14:04:50 +01:00 · 2021-09-16 13:54:27 +01:00 · 2021-09-14 19:13:15 +03:00 · 2021-09-14 16:08:30 +01:00
26 changed files with 1638 additions and 358 deletions
--- a/all.sas
+++ b/all.sas
@@ -160,21 +160,23 @@ options noquotelenmax;
  Full credit to [Bart](https://sasensei.com/user/305) for the vfunc pointer
  and the tidy approach for pure macro data set filtering.
  Check out his [SAS Packages](https://github.com/yabwon/SAS_PACKAGES)
-  framework!
+  framework!  Where you can find the same [function](
+https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionexists-macro
+  ).

  @param [in] name (positional) - function name

  @author Allan Bowe
 **/
 /** @cond */
-
 %macro mf_existfunction(name
 )/*/STORE SOURCE*/;

  %local dsid rc exist;
  %let dsid=%sysfunc(open(sashelp.vfunc(where=(fncname="%upcase(&name)"))));
-  %let exist = %sysfunc(fetch(&dsid, NOSET));
-  %let rc = %sysfunc(close(&dsid));
+  %let exist=1;
+  %let exist=%sysfunc(fetch(&dsid, NOSET));
+  %let rc=%sysfunc(close(&dsid));

  %sysevalf(0 = &exist)

@@ -310,13 +312,25 @@ options noquotelenmax;

 %macro mf_getapploc(pgm);
 %if "&pgm"="" %then %do;
-  %put &sysmacroname: No value provided;
-  %return;
+  %if %symexist(_program) %then %let pgm=&_program;
+  %else %do;
+    %put &sysmacroname: No value provided and no _program variable available;
+    %return;
+  %end;
 %end;
 %local root;

 /**
-  * move up two levels to avoid matches on subfolder or service name
+  * First check we are not in the tests/macros folder (which has no subfolders)
+  */
+%if %index(&pgm,/tests/macros/) %then %do;
+  %let root=%substr(&pgm,1,%index(&pgm,/tests/macros)-1);
+  &root
+  %return;
+%end;
+
+/**
+  * Next, move up two levels to avoid matches on subfolder or service name
  */
 %let root=%substr(&pgm,1,%length(&pgm)-%length(%scan(&pgm,-1,/))-1);
 %let root=%substr(&root,1,%length(&root)-%length(%scan(&root,-1,/))-1);
@@ -689,39 +703,60 @@ options noquotelenmax;
 /**
  @file
  @brief Assigns and returns an unused fileref
-  @details
+  @details  Using the native approach for assigning filerefs fails as some
+  procedures (such as proc http) do not recognise the temporary names (starting
+  with a hash), returning a message such as:
+
+  > ERROR 22-322: Expecting a name.
+
+  This macro works by attempting a random fileref (with a prefix), seeing if it
+  is already assigned, and if not - returning the fileref.
+
+  If your process can accept filerefs with the hash (#) prefix, then set
+  `prefix=0` to revert to the native approach - which is significantly faster
+  when there are a lot of filerefs in a session.
+
  Use as follows:

      %let fileref1=%mf_getuniquefileref();
-      %let fileref2=%mf_getuniquefileref();
+      %let fileref2=%mf_getuniquefileref(prefix=0);
      %put &fileref1 &fileref2;

-  which returns:
+  which returns filerefs similar to:

-> mcref0 mcref1
+> _7432233 #LN00070

-  @param prefix= first part of fileref. Remember that filerefs can only be 8
-    characters, so a 7 letter prefix would mean that `maxtries` should be 10.
-  @param maxtries= the last part of the libref.  Provide an integer value.
+  @param [in] prefix= (_) first part of fileref. Remember that filerefs can only
+    be 8 characters, so a 7 letter prefix would mean `maxtries` should be 10.
+    if using zero (0) as the prefix, a native assignment is used.
+  @param [in] maxtries= (1000) the last part of the libref. Must be an integer.

  @version 9.2
  @author Allan Bowe
 **/

-%macro mf_getuniquefileref(prefix=mcref,maxtries=1000);
-  %local x fname;
-  %let x=0;
-  %do x=0 %to &maxtries;
-  %if %sysfunc(fileref(&prefix&x)) > 0 %then %do;
-    %let fname=&prefix&x;
+%macro mf_getuniquefileref(prefix=_,maxtries=1000);
+  %local rc fname;
+  %if &prefix=0 %then %do;
    %let rc=%sysfunc(filename(fname,,temp));
    %if &rc %then %put %sysfunc(sysmsg());
-    &prefix&x
-    %*put &sysmacroname: Fileref &prefix&x was assigned and returned;
-    %return;
+    &fname
  %end;
+  %else %do;
+    %local x len;
+    %let len=%eval(8-%length(&prefix));
+    %let x=0;
+    %do x=0 %to &maxtries;
+      %let fname=&prefix%substr(%sysfunc(ranuni(0)),3,&len);
+      %if %sysfunc(fileref(&fname)) > 0 %then %do;
+        %let rc=%sysfunc(filename(fname,,temp));
+        %if &rc %then %put %sysfunc(sysmsg());
+        &fname
+        %return;
+      %end;
+    %end;
+    %put unable to find available fileref after &maxtries attempts;
  %end;
-  %put unable to find available fileref in range &prefix.0-&maxtries;
 %mend mf_getuniquefileref;/**
  @file
  @brief Returns an unused libref
@@ -1628,24 +1663,47 @@ Usage:
    recognise this and fetch the log of the parent session instead)
  @li STP environments must finish cleanly to avoid the log being sent to
    _webout.  To assist with this, we also run stpsrvset('program error', 0)
-    and set SYSCC=0.  For 9.4m3 we take a unique approach - we open a macro
-    but don't close it!  This provides a graceful abort, EXCEPT when called
-    called within a %include within a macro (and that macro contains additional
-    logic).  See mp_abort.test.nofix.sas for the example case.
-    If you know of another way to gracefully abort a 9.4m3 STP session, we'd
-    love to hear about it!
+    and set SYSCC=0.  We take a unique "soft abort" approach - we open a macro
+    but don't close it!  This works everywhere EXCEPT inside a \%include inside
+    a macro.  For that, we recommend you use mp_include.sas to perform the
+    include, and then call \%mp_abort(mode=INCLUDE) from the source program (ie,
+    OUTSIDE of the top-parent macro).


  @param mac= to contain the name of the calling macro
  @param msg= message to be returned
  @param iftrue= supply a condition under which the macro should be executed.
+  @param errds= (work.mp_abort_errds) There is no clean way to end a process
+    within a %include called within a macro.  Furthermore, there is no way to
+    test if a macro is called within a %include.  To handle this particular
+    scenario, the %include should be switched for the mp_include.sas macro.
+    This provides an indicator that we are running a macro within a \%include
+    (`_SYSINCLUDEFILEDEVICE`) and allows us to provide a dataset with the abort
+    values (msg, mac).
+    We can then run an abort cancel FILE to stop the include running, and pass
+    the dataset back to the calling program to run a regular \%mp_abort().
+    The dataset will contain the following fields:
+    @li iftrue (1=1)
+    @li msg (the message)
+    @li mac (the mac param)

-  @version 9.4M3
+  @param mode= (REGULAR) If mode=INCLUDE then the &errds dataset is checked for
+    an abort status.
+    Valid values:
+    @li REGULAR (default)
+    @li INCLUDE
+
+  <h4> Related Macros </h4>
+  @li mp_include.sas
+
+  @version 9.4
  @author Allan Bowe
  @cond
 **/

 %macro mp_abort(mac=mp_abort.sas, type=, msg=, iftrue=%str(1=1)
+  , errds=work.mp_abort_errds
+  , mode=REGULAR
 )/*/STORE SOURCE*/;

  %global sysprocessmode sysprocessname;
@@ -1656,9 +1714,44 @@ Usage:
  %if %length(&mac)>0 %then %put NOTE- called by &mac;
  %put NOTE - &msg;

+  %if %symexist(_SYSINCLUDEFILEDEVICE) %then %do;
+    %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do;
+      data &errds;
+        iftrue='1=1';
+        length mac $100 msg $5000;
+        mac=symget('mac');
+        msg=symget('msg');
+      run;
+      data _null_;
+        abort cancel FILE;
+      run;
+      %return;
+    %end;
+  %end;
+
  /* Stored Process Server web app context */
-  %if %symexist(_metaperson) or "&SYSPROCESSNAME "="Compute Server " %then %do;
+  %if %symexist(_metaperson)
+    or "&SYSPROCESSNAME "="Compute Server "
+    or &mode=INCLUDE
+  %then %do;
    options obs=max replace nosyntaxcheck mprint;
+    %if &mode=INCLUDE %then %do;
+      %if %sysfunc(exist(&errds))=1 %then %do;
+        data _null_;
+          set &errds;
+          call symputx('iftrue',iftrue,'l');
+          call symputx('mac',mac,'l');
+          call symputx('msg',msg,'l');
+          putlog (_all_)(=);
+        run;
+        %if (&iftrue)=0 %then %return;
+      %end;
+      %else %do;
+        %put &sysmacroname: No include errors found;
+        %return;
+      %end;
+    %end;
+
    /* extract log errs / warns, if exist */
    %local logloc logline;
    %global logmsg; /* capture global messages */
@@ -1713,7 +1806,7 @@ Usage:
    /* send response in SASjs JSON format */
    data _null_;
      file _webout mod lrecl=32000 encoding='utf-8';
-      length msg $32767 debug $8;
+      length msg $32767 ;
      sasdatetime=datetime();
      msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
      /* escape the quotes */
@@ -1744,11 +1837,15 @@ Usage:
      _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
      put ',"_PROGRAM" : ' _PROGRAM ;
      put ",""SYSCC"" : ""&syscc"" ";
-      put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+      syserrortext=quote(trim(symget('syserrortext')));
+      put ",""SYSERRORTEXT"" : " syserrortext;
+      put ",""SYSHOSTNAME"" : ""&syshostname"" ";
      put ",""SYSJOBID"" : ""&sysjobid"" ";
+      put ",""SYSSITE"" : ""&syssite"" ";
      sysvlong=quote(trim(symget('sysvlong')));
      put ',"SYSVLONG" : ' sysvlong;
-      put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+      syswarningtext=quote(trim(symget('syswarningtext')));
+      put ",""SYSWARNINGTEXT"" : " syswarningtext;
      put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
      put "}" @;
      if debug ge '"131"' then put '>>weboutEND<<';
@@ -1762,11 +1859,22 @@ Usage:
        rc=stpsrvset('program error', 0);
        call symputx("syscc",0,"g");
      run;
-      %if "%substr(&sysvlong.xxxxxxxxx,1,9)" ne "9.04.01M3" %then %do;
-        %put NOTE: Ending SAS session due to:;
-        %put NOTE- &msg;
-        endsas;
-      %end;
+      /**
+        * endsas kills 9.4m3 deployments by orphaning multibridges.
+        * Abort variants are ungraceful (non zero return code)
+        * This approach lets SAS run silently until the end :-)
+        * Caution - fails when called within a %include within a macro
+        * Use mp_include() to handle this.
+        */
+      filename skip temp;
+      data _null_;
+        file skip;
+        put '%macro skip();';
+        comment '%mend skip; -> fix lint ';
+        put '%macro skippy();';
+        comment '%mend skippy; -> fix lint ';
+      run;
+      %inc skip;
    %end;
    %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
      /* endsas kills the session making it harder to fetch results */
@@ -1782,24 +1890,6 @@ Usage:
        abort cancel nolist;
      run;
    %end;
-    %else %if "%substr(&sysvlong.xxxxxxxxx,1,9)" = "9.04.01M3" %then %do;
-      /**
-        * endsas kills 9.4m3 deployments by orphaning multibridges.
-        * Abort variants are ungraceful (non zero return code)
-        * This approach lets SAS run silently until the end :-)
-        * Caution - fails when called within a %include within a macro
-        * See tests/mp_abort.test.1 for an example case.
-        */
-      filename skip temp;
-      data _null_;
-        file skip;
-        put '%macro skip();';
-        comment '%mend skip; -> fix lint ';
-        put '%macro skippy();';
-        comment '%mend skippy; -> fix lint ';
-      run;
-      %inc skip;
-    %end;
    %else %do;
      %abort cancel;
    %end;
@@ -2399,10 +2489,29 @@ run;

        %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)

-  @param inloc full, quoted "path/and/filename.ext" of the object to be copied
-  @param outloc full, quoted "path/and/filename.ext" of object to be created
-  @param inref can override default input fileref to avoid naming clash
-  @param outref an override default output fileref to avoid naming clash
+  To append to a file, use the mode option, eg:
+
+      filename tmp1 temp;
+      filename tmp2 temp;
+      data _null_;
+        file tmp1;
+        put 'stacking';
+      run;
+
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+
+
+  @param [in] inloc quoted "path/and/filename.ext" of the file to be copied
+  @param [out] outloc quoted "path/and/filename.ext" of the file to be created
+  @param [in] inref (____in) If provided, this fileref will take precedence over
+    the `inloc` parameter
+  @param [out] outref (____in) If provided, this fileref will take precedence
+    over the `outloc` parameter.  It must already exist!
+  @param [in] mode (CREATE) Valid values:
+    @li CREATE - Create the file (even if it already exists)
+    @li APPEND - Append to the file (don't overwrite)
+
  @returns nothing

  @version 9.2
@@ -2414,20 +2523,29 @@ run;
    ,outloc=          /* full path and filename of object to be created */
    ,inref=____in   /* override default to use own filerefs */
    ,outref=____out /* override default to use own filerefs */
+    ,mode=CREATE
 )/*/STORE SOURCE*/;
+  %local mod outmode;
+  %if &mode=APPEND %then %do;
+    %let mod=mod;
+    %let outmode='a';
+  %end;
+  %else %do;
+    %let outmode='o';
+  %end;
  /* these IN and OUT filerefs can point to anything */
  %if &inref = ____in %then %do;
    filename &inref &inloc lrecl=1048576 ;
  %end;
  %if &outref=____out %then %do;
-    filename &outref &outloc lrecl=1048576 ;
+    filename &outref &outloc lrecl=1048576 &mod;
  %end;

  /* copy the file byte-for-byte  */
  data _null_;
    length filein 8 fileid 8;
    filein = fopen("&inref",'I',1,'B');
-    fileid = fopen("&outref",'O',1,'B');
+    fileid = fopen("&outref",&outmode,1,'B');
    rec = '20'x;
    do while(fread(filein)=0);
      rc = fget(filein,rec,1);
@@ -3549,10 +3667,13 @@ run;
    options:
    @li SAS (default) - suitable for regular proc sql
    @li PGSQL - Used for Postgres databases
+  @param [in] applydttm= (YES) If YES, any columns using datetime formats will
+    be converted to native DB datetime literals

  <h4> SAS Macros </h4>
  @li mf_existfileref.sas
  @li mf_getvarcount.sas
+  @li mf_getvarformat.sas
  @li mf_getvarlist.sas
  @li mf_getvartype.sas

@@ -3561,6 +3682,7 @@ run;
 **/

 %macro mp_ds2inserts(ds, outref=0,schema=0,outds=0,flavour=SAS,maxobs=max
+  ,applydttm=YES
 )/*/STORE SOURCE*/;

 %if not %sysfunc(exist(&ds)) %then %do;
@@ -3638,10 +3760,11 @@ data _null_;
  length _____str $32767;
  format _numeric_ best.;
  format _character_ ;
-  %local i comma var vtype;
+  %local i comma var vtype vfmt;
  %do i=1 %to %sysfunc(countw(&varlist));
    %let var=%scan(&varlist,&i);
    %let vtype=%mf_getvartype(&ds,&var);
+    %let vfmt=%upcase(%mf_getvarformat(&ds,&var,force=1));
    %if &i=1 %then %do;
      %if &flavour=SAS %then %do;
        put "insert into &schema.&outds set ";
@@ -3671,7 +3794,13 @@ data _null_;
      %end;
      %else %if &flavour=PGSQL %then %do;
        if missing(&var) then put 'NULL';
-        else put &var;
+        %if &applydttm=YES and "%substr(&vfmt.xxxxxxxx,1,8)"="DATETIME"
+        %then %do;
+          else put "TIMESTAMP '" &var E8601DT25.6 "'";
+        %end;
+        %else %do;
+          else put &var;
+        %end;
      %end;
    %end;
    %else %do;
@@ -4074,7 +4203,7 @@ run;

 data &outds;
  if &sqlrc or &syscc or &syserr then do;
-    REASON_CD='VALIDATION_ERROR: '!!
+    REASON_CD='VALIDATION_ERR'!!'OR: '!!
      coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
    output;
  end;
@@ -4965,6 +5094,59 @@ create table &outds (rename=(
  run;

 %mend mp_getmaxvarlengths;/**
+  @file
+  @brief Performs a text substitution on a file
+  @details Makes use of the GSUB function in LUA to perform a text substitution
+  in a file - either in-place, or writing to a new location.  The benefit of
+  using LUA is that the entire file can be loaded into a single variable,
+  thereby side stepping the 32767 character limit in a data step.
+
+  Usage:
+
+      %let file=%sysfunc(pathname(work))/file.txt;
+      %let str=replace/me;
+      %let rep=with/this;
+      data _null_;
+        file "&file";
+        put "&str";
+      run;
+      %mp_gsubfile(file=&file, patternvar=str, replacevar=rep)
+      data _null_;
+        infile "&file";
+        input;
+        list;
+      run;
+
+  @param file= (0) The file to perform the substitution on
+  @param patternvar= A macro variable containing the Lua
+    [pattern](https://www.lua.org/pil/20.2.html) to search for.  Due to the use
+    of special (magic) characters in Lua patterns, it is safer to pass the NAME
+    of the macro variable containing the string, rather than the value itself.
+  @param replacevar= The name of the macro variable containing the replacement
+    _string_.
+  @param outfile= (0) The file to write the output to. If zero, then the file
+    is overwritten in-place.
+
+  <h4> SAS Macros </h4>
+  @li ml_gsubfile.sas
+
+  <h4> Related Macros </h4>
+  @li mp_gsubfile.test.sas
+
+  @version 9.4
+  @author Allan Bowe
+**/
+
+%macro mp_gsubfile(file=0,
+  patternvar=,
+  replacevar=,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+  %ml_gsubfile()
+
+%mend mp_gsubfile;
+/**
  @file mp_guesspk.sas
  @brief Guess the primary key of a table
  @details Tries to guess the primary key of a table based on the following logic:
@@ -5344,12 +5526,115 @@ create table &outds (rename=(
    run;
  %end;
 %mend mp_hashdataset;/**
+  @file
+  @brief Performs a wrapped \%include
+  @details This macro wrapper is necessary if you need your included code to
+  know that it is being \%included.
+
+  If you are using %include in a regular program, you could make use of the
+  following macro variables:
+
+  @li SYSINCLUDEFILEDEVICE
+  @li SYSINCLUDEFILEDIR
+  @li SYSINCLUDEFILEFILEREF
+  @li SYSINCLUDEFILENAME
+
+  However these variables are NOT available inside a macro, as documented here:
+https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/mcrolref/n1j5tcc0n2xczyn1kg1o0606gsv9.htm
+
+  This macro can be used in place of the %include statement, and will insert
+  the following (equivalent) global variables:
+
+  @li _SYSINCLUDEFILEDEVICE
+  @li _SYSINCLUDEFILEDIR
+  @li _SYSINCLUDEFILEFILEREF
+  @li _SYSINCLUDEFILENAME
+
+  These can be used whenever testing _within a macro_.  Outside of the macro,
+  the regular automatic variables will still be available (thanks to a
+  concatenated file list in the include statement).
+
+  Example usage:
+
+      filename example temp;
+      data _null_;
+        file example;
+        put '%macro test();';
+        put '%put &=_SYSINCLUDEFILEFILEREF;';
+        put '%put &=SYSINCLUDEFILEFILEREF;';
+        put '%mend; %test()';
+        put '%put &=SYSINCLUDEFILEFILEREF;';
+      run;
+      %mp_include(example)
+
+  @param [in] fileref The fileref of the file to be included. Must be provided.
+  @param [in] prefix= (_) The prefix to apply to the global variables.
+  @param [in] opts= (SOURCE2) The options to apply to the %inc statement
+  @param [in] errds= (work.mp_abort_errds) There is no clean way to end a
+    process within a %include called within a macro.  Furthermore, there is no
+    way to test if a macro is called within a %include.  To handle this
+    particular scenario, the %mp_abort() macro will test for the existence of
+    the `_SYSINCLUDEFILEDEVICE` variable and return the outputs (msg,mac) inside
+    this dataset.
+    It will then run an abort cancel FILE to stop the include running, and pass
+    the dataset back.
+    NOTE - it is NOT possible to read this dataset as part of _this_ macro -
+    when running abort cancel FILE, ALL macros are closed, so instead it is
+    necessary to invoke "%mp_abort(mode=INCLUDE)" OUTSIDE of any macro wrappers.
+
+
+  @version 9.4
+  @author Allan Bowe
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_abort.sas
+
+**/
+
+%macro mp_include(fileref
+  ,prefix=_
+  ,opts=SOURCE2
+  ,errds=work.mp_abort_errds
+)/*/STORE SOURCE*/;
+
+/* prepare precode */
+%local tempref;
+%let tempref=%mf_getuniquefileref();
+data _null_;
+  file &tempref;
+  set sashelp.vextfl(where=(fileref="%upcase(&fileref)"));
+  put '%let _SYSINCLUDEFILEDEVICE=' xengine ';';
+  name=scan(xpath,-1,'/\');
+  put '%let _SYSINCLUDEFILENAME=' name ';';
+  path=subpad(xpath,1,length(xpath)-length(name)-1);
+  put '%let _SYSINCLUDEFILEDIR=' path ';';
+  put '%let _SYSINCLUDEFILEFILEREF=' "&fileref;";
+run;
+
+/* prepare the errds */
+data &errds;
+  length msg mac $1000;
+  iftrue='1=0';
+run;
+
+/* include the include */
+%inc &tempref &fileref/&opts;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=%str(&_SYSINCLUDEFILEDIR/&_SYSINCLUDEFILENAME)
+  ,msg=%str(syscc=&syscc after executing &_SYSINCLUDEFILENAME)
+)
+
+filename &tempref clear;
+
+%mend mp_include;/**
  @file mp_jsonout.sas
  @brief Writes JSON in SASjs format to a fileref
  @details PROC JSON is faster but will produce errs like the ones below if
  special chars are encountered.

-  > ERROR: Some code points did not transcode.
+  > (ERR)OR: Some code points did not transcode.

  > An object or array close is not valid at this point in the JSON text.

@@ -5683,6 +5968,8 @@ select distinct lowcase(memname)
  @param [out] outref= Output fileref in which to create the insert statements.
    If it exists, it will be appended to, otherwise it will be created.
  @param [out] schema= (0) The schema of the target database, or the libref.
+  @param [in] applydttm= (YES) If YES, any columns using datetime formats will
+    be converted to native DB datetime literals

  @version 9.2
  @author Allan Bowe
@@ -5693,6 +5980,7 @@ select distinct lowcase(memname)
    ,outref=0
    ,schema=0
    ,maxobs=max
+    ,applydttm=YES
 )/*/STORE SOURCE*/;

 /* Find the tables */
@@ -5722,6 +6010,7 @@ select distinct lowcase(memname)
    ,outds=&ds
    ,flavour=&flavour
    ,maxobs=&maxobs
+    ,applydttm=&applydttm
  )
 %end;

@@ -7310,6 +7599,64 @@ alter table &libds modify &var char(&len);

 %mend mp_validatecol;
 /**
+  @file
+  @brief Fix the `_WEBIN` variables provided to SAS web services
+  @details When uploading files to SAS Stored Processes or Viya Jobs a number
+  of global macro variables are automatically created - however there are some
+  differences in behaviour both between SAS 9 and Viya, and also between a
+  single file upload and a multi-file upload.
+
+  This macro "straightens" up the global macro variables to make it easier /
+  simpler to write code that works in both environments and with a variable
+  number of file inputs.
+
+  After running this macro, the following global variables will *always* exist:
+  @li `_WEBIN_FILE_COUNT`
+  @li `_WEBIN_FILENAME1`
+  @li `_WEBIN_FILEREF1`
+  @li `_WEBIN_NAME1`
+
+  Usage:
+
+    %mp_webin()
+
+  This was created as a macro procedure (over a macro function) as it will also
+  use the filename statement in Viya environments (where `_webin_fileuri` is
+  provided).
+
+  <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+
+**/
+
+%macro mp_webin();
+
+/* prepare global variables */
+%global _webin_file_count
+  _webin_filename _webin_filename1
+  _webin_fileref _webin_fileref1
+  _webin_fileuri _webin_fileuri1
+  _webin_name _webin_name1
+  ;
+
+/* create initial versions */
+%let _webin_file_count=%eval(&_webin_file_count+0);
+%let _webin_filename1=%sysfunc(coalescec(&_webin_filename1,&_webin_filename));
+%let _webin_fileref1=%sysfunc(coalescec(&_webin_fileref1,&_webin_fileref));
+%let _webin_fileuri1=%sysfunc(coalescec(&_webin_fileuri1,&_webin_fileuri));
+%let _webin_name1=%sysfunc(coalescec(&_webin_name1,&_webin_name));
+
+
+/* If Viya, create temporary fileref(s) */
+%local i;
+%if %mf_getplatform()=SASVIYA %then %do i=1 %to &_webin_file_count;
+  %let _webin_fileref&i=%mf_getuniquefileref();
+  filename &&_webin_fileref&i filesrvc "&&_webin_fileuri&i";
+%end;
+
+
+%mend mp_webin;/**
  @file
  @brief Creates a zip file
  @details For DIRECTORY usage, will ignore subfolders. For DATASET usage,
@@ -15380,24 +15727,27 @@ filename &fname1 clear;
 libname &libref1 clear;
 */
 %mend mv_getclients;/**
-  @file mv_getfoldermembers.sas
-  @brief Gets a list of folders (and ids) for a given root
-  @details Works for both root level and below, oauth or password. Default is
-    oauth, and the token is expected in a global ACCESS_TOKEN variable.
+  @file
+  @brief Gets a list of folder members (and ids) for a given root
+  @details Returns all members for a particular Viya folder.  Works at both root
+  level and below, and results are created in an output dataset.

-        %mv_getfoldermembers(root=/Public)
+        %mv_getfoldermembers(root=/Public, outds=work.mymembers)


-  @param root= The path for which to return the list of folders
-  @param outds= The output dataset to create (default is work.mv_getfolders).  Format:
+  @param [in] root= (/) The path for which to return the list of folders
+  @param [out] outds= (work.mv_getfolders) The output dataset to create. Format:
  |ordinal_root|ordinal_items|creationTimeStamp| modifiedTimeStamp|createdBy|modifiedBy|id| uri|added| type|name|description|
  |---|---|---|---|---|---|---|---|---|---|---|---|
  |1|1|2021-05-25T11:15:04.204Z|2021-05-25T11:15:04.204Z|allbow|allbow|4f1e3945-9655-462b-90f2-c31534b3ca47|/folders/folders/ed701ff3-77e8-468d-a4f5-8c43dec0fd9e|2021-05-25T11:15:04.212Z|child|my_folder_name|My folder Description|

-  @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values are "password" or "authorization_code" (unquoted).
-    The default is authorization_code.
-
+  @param [in] access_token_var= (ACCESS_TOKEN) The global macro variable to
+    contain the access token
+  @param [in] grant_type= (sas_services) Valid values are:
+    @li password
+    @li authorization_code
+    @li detect
+    @li sas_services

  @version VIYA V.03.04
  @author Allan Bowe, source: https://github.com/sasjs/core
@@ -15409,6 +15759,12 @@ libname &libref1 clear;
  @li mf_getuniquelibref.sas
  @li mf_isblank.sas

+  <h4> Related Macros </h4>
+  @li mv_createfolder.sas
+  @li mv_deletefoldermember.sas
+  @li mv_deleteviyafolder.sas
+  @li mv_getfoldermembers.test.sas
+
 **/

 %macro mv_getfoldermembers(root=/
@@ -15448,7 +15804,7 @@ options noquotelenmax;
 %if "&root"="/" %then %do;
  /* if root just list root folders */
  proc http method='GET' out=&fname1 &oauth_bearer
-      url="&base_uri/folders/rootFolders";
+      url="&base_uri/folders/rootFolders?limit=1000";
  %if &grant_type=authorization_code %then %do;
      headers "Authorization"="Bearer &&&access_token_var";
  %end;
@@ -15469,13 +15825,17 @@ options noquotelenmax;
  /*data _null_;infile &fname1;input;putlog _infile_;run;*/
  libname &libref1 JSON fileref=&fname1;
  /* now get the followon link to list members */
-  %local href;
-  %let href=0;
+  %local href cnt;
+  %let cnt=0;
  data _null_;
    set &libref1..links;
-    if rel='members' then call symputx('href',quote("&base_uri"!!trim(href)),'l');
+    if rel='members' then do;
+      url=cats("'","&base_uri",href,"?limit=10000'");
+      call symputx('href',url,'l');
+      call symputx('cnt',1,'l');
+    end;
  run;
-  %if &href=0 %then %do;
+  %if &cnt=0 %then %do;
    %put NOTE:;%put NOTE-  No members found in &root!!;%put NOTE-;
    %return;
  %end;
@@ -15490,6 +15850,7 @@ options noquotelenmax;
  run;
  libname &libref2 JSON fileref=&fname2;
  data &outds;
+    length id $36 name $128 uri $64 type $32 description $256;
    set &libref2..items;
  run;
  filename &fname2 clear;
@@ -18444,6 +18805,50 @@ filename &fref1 clear;
 %end;

 %mend mv_webout;
+/**
+  @file ml_gsubfile.sas
+  @brief Compiles the gsubfile.lua lua file
+  @details Writes gsubfile.lua to the work directory
+  and then includes it.
+  Usage:
+
+      %ml_gsubfile()
+
+**/
+
+%macro ml_gsubfile();
+data _null_;
+  file "%sysfunc(pathname(work))/ml_gsubfile.lua";
+  put 'local fpath, outpath, file, fcontent ';
+  put ' ';
+  put '-- configure in / out paths ';
+  put 'fpath = sas.symget("file") ';
+  put 'outpath = sas.symget("outfile") ';
+  put 'if ( outpath == 0 ) ';
+  put 'then ';
+  put '   outpath=fpath ';
+  put 'end ';
+  put ' ';
+  put '-- open file and perform the substitution ';
+  put 'file = io.open(fpath,"r") ';
+  put 'fcontent = file:read("*all") ';
+  put 'file:close() ';
+  put 'fcontent = string.gsub( ';
+  put '  fcontent, ';
+  put '  sas.symget(sas.symget("patternvar")), ';
+  put '  sas.symget(sas.symget("replacevar")) ';
+  put ') ';
+  put ' ';
+  put '-- write the file back out ';
+  put 'file = io.open(outpath, "w+") ';
+  put 'io.output(file) ';
+  put 'io.write(fcontent) ';
+  put 'io.close(file) ';
+run;
+
+%inc "%sysfunc(pathname(work))/ml_gsubfile.lua" /source2;
+
+%mend ml_gsubfile;
 /**
  @file ml_json.sas
  @brief Compiles the json.lua lua file
@@ -18835,7 +19240,7 @@ data _null_;
  put '-- JSON.LUA ENDS HERE ';
 run;

-%inc "%sysfunc(pathname(work))/ml_json.lua";
+%inc "%sysfunc(pathname(work))/ml_json.lua" /source2;

 %mend ml_json;
 /**
--- a/base/mf_existfunction.sas
+++ b/base/mf_existfunction.sas
@@ -12,21 +12,23 @@
  Full credit to [Bart](https://sasensei.com/user/305) for the vfunc pointer
  and the tidy approach for pure macro data set filtering.
  Check out his [SAS Packages](https://github.com/yabwon/SAS_PACKAGES)
-  framework!
+  framework!  Where you can find the same [function](
+https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionexists-macro
+  ).

  @param [in] name (positional) - function name

  @author Allan Bowe
 **/
 /** @cond */
-
 %macro mf_existfunction(name
 )/*/STORE SOURCE*/;

  %local dsid rc exist;
  %let dsid=%sysfunc(open(sashelp.vfunc(where=(fncname="%upcase(&name)"))));
-  %let exist = %sysfunc(fetch(&dsid, NOSET));
-  %let rc = %sysfunc(close(&dsid));
+  %let exist=1;
+  %let exist=%sysfunc(fetch(&dsid, NOSET));
+  %let rc=%sysfunc(close(&dsid));

  %sysevalf(0 = &exist)

--- a/base/mf_getapploc.sas
+++ b/base/mf_getapploc.sas
@@ -36,13 +36,25 @@

 %macro mf_getapploc(pgm);
 %if "&pgm"="" %then %do;
-  %put &sysmacroname: No value provided;
-  %return;
+  %if %symexist(_program) %then %let pgm=&_program;
+  %else %do;
+    %put &sysmacroname: No value provided and no _program variable available;
+    %return;
+  %end;
 %end;
 %local root;

 /**
-  * move up two levels to avoid matches on subfolder or service name
+  * First check we are not in the tests/macros folder (which has no subfolders)
+  */
+%if %index(&pgm,/tests/macros/) %then %do;
+  %let root=%substr(&pgm,1,%index(&pgm,/tests/macros)-1);
+  &root
+  %return;
+%end;
+
+/**
+  * Next, move up two levels to avoid matches on subfolder or service name
  */
 %let root=%substr(&pgm,1,%length(&pgm)-%length(%scan(&pgm,-1,/))-1);
 %let root=%substr(&root,1,%length(&root)-%length(%scan(&root,-1,/))-1);
--- a/base/mf_getuniquefileref.sas
+++ b/base/mf_getuniquefileref.sas
@@ -1,37 +1,58 @@
 /**
  @file
  @brief Assigns and returns an unused fileref
-  @details
+  @details  Using the native approach for assigning filerefs fails as some
+  procedures (such as proc http) do not recognise the temporary names (starting
+  with a hash), returning a message such as:
+
+  > ERROR 22-322: Expecting a name.
+
+  This macro works by attempting a random fileref (with a prefix), seeing if it
+  is already assigned, and if not - returning the fileref.
+
+  If your process can accept filerefs with the hash (#) prefix, then set
+  `prefix=0` to revert to the native approach - which is significantly faster
+  when there are a lot of filerefs in a session.
+
  Use as follows:

      %let fileref1=%mf_getuniquefileref();
-      %let fileref2=%mf_getuniquefileref();
+      %let fileref2=%mf_getuniquefileref(prefix=0);
      %put &fileref1 &fileref2;

-  which returns:
+  which returns filerefs similar to:

-> mcref0 mcref1
+> _7432233 #LN00070

-  @param prefix= first part of fileref. Remember that filerefs can only be 8
-    characters, so a 7 letter prefix would mean that `maxtries` should be 10.
-  @param maxtries= the last part of the libref.  Provide an integer value.
+  @param [in] prefix= (_) first part of fileref. Remember that filerefs can only
+    be 8 characters, so a 7 letter prefix would mean `maxtries` should be 10.
+    if using zero (0) as the prefix, a native assignment is used.
+  @param [in] maxtries= (1000) the last part of the libref. Must be an integer.

  @version 9.2
  @author Allan Bowe
 **/

-%macro mf_getuniquefileref(prefix=mcref,maxtries=1000);
-  %local x fname;
-  %let x=0;
-  %do x=0 %to &maxtries;
-  %if %sysfunc(fileref(&prefix&x)) > 0 %then %do;
-    %let fname=&prefix&x;
+%macro mf_getuniquefileref(prefix=_,maxtries=1000);
+  %local rc fname;
+  %if &prefix=0 %then %do;
    %let rc=%sysfunc(filename(fname,,temp));
    %if &rc %then %put %sysfunc(sysmsg());
-    &prefix&x
-    %*put &sysmacroname: Fileref &prefix&x was assigned and returned;
-    %return;
+    &fname
  %end;
+  %else %do;
+    %local x len;
+    %let len=%eval(8-%length(&prefix));
+    %let x=0;
+    %do x=0 %to &maxtries;
+      %let fname=&prefix%substr(%sysfunc(ranuni(0)),3,&len);
+      %if %sysfunc(fileref(&fname)) > 0 %then %do;
+        %let rc=%sysfunc(filename(fname,,temp));
+        %if &rc %then %put %sysfunc(sysmsg());
+        &fname
+        %return;
+      %end;
+    %end;
+    %put unable to find available fileref after &maxtries attempts;
  %end;
-  %put unable to find available fileref in range &prefix.0-&maxtries;
 %mend mf_getuniquefileref;
--- a/base/mp_abort.sas
+++ b/base/mp_abort.sas
@@ -15,24 +15,47 @@
    recognise this and fetch the log of the parent session instead)
  @li STP environments must finish cleanly to avoid the log being sent to
    _webout.  To assist with this, we also run stpsrvset('program error', 0)
-    and set SYSCC=0.  For 9.4m3 we take a unique approach - we open a macro
-    but don't close it!  This provides a graceful abort, EXCEPT when called
-    called within a %include within a macro (and that macro contains additional
-    logic).  See mp_abort.test.nofix.sas for the example case.
-    If you know of another way to gracefully abort a 9.4m3 STP session, we'd
-    love to hear about it!
+    and set SYSCC=0.  We take a unique "soft abort" approach - we open a macro
+    but don't close it!  This works everywhere EXCEPT inside a \%include inside
+    a macro.  For that, we recommend you use mp_include.sas to perform the
+    include, and then call \%mp_abort(mode=INCLUDE) from the source program (ie,
+    OUTSIDE of the top-parent macro).


  @param mac= to contain the name of the calling macro
  @param msg= message to be returned
  @param iftrue= supply a condition under which the macro should be executed.
+  @param errds= (work.mp_abort_errds) There is no clean way to end a process
+    within a %include called within a macro.  Furthermore, there is no way to
+    test if a macro is called within a %include.  To handle this particular
+    scenario, the %include should be switched for the mp_include.sas macro.
+    This provides an indicator that we are running a macro within a \%include
+    (`_SYSINCLUDEFILEDEVICE`) and allows us to provide a dataset with the abort
+    values (msg, mac).
+    We can then run an abort cancel FILE to stop the include running, and pass
+    the dataset back to the calling program to run a regular \%mp_abort().
+    The dataset will contain the following fields:
+    @li iftrue (1=1)
+    @li msg (the message)
+    @li mac (the mac param)

-  @version 9.4M3
+  @param mode= (REGULAR) If mode=INCLUDE then the &errds dataset is checked for
+    an abort status.
+    Valid values:
+    @li REGULAR (default)
+    @li INCLUDE
+
+  <h4> Related Macros </h4>
+  @li mp_include.sas
+
+  @version 9.4
  @author Allan Bowe
  @cond
 **/

 %macro mp_abort(mac=mp_abort.sas, type=, msg=, iftrue=%str(1=1)
+  , errds=work.mp_abort_errds
+  , mode=REGULAR
 )/*/STORE SOURCE*/;

  %global sysprocessmode sysprocessname;
@@ -43,9 +66,44 @@
  %if %length(&mac)>0 %then %put NOTE- called by &mac;
  %put NOTE - &msg;

+  %if %symexist(_SYSINCLUDEFILEDEVICE) %then %do;
+    %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do;
+      data &errds;
+        iftrue='1=1';
+        length mac $100 msg $5000;
+        mac=symget('mac');
+        msg=symget('msg');
+      run;
+      data _null_;
+        abort cancel FILE;
+      run;
+      %return;
+    %end;
+  %end;
+
  /* Stored Process Server web app context */
-  %if %symexist(_metaperson) or "&SYSPROCESSNAME "="Compute Server " %then %do;
+  %if %symexist(_metaperson)
+    or "&SYSPROCESSNAME "="Compute Server "
+    or &mode=INCLUDE
+  %then %do;
    options obs=max replace nosyntaxcheck mprint;
+    %if &mode=INCLUDE %then %do;
+      %if %sysfunc(exist(&errds))=1 %then %do;
+        data _null_;
+          set &errds;
+          call symputx('iftrue',iftrue,'l');
+          call symputx('mac',mac,'l');
+          call symputx('msg',msg,'l');
+          putlog (_all_)(=);
+        run;
+        %if (&iftrue)=0 %then %return;
+      %end;
+      %else %do;
+        %put &sysmacroname: No include errors found;
+        %return;
+      %end;
+    %end;
+
    /* extract log errs / warns, if exist */
    %local logloc logline;
    %global logmsg; /* capture global messages */
@@ -100,7 +158,7 @@
    /* send response in SASjs JSON format */
    data _null_;
      file _webout mod lrecl=32000 encoding='utf-8';
-      length msg $32767 debug $8;
+      length msg $32767 ;
      sasdatetime=datetime();
      msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
      /* escape the quotes */
@@ -131,11 +189,15 @@
      _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
      put ',"_PROGRAM" : ' _PROGRAM ;
      put ",""SYSCC"" : ""&syscc"" ";
-      put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+      syserrortext=quote(trim(symget('syserrortext')));
+      put ",""SYSERRORTEXT"" : " syserrortext;
+      put ",""SYSHOSTNAME"" : ""&syshostname"" ";
      put ",""SYSJOBID"" : ""&sysjobid"" ";
+      put ",""SYSSITE"" : ""&syssite"" ";
      sysvlong=quote(trim(symget('sysvlong')));
      put ',"SYSVLONG" : ' sysvlong;
-      put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+      syswarningtext=quote(trim(symget('syswarningtext')));
+      put ",""SYSWARNINGTEXT"" : " syswarningtext;
      put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
      put "}" @;
      if debug ge '"131"' then put '>>weboutEND<<';
@@ -149,11 +211,22 @@
        rc=stpsrvset('program error', 0);
        call symputx("syscc",0,"g");
      run;
-      %if "%substr(&sysvlong.xxxxxxxxx,1,9)" ne "9.04.01M3" %then %do;
-        %put NOTE: Ending SAS session due to:;
-        %put NOTE- &msg;
-        endsas;
-      %end;
+      /**
+        * endsas kills 9.4m3 deployments by orphaning multibridges.
+        * Abort variants are ungraceful (non zero return code)
+        * This approach lets SAS run silently until the end :-)
+        * Caution - fails when called within a %include within a macro
+        * Use mp_include() to handle this.
+        */
+      filename skip temp;
+      data _null_;
+        file skip;
+        put '%macro skip();';
+        comment '%mend skip; -> fix lint ';
+        put '%macro skippy();';
+        comment '%mend skippy; -> fix lint ';
+      run;
+      %inc skip;
    %end;
    %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
      /* endsas kills the session making it harder to fetch results */
@@ -169,24 +242,6 @@
        abort cancel nolist;
      run;
    %end;
-    %else %if "%substr(&sysvlong.xxxxxxxxx,1,9)" = "9.04.01M3" %then %do;
-      /**
-        * endsas kills 9.4m3 deployments by orphaning multibridges.
-        * Abort variants are ungraceful (non zero return code)
-        * This approach lets SAS run silently until the end :-)
-        * Caution - fails when called within a %include within a macro
-        * See tests/mp_abort.test.1 for an example case.
-        */
-      filename skip temp;
-      data _null_;
-        file skip;
-        put '%macro skip();';
-        comment '%mend skip; -> fix lint ';
-        put '%macro skippy();';
-        comment '%mend skippy; -> fix lint ';
-      run;
-      %inc skip;
-    %end;
    %else %do;
      %abort cancel;
    %end;
--- a/base/mp_binarycopy.sas
+++ b/base/mp_binarycopy.sas
@@ -9,10 +9,29 @@

        %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)

-  @param inloc full, quoted "path/and/filename.ext" of the object to be copied
-  @param outloc full, quoted "path/and/filename.ext" of object to be created
-  @param inref can override default input fileref to avoid naming clash
-  @param outref an override default output fileref to avoid naming clash
+  To append to a file, use the mode option, eg:
+
+      filename tmp1 temp;
+      filename tmp2 temp;
+      data _null_;
+        file tmp1;
+        put 'stacking';
+      run;
+
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+
+
+  @param [in] inloc quoted "path/and/filename.ext" of the file to be copied
+  @param [out] outloc quoted "path/and/filename.ext" of the file to be created
+  @param [in] inref (____in) If provided, this fileref will take precedence over
+    the `inloc` parameter
+  @param [out] outref (____in) If provided, this fileref will take precedence
+    over the `outloc` parameter.  It must already exist!
+  @param [in] mode (CREATE) Valid values:
+    @li CREATE - Create the file (even if it already exists)
+    @li APPEND - Append to the file (don't overwrite)
+
  @returns nothing

  @version 9.2
@@ -24,20 +43,29 @@
    ,outloc=          /* full path and filename of object to be created */
    ,inref=____in   /* override default to use own filerefs */
    ,outref=____out /* override default to use own filerefs */
+    ,mode=CREATE
 )/*/STORE SOURCE*/;
+  %local mod outmode;
+  %if &mode=APPEND %then %do;
+    %let mod=mod;
+    %let outmode='a';
+  %end;
+  %else %do;
+    %let outmode='o';
+  %end;
  /* these IN and OUT filerefs can point to anything */
  %if &inref = ____in %then %do;
    filename &inref &inloc lrecl=1048576 ;
  %end;
  %if &outref=____out %then %do;
-    filename &outref &outloc lrecl=1048576 ;
+    filename &outref &outloc lrecl=1048576 &mod;
  %end;

  /* copy the file byte-for-byte  */
  data _null_;
    length filein 8 fileid 8;
    filein = fopen("&inref",'I',1,'B');
-    fileid = fopen("&outref",'O',1,'B');
+    fileid = fopen("&outref",&outmode,1,'B');
    rec = '20'x;
    do while(fread(filein)=0);
      rc = fget(filein,rec,1);
--- a/base/mp_ds2inserts.sas
+++ b/base/mp_ds2inserts.sas
@@ -25,10 +25,13 @@
    options:
    @li SAS (default) - suitable for regular proc sql
    @li PGSQL - Used for Postgres databases
+  @param [in] applydttm= (YES) If YES, any columns using datetime formats will
+    be converted to native DB datetime literals

  <h4> SAS Macros </h4>
  @li mf_existfileref.sas
  @li mf_getvarcount.sas
+  @li mf_getvarformat.sas
  @li mf_getvarlist.sas
  @li mf_getvartype.sas

@@ -37,6 +40,7 @@
 **/

 %macro mp_ds2inserts(ds, outref=0,schema=0,outds=0,flavour=SAS,maxobs=max
+  ,applydttm=YES
 )/*/STORE SOURCE*/;

 %if not %sysfunc(exist(&ds)) %then %do;
@@ -114,10 +118,11 @@ data _null_;
  length _____str $32767;
  format _numeric_ best.;
  format _character_ ;
-  %local i comma var vtype;
+  %local i comma var vtype vfmt;
  %do i=1 %to %sysfunc(countw(&varlist));
    %let var=%scan(&varlist,&i);
    %let vtype=%mf_getvartype(&ds,&var);
+    %let vfmt=%upcase(%mf_getvarformat(&ds,&var,force=1));
    %if &i=1 %then %do;
      %if &flavour=SAS %then %do;
        put "insert into &schema.&outds set ";
@@ -147,7 +152,13 @@ data _null_;
      %end;
      %else %if &flavour=PGSQL %then %do;
        if missing(&var) then put 'NULL';
-        else put &var;
+        %if &applydttm=YES and "%substr(&vfmt.xxxxxxxx,1,8)"="DATETIME"
+        %then %do;
+          else put "TIMESTAMP '" &var E8601DT25.6 "'";
+        %end;
+        %else %do;
+          else put &var;
+        %end;
      %end;
    %end;
    %else %do;
--- a/base/mp_filtervalidate.sas
+++ b/base/mp_filtervalidate.sas
@@ -78,7 +78,7 @@ run;

 data &outds;
  if &sqlrc or &syscc or &syserr then do;
-    REASON_CD='VALIDATION_ERROR: '!!
+    REASON_CD='VALIDATION_ERR'!!'OR: '!!
      coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
    output;
  end;
--- a/base/mp_gsubfile.sas
+++ b/base/mp_gsubfile.sas
@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Performs a text substitution on a file
+  @details Makes use of the GSUB function in LUA to perform a text substitution
+  in a file - either in-place, or writing to a new location.  The benefit of
+  using LUA is that the entire file can be loaded into a single variable,
+  thereby side stepping the 32767 character limit in a data step.
+
+  Usage:
+
+      %let file=%sysfunc(pathname(work))/file.txt;
+      %let str=replace/me;
+      %let rep=with/this;
+      data _null_;
+        file "&file";
+        put "&str";
+      run;
+      %mp_gsubfile(file=&file, patternvar=str, replacevar=rep)
+      data _null_;
+        infile "&file";
+        input;
+        list;
+      run;
+
+  @param file= (0) The file to perform the substitution on
+  @param patternvar= A macro variable containing the Lua
+    [pattern](https://www.lua.org/pil/20.2.html) to search for.  Due to the use
+    of special (magic) characters in Lua patterns, it is safer to pass the NAME
+    of the macro variable containing the string, rather than the value itself.
+  @param replacevar= The name of the macro variable containing the replacement
+    _string_.
+  @param outfile= (0) The file to write the output to. If zero, then the file
+    is overwritten in-place.
+
+  <h4> SAS Macros </h4>
+  @li ml_gsubfile.sas
+
+  <h4> Related Macros </h4>
+  @li mp_gsubfile.test.sas
+
+  @version 9.4
+  @author Allan Bowe
+**/
+
+%macro mp_gsubfile(file=0,
+  patternvar=,
+  replacevar=,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+  %ml_gsubfile()
+
+%mend mp_gsubfile;
--- a/base/mp_include.sas
+++ b/base/mp_include.sas
@@ -0,0 +1,104 @@
+/**
+  @file
+  @brief Performs a wrapped \%include
+  @details This macro wrapper is necessary if you need your included code to
+  know that it is being \%included.
+
+  If you are using %include in a regular program, you could make use of the
+  following macro variables:
+
+  @li SYSINCLUDEFILEDEVICE
+  @li SYSINCLUDEFILEDIR
+  @li SYSINCLUDEFILEFILEREF
+  @li SYSINCLUDEFILENAME
+
+  However these variables are NOT available inside a macro, as documented here:
+https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/mcrolref/n1j5tcc0n2xczyn1kg1o0606gsv9.htm
+
+  This macro can be used in place of the %include statement, and will insert
+  the following (equivalent) global variables:
+
+  @li _SYSINCLUDEFILEDEVICE
+  @li _SYSINCLUDEFILEDIR
+  @li _SYSINCLUDEFILEFILEREF
+  @li _SYSINCLUDEFILENAME
+
+  These can be used whenever testing _within a macro_.  Outside of the macro,
+  the regular automatic variables will still be available (thanks to a
+  concatenated file list in the include statement).
+
+  Example usage:
+
+      filename example temp;
+      data _null_;
+        file example;
+        put '%macro test();';
+        put '%put &=_SYSINCLUDEFILEFILEREF;';
+        put '%put &=SYSINCLUDEFILEFILEREF;';
+        put '%mend; %test()';
+        put '%put &=SYSINCLUDEFILEFILEREF;';
+      run;
+      %mp_include(example)
+
+  @param [in] fileref The fileref of the file to be included. Must be provided.
+  @param [in] prefix= (_) The prefix to apply to the global variables.
+  @param [in] opts= (SOURCE2) The options to apply to the %inc statement
+  @param [in] errds= (work.mp_abort_errds) There is no clean way to end a
+    process within a %include called within a macro.  Furthermore, there is no
+    way to test if a macro is called within a %include.  To handle this
+    particular scenario, the %mp_abort() macro will test for the existence of
+    the `_SYSINCLUDEFILEDEVICE` variable and return the outputs (msg,mac) inside
+    this dataset.
+    It will then run an abort cancel FILE to stop the include running, and pass
+    the dataset back.
+    NOTE - it is NOT possible to read this dataset as part of _this_ macro -
+    when running abort cancel FILE, ALL macros are closed, so instead it is
+    necessary to invoke "%mp_abort(mode=INCLUDE)" OUTSIDE of any macro wrappers.
+
+
+  @version 9.4
+  @author Allan Bowe
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_abort.sas
+
+**/
+
+%macro mp_include(fileref
+  ,prefix=_
+  ,opts=SOURCE2
+  ,errds=work.mp_abort_errds
+)/*/STORE SOURCE*/;
+
+/* prepare precode */
+%local tempref;
+%let tempref=%mf_getuniquefileref();
+data _null_;
+  file &tempref;
+  set sashelp.vextfl(where=(fileref="%upcase(&fileref)"));
+  put '%let _SYSINCLUDEFILEDEVICE=' xengine ';';
+  name=scan(xpath,-1,'/\');
+  put '%let _SYSINCLUDEFILENAME=' name ';';
+  path=subpad(xpath,1,length(xpath)-length(name)-1);
+  put '%let _SYSINCLUDEFILEDIR=' path ';';
+  put '%let _SYSINCLUDEFILEFILEREF=' "&fileref;";
+run;
+
+/* prepare the errds */
+data &errds;
+  length msg mac $1000;
+  iftrue='1=0';
+run;
+
+/* include the include */
+%inc &tempref &fileref/&opts;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=%str(&_SYSINCLUDEFILEDIR/&_SYSINCLUDEFILENAME)
+  ,msg=%str(syscc=&syscc after executing &_SYSINCLUDEFILENAME)
+)
+
+filename &tempref clear;
+
+%mend mp_include;
--- a/base/mp_jsonout.sas
+++ b/base/mp_jsonout.sas
@@ -4,7 +4,7 @@
  @details PROC JSON is faster but will produce errs like the ones below if
  special chars are encountered.

-  > ERROR: Some code points did not transcode.
+  > (ERR)OR: Some code points did not transcode.

  > An object or array close is not valid at this point in the JSON text.

--- a/base/mp_lib2inserts.sas
+++ b/base/mp_lib2inserts.sas
@@ -28,6 +28,8 @@
  @param [out] outref= Output fileref in which to create the insert statements.
    If it exists, it will be appended to, otherwise it will be created.
  @param [out] schema= (0) The schema of the target database, or the libref.
+  @param [in] applydttm= (YES) If YES, any columns using datetime formats will
+    be converted to native DB datetime literals

  @version 9.2
  @author Allan Bowe
@@ -38,6 +40,7 @@
    ,outref=0
    ,schema=0
    ,maxobs=max
+    ,applydttm=YES
 )/*/STORE SOURCE*/;

 /* Find the tables */
@@ -67,6 +70,7 @@ select distinct lowcase(memname)
    ,outds=&ds
    ,flavour=&flavour
    ,maxobs=&maxobs
+    ,applydttm=&applydttm
  )
 %end;

--- a/base/mp_webin.sas
+++ b/base/mp_webin.sas
@@ -0,0 +1,59 @@
+/**
+  @file
+  @brief Fix the `_WEBIN` variables provided to SAS web services
+  @details When uploading files to SAS Stored Processes or Viya Jobs a number
+  of global macro variables are automatically created - however there are some
+  differences in behaviour both between SAS 9 and Viya, and also between a
+  single file upload and a multi-file upload.
+
+  This macro "straightens" up the global macro variables to make it easier /
+  simpler to write code that works in both environments and with a variable
+  number of file inputs.
+
+  After running this macro, the following global variables will *always* exist:
+  @li `_WEBIN_FILE_COUNT`
+  @li `_WEBIN_FILENAME1`
+  @li `_WEBIN_FILEREF1`
+  @li `_WEBIN_NAME1`
+
+  Usage:
+
+    %mp_webin()
+
+  This was created as a macro procedure (over a macro function) as it will also
+  use the filename statement in Viya environments (where `_webin_fileuri` is
+  provided).
+
+  <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+
+**/
+
+%macro mp_webin();
+
+/* prepare global variables */
+%global _webin_file_count
+  _webin_filename _webin_filename1
+  _webin_fileref _webin_fileref1
+  _webin_fileuri _webin_fileuri1
+  _webin_name _webin_name1
+  ;
+
+/* create initial versions */
+%let _webin_file_count=%eval(&_webin_file_count+0);
+%let _webin_filename1=%sysfunc(coalescec(&_webin_filename1,&_webin_filename));
+%let _webin_fileref1=%sysfunc(coalescec(&_webin_fileref1,&_webin_fileref));
+%let _webin_fileuri1=%sysfunc(coalescec(&_webin_fileuri1,&_webin_fileuri));
+%let _webin_name1=%sysfunc(coalescec(&_webin_name1,&_webin_name));
+
+
+/* If Viya, create temporary fileref(s) */
+%local i;
+%if %mf_getplatform()=SASVIYA %then %do i=1 %to &_webin_file_count;
+  %let _webin_fileref&i=%mf_getuniquefileref();
+  filename &&_webin_fileref&i filesrvc "&&_webin_fileuri&i";
+%end;
+
+
+%mend mp_webin;
--- a/build.py
+++ b/build.py
@@ -22,7 +22,7 @@ for file in files:
        for line in infile:
          ml.write("  put '" + line.rstrip().replace("'","''") + " ';\n")
    ml.write("run;\n\n")
-    ml.write("%inc \"%sysfunc(pathname(work))/" + name + ".lua\";\n\n")
+    ml.write("%inc \"%sysfunc(pathname(work))/" + name + ".lua\" /source2;\n\n")
    ml.write("%mend " + name + ";\n")

 ml.close()
--- a/lua/gsubfile.lua
+++ b/lua/gsubfile.lua
@@ -0,0 +1,25 @@
+local fpath, outpath, file, fcontent
+
+-- configure in / out paths
+fpath = sas.symget("file")
+outpath = sas.symget("outfile")
+if ( outpath == 0 )
+then
+   outpath=fpath
+end
+
+-- open file and perform the substitution
+file = io.open(fpath,"r")
+fcontent = file:read("*all")
+file:close()
+fcontent = string.gsub(
+  fcontent,
+  sas.symget(sas.symget("patternvar")),
+  sas.symget(sas.symget("replacevar"))
+)
+
+-- write the file back out
+file = io.open(outpath, "w+")
+io.output(file)
+io.write(fcontent)
+io.close(file)
--- a/lua/ml_gsubfile.sas
+++ b/lua/ml_gsubfile.sas
@@ -0,0 +1,44 @@
+/**
+  @file ml_gsubfile.sas
+  @brief Compiles the gsubfile.lua lua file
+  @details Writes gsubfile.lua to the work directory
+  and then includes it.
+  Usage:
+
+      %ml_gsubfile()
+
+**/
+
+%macro ml_gsubfile();
+data _null_;
+  file "%sysfunc(pathname(work))/ml_gsubfile.lua";
+  put 'local fpath, outpath, file, fcontent ';
+  put ' ';
+  put '-- configure in / out paths ';
+  put 'fpath = sas.symget("file") ';
+  put 'outpath = sas.symget("outfile") ';
+  put 'if ( outpath == 0 ) ';
+  put 'then ';
+  put '   outpath=fpath ';
+  put 'end ';
+  put ' ';
+  put '-- open file and perform the substitution ';
+  put 'file = io.open(fpath,"r") ';
+  put 'fcontent = file:read("*all") ';
+  put 'file:close() ';
+  put 'fcontent = string.gsub( ';
+  put '  fcontent, ';
+  put '  sas.symget(sas.symget("patternvar")), ';
+  put '  sas.symget(sas.symget("replacevar")) ';
+  put ') ';
+  put ' ';
+  put '-- write the file back out ';
+  put 'file = io.open(outpath, "w+") ';
+  put 'io.output(file) ';
+  put 'io.write(fcontent) ';
+  put 'io.close(file) ';
+run;
+
+%inc "%sysfunc(pathname(work))/ml_gsubfile.lua" /source2;
+
+%mend ml_gsubfile;
--- a/lua/ml_json.sas
+++ b/lua/ml_json.sas
@@ -389,6 +389,6 @@ data _null_;
  put '-- JSON.LUA ENDS HERE ';
 run;

-%inc "%sysfunc(pathname(work))/ml_json.lua";
+%inc "%sysfunc(pathname(work))/ml_json.lua" /source2;

 %mend ml_json;
--- a/package-lock.json
+++ b/package-lock.json
@@ -7,13 +7,13 @@
      "name": "@sasjs/core",
      "license": "MIT",
      "devDependencies": {
-        "@sasjs/cli": "^2.37.2"
+        "@sasjs/cli": "^2.37.8"
      }
    },
    "node_modules/@sasjs/adapter": {
-      "version": "2.10.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.10.0.tgz",
-      "integrity": "sha512-GbvyIgbODnAJaBaz/Tz8/IwcujNOsZcwzbIuKQtG5y13BzgVPtx/e7b2TAhdoBqd+8uVh0CdrSDfOV/SuvPurg==",
+      "version": "2.10.4",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.10.4.tgz",
+      "integrity": "sha512-9b3WGZUzRgszZVyPfrabSx6TycL4JFFXQR/RQKsFCDDwT8UgLGfJ4JmgaGCSmGCcsqML/Q41QipdgWu1M/QA3g==",
      "dev": true,
      "dependencies": {
        "@sasjs/utils": "^2.27.1",
@@ -28,23 +28,23 @@
      }
    },
    "node_modules/@sasjs/cli": {
-      "version": "2.37.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.37.2.tgz",
-      "integrity": "sha512-Bcm8UFN9Y/ZON4T31Gu9mf1REn1pZoStHFVrix/yp7mchFt5rrrY5RbIqO/AI1jzhShSVg9P7oU4VPJrqki+SA==",
+      "version": "2.37.8",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.37.8.tgz",
+      "integrity": "sha512-V1FGK0ByS5v5+xOX5J1rlQDOXSA37YLL18etEUQOmWK5X9R5tbK4LPIx/pEjLyfZcCHddrm+0nGzRroC715Ilg==",
      "dev": true,
      "dependencies": {
-        "@sasjs/adapter": "2.10.0",
-        "@sasjs/core": "2.35.4",
+        "@sasjs/adapter": "2.10.4",
+        "@sasjs/core": "2.42.0",
        "@sasjs/lint": "1.11.2",
-        "@sasjs/utils": "2.27.1",
-        "chalk": "4.1.1",
+        "@sasjs/utils": "2.28.0",
+        "chalk": "4.1.2",
        "csv-stringify": "5.6.2",
        "dotenv": "10.0.0",
        "esm": "3.2.25",
        "find": "0.3.0",
        "get-installed-path": "4.0.8",
        "js-base64": "3.6.1",
-        "jsdom": "16.6.0",
+        "jsdom": "17.0.0",
        "jwt-decode": "3.1.2",
        "lodash.groupby": "4.6.0",
        "lodash.uniqby": "4.7.0",
@@ -52,16 +52,17 @@
        "ora": "5.4.1",
        "rimraf": "3.0.2",
        "shelljs": "0.8.4",
-        "xml": "1.0.1"
+        "xml": "1.0.1",
+        "yargs": "17.1.1"
      },
      "bin": {
        "sasjs": "build/index.js"
      }
    },
    "node_modules/@sasjs/core": {
-      "version": "2.35.4",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.35.4.tgz",
-      "integrity": "sha512-Exr3+yRdvacKvbXQoi1RfGHi5NtZUkc7RwFNkemHTFXLYaIzPI8CGSCaQmKkwM1UteOJly2e2pw2YT6kHNY1NA==",
+      "version": "2.42.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.42.0.tgz",
+      "integrity": "sha512-EISPPCEv+vB3Nqp03NUaIvMZwEnggzGJeEpVfm2Sb504ySN1I2xiJ8bOHXIzvvYP5N/V9X8bXe1raLYnnt8HGA==",
      "dev": true
    },
    "node_modules/@sasjs/lint": {
@@ -74,9 +75,9 @@
      }
    },
    "node_modules/@sasjs/utils": {
-      "version": "2.27.1",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.27.1.tgz",
-      "integrity": "sha512-CYTQwEj89cc7H3tGiQQcyDkZYaWRc1HZJpOF8o2RHYS37fIAOy0SyyJdq6mcQ74Nb1u5AmFXPFIvnRCMEcTYeQ==",
+      "version": "2.28.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.28.0.tgz",
+      "integrity": "sha512-aZAbBDSjvW2TCY1lkuGqqiyWtR4W8GFpCNCUKa72VIkl4zwTE/pnSHQ+v8QilGDSHSPbPEsJGaBaldMGwoI12w==",
      "dev": true,
      "dependencies": {
        "@types/fs-extra": "^9.0.11",
@@ -141,9 +142,9 @@
      "dev": true
    },
    "node_modules/acorn": {
-      "version": "8.4.0",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.4.0.tgz",
-      "integrity": "sha512-ULr0LDaEqQrMFGyQ3bhJkLsbtrQ8QibAseGZeaSUiT/6zb9IvIkomWHJIvgvwad+hinRAgsI51JcWk2yvwyL+w==",
+      "version": "8.5.0",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.5.0.tgz",
+      "integrity": "sha512-yXbYeFy+jUuYd3/CDcg2NkIYE991XYX/bje7LmjJigUciaeO1JR4XxXgCIV1/Zc/dRuFEyw1L0pbA+qynJkW5Q==",
      "dev": true,
      "bin": {
        "acorn": "bin/acorn"
@@ -226,12 +227,12 @@
      "dev": true
    },
    "node_modules/axios": {
-      "version": "0.21.1",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz",
-      "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==",
+      "version": "0.21.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
+      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
      "dev": true,
      "dependencies": {
-        "follow-redirects": "^1.10.0"
+        "follow-redirects": "^1.14.0"
      }
    },
    "node_modules/axios-cookiejar-support": {
@@ -330,9 +331,9 @@
      }
    },
    "node_modules/chalk": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz",
-      "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==",
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
      "dev": true,
      "dependencies": {
        "ansi-styles": "^4.1.0",
@@ -381,6 +382,17 @@
        "node": ">= 0.2.0"
      }
    },
+    "node_modules/cliui": {
+      "version": "7.0.4",
+      "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
+      "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
+      "dev": true,
+      "dependencies": {
+        "string-width": "^4.2.0",
+        "strip-ansi": "^6.0.0",
+        "wrap-ansi": "^7.0.0"
+      }
+    },
    "node_modules/clone": {
      "version": "1.0.4",
      "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
@@ -442,9 +454,9 @@
      "dev": true
    },
    "node_modules/cssom": {
-      "version": "0.4.4",
-      "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz",
-      "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==",
+      "version": "0.5.0",
+      "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
+      "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==",
      "dev": true
    },
    "node_modules/cssstyle": {
@@ -472,23 +484,23 @@
      "dev": true
    },
    "node_modules/data-urls": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz",
-      "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==",
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.0.tgz",
+      "integrity": "sha512-4AefxbTTdFtxDUdh0BuMBs2qJVL25Mow2zlcuuePegQwgD6GEmQao42LLEeksOui8nL4RcNEugIpFP7eRd33xg==",
      "dev": true,
      "dependencies": {
        "abab": "^2.0.3",
        "whatwg-mimetype": "^2.3.0",
-        "whatwg-url": "^8.0.0"
+        "whatwg-url": "^9.0.0"
      },
      "engines": {
-        "node": ">=10"
+        "node": ">=12"
      }
    },
    "node_modules/debug": {
-      "version": "4.3.1",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
-      "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==",
+      "version": "4.3.2",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz",
+      "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==",
      "dev": true,
      "dependencies": {
        "ms": "2.1.2"
@@ -503,15 +515,15 @@
      }
    },
    "node_modules/decimal.js": {
-      "version": "10.2.1",
-      "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.2.1.tgz",
-      "integrity": "sha512-KaL7+6Fw6i5A2XSnsbhm/6B+NuEA7TZ4vqxnd5tXz9sbKtrN9Srj8ab4vKVdK8YAqZO9P1kg45Y6YLoduPf+kw==",
+      "version": "10.3.1",
+      "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz",
+      "integrity": "sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ==",
      "dev": true
    },
    "node_modules/deep-is": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz",
-      "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=",
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
+      "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==",
      "dev": true
    },
    "node_modules/defaults": {
@@ -562,6 +574,21 @@
        "node": ">=10"
      }
    },
+    "node_modules/emoji-regex": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
+      "dev": true
+    },
+    "node_modules/escalade": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
+      "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
+      "dev": true,
+      "engines": {
+        "node": ">=6"
+      }
+    },
    "node_modules/escodegen": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.0.0.tgz",
@@ -652,9 +679,9 @@
      }
    },
    "node_modules/follow-redirects": {
-      "version": "1.14.2",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.2.tgz",
-      "integrity": "sha512-yLR6WaE2lbF0x4K2qE2p9PEXKLDjUjnR/xmjS3wHAYxtlsI9MLLBJUZirAHKzUZDGLxje7w/cXR49WOUo4rbsA==",
+      "version": "1.14.3",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.3.tgz",
+      "integrity": "sha512-3MkHxknWMUtb23apkgz/83fDoe+y+qr0TdgacGIA7bew+QLBo3vdgEN2xEsuXNivpFy4CyDhBBZnNZOtalmenw==",
      "dev": true,
      "funding": [
        {
@@ -711,6 +738,15 @@
      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==",
      "dev": true
    },
+    "node_modules/get-caller-file": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
+      "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
+      "dev": true,
+      "engines": {
+        "node": "6.* || 8.* || >= 10.*"
+      }
+    },
    "node_modules/get-installed-path": {
      "version": "4.0.8",
      "resolved": "https://registry.npmjs.org/get-installed-path/-/get-installed-path-4.0.8.tgz",
@@ -934,6 +970,15 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
+    "node_modules/is-fullwidth-code-point": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+      "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
    "node_modules/is-interactive": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz",
@@ -992,21 +1037,21 @@
      "dev": true
    },
    "node_modules/jsdom": {
-      "version": "16.6.0",
-      "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.6.0.tgz",
-      "integrity": "sha512-Ty1vmF4NHJkolaEmdjtxTfSfkdb8Ywarwf63f+F8/mDD1uLSSWDxDuMiZxiPhwunLrn9LOSVItWj4bLYsLN3Dg==",
+      "version": "17.0.0",
+      "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-17.0.0.tgz",
+      "integrity": "sha512-MUq4XdqwtNurZDVeKScENMPHnkgmdIvMzZ1r1NSwHkDuaqI6BouPjr+17COo4/19oLNnmdpFDPOHVpgIZmZ+VA==",
      "dev": true,
      "dependencies": {
        "abab": "^2.0.5",
-        "acorn": "^8.2.4",
+        "acorn": "^8.4.1",
        "acorn-globals": "^6.0.0",
-        "cssom": "^0.4.4",
+        "cssom": "^0.5.0",
        "cssstyle": "^2.3.0",
-        "data-urls": "^2.0.0",
-        "decimal.js": "^10.2.1",
+        "data-urls": "^3.0.0",
+        "decimal.js": "^10.3.1",
        "domexception": "^2.0.1",
        "escodegen": "^2.0.0",
-        "form-data": "^3.0.0",
+        "form-data": "^4.0.0",
        "html-encoding-sniffer": "^2.0.1",
        "http-proxy-agent": "^4.0.1",
        "https-proxy-agent": "^5.0.0",
@@ -1021,12 +1066,12 @@
        "webidl-conversions": "^6.1.0",
        "whatwg-encoding": "^1.0.5",
        "whatwg-mimetype": "^2.3.0",
-        "whatwg-url": "^8.5.0",
-        "ws": "^7.4.5",
+        "whatwg-url": "^9.0.0",
+        "ws": "^8.0.0",
        "xml-name-validator": "^3.0.0"
      },
      "engines": {
-        "node": ">=10"
+        "node": ">=12"
      },
      "peerDependencies": {
        "canvas": "^2.5.0"
@@ -1037,20 +1082,6 @@
        }
      }
    },
-    "node_modules/jsdom/node_modules/form-data": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
-      "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
-      "dev": true,
-      "dependencies": {
-        "asynckit": "^0.4.0",
-        "combined-stream": "^1.0.8",
-        "mime-types": "^2.1.12"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
    "node_modules/jsonfile": {
      "version": "6.1.0",
      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
@@ -1091,12 +1122,6 @@
        "node": ">= 0.8.0"
      }
    },
-    "node_modules/lodash": {
-      "version": "4.17.21",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
-      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==",
-      "dev": true
-    },
    "node_modules/lodash.groupby": {
      "version": "4.6.0",
      "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz",
@@ -1126,21 +1151,21 @@
      }
    },
    "node_modules/mime-db": {
-      "version": "1.48.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.48.0.tgz",
-      "integrity": "sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ==",
+      "version": "1.49.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz",
+      "integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==",
      "dev": true,
      "engines": {
        "node": ">= 0.6"
      }
    },
    "node_modules/mime-types": {
-      "version": "2.1.31",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.31.tgz",
-      "integrity": "sha512-XGZnNzm3QvgKxa8dpzyhFTHmpP3l5YNusmne07VUOXxou9CqUqYa/HBy124RqtVh/O2pECas/MOcsDgpilPOPg==",
+      "version": "2.1.32",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz",
+      "integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==",
      "dev": true,
      "dependencies": {
-        "mime-db": "1.48.0"
+        "mime-db": "1.49.0"
      },
      "engines": {
        "node": ">= 0.6"
@@ -1354,6 +1379,15 @@
        "node": ">= 0.10"
      }
    },
+    "node_modules/require-directory": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
+      "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
    "node_modules/resolve": {
      "version": "1.20.0",
      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz",
@@ -1494,6 +1528,20 @@
        "safe-buffer": "~5.2.0"
      }
    },
+    "node_modules/string-width": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
+      "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
+      "dev": true,
+      "dependencies": {
+        "emoji-regex": "^8.0.0",
+        "is-fullwidth-code-point": "^3.0.0",
+        "strip-ansi": "^6.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
    "node_modules/strip-ansi": {
      "version": "6.0.0",
      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
@@ -1653,17 +1701,16 @@
      "dev": true
    },
    "node_modules/whatwg-url": {
-      "version": "8.6.0",
-      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.6.0.tgz",
-      "integrity": "sha512-os0KkeeqUOl7ccdDT1qqUcS4KH4tcBTSKK5Nl5WKb2lyxInIZ/CpjkqKa1Ss12mjfdcRX9mHmPPs7/SxG1Hbdw==",
+      "version": "9.1.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-9.1.0.tgz",
+      "integrity": "sha512-CQ0UcrPHyomtlOCot1TL77WyMIm/bCwrJ2D6AOKGwEczU9EpyoqAokfqrf/MioU9kHcMsmJZcg1egXix2KYEsA==",
      "dev": true,
      "dependencies": {
-        "lodash": "^4.7.0",
        "tr46": "^2.1.0",
        "webidl-conversions": "^6.1.0"
      },
      "engines": {
-        "node": ">=10"
+        "node": ">=12"
      }
    },
    "node_modules/which": {
@@ -1687,6 +1734,23 @@
        "node": ">=0.10.0"
      }
    },
+    "node_modules/wrap-ansi": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
+      "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
+      "dev": true,
+      "dependencies": {
+        "ansi-styles": "^4.0.0",
+        "string-width": "^4.1.0",
+        "strip-ansi": "^6.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
+      }
+    },
    "node_modules/wrappy": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
@@ -1694,12 +1758,12 @@
      "dev": true
    },
    "node_modules/ws": {
-      "version": "7.5.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.0.tgz",
-      "integrity": "sha512-6ezXvzOZupqKj4jUqbQ9tXuJNo+BR2gU8fFRk3XCP3e0G6WT414u5ELe6Y0vtp7kmSJ3F7YWObSNr1ESsgi4vw==",
+      "version": "8.2.1",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.2.1.tgz",
+      "integrity": "sha512-XkgWpJU3sHU7gX8f13NqTn6KQ85bd1WU7noBHTT8fSohx7OS1TPY8k+cyRPCzFkia7C4mM229yeHr1qK9sM4JQ==",
      "dev": true,
      "engines": {
-        "node": ">=8.3.0"
+        "node": ">=10.0.0"
      },
      "peerDependencies": {
        "bufferutil": "^4.0.1",
@@ -1731,13 +1795,49 @@
      "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
      "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==",
      "dev": true
+    },
+    "node_modules/y18n": {
+      "version": "5.0.8",
+      "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
+      "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
+      "dev": true,
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/yargs": {
+      "version": "17.1.1",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.1.1.tgz",
+      "integrity": "sha512-c2k48R0PwKIqKhPMWjeiF6y2xY/gPMUlro0sgxqXpbOIohWiLNXWslsootttv7E1e73QPAMQSg5FeySbVcpsPQ==",
+      "dev": true,
+      "dependencies": {
+        "cliui": "^7.0.2",
+        "escalade": "^3.1.1",
+        "get-caller-file": "^2.0.5",
+        "require-directory": "^2.1.1",
+        "string-width": "^4.2.0",
+        "y18n": "^5.0.5",
+        "yargs-parser": "^20.2.2"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/yargs-parser": {
+      "version": "20.2.9",
+      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz",
+      "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==",
+      "dev": true,
+      "engines": {
+        "node": ">=10"
+      }
    }
  },
  "dependencies": {
    "@sasjs/adapter": {
-      "version": "2.10.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.10.0.tgz",
-      "integrity": "sha512-GbvyIgbODnAJaBaz/Tz8/IwcujNOsZcwzbIuKQtG5y13BzgVPtx/e7b2TAhdoBqd+8uVh0CdrSDfOV/SuvPurg==",
+      "version": "2.10.4",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.10.4.tgz",
+      "integrity": "sha512-9b3WGZUzRgszZVyPfrabSx6TycL4JFFXQR/RQKsFCDDwT8UgLGfJ4JmgaGCSmGCcsqML/Q41QipdgWu1M/QA3g==",
      "dev": true,
      "requires": {
        "@sasjs/utils": "^2.27.1",
@@ -1749,23 +1849,23 @@
      }
    },
    "@sasjs/cli": {
-      "version": "2.37.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.37.2.tgz",
-      "integrity": "sha512-Bcm8UFN9Y/ZON4T31Gu9mf1REn1pZoStHFVrix/yp7mchFt5rrrY5RbIqO/AI1jzhShSVg9P7oU4VPJrqki+SA==",
+      "version": "2.37.8",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.37.8.tgz",
+      "integrity": "sha512-V1FGK0ByS5v5+xOX5J1rlQDOXSA37YLL18etEUQOmWK5X9R5tbK4LPIx/pEjLyfZcCHddrm+0nGzRroC715Ilg==",
      "dev": true,
      "requires": {
-        "@sasjs/adapter": "2.10.0",
-        "@sasjs/core": "2.35.4",
+        "@sasjs/adapter": "2.10.4",
+        "@sasjs/core": "2.42.0",
        "@sasjs/lint": "1.11.2",
-        "@sasjs/utils": "2.27.1",
-        "chalk": "4.1.1",
+        "@sasjs/utils": "2.28.0",
+        "chalk": "4.1.2",
        "csv-stringify": "5.6.2",
        "dotenv": "10.0.0",
        "esm": "3.2.25",
        "find": "0.3.0",
        "get-installed-path": "4.0.8",
        "js-base64": "3.6.1",
-        "jsdom": "16.6.0",
+        "jsdom": "17.0.0",
        "jwt-decode": "3.1.2",
        "lodash.groupby": "4.6.0",
        "lodash.uniqby": "4.7.0",
@@ -1773,13 +1873,14 @@
        "ora": "5.4.1",
        "rimraf": "3.0.2",
        "shelljs": "0.8.4",
-        "xml": "1.0.1"
+        "xml": "1.0.1",
+        "yargs": "17.1.1"
      }
    },
    "@sasjs/core": {
-      "version": "2.35.4",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.35.4.tgz",
-      "integrity": "sha512-Exr3+yRdvacKvbXQoi1RfGHi5NtZUkc7RwFNkemHTFXLYaIzPI8CGSCaQmKkwM1UteOJly2e2pw2YT6kHNY1NA==",
+      "version": "2.42.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.42.0.tgz",
+      "integrity": "sha512-EISPPCEv+vB3Nqp03NUaIvMZwEnggzGJeEpVfm2Sb504ySN1I2xiJ8bOHXIzvvYP5N/V9X8bXe1raLYnnt8HGA==",
      "dev": true
    },
    "@sasjs/lint": {
@@ -1792,9 +1893,9 @@
      }
    },
    "@sasjs/utils": {
-      "version": "2.27.1",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.27.1.tgz",
-      "integrity": "sha512-CYTQwEj89cc7H3tGiQQcyDkZYaWRc1HZJpOF8o2RHYS37fIAOy0SyyJdq6mcQ74Nb1u5AmFXPFIvnRCMEcTYeQ==",
+      "version": "2.28.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.28.0.tgz",
+      "integrity": "sha512-aZAbBDSjvW2TCY1lkuGqqiyWtR4W8GFpCNCUKa72VIkl4zwTE/pnSHQ+v8QilGDSHSPbPEsJGaBaldMGwoI12w==",
      "dev": true,
      "requires": {
        "@types/fs-extra": "^9.0.11",
@@ -1853,9 +1954,9 @@
      "dev": true
    },
    "acorn": {
-      "version": "8.4.0",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.4.0.tgz",
-      "integrity": "sha512-ULr0LDaEqQrMFGyQ3bhJkLsbtrQ8QibAseGZeaSUiT/6zb9IvIkomWHJIvgvwad+hinRAgsI51JcWk2yvwyL+w==",
+      "version": "8.5.0",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.5.0.tgz",
+      "integrity": "sha512-yXbYeFy+jUuYd3/CDcg2NkIYE991XYX/bje7LmjJigUciaeO1JR4XxXgCIV1/Zc/dRuFEyw1L0pbA+qynJkW5Q==",
      "dev": true
    },
    "acorn-globals": {
@@ -1913,12 +2014,12 @@
      "dev": true
    },
    "axios": {
-      "version": "0.21.1",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz",
-      "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==",
+      "version": "0.21.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
+      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
      "dev": true,
      "requires": {
-        "follow-redirects": "^1.10.0"
+        "follow-redirects": "^1.14.0"
      }
    },
    "axios-cookiejar-support": {
@@ -1981,9 +2082,9 @@
      }
    },
    "chalk": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz",
-      "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==",
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
      "dev": true,
      "requires": {
        "ansi-styles": "^4.1.0",
@@ -2014,6 +2115,17 @@
        "colors": "1.0.3"
      }
    },
+    "cliui": {
+      "version": "7.0.4",
+      "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
+      "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
+      "dev": true,
+      "requires": {
+        "string-width": "^4.2.0",
+        "strip-ansi": "^6.0.0",
+        "wrap-ansi": "^7.0.0"
+      }
+    },
    "clone": {
      "version": "1.0.4",
      "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
@@ -2063,9 +2175,9 @@
      "dev": true
    },
    "cssom": {
-      "version": "0.4.4",
-      "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz",
-      "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==",
+      "version": "0.5.0",
+      "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
+      "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==",
      "dev": true
    },
    "cssstyle": {
@@ -2092,35 +2204,35 @@
      "dev": true
    },
    "data-urls": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz",
-      "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==",
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.0.tgz",
+      "integrity": "sha512-4AefxbTTdFtxDUdh0BuMBs2qJVL25Mow2zlcuuePegQwgD6GEmQao42LLEeksOui8nL4RcNEugIpFP7eRd33xg==",
      "dev": true,
      "requires": {
        "abab": "^2.0.3",
        "whatwg-mimetype": "^2.3.0",
-        "whatwg-url": "^8.0.0"
+        "whatwg-url": "^9.0.0"
      }
    },
    "debug": {
-      "version": "4.3.1",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
-      "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==",
+      "version": "4.3.2",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz",
+      "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==",
      "dev": true,
      "requires": {
        "ms": "2.1.2"
      }
    },
    "decimal.js": {
-      "version": "10.2.1",
-      "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.2.1.tgz",
-      "integrity": "sha512-KaL7+6Fw6i5A2XSnsbhm/6B+NuEA7TZ4vqxnd5tXz9sbKtrN9Srj8ab4vKVdK8YAqZO9P1kg45Y6YLoduPf+kw==",
+      "version": "10.3.1",
+      "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz",
+      "integrity": "sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ==",
      "dev": true
    },
    "deep-is": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz",
-      "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=",
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
+      "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==",
      "dev": true
    },
    "defaults": {
@@ -2161,6 +2273,18 @@
      "integrity": "sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==",
      "dev": true
    },
+    "emoji-regex": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
+      "dev": true
+    },
+    "escalade": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
+      "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
+      "dev": true
+    },
    "escodegen": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.0.0.tgz",
@@ -2223,9 +2347,9 @@
      }
    },
    "follow-redirects": {
-      "version": "1.14.2",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.2.tgz",
-      "integrity": "sha512-yLR6WaE2lbF0x4K2qE2p9PEXKLDjUjnR/xmjS3wHAYxtlsI9MLLBJUZirAHKzUZDGLxje7w/cXR49WOUo4rbsA==",
+      "version": "1.14.3",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.3.tgz",
+      "integrity": "sha512-3MkHxknWMUtb23apkgz/83fDoe+y+qr0TdgacGIA7bew+QLBo3vdgEN2xEsuXNivpFy4CyDhBBZnNZOtalmenw==",
      "dev": true
    },
    "form-data": {
@@ -2262,6 +2386,12 @@
      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==",
      "dev": true
    },
+    "get-caller-file": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
+      "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
+      "dev": true
+    },
    "get-installed-path": {
      "version": "4.0.8",
      "resolved": "https://registry.npmjs.org/get-installed-path/-/get-installed-path-4.0.8.tgz",
@@ -2427,6 +2557,12 @@
        "has": "^1.0.3"
      }
    },
+    "is-fullwidth-code-point": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+      "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+      "dev": true
+    },
    "is-interactive": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz",
@@ -2470,21 +2606,21 @@
      "dev": true
    },
    "jsdom": {
-      "version": "16.6.0",
-      "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.6.0.tgz",
-      "integrity": "sha512-Ty1vmF4NHJkolaEmdjtxTfSfkdb8Ywarwf63f+F8/mDD1uLSSWDxDuMiZxiPhwunLrn9LOSVItWj4bLYsLN3Dg==",
+      "version": "17.0.0",
+      "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-17.0.0.tgz",
+      "integrity": "sha512-MUq4XdqwtNurZDVeKScENMPHnkgmdIvMzZ1r1NSwHkDuaqI6BouPjr+17COo4/19oLNnmdpFDPOHVpgIZmZ+VA==",
      "dev": true,
      "requires": {
        "abab": "^2.0.5",
-        "acorn": "^8.2.4",
+        "acorn": "^8.4.1",
        "acorn-globals": "^6.0.0",
-        "cssom": "^0.4.4",
+        "cssom": "^0.5.0",
        "cssstyle": "^2.3.0",
-        "data-urls": "^2.0.0",
-        "decimal.js": "^10.2.1",
+        "data-urls": "^3.0.0",
+        "decimal.js": "^10.3.1",
        "domexception": "^2.0.1",
        "escodegen": "^2.0.0",
-        "form-data": "^3.0.0",
+        "form-data": "^4.0.0",
        "html-encoding-sniffer": "^2.0.1",
        "http-proxy-agent": "^4.0.1",
        "https-proxy-agent": "^5.0.0",
@@ -2499,22 +2635,9 @@
        "webidl-conversions": "^6.1.0",
        "whatwg-encoding": "^1.0.5",
        "whatwg-mimetype": "^2.3.0",
-        "whatwg-url": "^8.5.0",
-        "ws": "^7.4.5",
+        "whatwg-url": "^9.0.0",
+        "ws": "^8.0.0",
        "xml-name-validator": "^3.0.0"
-      },
-      "dependencies": {
-        "form-data": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
-          "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
-          "dev": true,
-          "requires": {
-            "asynckit": "^0.4.0",
-            "combined-stream": "^1.0.8",
-            "mime-types": "^2.1.12"
-          }
-        }
      }
    },
    "jsonfile": {
@@ -2549,12 +2672,6 @@
        "type-check": "~0.3.2"
      }
    },
-    "lodash": {
-      "version": "4.17.21",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
-      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==",
-      "dev": true
-    },
    "lodash.groupby": {
      "version": "4.6.0",
      "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz",
@@ -2578,18 +2695,18 @@
      }
    },
    "mime-db": {
-      "version": "1.48.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.48.0.tgz",
-      "integrity": "sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ==",
+      "version": "1.49.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz",
+      "integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==",
      "dev": true
    },
    "mime-types": {
-      "version": "2.1.31",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.31.tgz",
-      "integrity": "sha512-XGZnNzm3QvgKxa8dpzyhFTHmpP3l5YNusmne07VUOXxou9CqUqYa/HBy124RqtVh/O2pECas/MOcsDgpilPOPg==",
+      "version": "2.1.32",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz",
+      "integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==",
      "dev": true,
      "requires": {
-        "mime-db": "1.48.0"
+        "mime-db": "1.49.0"
      }
    },
    "mimic-fn": {
@@ -2752,6 +2869,12 @@
        "resolve": "^1.1.6"
      }
    },
+    "require-directory": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
+      "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=",
+      "dev": true
+    },
    "resolve": {
      "version": "1.20.0",
      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz",
@@ -2851,6 +2974,17 @@
        "safe-buffer": "~5.2.0"
      }
    },
+    "string-width": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
+      "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
+      "dev": true,
+      "requires": {
+        "emoji-regex": "^8.0.0",
+        "is-fullwidth-code-point": "^3.0.0",
+        "strip-ansi": "^6.0.0"
+      }
+    },
    "strip-ansi": {
      "version": "6.0.0",
      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
@@ -2985,12 +3119,11 @@
      "dev": true
    },
    "whatwg-url": {
-      "version": "8.6.0",
-      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.6.0.tgz",
-      "integrity": "sha512-os0KkeeqUOl7ccdDT1qqUcS4KH4tcBTSKK5Nl5WKb2lyxInIZ/CpjkqKa1Ss12mjfdcRX9mHmPPs7/SxG1Hbdw==",
+      "version": "9.1.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-9.1.0.tgz",
+      "integrity": "sha512-CQ0UcrPHyomtlOCot1TL77WyMIm/bCwrJ2D6AOKGwEczU9EpyoqAokfqrf/MioU9kHcMsmJZcg1egXix2KYEsA==",
      "dev": true,
      "requires": {
-        "lodash": "^4.7.0",
        "tr46": "^2.1.0",
        "webidl-conversions": "^6.1.0"
      }
@@ -3010,6 +3143,17 @@
      "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
      "dev": true
    },
+    "wrap-ansi": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
+      "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
+      "dev": true,
+      "requires": {
+        "ansi-styles": "^4.0.0",
+        "string-width": "^4.1.0",
+        "strip-ansi": "^6.0.0"
+      }
+    },
    "wrappy": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
@@ -3017,9 +3161,9 @@
      "dev": true
    },
    "ws": {
-      "version": "7.5.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.0.tgz",
-      "integrity": "sha512-6ezXvzOZupqKj4jUqbQ9tXuJNo+BR2gU8fFRk3XCP3e0G6WT414u5ELe6Y0vtp7kmSJ3F7YWObSNr1ESsgi4vw==",
+      "version": "8.2.1",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.2.1.tgz",
+      "integrity": "sha512-XkgWpJU3sHU7gX8f13NqTn6KQ85bd1WU7noBHTT8fSohx7OS1TPY8k+cyRPCzFkia7C4mM229yeHr1qK9sM4JQ==",
      "dev": true,
      "requires": {}
    },
@@ -3040,6 +3184,33 @@
      "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
      "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==",
      "dev": true
+    },
+    "y18n": {
+      "version": "5.0.8",
+      "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
+      "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
+      "dev": true
+    },
+    "yargs": {
+      "version": "17.1.1",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.1.1.tgz",
+      "integrity": "sha512-c2k48R0PwKIqKhPMWjeiF6y2xY/gPMUlro0sgxqXpbOIohWiLNXWslsootttv7E1e73QPAMQSg5FeySbVcpsPQ==",
+      "dev": true,
+      "requires": {
+        "cliui": "^7.0.2",
+        "escalade": "^3.1.1",
+        "get-caller-file": "^2.0.5",
+        "require-directory": "^2.1.1",
+        "string-width": "^4.2.0",
+        "y18n": "^5.0.5",
+        "yargs-parser": "^20.2.2"
+      }
+    },
+    "yargs-parser": {
+      "version": "20.2.9",
+      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz",
+      "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==",
+      "dev": true
    }
  }
 }
--- a/package.json
+++ b/package.json
@@ -33,6 +33,6 @@
    "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
  },
  "devDependencies": {
-    "@sasjs/cli": "^2.37.2"
+    "@sasjs/cli": "^2.37.8"
  }
 }
--- a/tests/crossplatform/mf_getapploc.test.sas
+++ b/tests/crossplatform/mf_getapploc.test.sas
@@ -38,4 +38,12 @@
  ),
  desc=Checking jobs appLoc matches,
  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/tests/macros/somemacro.sas)"="/some/area"
+  ),
+  desc=Checking tests/macros appLoc matches (which has no subfolder),
+  outds=work.test_results
 )
--- a/tests/crossplatform/mf_getuniquefileref.test.sas
+++ b/tests/crossplatform/mf_getuniquefileref.test.sas
@@ -0,0 +1,36 @@
+/**
+  @file
+  @brief Testing mf_getuniquefileref macro
+  @details To test performance you can also use the following macro:
+
+      %macro x(prefix);
+      %let now=%sysfunc(datetime());
+      %do x=1 %to 1000;
+        %let rc=%mf_getuniquefileref(prefix=&prefix);
+      %end;
+      %put %sysevalf(%sysfunc(datetime())-&now);
+      %mend;
+      %x(_)
+      %x(0)
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(
+    "%substr(%mf_getuniquefileref(prefix=0),1,1)"="#"
+  ),
+  desc=Checking for a natively assigned fileref,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%substr(%mf_getuniquefileref(),1,1)"="_"
+  ),
+  desc=Checking for a default fileref,
+  outds=work.test_results
+)
--- a/tests/crossplatform/mp_binarycopy.test.sas
+++ b/tests/crossplatform/mp_binarycopy.test.sas
@@ -0,0 +1,99 @@
+/**
+  @file
+  @brief Testing mp_binarycopy.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_binarycopy.sas
+  @li mp_assert.sas
+
+**/
+
+
+/* TEST 1 - regular file copy */
+%let string1=test1;
+filename tmp temp;
+filename myref temp;
+data _null_;
+  file tmp;
+  put "&string1";
+run;
+%mp_binarycopy(inref=tmp, outref=myref)
+data _null_;
+  infile myref;
+  input;
+  put _infile_;
+  call symputx('string1_check',_infile_);
+  stop;
+run;
+%mp_assert(
+  iftrue=("&string1"="&string1_check"),
+  desc=Basic String Compare,
+  outds=work.test_results
+)
+
+
+/* TEST 2 - File append */
+%let string2=test2;
+%let path2=%sysfunc(pathname(work))/somefile.txt;
+data _null_;
+  file "&path2";
+  put "&string2";
+run;
+%mp_binarycopy(inloc="&path2", outref=myref, mode=APPEND)
+data _null_;
+  infile myref;
+  input;
+  put _infile_;
+  if _n_=2 then call symputx('string2_check',_infile_);
+run;
+%mp_assert(
+  iftrue=("&string2"="&string2_check"),
+  desc=Append Check (file to ref),
+  outds=work.test_results
+)
+
+/* TEST 3 - File create (ref to existing file) */
+%let string3=test3;
+%let path3=%sysfunc(pathname(work))/somefile3.txt;
+filename tmp3 temp;
+data _null_;
+  file tmp3;
+  put "&string3";
+run;
+data _null_;
+  file "&path3";
+  put "this should not be returned";
+run;
+%mp_binarycopy(inref=tmp3, outloc="&path3")
+data _null_;
+  infile "&path3";
+  input;
+  put _infile_;
+  if _n_=1 then call symputx('string3_check',_infile_);
+run;
+%mp_assert(
+  iftrue=("&string3"="&string3_check"),
+  desc=Append Check (ref to existing file),
+  outds=work.test_results
+)
+
+/* TEST 4 - File append (ref to file) */
+%let string4=test4;
+%let string4_check=;
+filename tmp4 temp;
+data _null_;
+  file tmp4;
+  put "&string4";
+run;
+%mp_binarycopy(inref=tmp4, outloc="&path3",mode=APPEND)
+data _null_;
+  infile "&path3";
+  input;
+  put _infile_;
+  if _n_=2 then call symputx('string4_check',_infile_);
+run;
+%mp_assert(
+  iftrue=("&string4"="&string4_check"),
+  desc=Append Check (ref to file),
+  outds=work.test_results
+)
--- a/tests/crossplatform/mp_gsubfile.test.sas
+++ b/tests/crossplatform/mp_gsubfile.test.sas
@@ -0,0 +1,66 @@
+/**
+  @file
+  @brief Testing mp_gsubfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_gsubfile.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * test 1 - simple replace
+  */
+%global str1;
+%let file=%sysfunc(pathname(work))/file.txt;
+%let pat=replace/me;
+%let str=with/this;
+data _null_;
+  file "&file";
+  put "&pat";
+run;
+%mp_gsubfile(file=&file, patternvar=pat, replacevar=str)
+data _null_;
+  infile "&file";
+  input;
+  call symputx('str1',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&str1"="&str"),
+  desc=Check that simple replacement was successful,
+  outds=work.test_results
+)
+
+/**
+  * test 2 - replace from additional line
+  */
+%global str2 strcheck2 strcheck2b;
+%let file2=%sysfunc(pathname(work))/file2.txt;
+%let pat2=replace/me;
+%let str2=with/this;
+data _null_;
+  file "&file2";
+  put 'line1';output;
+  put "&pat2";output;
+  put "&pat2";output;
+run;
+%mp_gsubfile(file=&file2, patternvar=pat2, replacevar=str2)
+data _null_;
+  infile "&file2";
+  input;
+  if _n_=2 then call symputx('strcheck2',_infile_);
+  if _n_=3 then call symputx('strcheck2b',_infile_);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=("&strcheck2"="&str2"),
+  desc=Check that multi line replacement was successful (line2),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&strcheck2b"="&str2"),
+  desc=Check that multi line replacement was successful (line3),
+  outds=work.test_results
+)
--- a/tests/crossplatform/mp_webin.test.sas
+++ b/tests/crossplatform/mp_webin.test.sas
@@ -0,0 +1,39 @@
+/**
+  @file
+  @brief Testing mp_webin macro
+
+  <h4> SAS Macros </h4>
+  @li mp_webin.sas
+  @li mp_assert.sas
+
+**/
+
+/* force SAS9 tests as we don't have a valid URI available */
+%macro mf_getplatform();
+  SAS9
+%mend mf_getplatform;
+
+/* TEST 1 */
+%let _webin_file_count=1;
+%let _webin_filename=test;
+%mp_webin()
+
+%mp_assert(
+  iftrue=(
+    %symexist(_WEBIN_FILEREF1)
+  ),
+  desc=Checking if the macvar exists,
+  outds=work.test_results
+)
+
+/* TEST 2 */
+%global _WEBIN_FILENAME1;
+%mp_assert(
+  iftrue=(
+    %str(&_WEBIN_FILENAME1)=%str(test)
+  ),
+  desc=Checking if the macvar exists,
+  outds=work.test_results
+)
+
+
--- a/tests/viyaonly/mv_getfoldermembers.test.sas
+++ b/tests/viyaonly/mv_getfoldermembers.test.sas
@@ -0,0 +1,24 @@
+/**
+  @file
+  @brief Testing mv_getfoldermembers macro
+  @details Testing is performed by ensuring that the tests/macros folder
+  contains more than 20 results (which also means the default was increased)
+
+  <h4> SAS Macros </h4>
+  @li mf_getapploc.sas
+  @li mp_assertdsobs.sas
+  @li mv_getfoldermembers.sas
+
+**/
+options mprint;
+
+%mv_getfoldermembers(
+  root=%mf_getapploc()/tests/macros,
+  outds=work.results
+)
+
+%mp_assertdsobs(work.results,
+  desc=%str(Ensuring over 20 results found in %mf_getapploc()/tests/macros),
+  test=ATLEAST 21,
+  outds=work.test_results
+)
--- a/viya/mv_getfoldermembers.sas
+++ b/viya/mv_getfoldermembers.sas
@@ -1,22 +1,25 @@
 /**
-  @file mv_getfoldermembers.sas
-  @brief Gets a list of folders (and ids) for a given root
-  @details Works for both root level and below, oauth or password. Default is
-    oauth, and the token is expected in a global ACCESS_TOKEN variable.
+  @file
+  @brief Gets a list of folder members (and ids) for a given root
+  @details Returns all members for a particular Viya folder.  Works at both root
+  level and below, and results are created in an output dataset.

-        %mv_getfoldermembers(root=/Public)
+        %mv_getfoldermembers(root=/Public, outds=work.mymembers)


-  @param root= The path for which to return the list of folders
-  @param outds= The output dataset to create (default is work.mv_getfolders).  Format:
+  @param [in] root= (/) The path for which to return the list of folders
+  @param [out] outds= (work.mv_getfolders) The output dataset to create. Format:
  |ordinal_root|ordinal_items|creationTimeStamp| modifiedTimeStamp|createdBy|modifiedBy|id| uri|added| type|name|description|
  |---|---|---|---|---|---|---|---|---|---|---|---|
  |1|1|2021-05-25T11:15:04.204Z|2021-05-25T11:15:04.204Z|allbow|allbow|4f1e3945-9655-462b-90f2-c31534b3ca47|/folders/folders/ed701ff3-77e8-468d-a4f5-8c43dec0fd9e|2021-05-25T11:15:04.212Z|child|my_folder_name|My folder Description|

-  @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values are "password" or "authorization_code" (unquoted).
-    The default is authorization_code.
-
+  @param [in] access_token_var= (ACCESS_TOKEN) The global macro variable to
+    contain the access token
+  @param [in] grant_type= (sas_services) Valid values are:
+    @li password
+    @li authorization_code
+    @li detect
+    @li sas_services

  @version VIYA V.03.04
  @author Allan Bowe, source: https://github.com/sasjs/core
@@ -28,6 +31,12 @@
  @li mf_getuniquelibref.sas
  @li mf_isblank.sas

+  <h4> Related Macros </h4>
+  @li mv_createfolder.sas
+  @li mv_deletefoldermember.sas
+  @li mv_deleteviyafolder.sas
+  @li mv_getfoldermembers.test.sas
+
 **/

 %macro mv_getfoldermembers(root=/
@@ -67,7 +76,7 @@ options noquotelenmax;
 %if "&root"="/" %then %do;
  /* if root just list root folders */
  proc http method='GET' out=&fname1 &oauth_bearer
-      url="&base_uri/folders/rootFolders";
+      url="&base_uri/folders/rootFolders?limit=1000";
  %if &grant_type=authorization_code %then %do;
      headers "Authorization"="Bearer &&&access_token_var";
  %end;
@@ -88,13 +97,17 @@ options noquotelenmax;
  /*data _null_;infile &fname1;input;putlog _infile_;run;*/
  libname &libref1 JSON fileref=&fname1;
  /* now get the followon link to list members */
-  %local href;
-  %let href=0;
+  %local href cnt;
+  %let cnt=0;
  data _null_;
    set &libref1..links;
-    if rel='members' then call symputx('href',quote("&base_uri"!!trim(href)),'l');
+    if rel='members' then do;
+      url=cats("'","&base_uri",href,"?limit=10000'");
+      call symputx('href',url,'l');
+      call symputx('cnt',1,'l');
+    end;
  run;
-  %if &href=0 %then %do;
+  %if &cnt=0 %then %do;
    %put NOTE:;%put NOTE-  No members found in &root!!;%put NOTE-;
    %return;
  %end;
@@ -109,6 +122,7 @@ options noquotelenmax;
  run;
  libname &libref2 JSON fileref=&fname2;
  data &outds;
+    length id $36 name $128 uri $64 type $32 description $256;
    set &libref2..items;
  run;
  filename &fname2 clear;
Author	SHA1	Message	Date
Allan Bowe	7406288d79	fix: mp_gsubfile() now works with multiline files (and we have a multiline test to go with it)	2021-09-16 18:30:17 +01:00
Allan Bowe	2e7fcbe5b8	fix: prevening truncation of _debug in mp_abort.sas and more reliable way to fetch syswarningtext and syserrortext	2021-09-16 14:04:50 +01:00
Allan Bowe	3e7b9f8c14	chore: fixing example in header for mp_gsubfile()	2021-09-16 13:54:27 +01:00
Allan Bowe	e9189ccc06	Merge pull request #74 from sasjs/gsub feat: adding mp_gsubfile.sas - a SAS macro that uses Lua to perform a…	2021-09-14 19:13:15 +03:00
Allan Bowe	8c00d715c2	chore: formatting	2021-09-14 16:08:30 +01:00
Allan Bowe	d47a369cdf	feat: adding mp_gsubfile.sas - a SAS macro that uses Lua to perform a full text find and replace of an entire file. Not restricted by number of characters (only memory). IIncludes a test.	2021-09-14 16:07:12 +01:00
Allan Bowe	52bf6019fd	Merge pull request #73 from sasjs/defaultvars fix: adding default lengths to vars in mv_getfoldermembers to cover u…	2021-09-11 19:26:06 +03:00
Allan Bowe	25e61fd8ef	chore: updating all.sas	2021-09-11 19:25:28 +03:00
Allan Bowe	3038be83a0	fix: adding default lengths to vars in mv_getfoldermembers to cover use case where no members found (and downstream process expects an empty table with those vars. Also fixing mp_webin to cover a case where native temp filerefs (starting with a #hash) are not supported.	2021-09-11 19:24:51 +03:00
Allan Bowe	6e2447c70a	fix: missing dependency in mp_webin()	2021-09-11 19:00:04 +03:00
Allan Bowe	486aba84ca	Merge pull request #72 from sasjs/webinnings New mp_webin() macro and performance improvement to mf_getuniquefileref()	2021-09-08 19:51:36 +03:00
Allan Bowe	b5944181e1	chore(dependencies): bumping cli	2021-09-08 19:35:27 +03:00
Allan Bowe	3f69cf506a	feat: mp_webin() macro for handling the _webin_xxx macro variables in SAS web services	2021-09-08 19:34:28 +03:00
Allan Bowe	6013897c50	feat: updating mf_getuniquefileref() to use native approach to get a unique fileref (which is 100 times faster than the old approach)	2021-09-08 15:30:51 +03:00
Allan Bowe	27cf2a2532	Merge pull request #71 from sasjs/including fix: conditional logic around mp_abort(mode=INCLUDE) to cover case wh…	2021-09-05 20:56:40 +03:00
Allan Bowe	d096cbddeb	fix: conditional logic around mp_abort(mode=INCLUDE) to cover case when no mp_include was executed	2021-09-05 20:55:44 +03:00
Allan Bowe	117503f214	Merge pull request #70 from sasjs/including feat: new mp_include() feature for handling %includes with macros	2021-09-03 11:17:21 +03:00
Allan Bowe	5cc5fae750	chore: adding mp_abort to mp_include sas macros list	2021-09-03 11:06:12 +03:00
Allan Bowe	d93032e1a9	chore: docs	2021-09-03 10:54:21 +03:00
Allan Bowe	507557b2cb	feat: new mp_include() feature for handling %includes with macros	2021-09-03 00:40:10 +03:00
Allan Bowe	ee5c3c185a	chore: merge	2021-08-28 14:22:16 +03:00
Allan Bowe	e5592a2eb2	chore: avoiding the string 'ERROR:' in logs	2021-08-28 14:21:36 +03:00
Allan Bowe	59200a6e73	Merge pull request #69 from sasjs/issue68 feat: supporting postgres timestamps for mp_ds2inserts and mp_lib2inserts	2021-08-24 21:03:09 +03:00
Allan Bowe	f468f60ae1	fix: substr issue in mp_ds2inserts. Closes #68	2021-08-24 20:51:29 +03:00
Allan Bowe	9f60d827b6	feat: supporting postgres timestamps for mp_ds2inserts and mp_lib2inserts	2021-08-24 20:39:02 +03:00
Allan Bowe	5c936ddb65	Merge pull request #67 from sasjs/issue66 feat: adding APPEND option to mp_binarycopy.sas, and a new test (mp_b…	2021-08-24 14:04:25 +03:00
Allan Bowe	d0bde62594	feat: adding APPEND option to mp_binarycopy.sas, and a new test (mp_binarycopy.test.sas). Closes #66	2021-08-24 13:43:31 +03:00
Allan Bowe	ada9192337	Merge pull request #65 from sasjs/issue64 fix: increasing limit for mv_getfoldermember.sas, also adding a test,…	2021-08-20 22:50:27 +03:00
Allan Bowe	6161f588a9	fix: increasing limit for mv_getfoldermember.sas, also adding a test, and updating mf_getapploc to search _program by default (and fixing it to work with macro tests, and updating the test for that also)	2021-08-20 22:38:56 +03:00
Allan Bowe	67079d8c17	fix: adding default value for exists in mf_existfunction	2021-08-19 00:20:49 +03:00