Merge pull request #71 from sasjs/including

fix: conditional logic around mp_abort(mode=INCLUDE) to cover case wh…
fix: conditional logic around mp_abort(mode=INCLUDE) to cover case when no mp_include was executed
2025-12-11 06:24:35 +00:00 · 2021-09-05 20:56:40 +03:00 · 2021-09-05 20:55:44 +03:00 · 2021-09-03 11:17:21 +03:00 · 2021-09-03 11:06:12 +03:00 · 2021-09-03 10:54:21 +03:00
59 changed files with 1593 additions and 288 deletions
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -39,7 +39,7 @@ jobs:
          sudo apt install apt-transport-https
          sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
          sudo apt-key add openvpn-repo-pkg-key.pub
-          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-bionic.list
+          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-focal.list
          sudo apt update
          sudo apt install openvpn3

--- a/README.md
+++ b/README.md
@@ -41,6 +41,13 @@ Documentation: https://core.sasjs.io
 - No X command
 - Prefixes: _mf_, _mp_

+**fcmp** library (SAS9/Viya)
+- Function and macro names are identical, except for special cases
+- Prefixes: _mcf_
+
+The fcmp macros are used to generate fcmp functions, and can be used with or
+without the `proc fcmp` wrapper.
+
 **meta** library (SAS9 only)

 - OS independent
@@ -84,7 +91,7 @@ run;

 # Installation

-First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available,eg:
+First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available, eg:

 ```sas
 options insert=(sasautos="/your/path/macrocore/base");
@@ -209,4 +216,4 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d

 <!-- ALL-CONTRIBUTORS-LIST:END -->

-This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!
+This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!
--- a/all.sas
+++ b/all.sas
@@ -133,7 +133,13 @@ options noquotelenmax;
 %macro mf_existfileref(fref
 )/*/STORE SOURCE*/;

-  %if %sysfunc(fileref(&fref))=0 %then %do;
+  %local rc;
+  %let rc=%sysfunc(fileref(&fref));
+  %if &rc=0 %then %do;
+    1
+  %end;
+  %else %if &rc<0 %then %do;
+    %put &sysmacroname: Fileref &fref exists but the underlying file does not;
    1
  %end;
  %else %do;
@@ -141,6 +147,42 @@ options noquotelenmax;
  %end;

 %mend mf_existfileref;/**
+  @file
+  @brief Checks if a function exists
+  @details Returns 1 if the function exists, else 0.  Note that this function
+  can be slow as it needs to open the sashelp.vfuncs table.
+
+  Usage:
+
+      %put %mf_existfunction(CAT);
+      %put %mf_existfunction(DOG);
+
+  Full credit to [Bart](https://sasensei.com/user/305) for the vfunc pointer
+  and the tidy approach for pure macro data set filtering.
+  Check out his [SAS Packages](https://github.com/yabwon/SAS_PACKAGES)
+  framework!  Where you can find the same [function](
+https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionexists-macro
+  ).
+
+  @param [in] name (positional) - function name
+
+  @author Allan Bowe
+**/
+/** @cond */
+%macro mf_existfunction(name
+)/*/STORE SOURCE*/;
+
+  %local dsid rc exist;
+  %let dsid=%sysfunc(open(sashelp.vfunc(where=(fncname="%upcase(&name)"))));
+  %let exist=1;
+  %let exist=%sysfunc(fetch(&dsid, NOSET));
+  %let rc=%sysfunc(close(&dsid));
+
+  %sysevalf(0 = &exist)
+
+%mend mf_existfunction;
+
+/** @endcond *//**
  @file
  @brief Checks if a variable exists in a data set.
  @details Returns 0 if the variable does NOT exist, and return the position of
@@ -233,6 +275,78 @@ options noquotelenmax;
 %mend mf_existvarlist;

 /** @endcond *//**
+  @file
+  @brief Returns the appLoc from the _program variable
+  @details When working with SASjs apps, web services / tests / jobs are always
+  deployed to a root (app) location in the SAS logical folder tree.
+
+  When building apps for use in other environments, you do not necessarily know
+  where the backend services will be deployed.  Therefore a function like this
+  is handy in order to dynamically figure out the appLoc, and enable other
+  services to be connected by a relative reference.
+
+  SASjs apps always have the same immediate substructure (one or more of the
+  following):
+
+  @li /data
+  @li /jobs
+  @li /services
+  @li /tests/jobs
+  @li /tests/services
+  @li /tests/macros
+
+  This function works by testing for the existence of any of the above in the
+  automatic _program variable, and returning the part to the left of it.
+
+  Usage:
+
+      %put %mf_getapploc(&_program)
+
+      %put %mf_getapploc(/some/location/services/admin/myservice);
+      %put %mf_getapploc(/some/location/jobs/extract/somejob/);
+      %put %mf_getapploc(/some/location/tests/jobs/somejob/);
+
+
+  @author Allan Bowe
+**/
+
+%macro mf_getapploc(pgm);
+%if "&pgm"="" %then %do;
+  %if %symexist(_program) %then %let pgm=&_program;
+  %else %do;
+    %put &sysmacroname: No value provided and no _program variable available;
+    %return;
+  %end;
+%end;
+%local root;
+
+/**
+  * First check we are not in the tests/macros folder (which has no subfolders)
+  */
+%if %index(&pgm,/tests/macros/) %then %do;
+  %let root=%substr(&pgm,1,%index(&pgm,/tests/macros)-1);
+  &root
+  %return;
+%end;
+
+/**
+  * Next, move up two levels to avoid matches on subfolder or service name
+  */
+%let root=%substr(&pgm,1,%length(&pgm)-%length(%scan(&pgm,-1,/))-1);
+%let root=%substr(&root,1,%length(&root)-%length(%scan(&root,-1,/))-1);
+
+%if %index(&root,/tests/) %then %do;
+  %let root=%substr(&root,1,%index(&root,/tests/)-1);
+%end;
+%else %if %index(&root,/services) %then %do;
+  %let root=%substr(&root,1,%index(&root,/services)-1);
+%end;
+%else %if %index(&root,/jobs) %then %do;
+  %let root=%substr(&root,1,%index(&root,/jobs)-1);
+%end;
+%else %put &sysmacroname: Could not find an app location from &pgm;
+  &root
+%mend mf_getapploc ;/**
  @file
  @brief Returns a character attribute of a dataset.
  @details Can be used in open code, eg as follows:
@@ -1528,24 +1642,47 @@ Usage:
    recognise this and fetch the log of the parent session instead)
  @li STP environments must finish cleanly to avoid the log being sent to
    _webout.  To assist with this, we also run stpsrvset('program error', 0)
-    and set SYSCC=0.  For 9.4m3 we take a unique approach - we open a macro
-    but don't close it!  This provides a graceful abort, EXCEPT when called
-    called within a %include within a macro (and that macro contains additional
-    logic).  See mp_abort.test.nofix.sas for the example case.
-    If you know of another way to gracefully abort a 9.4m3 STP session, we'd
-    love to hear about it!
+    and set SYSCC=0.  We take a unique "soft abort" approach - we open a macro
+    but don't close it!  This works everywhere EXCEPT inside a \%include inside
+    a macro.  For that, we recommend you use mp_include.sas to perform the
+    include, and then call \%mp_abort(mode=INCLUDE) from the source program (ie,
+    OUTSIDE of the top-parent macro).


  @param mac= to contain the name of the calling macro
  @param msg= message to be returned
  @param iftrue= supply a condition under which the macro should be executed.
+  @param errds= (work.mp_abort_errds) There is no clean way to end a process
+    within a %include called within a macro.  Furthermore, there is no way to
+    test if a macro is called within a %include.  To handle this particular
+    scenario, the %include should be switched for the mp_include.sas macro.
+    This provides an indicator that we are running a macro within a \%include
+    (`_SYSINCLUDEFILEDEVICE`) and allows us to provide a dataset with the abort
+    values (msg, mac).
+    We can then run an abort cancel FILE to stop the include running, and pass
+    the dataset back to the calling program to run a regular \%mp_abort().
+    The dataset will contain the following fields:
+    @li iftrue (1=1)
+    @li msg (the message)
+    @li mac (the mac param)

-  @version 9.4M3
+  @param mode= (REGULAR) If mode=INCLUDE then the &errds dataset is checked for
+    an abort status.
+    Valid values:
+    @li REGULAR (default)
+    @li INCLUDE
+
+  <h4> Related Macros </h4>
+  @li mp_include.sas
+
+  @version 9.4
  @author Allan Bowe
  @cond
 **/

 %macro mp_abort(mac=mp_abort.sas, type=, msg=, iftrue=%str(1=1)
+  , errds=work.mp_abort_errds
+  , mode=REGULAR
 )/*/STORE SOURCE*/;

  %global sysprocessmode sysprocessname;
@@ -1556,9 +1693,44 @@ Usage:
  %if %length(&mac)>0 %then %put NOTE- called by &mac;
  %put NOTE - &msg;

+  %if %symexist(_SYSINCLUDEFILEDEVICE) %then %do;
+    %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do;
+      data &errds;
+        iftrue='1=1';
+        length mac $100 msg $5000;
+        mac=symget('mac');
+        msg=symget('msg');
+      run;
+      data _null_;
+        abort cancel FILE;
+      run;
+      %return;
+    %end;
+  %end;
+
  /* Stored Process Server web app context */
-  %if %symexist(_metaperson) or "&SYSPROCESSNAME "="Compute Server " %then %do;
+  %if %symexist(_metaperson)
+    or "&SYSPROCESSNAME "="Compute Server "
+    or &mode=INCLUDE
+  %then %do;
    options obs=max replace nosyntaxcheck mprint;
+    %if &mode=INCLUDE %then %do;
+      %if %sysfunc(exist(&errds))=1 %then %do;
+        data _null_;
+          set &errds;
+          call symputx('iftrue',iftrue,'l');
+          call symputx('mac',mac,'l');
+          call symputx('msg',msg,'l');
+          putlog (_all_)(=);
+        run;
+        %if (&iftrue)=0 %then %return;
+      %end;
+      %else %do;
+        %put &sysmacroname: No include errors found;
+        %return;
+      %end;
+    %end;
+
    /* extract log errs / warns, if exist */
    %local logloc logline;
    %global logmsg; /* capture global messages */
@@ -1645,7 +1817,9 @@ Usage:
      put ',"_PROGRAM" : ' _PROGRAM ;
      put ",""SYSCC"" : ""&syscc"" ";
      put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+      put ",""SYSHOSTNAME"" : ""&syshostname"" ";
      put ",""SYSJOBID"" : ""&sysjobid"" ";
+      put ",""SYSSITE"" : ""&syssite"" ";
      sysvlong=quote(trim(symget('sysvlong')));
      put ',"SYSVLONG" : ' sysvlong;
      put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
@@ -1662,11 +1836,22 @@ Usage:
        rc=stpsrvset('program error', 0);
        call symputx("syscc",0,"g");
      run;
-      %if "%substr(&sysvlong.xxxxxxxxx,1,9)" ne "9.04.01M3" %then %do;
-        %put NOTE: Ending SAS session due to:;
-        %put NOTE- &msg;
-        endsas;
-      %end;
+      /**
+        * endsas kills 9.4m3 deployments by orphaning multibridges.
+        * Abort variants are ungraceful (non zero return code)
+        * This approach lets SAS run silently until the end :-)
+        * Caution - fails when called within a %include within a macro
+        * Use mp_include() to handle this.
+        */
+      filename skip temp;
+      data _null_;
+        file skip;
+        put '%macro skip();';
+        comment '%mend skip; -> fix lint ';
+        put '%macro skippy();';
+        comment '%mend skippy; -> fix lint ';
+      run;
+      %inc skip;
    %end;
    %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
      /* endsas kills the session making it harder to fetch results */
@@ -1682,24 +1867,6 @@ Usage:
        abort cancel nolist;
      run;
    %end;
-    %else %if "%substr(&sysvlong.xxxxxxxxx,1,9)" = "9.04.01M3" %then %do;
-      /**
-        * endsas kills 9.4m3 deployments by orphaning multibridges.
-        * Abort variants are ungraceful (non zero return code)
-        * This approach lets SAS run silently until the end :-)
-        * Caution - fails when called within a %include within a macro
-        * See tests/mp_abort.test.1 for an example case.
-        */
-      filename skip temp;
-      data _null_;
-        file skip;
-        put '%macro skip();';
-        comment '%mend skip; -> fix lint ';
-        put '%macro skippy();';
-        comment '%mend skippy; -> fix lint ';
-      run;
-      %inc skip;
-    %end;
    %else %do;
      %abort cancel;
    %end;
@@ -2299,10 +2466,29 @@ run;

        %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)

-  @param inloc full, quoted "path/and/filename.ext" of the object to be copied
-  @param outloc full, quoted "path/and/filename.ext" of object to be created
-  @param inref can override default input fileref to avoid naming clash
-  @param outref an override default output fileref to avoid naming clash
+  To append to a file, use the mode option, eg:
+
+      filename tmp1 temp;
+      filename tmp2 temp;
+      data _null_;
+        file tmp1;
+        put 'stacking';
+      run;
+
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+
+
+  @param [in] inloc quoted "path/and/filename.ext" of the file to be copied
+  @param [out] outloc quoted "path/and/filename.ext" of the file to be created
+  @param [in] inref (____in) If provided, this fileref will take precedence over
+    the `inloc` parameter
+  @param [out] outref (____in) If provided, this fileref will take precedence
+    over the `outloc` parameter.  It must already exist!
+  @param [in] mode (CREATE) Valid values:
+    @li CREATE - Create the file (even if it already exists)
+    @li APPEND - Append to the file (don't overwrite)
+
  @returns nothing

  @version 9.2
@@ -2314,20 +2500,29 @@ run;
    ,outloc=          /* full path and filename of object to be created */
    ,inref=____in   /* override default to use own filerefs */
    ,outref=____out /* override default to use own filerefs */
+    ,mode=CREATE
 )/*/STORE SOURCE*/;
+  %local mod outmode;
+  %if &mode=APPEND %then %do;
+    %let mod=mod;
+    %let outmode='a';
+  %end;
+  %else %do;
+    %let outmode='o';
+  %end;
  /* these IN and OUT filerefs can point to anything */
  %if &inref = ____in %then %do;
    filename &inref &inloc lrecl=1048576 ;
  %end;
  %if &outref=____out %then %do;
-    filename &outref &outloc lrecl=1048576 ;
+    filename &outref &outloc lrecl=1048576 &mod;
  %end;

  /* copy the file byte-for-byte  */
  data _null_;
    length filein 8 fileid 8;
    filein = fopen("&inref",'I',1,'B');
-    fileid = fopen("&outref",'O',1,'B');
+    fileid = fopen("&outref",&outmode,1,'B');
    rec = '20'x;
    do while(fread(filein)=0);
      rc = fget(filein,rec,1);
@@ -3449,10 +3644,13 @@ run;
    options:
    @li SAS (default) - suitable for regular proc sql
    @li PGSQL - Used for Postgres databases
+  @param [in] applydttm= (YES) If YES, any columns using datetime formats will
+    be converted to native DB datetime literals

  <h4> SAS Macros </h4>
  @li mf_existfileref.sas
  @li mf_getvarcount.sas
+  @li mf_getvarformat.sas
  @li mf_getvarlist.sas
  @li mf_getvartype.sas

@@ -3461,6 +3659,7 @@ run;
 **/

 %macro mp_ds2inserts(ds, outref=0,schema=0,outds=0,flavour=SAS,maxobs=max
+  ,applydttm=YES
 )/*/STORE SOURCE*/;

 %if not %sysfunc(exist(&ds)) %then %do;
@@ -3538,10 +3737,11 @@ data _null_;
  length _____str $32767;
  format _numeric_ best.;
  format _character_ ;
-  %local i comma var vtype;
+  %local i comma var vtype vfmt;
  %do i=1 %to %sysfunc(countw(&varlist));
    %let var=%scan(&varlist,&i);
    %let vtype=%mf_getvartype(&ds,&var);
+    %let vfmt=%upcase(%mf_getvarformat(&ds,&var,force=1));
    %if &i=1 %then %do;
      %if &flavour=SAS %then %do;
        put "insert into &schema.&outds set ";
@@ -3571,7 +3771,13 @@ data _null_;
      %end;
      %else %if &flavour=PGSQL %then %do;
        if missing(&var) then put 'NULL';
-        else put &var;
+        %if &applydttm=YES and "%substr(&vfmt.xxxxxxxx,1,8)"="DATETIME"
+        %then %do;
+          else put "TIMESTAMP '" &var E8601DT25.6 "'";
+        %end;
+        %else %do;
+          else put &var;
+        %end;
      %end;
    %end;
    %else %do;
@@ -3974,7 +4180,7 @@ run;

 data &outds;
  if &sqlrc or &syscc or &syserr then do;
-    REASON_CD='VALIDATION_ERROR: '!!
+    REASON_CD='VALIDATION_ERR'!!'OR: '!!
      coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
    output;
  end;
@@ -5244,12 +5450,115 @@ create table &outds (rename=(
    run;
  %end;
 %mend mp_hashdataset;/**
+  @file
+  @brief Performs a wrapped \%include
+  @details This macro wrapper is necessary if you need your included code to
+  know that it is being \%included.
+
+  If you are using %include in a regular program, you could make use of the
+  following macro variables:
+
+  @li SYSINCLUDEFILEDEVICE
+  @li SYSINCLUDEFILEDIR
+  @li SYSINCLUDEFILEFILEREF
+  @li SYSINCLUDEFILENAME
+
+  However these variables are NOT available inside a macro, as documented here:
+https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/mcrolref/n1j5tcc0n2xczyn1kg1o0606gsv9.htm
+
+  This macro can be used in place of the %include statement, and will insert
+  the following (equivalent) global variables:
+
+  @li _SYSINCLUDEFILEDEVICE
+  @li _SYSINCLUDEFILEDIR
+  @li _SYSINCLUDEFILEFILEREF
+  @li _SYSINCLUDEFILENAME
+
+  These can be used whenever testing _within a macro_.  Outside of the macro,
+  the regular automatic variables will still be available (thanks to a
+  concatenated file list in the include statement).
+
+  Example usage:
+
+      filename example temp;
+      data _null_;
+        file example;
+        put '%macro test();';
+        put '%put &=_SYSINCLUDEFILEFILEREF;';
+        put '%put &=SYSINCLUDEFILEFILEREF;';
+        put '%mend; %test()';
+        put '%put &=SYSINCLUDEFILEFILEREF;';
+      run;
+      %mp_include(example)
+
+  @param [in] fileref The fileref of the file to be included. Must be provided.
+  @param [in] prefix= (_) The prefix to apply to the global variables.
+  @param [in] opts= (SOURCE2) The options to apply to the %inc statement
+  @param [in] errds= (work.mp_abort_errds) There is no clean way to end a
+    process within a %include called within a macro.  Furthermore, there is no
+    way to test if a macro is called within a %include.  To handle this
+    particular scenario, the %mp_abort() macro will test for the existence of
+    the `_SYSINCLUDEFILEDEVICE` variable and return the outputs (msg,mac) inside
+    this dataset.
+    It will then run an abort cancel FILE to stop the include running, and pass
+    the dataset back.
+    NOTE - it is NOT possible to read this dataset as part of _this_ macro -
+    when running abort cancel FILE, ALL macros are closed, so instead it is
+    necessary to invoke "%mp_abort(mode=INCLUDE)" OUTSIDE of any macro wrappers.
+
+
+  @version 9.4
+  @author Allan Bowe
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_abort.sas
+
+**/
+
+%macro mp_include(fileref
+  ,prefix=_
+  ,opts=SOURCE2
+  ,errds=work.mp_abort_errds
+)/*/STORE SOURCE*/;
+
+/* prepare precode */
+%local tempref;
+%let tempref=%mf_getuniquefileref();
+data _null_;
+  file &tempref;
+  set sashelp.vextfl(where=(fileref="%upcase(&fileref)"));
+  put '%let _SYSINCLUDEFILEDEVICE=' xengine ';';
+  name=scan(xpath,-1,'/\');
+  put '%let _SYSINCLUDEFILENAME=' name ';';
+  path=subpad(xpath,1,length(xpath)-length(name)-1);
+  put '%let _SYSINCLUDEFILEDIR=' path ';';
+  put '%let _SYSINCLUDEFILEFILEREF=' "&fileref;";
+run;
+
+/* prepare the errds */
+data &errds;
+  length msg mac $1000;
+  iftrue='1=0';
+run;
+
+/* include the include */
+%inc &tempref &fileref/&opts;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=%str(&_SYSINCLUDEFILEDIR/&_SYSINCLUDEFILENAME)
+  ,msg=%str(syscc=&syscc after executing &_SYSINCLUDEFILENAME)
+)
+
+filename &tempref clear;
+
+%mend mp_include;/**
  @file mp_jsonout.sas
  @brief Writes JSON in SASjs format to a fileref
  @details PROC JSON is faster but will produce errs like the ones below if
  special chars are encountered.

-  > ERROR: Some code points did not transcode.
+  > (ERR)OR: Some code points did not transcode.

  > An object or array close is not valid at this point in the JSON text.

@@ -5583,6 +5892,8 @@ select distinct lowcase(memname)
  @param [out] outref= Output fileref in which to create the insert statements.
    If it exists, it will be appended to, otherwise it will be created.
  @param [out] schema= (0) The schema of the target database, or the libref.
+  @param [in] applydttm= (YES) If YES, any columns using datetime formats will
+    be converted to native DB datetime literals

  @version 9.2
  @author Allan Bowe
@@ -5593,6 +5904,7 @@ select distinct lowcase(memname)
    ,outref=0
    ,schema=0
    ,maxobs=max
+    ,applydttm=YES
 )/*/STORE SOURCE*/;

 /* Find the tables */
@@ -5622,6 +5934,7 @@ select distinct lowcase(memname)
    ,outds=&ds
    ,flavour=&flavour
    ,maxobs=&maxobs
+    ,applydttm=&applydttm
  )
 %end;

@@ -8301,10 +8614,10 @@ run;

  usage:

-    %mm_createfolder(path=/some/meta/folder)
+      %mm_createfolder(path=/some/meta/folder)

-  @param path= Name of the folder to create.
-  @param mdebug= set DBG to 1 to disable DEBUG messages
+  @param [in] path= Name of the folder to create.
+  @param [in] mdebug= set DBG to 1 to disable DEBUG messages

  @version 9.4
  @author Allan Bowe
@@ -11665,18 +11978,20 @@ run;
 %mend mm_gettableid;/**
  @file
  @brief Creates a dataset with all metadata tables for a particular library
-  @details Will only show the tables to which a user has the requisite
-    metadata access.
+  @details Will only show the tables for which the executing user has the
+    requisite metadata access.

  usage:

-    %mm_gettables(uri=A5X8AHW1.B40001S5)
+      %mm_gettables(uri=A5X8AHW1.B40001S5)

-  @param outds the dataset to create that contains the list of tables
-  @param uri the uri of the library for which to return tables
-  @param getauth= YES|NO - fetch the authdomain used in database connections.
-  Set to NO to improve runtimes in larger environments, as there can be a
-  performance hit on the `metadata_getattr(domainuri, "Name", AuthDomain)` call.
+  @param [in] uri= the uri of the library for which to return tables
+  @param [out] outds= (work.mm_gettables) the dataset to contain the list of
+    tables
+  @param [in] getauth= (YES) Fetch the authdomain used in database connections.
+    Set to NO to improve runtimes in larger environments, as there can be a
+    performance hit on the `metadata_getattr(domainuri, "Name", AuthDomain)`
+    call.

  @returns outds  dataset containing all groups in a column named "metagroup"
    (defaults to work.mm_getlibs). The following columns are provided:
@@ -11704,8 +12019,8 @@ data &outds;
    libdesc $200 libref engine $8 IsDBMSLibname $1
    tablename $50 /* metadata table names can be longer than $32 */
    ;
-  keep libname libdesc libref engine ServerContext path_schema AuthDomain tableuri
-    tablename IsPreassigned IsDBMSLibname id;
+  keep libname libdesc libref engine ServerContext path_schema AuthDomain
+    tableuri tablename IsPreassigned IsDBMSLibname id;
  call missing (of _all_);

  uri=symget('uri');
@@ -15278,24 +15593,27 @@ filename &fname1 clear;
 libname &libref1 clear;
 */
 %mend mv_getclients;/**
-  @file mv_getfoldermembers.sas
-  @brief Gets a list of folders (and ids) for a given root
-  @details Works for both root level and below, oauth or password. Default is
-    oauth, and the token is expected in a global ACCESS_TOKEN variable.
+  @file
+  @brief Gets a list of folder members (and ids) for a given root
+  @details Returns all members for a particular Viya folder.  Works at both root
+  level and below, and results are created in an output dataset.

-        %mv_getfoldermembers(root=/Public)
+        %mv_getfoldermembers(root=/Public, outds=work.mymembers)


-  @param root= The path for which to return the list of folders
-  @param outds= The output dataset to create (default is work.mv_getfolders).  Format:
+  @param [in] root= (/) The path for which to return the list of folders
+  @param [out] outds= (work.mv_getfolders) The output dataset to create. Format:
  |ordinal_root|ordinal_items|creationTimeStamp| modifiedTimeStamp|createdBy|modifiedBy|id| uri|added| type|name|description|
  |---|---|---|---|---|---|---|---|---|---|---|---|
  |1|1|2021-05-25T11:15:04.204Z|2021-05-25T11:15:04.204Z|allbow|allbow|4f1e3945-9655-462b-90f2-c31534b3ca47|/folders/folders/ed701ff3-77e8-468d-a4f5-8c43dec0fd9e|2021-05-25T11:15:04.212Z|child|my_folder_name|My folder Description|

-  @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values are "password" or "authorization_code" (unquoted).
-    The default is authorization_code.
-
+  @param [in] access_token_var= (ACCESS_TOKEN) The global macro variable to
+    contain the access token
+  @param [in] grant_type= (sas_services) Valid values are:
+    @li password
+    @li authorization_code
+    @li detect
+    @li sas_services

  @version VIYA V.03.04
  @author Allan Bowe, source: https://github.com/sasjs/core
@@ -15307,6 +15625,12 @@ libname &libref1 clear;
  @li mf_getuniquelibref.sas
  @li mf_isblank.sas

+  <h4> Related Macros </h4>
+  @li mv_createfolder.sas
+  @li mv_deletefoldermember.sas
+  @li mv_deleteviyafolder.sas
+  @li mv_getfoldermembers.test.sas
+
 **/

 %macro mv_getfoldermembers(root=/
@@ -15346,7 +15670,7 @@ options noquotelenmax;
 %if "&root"="/" %then %do;
  /* if root just list root folders */
  proc http method='GET' out=&fname1 &oauth_bearer
-      url="&base_uri/folders/rootFolders";
+      url="&base_uri/folders/rootFolders?limit=1000";
  %if &grant_type=authorization_code %then %do;
      headers "Authorization"="Bearer &&&access_token_var";
  %end;
@@ -15367,13 +15691,17 @@ options noquotelenmax;
  /*data _null_;infile &fname1;input;putlog _infile_;run;*/
  libname &libref1 JSON fileref=&fname1;
  /* now get the followon link to list members */
-  %local href;
-  %let href=0;
+  %local href cnt;
+  %let cnt=0;
  data _null_;
    set &libref1..links;
-    if rel='members' then call symputx('href',quote("&base_uri"!!trim(href)),'l');
+    if rel='members' then do;
+      url=cats("'","&base_uri",href,"?limit=10000'");
+      call symputx('href',url,'l');
+      call symputx('cnt',1,'l');
+    end;
  run;
-  %if &href=0 %then %do;
+  %if &cnt=0 %then %do;
    %put NOTE:;%put NOTE-  No members found in &root!!;%put NOTE-;
    %return;
  %end;
@@ -18736,3 +19064,175 @@ run;
 %inc "%sysfunc(pathname(work))/ml_json.lua";

 %mend ml_json;
+/**
+  @file
+  @brief Provides a replacement for the stpsrv_header function
+  @details The stpsrv_header is normally a built-in function, used to set the
+  headers for SAS 9 Stored Processes as documented here:
+  https://go.documentation.sas.com/doc/en/itechcdc/9.4/stpug/srvhead.htm
+
+  The purpose of this custom function is to provide a replacement when running
+  similar code as a web service against
+  [sasjs/server](https://github.com/sasjs/server).  It operates by creating a
+  text file with the headers.  The location of this text file is determined by
+  a macro variable (`sasjs_stpsrv_header_loc`) which needs to be injected into
+  each service by the calling process, eg:
+
+      %let sasjs_stpsrv_header_loc = C:/temp/some_uuid/stpsrv_header.txt;
+
+  Note - the function works by appending headers to the file.  If multiple same-
+  named headers are provided, they will all be appended - the calling process
+  needs to pick up the last one.  This will mean removing the attribute if the
+  final record has an empty value.
+
+  The function takes the following (positional) parameters:
+
+  | PARAMETER | DESCRIPTION |
+  |------------|-------------|
+  | name $  | name of the header attribute to create|
+  | value  $  | value of the header attribute|
+
+  It returns 0 if successful, or -1 if an error occured.
+
+  Usage:
+
+      %let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/stpsrv_header.txt;
+
+      %mcf_stpsrv_header(wrap=YES, insert_cmplib=YES)
+
+      data _null_;
+        rc=stpsrv_header('Content-type','application/text');
+        rc=stpsrv_header('Content-disposition',"attachment; filename=file.txt");
+      run;
+
+      data _null_;
+        infile "&sasjs_stpsrv_header_loc";
+        input;
+        putlog _infile_;
+      run;
+
+
+  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
+  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
+    CMPLIB reference.
+  @param [out] lib= (work) The output library in which to create the catalog.
+  @param [out] cat= (sasjs) The output catalog in which to create the package.
+  @param [out] pkg= (utils) The output package in which to create the function.
+    Uses a 3 part format:  libref.catalog.package
+
+**/
+
+%macro mcf_stpsrv_header(wrap=NO
+  ,insert_cmplib=NO
+  ,lib=WORK
+  ,cat=SASJS
+  ,pkg=UTILS
+)/*/STORE SOURCE*/;
+
+%if &wrap=YES  %then %do;
+  proc fcmp outcat=&lib..&cat..&pkg;
+%end;
+
+function stpsrv_header(name $, value $);
+  length loc $128 val $512;
+  loc=symget('sasjs_stpsrv_header_loc');
+  val=trim(name)!!': '!!value;
+  length fref $8;
+  rc=filename(fref,loc);
+  if (rc ne 0) then return( -1 );
+  fid = fopen(fref,'a');
+  if (fid = 0) then return( -1 );
+  rc=fput(fid, val);
+  rc=fwrite(fid);
+  rc=fclose(fid);
+  rc=filename(fref);
+  return(0);
+endsub;
+
+%if &wrap=YES %then %do;
+  quit;
+%end;
+
+%if &insert_cmplib=YES %then %do;
+  options insert=(CMPLIB=(&lib..&cat));
+%end;
+
+%mend mcf_stpsrv_header;/**
+  @file
+  @brief Adds a string to a file
+  @details Creates an fcmp function for appending a string to an external file.
+  If the file does not exist, it is created.
+
+  The function itself takes the following (positional) parameters:
+
+  | PARAMETER | DESCRIPTION |
+  |------------|-------------|
+  | filepath $  | full path to the file|
+  | string  $  | string to add to the file |
+  | mode $     | mode of the output - either APPEND (default) or CREATE |
+
+  It returns 0 if successful, or -1 if an error occured.
+
+  Usage:
+
+      %mcf_string2file(wrap=YES, insert_cmplib=YES)
+
+      data _null_;
+        rc=mcf_string2file(
+          "%sysfunc(pathname(work))/newfile.txt"
+          , "This is a test"
+          , "CREATE");
+      run;
+
+      data _null_;
+        infile "%sysfunc(pathname(work))/newfile.txt";
+        input;
+        putlog _infile_;
+      run;
+
+  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
+  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
+    CMPLIB reference.
+  @param [out] lib= (work) The output library in which to create the catalog.
+  @param [out] cat= (sasjs) The output catalog in which to create the package.
+  @param [out] pkg= (utils) The output package in which to create the function.
+    Uses a 3 part format:  libref.catalog.package
+
+**/
+
+%macro mcf_string2file(wrap=NO
+  ,insert_cmplib=NO
+  ,lib=WORK
+  ,cat=SASJS
+  ,pkg=UTILS
+)/*/STORE SOURCE*/;
+
+%if &wrap=YES  %then %do;
+  proc fcmp outcat=&lib..&cat..&pkg;
+%end;
+
+function mcf_string2file(filepath $, string $, mode $);
+  if mode='APPEND' then fmode='a';
+  else fmode='o';
+  length fref $8;
+  rc=filename(fref,filepath);
+  if (rc ne 0) then return( -1 );
+  fid = fopen(fref,fmode);
+  if (fid = 0) then return( -1 );
+  rc=fput(fid, string);
+  rc=fwrite(fid);
+  rc=fclose(fid);
+  rc=filename(fref);
+  return(0);
+endsub;
+
+
+%if &wrap=YES %then %do;
+  quit;
+%end;
+
+%if &insert_cmplib=YES %then %do;
+  options insert=(CMPLIB=(&lib..&cat));
+%end;
+
+%mend mcf_string2file;
--- a/base/mf_existfileref.sas
+++ b/base/mf_existfileref.sas
@@ -17,7 +17,13 @@
 %macro mf_existfileref(fref
 )/*/STORE SOURCE*/;

-  %if %sysfunc(fileref(&fref))=0 %then %do;
+  %local rc;
+  %let rc=%sysfunc(fileref(&fref));
+  %if &rc=0 %then %do;
+    1
+  %end;
+  %else %if &rc<0 %then %do;
+    %put &sysmacroname: Fileref &fref exists but the underlying file does not;
    1
  %end;
  %else %do;
--- a/base/mf_existfunction.sas
+++ b/base/mf_existfunction.sas
@@ -0,0 +1,37 @@
+/**
+  @file
+  @brief Checks if a function exists
+  @details Returns 1 if the function exists, else 0.  Note that this function
+  can be slow as it needs to open the sashelp.vfuncs table.
+
+  Usage:
+
+      %put %mf_existfunction(CAT);
+      %put %mf_existfunction(DOG);
+
+  Full credit to [Bart](https://sasensei.com/user/305) for the vfunc pointer
+  and the tidy approach for pure macro data set filtering.
+  Check out his [SAS Packages](https://github.com/yabwon/SAS_PACKAGES)
+  framework!  Where you can find the same [function](
+https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionexists-macro
+  ).
+
+  @param [in] name (positional) - function name
+
+  @author Allan Bowe
+**/
+/** @cond */
+%macro mf_existfunction(name
+)/*/STORE SOURCE*/;
+
+  %local dsid rc exist;
+  %let dsid=%sysfunc(open(sashelp.vfunc(where=(fncname="%upcase(&name)"))));
+  %let exist=1;
+  %let exist=%sysfunc(fetch(&dsid, NOSET));
+  %let rc=%sysfunc(close(&dsid));
+
+  %sysevalf(0 = &exist)
+
+%mend mf_existfunction;
+
+/** @endcond */
--- a/base/mf_getapploc.sas
+++ b/base/mf_getapploc.sas
@@ -0,0 +1,73 @@
+/**
+  @file
+  @brief Returns the appLoc from the _program variable
+  @details When working with SASjs apps, web services / tests / jobs are always
+  deployed to a root (app) location in the SAS logical folder tree.
+
+  When building apps for use in other environments, you do not necessarily know
+  where the backend services will be deployed.  Therefore a function like this
+  is handy in order to dynamically figure out the appLoc, and enable other
+  services to be connected by a relative reference.
+
+  SASjs apps always have the same immediate substructure (one or more of the
+  following):
+
+  @li /data
+  @li /jobs
+  @li /services
+  @li /tests/jobs
+  @li /tests/services
+  @li /tests/macros
+
+  This function works by testing for the existence of any of the above in the
+  automatic _program variable, and returning the part to the left of it.
+
+  Usage:
+
+      %put %mf_getapploc(&_program)
+
+      %put %mf_getapploc(/some/location/services/admin/myservice);
+      %put %mf_getapploc(/some/location/jobs/extract/somejob/);
+      %put %mf_getapploc(/some/location/tests/jobs/somejob/);
+
+
+  @author Allan Bowe
+**/
+
+%macro mf_getapploc(pgm);
+%if "&pgm"="" %then %do;
+  %if %symexist(_program) %then %let pgm=&_program;
+  %else %do;
+    %put &sysmacroname: No value provided and no _program variable available;
+    %return;
+  %end;
+%end;
+%local root;
+
+/**
+  * First check we are not in the tests/macros folder (which has no subfolders)
+  */
+%if %index(&pgm,/tests/macros/) %then %do;
+  %let root=%substr(&pgm,1,%index(&pgm,/tests/macros)-1);
+  &root
+  %return;
+%end;
+
+/**
+  * Next, move up two levels to avoid matches on subfolder or service name
+  */
+%let root=%substr(&pgm,1,%length(&pgm)-%length(%scan(&pgm,-1,/))-1);
+%let root=%substr(&root,1,%length(&root)-%length(%scan(&root,-1,/))-1);
+
+%if %index(&root,/tests/) %then %do;
+  %let root=%substr(&root,1,%index(&root,/tests/)-1);
+%end;
+%else %if %index(&root,/services) %then %do;
+  %let root=%substr(&root,1,%index(&root,/services)-1);
+%end;
+%else %if %index(&root,/jobs) %then %do;
+  %let root=%substr(&root,1,%index(&root,/jobs)-1);
+%end;
+%else %put &sysmacroname: Could not find an app location from &pgm;
+  &root
+%mend mf_getapploc ;
--- a/base/mp_abort.sas
+++ b/base/mp_abort.sas
@@ -15,24 +15,47 @@
    recognise this and fetch the log of the parent session instead)
  @li STP environments must finish cleanly to avoid the log being sent to
    _webout.  To assist with this, we also run stpsrvset('program error', 0)
-    and set SYSCC=0.  For 9.4m3 we take a unique approach - we open a macro
-    but don't close it!  This provides a graceful abort, EXCEPT when called
-    called within a %include within a macro (and that macro contains additional
-    logic).  See mp_abort.test.nofix.sas for the example case.
-    If you know of another way to gracefully abort a 9.4m3 STP session, we'd
-    love to hear about it!
+    and set SYSCC=0.  We take a unique "soft abort" approach - we open a macro
+    but don't close it!  This works everywhere EXCEPT inside a \%include inside
+    a macro.  For that, we recommend you use mp_include.sas to perform the
+    include, and then call \%mp_abort(mode=INCLUDE) from the source program (ie,
+    OUTSIDE of the top-parent macro).


  @param mac= to contain the name of the calling macro
  @param msg= message to be returned
  @param iftrue= supply a condition under which the macro should be executed.
+  @param errds= (work.mp_abort_errds) There is no clean way to end a process
+    within a %include called within a macro.  Furthermore, there is no way to
+    test if a macro is called within a %include.  To handle this particular
+    scenario, the %include should be switched for the mp_include.sas macro.
+    This provides an indicator that we are running a macro within a \%include
+    (`_SYSINCLUDEFILEDEVICE`) and allows us to provide a dataset with the abort
+    values (msg, mac).
+    We can then run an abort cancel FILE to stop the include running, and pass
+    the dataset back to the calling program to run a regular \%mp_abort().
+    The dataset will contain the following fields:
+    @li iftrue (1=1)
+    @li msg (the message)
+    @li mac (the mac param)

-  @version 9.4M3
+  @param mode= (REGULAR) If mode=INCLUDE then the &errds dataset is checked for
+    an abort status.
+    Valid values:
+    @li REGULAR (default)
+    @li INCLUDE
+
+  <h4> Related Macros </h4>
+  @li mp_include.sas
+
+  @version 9.4
  @author Allan Bowe
  @cond
 **/

 %macro mp_abort(mac=mp_abort.sas, type=, msg=, iftrue=%str(1=1)
+  , errds=work.mp_abort_errds
+  , mode=REGULAR
 )/*/STORE SOURCE*/;

  %global sysprocessmode sysprocessname;
@@ -43,9 +66,44 @@
  %if %length(&mac)>0 %then %put NOTE- called by &mac;
  %put NOTE - &msg;

+  %if %symexist(_SYSINCLUDEFILEDEVICE) %then %do;
+    %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do;
+      data &errds;
+        iftrue='1=1';
+        length mac $100 msg $5000;
+        mac=symget('mac');
+        msg=symget('msg');
+      run;
+      data _null_;
+        abort cancel FILE;
+      run;
+      %return;
+    %end;
+  %end;
+
  /* Stored Process Server web app context */
-  %if %symexist(_metaperson) or "&SYSPROCESSNAME "="Compute Server " %then %do;
+  %if %symexist(_metaperson)
+    or "&SYSPROCESSNAME "="Compute Server "
+    or &mode=INCLUDE
+  %then %do;
    options obs=max replace nosyntaxcheck mprint;
+    %if &mode=INCLUDE %then %do;
+      %if %sysfunc(exist(&errds))=1 %then %do;
+        data _null_;
+          set &errds;
+          call symputx('iftrue',iftrue,'l');
+          call symputx('mac',mac,'l');
+          call symputx('msg',msg,'l');
+          putlog (_all_)(=);
+        run;
+        %if (&iftrue)=0 %then %return;
+      %end;
+      %else %do;
+        %put &sysmacroname: No include errors found;
+        %return;
+      %end;
+    %end;
+
    /* extract log errs / warns, if exist */
    %local logloc logline;
    %global logmsg; /* capture global messages */
@@ -132,7 +190,9 @@
      put ',"_PROGRAM" : ' _PROGRAM ;
      put ",""SYSCC"" : ""&syscc"" ";
      put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+      put ",""SYSHOSTNAME"" : ""&syshostname"" ";
      put ",""SYSJOBID"" : ""&sysjobid"" ";
+      put ",""SYSSITE"" : ""&syssite"" ";
      sysvlong=quote(trim(symget('sysvlong')));
      put ',"SYSVLONG" : ' sysvlong;
      put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
@@ -149,11 +209,22 @@
        rc=stpsrvset('program error', 0);
        call symputx("syscc",0,"g");
      run;
-      %if "%substr(&sysvlong.xxxxxxxxx,1,9)" ne "9.04.01M3" %then %do;
-        %put NOTE: Ending SAS session due to:;
-        %put NOTE- &msg;
-        endsas;
-      %end;
+      /**
+        * endsas kills 9.4m3 deployments by orphaning multibridges.
+        * Abort variants are ungraceful (non zero return code)
+        * This approach lets SAS run silently until the end :-)
+        * Caution - fails when called within a %include within a macro
+        * Use mp_include() to handle this.
+        */
+      filename skip temp;
+      data _null_;
+        file skip;
+        put '%macro skip();';
+        comment '%mend skip; -> fix lint ';
+        put '%macro skippy();';
+        comment '%mend skippy; -> fix lint ';
+      run;
+      %inc skip;
    %end;
    %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
      /* endsas kills the session making it harder to fetch results */
@@ -169,24 +240,6 @@
        abort cancel nolist;
      run;
    %end;
-    %else %if "%substr(&sysvlong.xxxxxxxxx,1,9)" = "9.04.01M3" %then %do;
-      /**
-        * endsas kills 9.4m3 deployments by orphaning multibridges.
-        * Abort variants are ungraceful (non zero return code)
-        * This approach lets SAS run silently until the end :-)
-        * Caution - fails when called within a %include within a macro
-        * See tests/mp_abort.test.1 for an example case.
-        */
-      filename skip temp;
-      data _null_;
-        file skip;
-        put '%macro skip();';
-        comment '%mend skip; -> fix lint ';
-        put '%macro skippy();';
-        comment '%mend skippy; -> fix lint ';
-      run;
-      %inc skip;
-    %end;
    %else %do;
      %abort cancel;
    %end;
--- a/base/mp_binarycopy.sas
+++ b/base/mp_binarycopy.sas
@@ -9,10 +9,29 @@

        %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)

-  @param inloc full, quoted "path/and/filename.ext" of the object to be copied
-  @param outloc full, quoted "path/and/filename.ext" of object to be created
-  @param inref can override default input fileref to avoid naming clash
-  @param outref an override default output fileref to avoid naming clash
+  To append to a file, use the mode option, eg:
+
+      filename tmp1 temp;
+      filename tmp2 temp;
+      data _null_;
+        file tmp1;
+        put 'stacking';
+      run;
+
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+
+
+  @param [in] inloc quoted "path/and/filename.ext" of the file to be copied
+  @param [out] outloc quoted "path/and/filename.ext" of the file to be created
+  @param [in] inref (____in) If provided, this fileref will take precedence over
+    the `inloc` parameter
+  @param [out] outref (____in) If provided, this fileref will take precedence
+    over the `outloc` parameter.  It must already exist!
+  @param [in] mode (CREATE) Valid values:
+    @li CREATE - Create the file (even if it already exists)
+    @li APPEND - Append to the file (don't overwrite)
+
  @returns nothing

  @version 9.2
@@ -24,20 +43,29 @@
    ,outloc=          /* full path and filename of object to be created */
    ,inref=____in   /* override default to use own filerefs */
    ,outref=____out /* override default to use own filerefs */
+    ,mode=CREATE
 )/*/STORE SOURCE*/;
+  %local mod outmode;
+  %if &mode=APPEND %then %do;
+    %let mod=mod;
+    %let outmode='a';
+  %end;
+  %else %do;
+    %let outmode='o';
+  %end;
  /* these IN and OUT filerefs can point to anything */
  %if &inref = ____in %then %do;
    filename &inref &inloc lrecl=1048576 ;
  %end;
  %if &outref=____out %then %do;
-    filename &outref &outloc lrecl=1048576 ;
+    filename &outref &outloc lrecl=1048576 &mod;
  %end;

  /* copy the file byte-for-byte  */
  data _null_;
    length filein 8 fileid 8;
    filein = fopen("&inref",'I',1,'B');
-    fileid = fopen("&outref",'O',1,'B');
+    fileid = fopen("&outref",&outmode,1,'B');
    rec = '20'x;
    do while(fread(filein)=0);
      rc = fget(filein,rec,1);
--- a/base/mp_ds2inserts.sas
+++ b/base/mp_ds2inserts.sas
@@ -25,10 +25,13 @@
    options:
    @li SAS (default) - suitable for regular proc sql
    @li PGSQL - Used for Postgres databases
+  @param [in] applydttm= (YES) If YES, any columns using datetime formats will
+    be converted to native DB datetime literals

  <h4> SAS Macros </h4>
  @li mf_existfileref.sas
  @li mf_getvarcount.sas
+  @li mf_getvarformat.sas
  @li mf_getvarlist.sas
  @li mf_getvartype.sas

@@ -37,6 +40,7 @@
 **/

 %macro mp_ds2inserts(ds, outref=0,schema=0,outds=0,flavour=SAS,maxobs=max
+  ,applydttm=YES
 )/*/STORE SOURCE*/;

 %if not %sysfunc(exist(&ds)) %then %do;
@@ -114,10 +118,11 @@ data _null_;
  length _____str $32767;
  format _numeric_ best.;
  format _character_ ;
-  %local i comma var vtype;
+  %local i comma var vtype vfmt;
  %do i=1 %to %sysfunc(countw(&varlist));
    %let var=%scan(&varlist,&i);
    %let vtype=%mf_getvartype(&ds,&var);
+    %let vfmt=%upcase(%mf_getvarformat(&ds,&var,force=1));
    %if &i=1 %then %do;
      %if &flavour=SAS %then %do;
        put "insert into &schema.&outds set ";
@@ -147,7 +152,13 @@ data _null_;
      %end;
      %else %if &flavour=PGSQL %then %do;
        if missing(&var) then put 'NULL';
-        else put &var;
+        %if &applydttm=YES and "%substr(&vfmt.xxxxxxxx,1,8)"="DATETIME"
+        %then %do;
+          else put "TIMESTAMP '" &var E8601DT25.6 "'";
+        %end;
+        %else %do;
+          else put &var;
+        %end;
      %end;
    %end;
    %else %do;
--- a/base/mp_filtervalidate.sas
+++ b/base/mp_filtervalidate.sas
@@ -78,7 +78,7 @@ run;

 data &outds;
  if &sqlrc or &syscc or &syserr then do;
-    REASON_CD='VALIDATION_ERROR: '!!
+    REASON_CD='VALIDATION_ERR'!!'OR: '!!
      coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
    output;
  end;
--- a/base/mp_include.sas
+++ b/base/mp_include.sas
@@ -0,0 +1,104 @@
+/**
+  @file
+  @brief Performs a wrapped \%include
+  @details This macro wrapper is necessary if you need your included code to
+  know that it is being \%included.
+
+  If you are using %include in a regular program, you could make use of the
+  following macro variables:
+
+  @li SYSINCLUDEFILEDEVICE
+  @li SYSINCLUDEFILEDIR
+  @li SYSINCLUDEFILEFILEREF
+  @li SYSINCLUDEFILENAME
+
+  However these variables are NOT available inside a macro, as documented here:
+https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/mcrolref/n1j5tcc0n2xczyn1kg1o0606gsv9.htm
+
+  This macro can be used in place of the %include statement, and will insert
+  the following (equivalent) global variables:
+
+  @li _SYSINCLUDEFILEDEVICE
+  @li _SYSINCLUDEFILEDIR
+  @li _SYSINCLUDEFILEFILEREF
+  @li _SYSINCLUDEFILENAME
+
+  These can be used whenever testing _within a macro_.  Outside of the macro,
+  the regular automatic variables will still be available (thanks to a
+  concatenated file list in the include statement).
+
+  Example usage:
+
+      filename example temp;
+      data _null_;
+        file example;
+        put '%macro test();';
+        put '%put &=_SYSINCLUDEFILEFILEREF;';
+        put '%put &=SYSINCLUDEFILEFILEREF;';
+        put '%mend; %test()';
+        put '%put &=SYSINCLUDEFILEFILEREF;';
+      run;
+      %mp_include(example)
+
+  @param [in] fileref The fileref of the file to be included. Must be provided.
+  @param [in] prefix= (_) The prefix to apply to the global variables.
+  @param [in] opts= (SOURCE2) The options to apply to the %inc statement
+  @param [in] errds= (work.mp_abort_errds) There is no clean way to end a
+    process within a %include called within a macro.  Furthermore, there is no
+    way to test if a macro is called within a %include.  To handle this
+    particular scenario, the %mp_abort() macro will test for the existence of
+    the `_SYSINCLUDEFILEDEVICE` variable and return the outputs (msg,mac) inside
+    this dataset.
+    It will then run an abort cancel FILE to stop the include running, and pass
+    the dataset back.
+    NOTE - it is NOT possible to read this dataset as part of _this_ macro -
+    when running abort cancel FILE, ALL macros are closed, so instead it is
+    necessary to invoke "%mp_abort(mode=INCLUDE)" OUTSIDE of any macro wrappers.
+
+
+  @version 9.4
+  @author Allan Bowe
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_abort.sas
+
+**/
+
+%macro mp_include(fileref
+  ,prefix=_
+  ,opts=SOURCE2
+  ,errds=work.mp_abort_errds
+)/*/STORE SOURCE*/;
+
+/* prepare precode */
+%local tempref;
+%let tempref=%mf_getuniquefileref();
+data _null_;
+  file &tempref;
+  set sashelp.vextfl(where=(fileref="%upcase(&fileref)"));
+  put '%let _SYSINCLUDEFILEDEVICE=' xengine ';';
+  name=scan(xpath,-1,'/\');
+  put '%let _SYSINCLUDEFILENAME=' name ';';
+  path=subpad(xpath,1,length(xpath)-length(name)-1);
+  put '%let _SYSINCLUDEFILEDIR=' path ';';
+  put '%let _SYSINCLUDEFILEFILEREF=' "&fileref;";
+run;
+
+/* prepare the errds */
+data &errds;
+  length msg mac $1000;
+  iftrue='1=0';
+run;
+
+/* include the include */
+%inc &tempref &fileref/&opts;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=%str(&_SYSINCLUDEFILEDIR/&_SYSINCLUDEFILENAME)
+  ,msg=%str(syscc=&syscc after executing &_SYSINCLUDEFILENAME)
+)
+
+filename &tempref clear;
+
+%mend mp_include;
--- a/base/mp_jsonout.sas
+++ b/base/mp_jsonout.sas
@@ -4,7 +4,7 @@
  @details PROC JSON is faster but will produce errs like the ones below if
  special chars are encountered.

-  > ERROR: Some code points did not transcode.
+  > (ERR)OR: Some code points did not transcode.

  > An object or array close is not valid at this point in the JSON text.

--- a/base/mp_lib2inserts.sas
+++ b/base/mp_lib2inserts.sas
@@ -28,6 +28,8 @@
  @param [out] outref= Output fileref in which to create the insert statements.
    If it exists, it will be appended to, otherwise it will be created.
  @param [out] schema= (0) The schema of the target database, or the libref.
+  @param [in] applydttm= (YES) If YES, any columns using datetime formats will
+    be converted to native DB datetime literals

  @version 9.2
  @author Allan Bowe
@@ -38,6 +40,7 @@
    ,outref=0
    ,schema=0
    ,maxobs=max
+    ,applydttm=YES
 )/*/STORE SOURCE*/;

 /* Find the tables */
@@ -67,6 +70,7 @@ select distinct lowcase(memname)
    ,outds=&ds
    ,flavour=&flavour
    ,maxobs=&maxobs
+    ,applydttm=&applydttm
  )
 %end;

--- a/build.py
+++ b/build.py
@@ -84,7 +84,7 @@ options noquotelenmax;
 """
 f = open('all.sas', "w")             # r / r+ / rb / rb+ / w / wb
 f.write(header)
-folders=['base','meta','metax','viya','lua']
+folders=['base','meta','metax','viya','lua','fcmp']
 for folder in folders:
    filenames = [fn for fn in Path('./' + folder).iterdir() if fn.match("*.sas")]
    filenames.sort()
--- a/fcmp/mcf_stpsrv_header.sas
+++ b/fcmp/mcf_stpsrv_header.sas
@@ -0,0 +1,94 @@
+/**
+  @file
+  @brief Provides a replacement for the stpsrv_header function
+  @details The stpsrv_header is normally a built-in function, used to set the
+  headers for SAS 9 Stored Processes as documented here:
+  https://go.documentation.sas.com/doc/en/itechcdc/9.4/stpug/srvhead.htm
+
+  The purpose of this custom function is to provide a replacement when running
+  similar code as a web service against
+  [sasjs/server](https://github.com/sasjs/server).  It operates by creating a
+  text file with the headers.  The location of this text file is determined by
+  a macro variable (`sasjs_stpsrv_header_loc`) which needs to be injected into
+  each service by the calling process, eg:
+
+      %let sasjs_stpsrv_header_loc = C:/temp/some_uuid/stpsrv_header.txt;
+
+  Note - the function works by appending headers to the file.  If multiple same-
+  named headers are provided, they will all be appended - the calling process
+  needs to pick up the last one.  This will mean removing the attribute if the
+  final record has an empty value.
+
+  The function takes the following (positional) parameters:
+
+  | PARAMETER | DESCRIPTION |
+  |------------|-------------|
+  | name $  | name of the header attribute to create|
+  | value  $  | value of the header attribute|
+
+  It returns 0 if successful, or -1 if an error occured.
+
+  Usage:
+
+      %let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/stpsrv_header.txt;
+
+      %mcf_stpsrv_header(wrap=YES, insert_cmplib=YES)
+
+      data _null_;
+        rc=stpsrv_header('Content-type','application/text');
+        rc=stpsrv_header('Content-disposition',"attachment; filename=file.txt");
+      run;
+
+      data _null_;
+        infile "&sasjs_stpsrv_header_loc";
+        input;
+        putlog _infile_;
+      run;
+
+
+  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
+  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
+    CMPLIB reference.
+  @param [out] lib= (work) The output library in which to create the catalog.
+  @param [out] cat= (sasjs) The output catalog in which to create the package.
+  @param [out] pkg= (utils) The output package in which to create the function.
+    Uses a 3 part format:  libref.catalog.package
+
+**/
+
+%macro mcf_stpsrv_header(wrap=NO
+  ,insert_cmplib=NO
+  ,lib=WORK
+  ,cat=SASJS
+  ,pkg=UTILS
+)/*/STORE SOURCE*/;
+
+%if &wrap=YES  %then %do;
+  proc fcmp outcat=&lib..&cat..&pkg;
+%end;
+
+function stpsrv_header(name $, value $);
+  length loc $128 val $512;
+  loc=symget('sasjs_stpsrv_header_loc');
+  val=trim(name)!!': '!!value;
+  length fref $8;
+  rc=filename(fref,loc);
+  if (rc ne 0) then return( -1 );
+  fid = fopen(fref,'a');
+  if (fid = 0) then return( -1 );
+  rc=fput(fid, val);
+  rc=fwrite(fid);
+  rc=fclose(fid);
+  rc=filename(fref);
+  return(0);
+endsub;
+
+%if &wrap=YES %then %do;
+  quit;
+%end;
+
+%if &insert_cmplib=YES %then %do;
+  options insert=(CMPLIB=(&lib..&cat));
+%end;
+
+%mend mcf_stpsrv_header;
--- a/fcmp/mcf_string2file.sas
+++ b/fcmp/mcf_string2file.sas
@@ -0,0 +1,79 @@
+/**
+  @file
+  @brief Adds a string to a file
+  @details Creates an fcmp function for appending a string to an external file.
+  If the file does not exist, it is created.
+
+  The function itself takes the following (positional) parameters:
+
+  | PARAMETER | DESCRIPTION |
+  |------------|-------------|
+  | filepath $  | full path to the file|
+  | string  $  | string to add to the file |
+  | mode $     | mode of the output - either APPEND (default) or CREATE |
+
+  It returns 0 if successful, or -1 if an error occured.
+
+  Usage:
+
+      %mcf_string2file(wrap=YES, insert_cmplib=YES)
+
+      data _null_;
+        rc=mcf_string2file(
+          "%sysfunc(pathname(work))/newfile.txt"
+          , "This is a test"
+          , "CREATE");
+      run;
+
+      data _null_;
+        infile "%sysfunc(pathname(work))/newfile.txt";
+        input;
+        putlog _infile_;
+      run;
+
+  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
+  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
+    CMPLIB reference.
+  @param [out] lib= (work) The output library in which to create the catalog.
+  @param [out] cat= (sasjs) The output catalog in which to create the package.
+  @param [out] pkg= (utils) The output package in which to create the function.
+    Uses a 3 part format:  libref.catalog.package
+
+**/
+
+%macro mcf_string2file(wrap=NO
+  ,insert_cmplib=NO
+  ,lib=WORK
+  ,cat=SASJS
+  ,pkg=UTILS
+)/*/STORE SOURCE*/;
+
+%if &wrap=YES  %then %do;
+  proc fcmp outcat=&lib..&cat..&pkg;
+%end;
+
+function mcf_string2file(filepath $, string $, mode $);
+  if mode='APPEND' then fmode='a';
+  else fmode='o';
+  length fref $8;
+  rc=filename(fref,filepath);
+  if (rc ne 0) then return( -1 );
+  fid = fopen(fref,fmode);
+  if (fid = 0) then return( -1 );
+  rc=fput(fid, string);
+  rc=fwrite(fid);
+  rc=fclose(fid);
+  rc=filename(fref);
+  return(0);
+endsub;
+
+
+%if &wrap=YES %then %do;
+  quit;
+%end;
+
+%if &insert_cmplib=YES %then %do;
+  options insert=(CMPLIB=(&lib..&cat));
+%end;
+
+%mend mcf_string2file;
--- a/main.dox
+++ b/main.dox
@@ -20,6 +20,19 @@

 */

+/*! \dir fcmp
+ *  \brief Macros for generating FCMP functions
+ *  \details These macros have the following attributes:
+
+        * Macro name matches compiled function / subroutine name
+        * Prefixes:  _mcf_, _mcs_
+
+    The macro part is just a wrapper for the underlying function / subroutine,
+    and has switches for including the proc fcmp / quit statements and whether
+    to insert the package into the CMPLIB option.
+
+ */
+
 /*! \dir meta
 *  \brief Metadata Aware Macros
 *  \details These macros have the following attributes:
--- a/meta/mm_createfolder.sas
+++ b/meta/mm_createfolder.sas
@@ -14,10 +14,10 @@

  usage:

-    %mm_createfolder(path=/some/meta/folder)
+      %mm_createfolder(path=/some/meta/folder)

-  @param path= Name of the folder to create.
-  @param mdebug= set DBG to 1 to disable DEBUG messages
+  @param [in] path= Name of the folder to create.
+  @param [in] mdebug= set DBG to 1 to disable DEBUG messages

  @version 9.4
  @author Allan Bowe
--- a/meta/mm_gettables.sas
+++ b/meta/mm_gettables.sas
@@ -1,18 +1,20 @@
 /**
  @file
  @brief Creates a dataset with all metadata tables for a particular library
-  @details Will only show the tables to which a user has the requisite
-    metadata access.
+  @details Will only show the tables for which the executing user has the
+    requisite metadata access.

  usage:

-    %mm_gettables(uri=A5X8AHW1.B40001S5)
+      %mm_gettables(uri=A5X8AHW1.B40001S5)

-  @param outds the dataset to create that contains the list of tables
-  @param uri the uri of the library for which to return tables
-  @param getauth= YES|NO - fetch the authdomain used in database connections.
-  Set to NO to improve runtimes in larger environments, as there can be a
-  performance hit on the `metadata_getattr(domainuri, "Name", AuthDomain)` call.
+  @param [in] uri= the uri of the library for which to return tables
+  @param [out] outds= (work.mm_gettables) the dataset to contain the list of
+    tables
+  @param [in] getauth= (YES) Fetch the authdomain used in database connections.
+    Set to NO to improve runtimes in larger environments, as there can be a
+    performance hit on the `metadata_getattr(domainuri, "Name", AuthDomain)`
+    call.

  @returns outds  dataset containing all groups in a column named "metagroup"
    (defaults to work.mm_getlibs). The following columns are provided:
@@ -40,8 +42,8 @@ data &outds;
    libdesc $200 libref engine $8 IsDBMSLibname $1
    tablename $50 /* metadata table names can be longer than $32 */
    ;
-  keep libname libdesc libref engine ServerContext path_schema AuthDomain tableuri
-    tablename IsPreassigned IsDBMSLibname id;
+  keep libname libdesc libref engine ServerContext path_schema AuthDomain
+    tableuri tablename IsPreassigned IsDBMSLibname id;
  call missing (of _all_);

  uri=symget('uri');
--- a/package-lock.json
+++ b/package-lock.json
@@ -7,44 +7,43 @@
      "name": "@sasjs/core",
      "license": "MIT",
      "devDependencies": {
-        "@sasjs/cli": "2.33.3"
+        "@sasjs/cli": "^2.37.2"
      }
    },
    "node_modules/@sasjs/adapter": {
-      "version": "2.8.9",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.8.9.tgz",
-      "integrity": "sha512-8UChcZlqqlmaMMaKOCr2Bc1h+i2KVDY0FINlPXQN5PdAgEMd8dbxI2p9bsiI1yjYjOBO9LuMl7B79/mwYCtyEw==",
+      "version": "2.10.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.10.0.tgz",
+      "integrity": "sha512-GbvyIgbODnAJaBaz/Tz8/IwcujNOsZcwzbIuKQtG5y13BzgVPtx/e7b2TAhdoBqd+8uVh0CdrSDfOV/SuvPurg==",
      "dev": true,
      "dependencies": {
-        "@sasjs/utils": "^2.21.0",
+        "@sasjs/utils": "^2.27.1",
        "axios": "^0.21.1",
        "axios-cookiejar-support": "^1.0.1",
        "form-data": "^4.0.0",
        "https": "^1.0.0",
-        "jwt-decode": "^3.1.2",
-        "tough-cookie": "^4.0.0",
-        "url": "^0.11.0"
+        "tough-cookie": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=15"
      }
    },
    "node_modules/@sasjs/cli": {
-      "version": "2.33.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.33.3.tgz",
-      "integrity": "sha512-y1uFM5MEE6eoKLrPUJbweDt4njSy9Y3CS5w5/U2xwNbiAyhiPXgkwCHjCQ8Qg+0rQnMvyNEn6qJuJZ3udc6T7w==",
+      "version": "2.37.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.37.2.tgz",
+      "integrity": "sha512-Bcm8UFN9Y/ZON4T31Gu9mf1REn1pZoStHFVrix/yp7mchFt5rrrY5RbIqO/AI1jzhShSVg9P7oU4VPJrqki+SA==",
      "dev": true,
      "dependencies": {
-        "@sasjs/adapter": "2.8.9",
-        "@sasjs/core": "2.35.3",
+        "@sasjs/adapter": "2.10.0",
+        "@sasjs/core": "2.35.4",
        "@sasjs/lint": "1.11.2",
-        "@sasjs/utils": "2.23.3",
-        "@types/url-parse": "1.4.3",
-        "btoa": "1.2.1",
+        "@sasjs/utils": "2.27.1",
        "chalk": "4.1.1",
        "csv-stringify": "5.6.2",
        "dotenv": "10.0.0",
        "esm": "3.2.25",
        "find": "0.3.0",
-        "fs-extra": "10.0.0",
        "get-installed-path": "4.0.8",
+        "js-base64": "3.6.1",
        "jsdom": "16.6.0",
        "jwt-decode": "3.1.2",
        "lodash.groupby": "4.6.0",
@@ -60,9 +59,9 @@
      }
    },
    "node_modules/@sasjs/core": {
-      "version": "2.35.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.35.3.tgz",
-      "integrity": "sha512-3o5PU6DkihpA+Aibt1lRy4USqJI0VFa+wNsKCD+bUD2DLZICU3JablZQxwAPH70VWJGXAUJtDFj0T/iRo5Devg==",
+      "version": "2.35.4",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.35.4.tgz",
+      "integrity": "sha512-Exr3+yRdvacKvbXQoi1RfGHi5NtZUkc7RwFNkemHTFXLYaIzPI8CGSCaQmKkwM1UteOJly2e2pw2YT6kHNY1NA==",
      "dev": true
    },
    "node_modules/@sasjs/lint": {
@@ -75,11 +74,12 @@
      }
    },
    "node_modules/@sasjs/utils": {
-      "version": "2.23.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.23.3.tgz",
-      "integrity": "sha512-tEh4mGG80eUxSLpbPivA0vl4akMdauL+yZrLn1uUM8EyiXPvlcWPkQTeN6oHbyyAH108D9cfEBidTePZh1p5VQ==",
+      "version": "2.27.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.27.1.tgz",
+      "integrity": "sha512-CYTQwEj89cc7H3tGiQQcyDkZYaWRc1HZJpOF8o2RHYS37fIAOy0SyyJdq6mcQ74Nb1u5AmFXPFIvnRCMEcTYeQ==",
      "dev": true,
      "dependencies": {
+        "@types/fs-extra": "^9.0.11",
        "@types/prompts": "^2.0.13",
        "chalk": "^4.1.1",
        "cli-table": "^0.3.6",
@@ -87,7 +87,11 @@
        "fs-extra": "^10.0.0",
        "jwt-decode": "^3.1.2",
        "prompts": "^2.4.1",
+        "rimraf": "^3.0.2",
        "valid-url": "^1.0.9"
+      },
+      "engines": {
+        "node": ">=15"
      }
    },
    "node_modules/@tootallnate/once": {
@@ -99,6 +103,15 @@
        "node": ">= 6"
      }
    },
+    "node_modules/@types/fs-extra": {
+      "version": "9.0.12",
+      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.12.tgz",
+      "integrity": "sha512-I+bsBr67CurCGnSenZZ7v94gd3tc3+Aj2taxMT4yu4ABLuOgOjeFxX3dokG24ztSRg5tnT00sL8BszO7gSMoIw==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
    "node_modules/@types/node": {
      "version": "16.3.1",
      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.3.1.tgz",
@@ -121,12 +134,6 @@
      "dev": true,
      "peer": true
    },
-    "node_modules/@types/url-parse": {
-      "version": "1.4.3",
-      "resolved": "https://registry.npmjs.org/@types/url-parse/-/url-parse-1.4.3.tgz",
-      "integrity": "sha512-4kHAkbV/OfW2kb5BLVUuUMoumB3CP8rHqlw48aHvFy5tf9ER0AfOonBlX29l/DD68G70DmyhRlSYfQPSYpC5Vw==",
-      "dev": true
-    },
    "node_modules/abab": {
      "version": "2.0.5",
      "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.5.tgz",
@@ -298,18 +305,6 @@
      "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==",
      "dev": true
    },
-    "node_modules/btoa": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz",
-      "integrity": "sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g==",
-      "dev": true,
-      "bin": {
-        "btoa": "bin/btoa.js"
-      },
-      "engines": {
-        "node": ">= 0.4.0"
-      }
-    },
    "node_modules/buffer": {
      "version": "5.7.1",
      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
@@ -657,9 +652,9 @@
      }
    },
    "node_modules/follow-redirects": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz",
-      "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg==",
+      "version": "1.14.2",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.2.tgz",
+      "integrity": "sha512-yLR6WaE2lbF0x4K2qE2p9PEXKLDjUjnR/xmjS3wHAYxtlsI9MLLBJUZirAHKzUZDGLxje7w/cXR49WOUo4rbsA==",
      "dev": true,
      "funding": [
        {
@@ -990,6 +985,12 @@
      "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
      "dev": true
    },
+    "node_modules/js-base64": {
+      "version": "3.6.1",
+      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.6.1.tgz",
+      "integrity": "sha512-Frdq2+tRRGLQUIQOgsIGSCd1VePCS2fsddTG5dTCqR0JHgltXWfsxnY0gIXPoMeRmdom6Oyq+UMOFg5suduOjQ==",
+      "dev": true
+    },
    "node_modules/jsdom": {
      "version": "16.6.0",
      "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.6.0.tgz",
@@ -1273,9 +1274,9 @@
      }
    },
    "node_modules/path-parse": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz",
-      "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==",
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
      "dev": true
    },
    "node_modules/pify": {
@@ -1327,16 +1328,6 @@
        "node": ">=6"
      }
    },
-    "node_modules/querystring": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
-      "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=",
-      "deprecated": "The",
-      "dev": true,
-      "engines": {
-        "node": ">=0.4.x"
-      }
-    },
    "node_modules/readable-stream": {
      "version": "3.6.0",
      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@@ -1595,22 +1586,6 @@
        "node": ">= 10.0.0"
      }
    },
-    "node_modules/url": {
-      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz",
-      "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=",
-      "dev": true,
-      "dependencies": {
-        "punycode": "1.3.2",
-        "querystring": "0.2.0"
-      }
-    },
-    "node_modules/url/node_modules/punycode": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
-      "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=",
-      "dev": true
-    },
    "node_modules/util-deprecate": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
@@ -1760,40 +1735,36 @@
  },
  "dependencies": {
    "@sasjs/adapter": {
-      "version": "2.8.9",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.8.9.tgz",
-      "integrity": "sha512-8UChcZlqqlmaMMaKOCr2Bc1h+i2KVDY0FINlPXQN5PdAgEMd8dbxI2p9bsiI1yjYjOBO9LuMl7B79/mwYCtyEw==",
+      "version": "2.10.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.10.0.tgz",
+      "integrity": "sha512-GbvyIgbODnAJaBaz/Tz8/IwcujNOsZcwzbIuKQtG5y13BzgVPtx/e7b2TAhdoBqd+8uVh0CdrSDfOV/SuvPurg==",
      "dev": true,
      "requires": {
-        "@sasjs/utils": "^2.21.0",
+        "@sasjs/utils": "^2.27.1",
        "axios": "^0.21.1",
        "axios-cookiejar-support": "^1.0.1",
        "form-data": "^4.0.0",
        "https": "^1.0.0",
-        "jwt-decode": "^3.1.2",
-        "tough-cookie": "^4.0.0",
-        "url": "^0.11.0"
+        "tough-cookie": "^4.0.0"
      }
    },
    "@sasjs/cli": {
-      "version": "2.33.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.33.3.tgz",
-      "integrity": "sha512-y1uFM5MEE6eoKLrPUJbweDt4njSy9Y3CS5w5/U2xwNbiAyhiPXgkwCHjCQ8Qg+0rQnMvyNEn6qJuJZ3udc6T7w==",
+      "version": "2.37.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.37.2.tgz",
+      "integrity": "sha512-Bcm8UFN9Y/ZON4T31Gu9mf1REn1pZoStHFVrix/yp7mchFt5rrrY5RbIqO/AI1jzhShSVg9P7oU4VPJrqki+SA==",
      "dev": true,
      "requires": {
-        "@sasjs/adapter": "2.8.9",
-        "@sasjs/core": "2.35.3",
+        "@sasjs/adapter": "2.10.0",
+        "@sasjs/core": "2.35.4",
        "@sasjs/lint": "1.11.2",
-        "@sasjs/utils": "2.23.3",
-        "@types/url-parse": "1.4.3",
-        "btoa": "1.2.1",
+        "@sasjs/utils": "2.27.1",
        "chalk": "4.1.1",
        "csv-stringify": "5.6.2",
        "dotenv": "10.0.0",
        "esm": "3.2.25",
        "find": "0.3.0",
-        "fs-extra": "10.0.0",
        "get-installed-path": "4.0.8",
+        "js-base64": "3.6.1",
        "jsdom": "16.6.0",
        "jwt-decode": "3.1.2",
        "lodash.groupby": "4.6.0",
@@ -1806,9 +1777,9 @@
      }
    },
    "@sasjs/core": {
-      "version": "2.35.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.35.3.tgz",
-      "integrity": "sha512-3o5PU6DkihpA+Aibt1lRy4USqJI0VFa+wNsKCD+bUD2DLZICU3JablZQxwAPH70VWJGXAUJtDFj0T/iRo5Devg==",
+      "version": "2.35.4",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.35.4.tgz",
+      "integrity": "sha512-Exr3+yRdvacKvbXQoi1RfGHi5NtZUkc7RwFNkemHTFXLYaIzPI8CGSCaQmKkwM1UteOJly2e2pw2YT6kHNY1NA==",
      "dev": true
    },
    "@sasjs/lint": {
@@ -1821,11 +1792,12 @@
      }
    },
    "@sasjs/utils": {
-      "version": "2.23.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.23.3.tgz",
-      "integrity": "sha512-tEh4mGG80eUxSLpbPivA0vl4akMdauL+yZrLn1uUM8EyiXPvlcWPkQTeN6oHbyyAH108D9cfEBidTePZh1p5VQ==",
+      "version": "2.27.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.27.1.tgz",
+      "integrity": "sha512-CYTQwEj89cc7H3tGiQQcyDkZYaWRc1HZJpOF8o2RHYS37fIAOy0SyyJdq6mcQ74Nb1u5AmFXPFIvnRCMEcTYeQ==",
      "dev": true,
      "requires": {
+        "@types/fs-extra": "^9.0.11",
        "@types/prompts": "^2.0.13",
        "chalk": "^4.1.1",
        "cli-table": "^0.3.6",
@@ -1833,6 +1805,7 @@
        "fs-extra": "^10.0.0",
        "jwt-decode": "^3.1.2",
        "prompts": "^2.4.1",
+        "rimraf": "^3.0.2",
        "valid-url": "^1.0.9"
      }
    },
@@ -1842,6 +1815,15 @@
      "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==",
      "dev": true
    },
+    "@types/fs-extra": {
+      "version": "9.0.12",
+      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.12.tgz",
+      "integrity": "sha512-I+bsBr67CurCGnSenZZ7v94gd3tc3+Aj2taxMT4yu4ABLuOgOjeFxX3dokG24ztSRg5tnT00sL8BszO7gSMoIw==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
    "@types/node": {
      "version": "16.3.1",
      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.3.1.tgz",
@@ -1864,12 +1846,6 @@
      "dev": true,
      "peer": true
    },
-    "@types/url-parse": {
-      "version": "1.4.3",
-      "resolved": "https://registry.npmjs.org/@types/url-parse/-/url-parse-1.4.3.tgz",
-      "integrity": "sha512-4kHAkbV/OfW2kb5BLVUuUMoumB3CP8rHqlw48aHvFy5tf9ER0AfOonBlX29l/DD68G70DmyhRlSYfQPSYpC5Vw==",
-      "dev": true
-    },
    "abab": {
      "version": "2.0.5",
      "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.5.tgz",
@@ -1994,12 +1970,6 @@
      "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==",
      "dev": true
    },
-    "btoa": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz",
-      "integrity": "sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g==",
-      "dev": true
-    },
    "buffer": {
      "version": "5.7.1",
      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
@@ -2253,9 +2223,9 @@
      }
    },
    "follow-redirects": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz",
-      "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg==",
+      "version": "1.14.2",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.2.tgz",
+      "integrity": "sha512-yLR6WaE2lbF0x4K2qE2p9PEXKLDjUjnR/xmjS3wHAYxtlsI9MLLBJUZirAHKzUZDGLxje7w/cXR49WOUo4rbsA==",
      "dev": true
    },
    "form-data": {
@@ -2493,6 +2463,12 @@
      "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
      "dev": true
    },
+    "js-base64": {
+      "version": "3.6.1",
+      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.6.1.tgz",
+      "integrity": "sha512-Frdq2+tRRGLQUIQOgsIGSCd1VePCS2fsddTG5dTCqR0JHgltXWfsxnY0gIXPoMeRmdom6Oyq+UMOFg5suduOjQ==",
+      "dev": true
+    },
    "jsdom": {
      "version": "16.6.0",
      "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.6.0.tgz",
@@ -2717,9 +2693,9 @@
      "dev": true
    },
    "path-parse": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz",
-      "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==",
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
      "dev": true
    },
    "pify": {
@@ -2756,12 +2732,6 @@
      "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
      "dev": true
    },
-    "querystring": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
-      "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=",
-      "dev": true
-    },
    "readable-stream": {
      "version": "3.6.0",
      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@@ -2954,24 +2924,6 @@
      "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==",
      "dev": true
    },
-    "url": {
-      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz",
-      "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=",
-      "dev": true,
-      "requires": {
-        "punycode": "1.3.2",
-        "querystring": "0.2.0"
-      },
-      "dependencies": {
-        "punycode": {
-          "version": "1.3.2",
-          "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
-          "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=",
-          "dev": true
-        }
-      }
-    },
    "util-deprecate": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
--- a/package.json
+++ b/package.json
@@ -27,12 +27,12 @@
  "main": "index.js",
  "scripts": {
    "build": "sasjs cbd -t viya",
-    "docs": "sasjs doc && ./sasjs/utils/build.sh",
+    "docs": "sasjs doc -t docsonly && ./sasjs/utils/build.sh",
    "test": "sasjs test -t viya",
    "lint": "sasjs lint",
    "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
  },
  "devDependencies": {
-    "@sasjs/cli": "2.33.3"
+    "@sasjs/cli": "^2.37.2"
  }
 }
--- a/sasjs/sasjsconfig.json
+++ b/sasjs/sasjsconfig.json
@@ -2,11 +2,12 @@
  "$schema": "https://cli.sasjs.io/sasjsconfig-schema.json",
  "macroFolders": [
    "base",
+    "fcmp",
    "meta",
    "metax",
    "viya",
    "lua",
-    "tests/base"
+    "tests/crossplatform"
  ],
  "docConfig": {
    "displayMacroCore": false,
@@ -33,7 +34,7 @@
      "allowInsecureRequests": false,
      "appLoc": "/Public/temp/macrocore",
      "macroFolders": [
-        "tests/viya"
+        "tests/viyaonly"
      ],
      "programFolders": [],
      "deployConfig": {
@@ -48,7 +49,15 @@
      "serverType": "SAS9",
      "appLoc": "/Shared Data/temp/macrocore",
      "macroFolders": [
-        "tests/meta"
+        "tests/sas9only"
+      ]
+    },
+    {
+      "name": "docsonly",
+      "serverType": "SAS9",
+      "macroFolders": [
+        "tests/sas9only",
+        "tests/viyaonly"
      ]
    }
  ]
--- a/tests/crossplatform/mcf_stpsrv_header.test.sas
+++ b/tests/crossplatform/mcf_stpsrv_header.test.sas
@@ -0,0 +1,39 @@
+/**
+  @file
+  @brief Testing mcf_stpsrv_header macro
+
+  <h4> SAS Macros </h4>
+  @li mcf_stpsrv_header.sas
+  @li mp_assert.sas
+
+**/
+
+%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/stpsrv_header.txt;
+
+%mcf_stpsrv_header(wrap=YES, insert_cmplib=YES)
+
+data _null_;
+  rc=stpsrv_header('Content-type','application/text');
+  rc=stpsrv_header('Content-disposition',"attachment; filename=file.txt");
+run;
+
+%let test1=FAIL;
+%let test2=FAIL;
+
+data _null_;
+  infile "&sasjs_stpsrv_header_loc";
+  input;
+  if _n_=1 and _infile_='Content-type: application/text'
+  then call symputx('test1','PASS');
+  else if _n_=2 & _infile_='Content-disposition: attachment; filename=file.txt'
+  then call symputx('test2','PASS');
+run;
+
+%mp_assert(
+  iftrue=(%str(&test1)=%str(PASS)),
+  desc=Check first header line
+)
+%mp_assert(
+  iftrue=(%str(&test2)=%str(PASS)),
+  desc=Check second header line
+)
--- a/tests/crossplatform/mcf_string2file.test.sas
+++ b/tests/crossplatform/mcf_string2file.test.sas
@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief Testing mcf_string2file macro
+
+  <h4> SAS Macros </h4>
+  @li mcf_string2file.sas
+  @li mp_assert.sas
+
+**/
+
+
+%mcf_string2file(wrap=YES, insert_cmplib=YES)
+
+data _null_;
+  rc=mcf_string2file(
+    "%sysfunc(pathname(work))/newfile.txt"
+    , "line1"
+    , "APPEND");
+  rc=mcf_string2file(
+    "%sysfunc(pathname(work))/newfile.txt"
+    , "line2"
+    , "APPEND");
+run;
+
+data _null_;
+  infile "%sysfunc(pathname(work))/newfile.txt";
+  input;
+  if _n_=2 then call symputx('val',_infile_);
+run;
+
+%mp_assert(
+  iftrue=(%str(&val)=%str(line2)),
+  desc=Check if APPEND works
+)
+
+data _null_;
+  rc=mcf_string2file(
+    "%sysfunc(pathname(work))/newfile.txt"
+    , "creating"
+    , "CREATE");
+run;
+
+data _null_;
+  infile "%sysfunc(pathname(work))/newfile.txt";
+  input;
+  if _n_=1 then call symputx('val2',_infile_);
+run;
+
+%mp_assert(
+  iftrue=(%str(&val2)=%str(creating)),
+  desc=Check if CREATE works
+)
--- a/tests/crossplatform/mf_existfileref.test.sas
+++ b/tests/crossplatform/mf_existfileref.test.sas
@@ -0,0 +1,35 @@
+/**
+  @file
+  @brief Testing mf_existfileref macro
+
+  <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mp_assert.sas
+
+**/
+
+filename ref1 temp;
+filename ref2 temp;
+
+data _null_;
+  file ref1;
+  put 'exists';
+run;
+
+%mp_assert(
+  iftrue=(%mf_existfileref(ref1)=1),
+  desc=Checking fileref WITH target file exists,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_existfileref(ref2)=1),
+  desc=Checking fileref WITHOUT target file exists,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_existfileref(ref3)=0),
+  desc=Checking non-existant fref does not exist,
+  outds=work.test_results
+)
--- a/tests/crossplatform/mf_existfunction.test.sas
+++ b/tests/crossplatform/mf_existfunction.test.sas
@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Testing mf_existfunction macro
+
+  <h4> SAS Macros </h4>
+  @li mf_existfunction.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(%mf_existfunction(CAT)=1),
+  desc=Checking if CAT function exists,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_existfunction(DOG)=0),
+  desc=Checking DOG function does not exist,
+  outds=work.test_results
+)
+
--- a/tests/crossplatform/mf_getapploc.test.sas
+++ b/tests/crossplatform/mf_getapploc.test.sas
@@ -0,0 +1,49 @@
+/**
+  @file
+  @brief Testing mf_getapploc macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getapploc.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/loc/tests/services/x/service)"="/some/loc"
+  ),
+  desc=Checking test appLoc matches,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/loc/tests/services/tests/service)"="/some/loc"
+  ),
+  desc=Checking nested services appLoc matches,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/services/admin/service)"="/some/area"
+  ),
+  desc=Checking services appLoc matches,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/jobs/jobs/job)"="/some/area"
+  ),
+  desc=Checking jobs appLoc matches,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/tests/macros/somemacro.sas)"="/some/area"
+  ),
+  desc=Checking tests/macros appLoc matches (which has no subfolder),
+  outds=work.test_results
+)
--- a/tests/crossplatform/mf_getvarlist.test.sas
+++ b/tests/crossplatform/mf_getvarlist.test.sas
--- a/tests/crossplatform/mp_abort.test.nofix.sas
+++ b/tests/crossplatform/mp_abort.test.nofix.sas
--- a/tests/crossplatform/mp_assertcolvals.test.sas
+++ b/tests/crossplatform/mp_assertcolvals.test.sas
--- a/tests/crossplatform/mp_assertdsobs.test.sas
+++ b/tests/crossplatform/mp_assertdsobs.test.sas
--- a/tests/crossplatform/mp_base64copy.test.sas
+++ b/tests/crossplatform/mp_base64copy.test.sas
--- a/tests/crossplatform/mp_binarycopy.test.sas
+++ b/tests/crossplatform/mp_binarycopy.test.sas
@@ -0,0 +1,99 @@
+/**
+  @file
+  @brief Testing mp_binarycopy.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_binarycopy.sas
+  @li mp_assert.sas
+
+**/
+
+
+/* TEST 1 - regular file copy */
+%let string1=test1;
+filename tmp temp;
+filename myref temp;
+data _null_;
+  file tmp;
+  put "&string1";
+run;
+%mp_binarycopy(inref=tmp, outref=myref)
+data _null_;
+  infile myref;
+  input;
+  put _infile_;
+  call symputx('string1_check',_infile_);
+  stop;
+run;
+%mp_assert(
+  iftrue=("&string1"="&string1_check"),
+  desc=Basic String Compare,
+  outds=work.test_results
+)
+
+
+/* TEST 2 - File append */
+%let string2=test2;
+%let path2=%sysfunc(pathname(work))/somefile.txt;
+data _null_;
+  file "&path2";
+  put "&string2";
+run;
+%mp_binarycopy(inloc="&path2", outref=myref, mode=APPEND)
+data _null_;
+  infile myref;
+  input;
+  put _infile_;
+  if _n_=2 then call symputx('string2_check',_infile_);
+run;
+%mp_assert(
+  iftrue=("&string2"="&string2_check"),
+  desc=Append Check (file to ref),
+  outds=work.test_results
+)
+
+/* TEST 3 - File create (ref to existing file) */
+%let string3=test3;
+%let path3=%sysfunc(pathname(work))/somefile3.txt;
+filename tmp3 temp;
+data _null_;
+  file tmp3;
+  put "&string3";
+run;
+data _null_;
+  file "&path3";
+  put "this should not be returned";
+run;
+%mp_binarycopy(inref=tmp3, outloc="&path3")
+data _null_;
+  infile "&path3";
+  input;
+  put _infile_;
+  if _n_=1 then call symputx('string3_check',_infile_);
+run;
+%mp_assert(
+  iftrue=("&string3"="&string3_check"),
+  desc=Append Check (ref to existing file),
+  outds=work.test_results
+)
+
+/* TEST 4 - File append (ref to file) */
+%let string4=test4;
+%let string4_check=;
+filename tmp4 temp;
+data _null_;
+  file tmp4;
+  put "&string4";
+run;
+%mp_binarycopy(inref=tmp4, outloc="&path3",mode=APPEND)
+data _null_;
+  infile "&path3";
+  input;
+  put _infile_;
+  if _n_=2 then call symputx('string4_check',_infile_);
+run;
+%mp_assert(
+  iftrue=("&string4"="&string4_check"),
+  desc=Append Check (ref to file),
+  outds=work.test_results
+)
--- a/tests/crossplatform/mp_ds2fmtds.test.sas
+++ b/tests/crossplatform/mp_ds2fmtds.test.sas
--- a/tests/crossplatform/mp_ds2inserts.test.sas
+++ b/tests/crossplatform/mp_ds2inserts.test.sas
--- a/tests/crossplatform/mp_filtercheck.test.sas
+++ b/tests/crossplatform/mp_filtercheck.test.sas
--- a/tests/crossplatform/mp_filtergenerate.test.sas
+++ b/tests/crossplatform/mp_filtergenerate.test.sas
--- a/tests/crossplatform/mp_filtervalidate.test.sas
+++ b/tests/crossplatform/mp_filtervalidate.test.sas
--- a/tests/crossplatform/mp_getddl.test.sas
+++ b/tests/crossplatform/mp_getddl.test.sas
--- a/tests/crossplatform/mp_jsonout.test.sas
+++ b/tests/crossplatform/mp_jsonout.test.sas
--- a/tests/crossplatform/mp_lib2inserts.test.sas
+++ b/tests/crossplatform/mp_lib2inserts.test.sas
--- a/tests/crossplatform/mp_searchcols.test.sas
+++ b/tests/crossplatform/mp_searchcols.test.sas
--- a/tests/crossplatform/mp_validatecol.test.sas
+++ b/tests/crossplatform/mp_validatecol.test.sas
--- a/tests/sas9only/mm_webout.test.sas
+++ b/tests/sas9only/mm_webout.test.sas
--- a/tests/viyaonly/mfv_existfile.test.sas
+++ b/tests/viyaonly/mfv_existfile.test.sas
--- a/tests/viyaonly/mfv_existfolder.test.sas
+++ b/tests/viyaonly/mfv_existfolder.test.sas
--- a/tests/viyaonly/mv_createfile.test.sas
+++ b/tests/viyaonly/mv_createfile.test.sas
--- a/tests/viyaonly/mv_createfolder.test.sas
+++ b/tests/viyaonly/mv_createfolder.test.sas
--- a/tests/viyaonly/mv_createwebservice.test.sas
+++ b/tests/viyaonly/mv_createwebservice.test.sas
--- a/tests/viyaonly/mv_getfoldermembers.test.sas
+++ b/tests/viyaonly/mv_getfoldermembers.test.sas
@@ -0,0 +1,24 @@
+/**
+  @file
+  @brief Testing mv_getfoldermembers macro
+  @details Testing is performed by ensuring that the tests/macros folder
+  contains more than 20 results (which also means the default was increased)
+
+  <h4> SAS Macros </h4>
+  @li mf_getapploc.sas
+  @li mp_assertdsobs.sas
+  @li mv_getfoldermembers.sas
+
+**/
+options mprint;
+
+%mv_getfoldermembers(
+  root=%mf_getapploc()/tests/macros,
+  outds=work.results
+)
+
+%mp_assertdsobs(work.results,
+  desc=%str(Ensuring over 20 results found in %mf_getapploc()/tests/macros),
+  test=ATLEAST 21,
+  outds=work.test_results
+)
--- a/tests/viyaonly/mv_getjobcode.test.sas
+++ b/tests/viyaonly/mv_getjobcode.test.sas
--- a/tests/viyaonly/mv_getjoblog.test.sas
+++ b/tests/viyaonly/mv_getjoblog.test.sas
--- a/tests/viyaonly/mv_getjobresult.test.sas
+++ b/tests/viyaonly/mv_getjobresult.test.sas
--- a/tests/viyaonly/mv_jobflow.test.1.sas
+++ b/tests/viyaonly/mv_jobflow.test.1.sas
--- a/tests/viyaonly/mv_jobflow.test.2.sas
+++ b/tests/viyaonly/mv_jobflow.test.2.sas
--- a/tests/viyaonly/mv_registerclient.test.1.sas
+++ b/tests/viyaonly/mv_registerclient.test.1.sas
--- a/tests/viyaonly/mv_registerclient.test.2.sas
+++ b/tests/viyaonly/mv_registerclient.test.2.sas
--- a/tests/viyaonly/mv_webout.test.sas
+++ b/tests/viyaonly/mv_webout.test.sas
--- a/viya/mv_getfoldermembers.sas
+++ b/viya/mv_getfoldermembers.sas
@@ -1,22 +1,25 @@
 /**
-  @file mv_getfoldermembers.sas
-  @brief Gets a list of folders (and ids) for a given root
-  @details Works for both root level and below, oauth or password. Default is
-    oauth, and the token is expected in a global ACCESS_TOKEN variable.
+  @file
+  @brief Gets a list of folder members (and ids) for a given root
+  @details Returns all members for a particular Viya folder.  Works at both root
+  level and below, and results are created in an output dataset.

-        %mv_getfoldermembers(root=/Public)
+        %mv_getfoldermembers(root=/Public, outds=work.mymembers)


-  @param root= The path for which to return the list of folders
-  @param outds= The output dataset to create (default is work.mv_getfolders).  Format:
+  @param [in] root= (/) The path for which to return the list of folders
+  @param [out] outds= (work.mv_getfolders) The output dataset to create. Format:
  |ordinal_root|ordinal_items|creationTimeStamp| modifiedTimeStamp|createdBy|modifiedBy|id| uri|added| type|name|description|
  |---|---|---|---|---|---|---|---|---|---|---|---|
  |1|1|2021-05-25T11:15:04.204Z|2021-05-25T11:15:04.204Z|allbow|allbow|4f1e3945-9655-462b-90f2-c31534b3ca47|/folders/folders/ed701ff3-77e8-468d-a4f5-8c43dec0fd9e|2021-05-25T11:15:04.212Z|child|my_folder_name|My folder Description|

-  @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values are "password" or "authorization_code" (unquoted).
-    The default is authorization_code.
-
+  @param [in] access_token_var= (ACCESS_TOKEN) The global macro variable to
+    contain the access token
+  @param [in] grant_type= (sas_services) Valid values are:
+    @li password
+    @li authorization_code
+    @li detect
+    @li sas_services

  @version VIYA V.03.04
  @author Allan Bowe, source: https://github.com/sasjs/core
@@ -28,6 +31,12 @@
  @li mf_getuniquelibref.sas
  @li mf_isblank.sas

+  <h4> Related Macros </h4>
+  @li mv_createfolder.sas
+  @li mv_deletefoldermember.sas
+  @li mv_deleteviyafolder.sas
+  @li mv_getfoldermembers.test.sas
+
 **/

 %macro mv_getfoldermembers(root=/
@@ -67,7 +76,7 @@ options noquotelenmax;
 %if "&root"="/" %then %do;
  /* if root just list root folders */
  proc http method='GET' out=&fname1 &oauth_bearer
-      url="&base_uri/folders/rootFolders";
+      url="&base_uri/folders/rootFolders?limit=1000";
  %if &grant_type=authorization_code %then %do;
      headers "Authorization"="Bearer &&&access_token_var";
  %end;
@@ -88,13 +97,17 @@ options noquotelenmax;
  /*data _null_;infile &fname1;input;putlog _infile_;run;*/
  libname &libref1 JSON fileref=&fname1;
  /* now get the followon link to list members */
-  %local href;
-  %let href=0;
+  %local href cnt;
+  %let cnt=0;
  data _null_;
    set &libref1..links;
-    if rel='members' then call symputx('href',quote("&base_uri"!!trim(href)),'l');
+    if rel='members' then do;
+      url=cats("'","&base_uri",href,"?limit=10000'");
+      call symputx('href',url,'l');
+      call symputx('cnt',1,'l');
+    end;
  run;
-  %if &href=0 %then %do;
+  %if &cnt=0 %then %do;
    %put NOTE:;%put NOTE-  No members found in &root!!;%put NOTE-;
    %return;
  %end;
Author	SHA1	Message	Date
Allan Bowe	27cf2a2532	Merge pull request #71 from sasjs/including fix: conditional logic around mp_abort(mode=INCLUDE) to cover case wh…	2021-09-05 20:56:40 +03:00
Allan Bowe	d096cbddeb	fix: conditional logic around mp_abort(mode=INCLUDE) to cover case when no mp_include was executed	2021-09-05 20:55:44 +03:00
Allan Bowe	117503f214	Merge pull request #70 from sasjs/including feat: new mp_include() feature for handling %includes with macros	2021-09-03 11:17:21 +03:00
Allan Bowe	5cc5fae750	chore: adding mp_abort to mp_include sas macros list	2021-09-03 11:06:12 +03:00
Allan Bowe	d93032e1a9	chore: docs	2021-09-03 10:54:21 +03:00
Allan Bowe	507557b2cb	feat: new mp_include() feature for handling %includes with macros	2021-09-03 00:40:10 +03:00
Allan Bowe	ee5c3c185a	chore: merge	2021-08-28 14:22:16 +03:00
Allan Bowe	e5592a2eb2	chore: avoiding the string 'ERROR:' in logs	2021-08-28 14:21:36 +03:00
Allan Bowe	59200a6e73	Merge pull request #69 from sasjs/issue68 feat: supporting postgres timestamps for mp_ds2inserts and mp_lib2inserts	2021-08-24 21:03:09 +03:00
Allan Bowe	f468f60ae1	fix: substr issue in mp_ds2inserts. Closes #68	2021-08-24 20:51:29 +03:00
Allan Bowe	9f60d827b6	feat: supporting postgres timestamps for mp_ds2inserts and mp_lib2inserts	2021-08-24 20:39:02 +03:00
Allan Bowe	5c936ddb65	Merge pull request #67 from sasjs/issue66 feat: adding APPEND option to mp_binarycopy.sas, and a new test (mp_b…	2021-08-24 14:04:25 +03:00
Allan Bowe	d0bde62594	feat: adding APPEND option to mp_binarycopy.sas, and a new test (mp_binarycopy.test.sas). Closes #66	2021-08-24 13:43:31 +03:00
Allan Bowe	ada9192337	Merge pull request #65 from sasjs/issue64 fix: increasing limit for mv_getfoldermember.sas, also adding a test,…	2021-08-20 22:50:27 +03:00
Allan Bowe	6161f588a9	fix: increasing limit for mv_getfoldermember.sas, also adding a test, and updating mf_getapploc to search _program by default (and fixing it to work with macro tests, and updating the test for that also)	2021-08-20 22:38:56 +03:00
Allan Bowe	67079d8c17	fix: adding default value for exists in mf_existfunction	2021-08-19 00:20:49 +03:00
Allan Bowe	75bd39adb0	Merge pull request #63 from sasjs/issue62 feat: adding FCMP capability	2021-08-19 00:00:15 +03:00
Allan Bowe	078bdbeecf	chore: bumping devDependency (sasjs cli)	2021-08-18 23:47:17 +03:00
Allan Bowe	8ddb86785c	feat: new fcmp stpsrv_header function	2021-08-18 23:45:45 +03:00
Allan Bowe	005af0ecf8	feat: new mf_existfunction macro	2021-08-18 23:36:12 +03:00
Allan Bowe	bc410a9135	chore: fixing docs for tests	2021-08-18 19:55:05 +03:00
Allan Bowe	fc8ba2e36c	chore: moving files to tidy up docs	2021-08-18 19:43:38 +03:00
Allan Bowe	756441384a	feat: adding first fcmp macro, mcf_string2file.sas	2021-08-18 18:35:51 +03:00
Allan Bowe	10f9eecf9e	Merge pull request #60 from sasjs/vpn-fix chore: vpn fix	2021-08-13 14:41:25 +03:00
Mihajlo	470ebb50a7	chore: vpn fix	2021-08-13 12:57:31 +02:00
Allan Bowe	26cd5d9d31	Merge pull request #59 from Stefan-Dimitrov-Stoyanov/patch-1 Update README.md	2021-08-12 11:59:25 +03:00
Stefan-Dimitrov-Stoyanov	0b694bb878	Update README.md Fix the missing space at the end of the first line under the Installation header: https://github.com/sasjs/core/blob/main/README.md#installation	2021-08-11 13:34:44 +01:00
Allan Bowe	b403c02bba	chore: docs for mm_createfolder	2021-08-06 15:36:55 +03:00
Allan Bowe	0b555bb31c	Merge pull request #58 from sasjs/apploc feat: new mf_getapploc macro	2021-08-04 22:04:55 +03:00
Allan Bowe	40b513a9e3	feat: new mf_getapploc macro	2021-08-04 22:00:18 +03:00
Allan Bowe	4eacf4deae	Merge pull request #57 from sasjs/mf_existfref fix: showing filerefs that exist (even when underlying does not) in m…	2021-08-03 14:32:23 +03:00
Allan Bowe	5824423c13	fix: showing filerefs that exist (even when underlying does not) in mf_existfileref, along with 3 tests	2021-08-03 14:16:52 +03:00
Allan Bowe	ce5bfd41dc	chore(docs): updating header for mm_gettables	2021-08-02 10:37:01 +03:00