Merge pull request #67 from sasjs/issue66

feat: adding APPEND option to mp_binarycopy.sas, and a new test (mp_b…

.all-contributorsrc

@@ -0,0 +1,104 @@
+{
+  "projectName": "core",
+  "projectOwner": "sasjs",
+  "repoType": "github",
+  "repoHost": "https://github.com",
+  "files": [
+    "README.md"
+  ],
+  "imageSize": 100,
+  "commit": false,
+  "commitConvention": "angular",
+  "contributors": [
+    {
+      "login": "allanbowe",
+      "name": "Allan Bowe",
+      "avatar_url": "https://avatars.githubusercontent.com/u/4420615?v=4",
+      "profile": "https://github.com/allanbowe",
+      "contributions": [
+        "business",
+        "code",
+        "content",
+        "doc",
+        "infra",
+        "maintenance",
+        "mentoring",
+        "question",
+        "review",
+        "test"
+      ]
+    },
+    {
+      "login": "rafgag",
+      "name": "rafgag",
+      "avatar_url": "https://avatars.githubusercontent.com/u/69139928?v=4",
+      "profile": "https://github.com/rafgag",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "tmoody",
+      "name": "Trevor Moody",
+      "avatar_url": "https://avatars.githubusercontent.com/u/79837106?v=4",
+      "profile": "https://github.com/tmoody",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "krishna-acondy",
+      "name": "Krishna Acondy",
+      "avatar_url": "https://avatars.githubusercontent.com/u/2980428?v=4",
+      "profile": "https://krishna-acondy.io/",
+      "contributions": [
+        "code",
+        "infra",
+        "blog",
+        "content",
+        "ideas",
+        "video"
+      ]
+    },
+    {
+      "login": "saadjutt01",
+      "name": "Muhammad Saad ",
+      "avatar_url": "https://avatars.githubusercontent.com/u/8914650?v=4",
+      "profile": "https://github.com/saadjutt01",
+      "contributions": [
+        "code",
+        "ideas"
+      ]
+    },
+    {
+      "login": "YuryShkoda",
+      "name": "Yury Shkoda",
+      "avatar_url": "https://avatars.githubusercontent.com/u/25773492?v=4",
+      "profile": "https://www.erudicat.com/",
+      "contributions": [
+        "code",
+        "infra",
+        "video"
+      ]
+    },
+    {
+      "login": "medjedovicm",
+      "name": "Mihajlo Medjedovic",
+      "avatar_url": "https://avatars.githubusercontent.com/u/18329105?v=4",
+      "profile": "https://github.com/medjedovicm",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "kkchandok",
+      "name": "kkchandok",
+      "avatar_url": "https://avatars.githubusercontent.com/u/46090627?v=4",
+      "profile": "https://github.com/kkchandok",
+      "contributions": [
+        "ideas"
+      ]
+    }
+  ],
+  "contributorsPerLine": 7
+}

.github/workflows/run-tests.yml

@@ -39,7 +39,7 @@ jobs:
           sudo apt install apt-transport-https
           sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
           sudo apt-key add openvpn-repo-pkg-key.pub
-          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-bionic.list
+          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-focal.list
           sudo apt update
           sudo apt install openvpn3
 

README.md

@@ -41,6 +41,13 @@ Documentation: https://core.sasjs.io
 - No X command
 - Prefixes: _mf_, _mp_
 
+**fcmp** library (SAS9/Viya)
+- Function and macro names are identical, except for special cases
+- Prefixes: _mcf_
+
+The fcmp macros are used to generate fcmp functions, and can be used with or
+without the `proc fcmp` wrapper.
+
 **meta** library (SAS9 only)
 
 - OS independent
@@ -84,7 +91,7 @@ run;
 
 # Installation
 
-First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available,eg:
+First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available, eg:
 
 ```sas
 options insert=(sasautos="/your/path/macrocore/base");
@@ -179,3 +186,34 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 
 
+
+## Contributors ✨
+<!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
+[![All Contributors](https://img.shields.io/badge/all_contributors-8-orange.svg?style=flat-square)](#contributors-)
+<!-- ALL-CONTRIBUTORS-BADGE:END -->
+Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
+
+<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-disable -->
+<table>
+  <tr>
+    <td align="center"><a href="https://github.com/allanbowe"><img src="https://avatars.githubusercontent.com/u/4420615?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Allan Bowe</b></sub></a><br /><a href="#business-allanbowe" title="Business development">💼</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Code">💻</a> <a href="#content-allanbowe" title="Content">🖋</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Documentation">📖</a> <a href="#infra-allanbowe" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#maintenance-allanbowe" title="Maintenance">🚧</a> <a href="#mentoring-allanbowe" title="Mentoring">🧑‍🏫</a> <a href="#question-allanbowe" title="Answering Questions">💬</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3Aallanbowe" title="Reviewed Pull Requests">👀</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Tests">⚠️</a></td>
+    <td align="center"><a href="https://github.com/rafgag"><img src="https://avatars.githubusercontent.com/u/69139928?v=4?s=100" width="100px;" alt=""/><br /><sub><b>rafgag</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rafgag" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/tmoody"><img src="https://avatars.githubusercontent.com/u/79837106?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Trevor Moody</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=tmoody" title="Code">💻</a></td>
+    <td align="center"><a href="https://krishna-acondy.io/"><img src="https://avatars.githubusercontent.com/u/2980428?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Krishna Acondy</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=krishna-acondy" title="Code">💻</a> <a href="#infra-krishna-acondy" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#blog-krishna-acondy" title="Blogposts">📝</a> <a href="#content-krishna-acondy" title="Content">🖋</a> <a href="#ideas-krishna-acondy" title="Ideas, Planning, & Feedback">🤔</a> <a href="#video-krishna-acondy" title="Videos">📹</a></td>
+    <td align="center"><a href="https://github.com/saadjutt01"><img src="https://avatars.githubusercontent.com/u/8914650?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Muhammad Saad </b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=saadjutt01" title="Code">💻</a> <a href="#ideas-saadjutt01" title="Ideas, Planning, & Feedback">🤔</a></td>
+    <td align="center"><a href="https://www.erudicat.com/"><img src="https://avatars.githubusercontent.com/u/25773492?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Yury Shkoda</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=YuryShkoda" title="Code">💻</a> <a href="#infra-YuryShkoda" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#video-YuryShkoda" title="Videos">📹</a></td>
+    <td align="center"><a href="https://github.com/medjedovicm"><img src="https://avatars.githubusercontent.com/u/18329105?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Mihajlo Medjedovic</b></sub></a><br /><a href="#infra-medjedovicm" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+  </tr>
+  <tr>
+    <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
+  </tr>
+</table>
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+
+<!-- ALL-CONTRIBUTORS-LIST:END -->
+
+This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!

base/mf_existfileref.sas

@@ -17,7 +17,13 @@
 %macro mf_existfileref(fref
 )/*/STORE SOURCE*/;
 
-  %if %sysfunc(fileref(&fref))=0 %then %do;
+  %local rc;
+  %let rc=%sysfunc(fileref(&fref));
+  %if &rc=0 %then %do;
+    1
+  %end;
+  %else %if &rc<0 %then %do;
+    %put &sysmacroname: Fileref &fref exists but the underlying file does not;
     1
   %end;
   %else %do;

base/mf_existfunction.sas

@@ -0,0 +1,37 @@
+/**
+  @file
+  @brief Checks if a function exists
+  @details Returns 1 if the function exists, else 0.  Note that this function
+  can be slow as it needs to open the sashelp.vfuncs table.
+
+  Usage:
+
+      %put %mf_existfunction(CAT);
+      %put %mf_existfunction(DOG);
+
+  Full credit to [Bart](https://sasensei.com/user/305) for the vfunc pointer
+  and the tidy approach for pure macro data set filtering.
+  Check out his [SAS Packages](https://github.com/yabwon/SAS_PACKAGES)
+  framework!  Where you can find the same [function](
+https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionexists-macro
+  ).
+
+  @param [in] name (positional) - function name
+
+  @author Allan Bowe
+**/
+/** @cond */
+%macro mf_existfunction(name
+)/*/STORE SOURCE*/;
+
+  %local dsid rc exist;
+  %let dsid=%sysfunc(open(sashelp.vfunc(where=(fncname="%upcase(&name)"))));
+  %let exist=1;
+  %let exist=%sysfunc(fetch(&dsid, NOSET));
+  %let rc=%sysfunc(close(&dsid));
+
+  %sysevalf(0 = &exist)
+
+%mend mf_existfunction;
+
+/** @endcond */

base/mf_getapploc.sas

@@ -0,0 +1,73 @@
+/**
+  @file
+  @brief Returns the appLoc from the _program variable
+  @details When working with SASjs apps, web services / tests / jobs are always
+  deployed to a root (app) location in the SAS logical folder tree.
+
+  When building apps for use in other environments, you do not necessarily know
+  where the backend services will be deployed.  Therefore a function like this
+  is handy in order to dynamically figure out the appLoc, and enable other
+  services to be connected by a relative reference.
+
+  SASjs apps always have the same immediate substructure (one or more of the
+  following):
+
+  @li /data
+  @li /jobs
+  @li /services
+  @li /tests/jobs
+  @li /tests/services
+  @li /tests/macros
+
+  This function works by testing for the existence of any of the above in the
+  automatic _program variable, and returning the part to the left of it.
+
+  Usage:
+
+      %put %mf_getapploc(&_program)
+
+      %put %mf_getapploc(/some/location/services/admin/myservice);
+      %put %mf_getapploc(/some/location/jobs/extract/somejob/);
+      %put %mf_getapploc(/some/location/tests/jobs/somejob/);
+
+
+  @author Allan Bowe
+**/
+
+%macro mf_getapploc(pgm);
+%if "&pgm"="" %then %do;
+  %if %symexist(_program) %then %let pgm=&_program;
+  %else %do;
+    %put &sysmacroname: No value provided and no _program variable available;
+    %return;
+  %end;
+%end;
+%local root;
+
+/**
+  * First check we are not in the tests/macros folder (which has no subfolders)
+  */
+%if %index(&pgm,/tests/macros/) %then %do;
+  %let root=%substr(&pgm,1,%index(&pgm,/tests/macros)-1);
+  &root
+  %return;
+%end;
+
+/**
+  * Next, move up two levels to avoid matches on subfolder or service name
+  */
+%let root=%substr(&pgm,1,%length(&pgm)-%length(%scan(&pgm,-1,/))-1);
+%let root=%substr(&root,1,%length(&root)-%length(%scan(&root,-1,/))-1);
+
+%if %index(&root,/tests/) %then %do;
+  %let root=%substr(&root,1,%index(&root,/tests/)-1);
+%end;
+%else %if %index(&root,/services) %then %do;
+  %let root=%substr(&root,1,%index(&root,/services)-1);
+%end;
+%else %if %index(&root,/jobs) %then %do;
+  %let root=%substr(&root,1,%index(&root,/jobs)-1);
+%end;
+%else %put &sysmacroname: Could not find an app location from &pgm;
+  &root
+%mend mf_getapploc ;

base/mp_base64copy.sas

@@ -76,7 +76,7 @@ run;
 
 %if &outfound=0 %then %do;
   filename &outref temp lrecl=2097088;
-%%end;
+%end;
 
 %if &action=ENCODE %then %do;
   data _null_;
@@ -84,12 +84,12 @@ run;
     retain line "";
     infile &inref recfm=F lrecl= 1 end=eof;
     input @1 stream $char1.;
-    file &outref lrecl=76;
+    file &outref recfm=N;
     substr(line,(_N_-(CEIL(_N_/57)-1)*57),1) = byte(rank(stream));
     if mod(_N_,57)=0 or EOF then do;
       if eof then b64=put(trim(line),$base64X76.);
       else b64=put(line, $base64X76.);
-      put b64;
+      put b64 + (-1) @;
       line="";
     end;
   run;
@@ -101,19 +101,13 @@ run;
     fileout = fopen("&outref",'O',3,'B');
     char= '20'x;
     do while(fread(filein)=0);
-      raw="1234";
+      length raw $4;
       do i=1 to 4;
         rc=fget(filein,char,1);
         substr(raw,i,1)=char;
       end;
-      val="123";
-      val=input(raw,$base64X4.);
-      do i=1 to 3;
-        length byte $1;
-        byte=byte(rank(substr(val,i,1)));
-        rc = fput(fileout, byte);
-      end;
-      rc =fwrite(fileout);
+      rc = fput(fileout,input(raw,$base64X4.));
+      rc = fwrite(fileout);
     end;
     rc = fclose(filein);
     rc = fclose(fileout);

base/mp_binarycopy.sas

@@ -9,10 +9,29 @@
 
         %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
 
-  @param inloc full, quoted "path/and/filename.ext" of the object to be copied
-  @param outloc full, quoted "path/and/filename.ext" of object to be created
-  @param inref can override default input fileref to avoid naming clash
-  @param outref an override default output fileref to avoid naming clash
+  To append to a file, use the mode option, eg:
+
+      filename tmp1 temp;
+      filename tmp2 temp;
+      data _null_;
+        file tmp1;
+        put 'stacking';
+      run;
+
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+
+
+  @param [in] inloc quoted "path/and/filename.ext" of the file to be copied
+  @param [out] outloc quoted "path/and/filename.ext" of the file to be created
+  @param [in] inref (____in) If provided, this fileref will take precedence over
+    the `inloc` parameter
+  @param [out] outref (____in) If provided, this fileref will take precedence
+    over the `outloc` parameter.  It must already exist!
+  @param [in] mode (CREATE) Valid values:
+    @li CREATE - Create the file (even if it already exists)
+    @li APPEND - Append to the file (don't overwrite)
+
   @returns nothing
 
   @version 9.2
@@ -24,20 +43,29 @@
     ,outloc=          /* full path and filename of object to be created */
     ,inref=____in   /* override default to use own filerefs */
     ,outref=____out /* override default to use own filerefs */
+    ,mode=CREATE
 )/*/STORE SOURCE*/;
+  %local mod outmode;
+  %if &mode=APPEND %then %do;
+    %let mod=mod;
+    %let outmode='a';
+  %end;
+  %else %do;
+    %let outmode='o';
+  %end;
   /* these IN and OUT filerefs can point to anything */
   %if &inref = ____in %then %do;
     filename &inref &inloc lrecl=1048576 ;
   %end;
   %if &outref=____out %then %do;
-    filename &outref &outloc lrecl=1048576 ;
+    filename &outref &outloc lrecl=1048576 &mod;
   %end;
 
   /* copy the file byte-for-byte  */
   data _null_;
     length filein 8 fileid 8;
     filein = fopen("&inref",'I',1,'B');
-    fileid = fopen("&outref",'O',1,'B');
+    fileid = fopen("&outref",&outmode,1,'B');
     rec = '20'x;
     do while(fread(filein)=0);
       rc = fget(filein,rec,1);

base/mp_ds2inserts.sas

@@ -0,0 +1,168 @@
+/**
+  @file
+  @brief Export a dataset to SQL insert statements
+  @details Converts dataset values to SQL insert statements for use across
+  multiple database types.
+
+  Usage:
+
+      %mp_ds2inserts(sashelp.class,outref=myref,outds=class)
+      data class;
+        set sashelp.class;
+        stop;
+      proc sql;
+      %inc myref;
+
+  @param [in] ds The dataset to be exported
+  @param [in] maxobs= (max) The max number of inserts to create
+  @param [out] outref= (0) The output fileref.  If it does not exist, it is
+    created. If it does exist, new records are APPENDED.
+  @param [out] schema= (0) The library (or schema) in which the target table is
+    located.  If not provided, is ignored.
+  @param [out] outds= (0) The output table to load.  If not provided, will
+    default to the table in the &ds parameter.
+  @param [in] flavour= (SAS) The SQL flavour to be applied to the output. Valid
+    options:
+    @li SAS (default) - suitable for regular proc sql
+    @li PGSQL - Used for Postgres databases
+
+  <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mf_getvarcount.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
+
+  @version 9.2
+  @author Allan Bowe (credit mjsq)
+**/
+
+%macro mp_ds2inserts(ds, outref=0,schema=0,outds=0,flavour=SAS,maxobs=max
+)/*/STORE SOURCE*/;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if %index(&ds,.)=0 %then %let ds=WORK.&ds;
+
+%let flavour=%upcase(&flavour);
+%if &flavour ne SAS and &flavour ne PGSQL %then %do;
+  %put %str(WAR)NING:  &flavour is not supported;
+  %return;
+%end;
+
+%if &outref=0 %then %do;
+  %put %str(WAR)NING:  Please provide a fileref;
+  %return;
+%end;
+%if %mf_existfileref(&outref)=0 %then %do;
+  filename &outref temp lrecl=66000;
+%end;
+
+%if &schema=0 %then %let schema=;
+%else %let schema=&schema..;
+
+%if &outds=0 %then %let outds=%scan(&ds,2,.);
+
+%local nobs;
+proc sql noprint;
+select count(*) into: nobs TRIMMED from &ds;
+%if &nobs=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No rows found in &ds */";
+  run;
+%end;
+
+%local vars;
+%let vars=%mf_getvarcount(&ds);
+%if &vars=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No columns found in &schema.&ds */";
+  run;
+  %return;
+%end;
+%else %if &vars>1600 and &flavour=PGSQL %then %do;
+  data _null_;
+    file &fref mod;
+    put "/* &schema.&ds contains &vars vars */";
+    put "/* Postgres cannot handle tables with over 1600 vars */";
+    put "/* No inserts will be generated for this table */";
+  run;
+  %return;
+%end;
+
+%local varlist varlistcomma;
+%let varlist=%mf_getvarlist(&ds);
+%let varlistcomma=%mf_getvarlist(&ds,dlm=%str(,),quote=double);
+
+/* next, export data */
+data _null_;
+  file &outref mod ;
+  if _n_=1 then put "/* &schema.&outds (&nobs rows, &vars columns) */";
+  set &ds;
+  %if &maxobs ne max %then %do;
+    if _n_>&maxobs then stop;
+  %end;
+  length _____str $32767;
+  format _numeric_ best.;
+  format _character_ ;
+  %local i comma var vtype;
+  %do i=1 %to %sysfunc(countw(&varlist));
+    %let var=%scan(&varlist,&i);
+    %let vtype=%mf_getvartype(&ds,&var);
+    %if &i=1 %then %do;
+      %if &flavour=SAS %then %do;
+        put "insert into &schema.&outds set ";
+        put "  &var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        _____str=cats(
+          "INSERT INTO &schema.&outds ("
+          ,symget('varlistcomma')
+          ,") VALUES ("
+        );
+        put _____str;
+        put "  "@;
+      %end;
+    %end;
+    %else %do;
+      %if &flavour=SAS %then %do;
+        put "  ,&var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        put "  ,"@;
+      %end;
+    %end;
+    %if &vtype=N %then %do;
+      %if &flavour=SAS %then %do;
+        put &var;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        if missing(&var) then put 'NULL';
+        else put &var;
+      %end;
+    %end;
+    %else %do;
+      _____str="'"!!trim(tranwrd(&var,"'","''"))!!"'";
+      put _____str;
+    %end;
+  %end;
+  %if &flavour=SAS %then %do;
+    put ';';
+  %end;
+  %else %if &flavour=PGSQL %then %do;
+    put ');';
+  %end;
+
+  if _n_=&nobs then put /;
+run;
+
+%mend mp_ds2inserts;

base/mp_getddl.sas

@@ -5,6 +5,8 @@
     to create tables in SAS or a database.  The macro can be used at table or
     library level.  The default behaviour is to create DDL in SAS format.
 
+    Note - views are not currently supported.
+
   Usage:
 
       data test(index=(pk=(x y)/unique /nomiss));
@@ -16,12 +18,14 @@
       %mp_getddl(work,test,flavour=tsql,showlog=YES)
 
   <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mf_getvarcount.sas
   @li mp_getconstraints.sas
 
   @param lib libref of the library to create DDL for.  Should be assigned.
   @param ds dataset to create ddl for (optional)
-  @param fref= the fileref to which to write the DDL.  If not preassigned, will
-    be assigned to TEMP.
+  @param fref= the fileref to which to _append_ the DDL.  If it does not exist,
+    it will be created.
   @param flavour= The type of DDL to create (default=SAS). Supported=TSQL
   @param showlog= Set to YES to show the DDL in the log
   @param schema= Choose a preferred schema name (default is to use actual schema
@@ -37,9 +41,10 @@
 )/*/STORE SOURCE*/;
 
 /* check fileref is assigned */
-%if %sysfunc(fileref(&fref)) > 0 %then %do;
-  filename &fref temp;
+%if %mf_existfileref(&fref)=0 %then %do;
+  filename &fref temp ;
 %end;
+
 %if %length(&libref)=0 %then %let libref=WORK;
 %let flavour=%upcase(&flavour);
 
@@ -47,6 +52,7 @@ proc sql noprint;
 create table _data_ as
   select * from dictionary.tables
   where upcase(libname)="%upcase(&libref)"
+    and memtype='DATA' /* views not currently supported */
   %if %length(&ds)>0 %then %do;
     and upcase(memname)="%upcase(&ds)"
   %end;
@@ -118,7 +124,7 @@ create table _data_ as
 %mend addConst;
 
 data _null_;
-  file &fref;
+  file &fref mod;
   put "/* DDL generated by &sysuserid on %sysfunc(datetime(),datetime19.) */";
 run;
 
@@ -141,13 +147,15 @@ run;
           put "create table &libref..&curds(";
         end;
         else do;
+          /* just a placeholder - we filter out views at the top */
           put "create view &libref..&curds(";
         end;
         put "    "@@;
       end;
       else put "   ,"@@;
       if length(format)>1 then fmt=" format="!!cats(format);
-      if length(label)>1 then lab=" label="!!quote(trim(label));
+      if length(label)>1 then
+        lab=" label="!!cats("'",tranwrd(label,"'","''"),"'");
       if notnull='yes' then notnul=' not null';
       if type='char' then typ=cats('char(',length,')');
       else if length ne 8 then typ='num length='!!left(length);
@@ -219,6 +227,7 @@ run;
           put "create table [&schema].[&curds](";
         end;
         else do;
+          /* just a placeholder - we filter out views at the top */
           put "create view [&schema].[&curds](";
         end;
         put "    "@@;
@@ -302,71 +311,81 @@ run;
     put "CREATE SCHEMA &schema;";
   %do x=1 %to %sysfunc(countw(&dsnlist));
     %let curds=%scan(&dsnlist,&x);
-    data _null_;
-      file &fref mod;
-      put "/* Postgres Flavour DDL for &schema..&curds */";
-    data _null_;
-      file &fref mod;
-      set &colinfo (where=(upcase(memname)="&curds")) end=last;
-      length fmt $32;
-      if _n_=1 then do;
-        if memtype='DATA' then do;
-          put "CREATE TABLE &schema..&curds (";
+    %local curdsvarcount;
+    %let curdsvarcount=%mf_getvarcount(&libref..&curds);
+    %if &curdsvarcount>1600 %then %do;
+      data _null_;
+        file &fref mod;
+        put "/* &libref..&curds contains &curdsvarcount vars */";
+        put "/* Postgres cannot create tables with over 1600 vars */";
+        put "/* No DDL will be generated for this table";
+      run;
+    %end;
+    %else %do;
+      data _null_;
+        file &fref mod;
+        put "/* Postgres Flavour DDL for &schema..&curds */";
+      data _null_;
+        file &fref mod;
+        set &colinfo (where=(upcase(memname)="&curds")) end=last;
+        length fmt $32;
+        if _n_=1 then do;
+          if memtype='DATA' then do;
+            put "CREATE TABLE &schema..&curds (";
+          end;
+          else do;
+            /* just a placeholder - we filter out views at the top */
+            put "CREATE VIEW &schema..&curds (";
+          end;
+          put "    "@@;
         end;
-        else do;
-          put "CREATE VIEW &schema..&curds (";
-        end;
-        put "    "@@;
-      end;
-      else put "   ,"@@;
-      format=upcase(format);
-      if 1=0 then; /* dummy if */
-      %if &applydttm=YES %then %do;
-        else if format=:'DATETIME' then fmt=' TIMESTAMP ';
-      %end;
-      else if type='num' then fmt=' DOUBLE PRECISION';
-      else fmt='VARCHAR('!!cats(length)!!')';
-      if notnull='yes' then notnul=' NOT NULL';
-      /* quote column names in case they represent reserved words */
-      name2=quote(trim(name));
-      put name2 fmt notnul;
-    run;
+        else put "   ,"@@;
+        format=upcase(format);
+        if 1=0 then; /* dummy if */
+        %if &applydttm=YES %then %do;
+          else if format=:'DATETIME' then fmt=' TIMESTAMP ';
+        %end;
+        else if type='num' then fmt=' DOUBLE PRECISION';
+        else fmt='VARCHAR('!!cats(length)!!')';
+        if notnull='yes' then notnul=' NOT NULL';
+        /* quote column names in case they represent reserved words */
+        name2=quote(trim(name));
+        put name2 fmt notnul;
+      run;
 
-    /* Extra step for data constraints */
-    %addConst()
+      /* Extra step for data constraints */
+      %addConst()
 
-    data _null_;
-      file &fref mod;
-      put ');';
-    run;
+      data _null_;
+        file &fref mod;
+        put ');';
+      run;
 
-    /* Create Unique Indexes, but only if they were not already defined within
-      the Constraints section. */
-    data _null_;
-      *length ds $128;
-      set &idxinfo(
-        where=(
-          memname="&curds"
-          and unique='yes'
-          and indxname not in (
-            %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+      /* Create Unique Indexes, but only if they were not already defined within
+        the Constraints section. */
+      data _null_;
+        *length ds $128;
+        set &idxinfo(
+          where=(
+            memname="&curds"
+            and unique='yes'
+            and indxname not in (
+              %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+            )
           )
-        )
-      );
-      file &fref mod;
-      by idxusage indxname;
-/*       ds=cats(libname,'.',memname); */
-      if first.indxname then do;
+        );
+        file &fref mod;
+        by idxusage indxname;
+        if first.indxname then do;
           put 'CREATE UNIQUE INDEX "' indxname +(-1) '" ' "ON &schema..&curds(";
           put '  "' name +(-1) '"' ;
-      end;
-      else put '  ,"' name +(-1) '"';
-      *else put '    ,' name ;
-      if last.indxname then do;
-        put ');';
-      end;
-    run;
-
+        end;
+        else put '  ,"' name +(-1) '"';
+        if last.indxname then do;
+          put ');';
+        end;
+      run;
+    %end;
   %end;
 %end;
 %if %upcase(&showlog)=YES %then %do;

base/mp_hashdataset.sas

@@ -20,6 +20,7 @@
   @li mf_getvartype.sas
 
   @param [in] libds dataset to hash
+  @param [in] salt= Provide a salt (could be, for instance, the dataset name)
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
   will contain one column (hashkey) with one observation (a hex32.
   representation of the input hash)
@@ -33,7 +34,8 @@
 
 %macro mp_hashdataset(
   libds,
-  outds=
+  outds=,
+  salt=
 )/*/STORE SOURCE*/;
   %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
     %put %str(WARN)ING: Dataset &libds is empty;, or is not a dataset;
@@ -53,7 +55,7 @@
     %let varlist=%mf_getvarlist(&libds);
     data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
       length &prevkeyvar &keyvar $32;
-      retain &prevkeyvar;
+      retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
       set &libds end=&lastvar;
       /* hash should include previous row */
       &keyvar=put(md5(&prevkeyvar

base/mp_lib2inserts.sas

@@ -0,0 +1,73 @@
+/**
+  @file
+  @brief Convert all data in a library to SQL insert statements
+  @details Gets list of members then calls the <code>%mp_ds2inserts()</code>
+  macro.
+  Usage:
+
+      %mp_getddl(sashelp, schema=work, fref=tempref)
+
+      %mp_lib2inserts(sashelp, schema=work, outref=tempref)
+
+      %inc tempref;
+
+
+  The output will be one file in the outref fileref.
+
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2inserts.sas
+
+
+  @param [in] lib Library in which to convert all datasets to inserts
+  @param [in] flavour= (SAS) The SQL flavour to be applied to the output. Valid
+    options:
+    @li SAS (default) - suitable for regular proc sql
+    @li PGSQL - Used for Postgres databases
+  @param [in] maxobs= (max) The max number of observations (per table) to create
+  @param [out] outref= Output fileref in which to create the insert statements.
+    If it exists, it will be appended to, otherwise it will be created.
+  @param [out] schema= (0) The schema of the target database, or the libref.
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_lib2inserts(lib
+    ,flavour=SAS
+    ,outref=0
+    ,schema=0
+    ,maxobs=max
+)/*/STORE SOURCE*/;
+
+/* Find the tables */
+%local x ds memlist;
+proc sql noprint;
+select distinct lowcase(memname)
+  into: memlist
+  separated by ' '
+  from dictionary.tables
+  where upcase(libname)="%upcase(&lib)"
+    and memtype='DATA'; /* exclude views */
+
+
+%let flavour=%upcase(&flavour);
+%if &flavour ne SAS and &flavour ne PGSQL %then %do;
+  %put %str(WAR)NING:  &flavour is not supported;
+  %return;
+%end;
+
+
+/* create the inserts */
+%do x=1 %to %sysfunc(countw(&memlist));
+  %let ds=%scan(&memlist,&x);
+  %mp_ds2inserts(&lib..&ds
+    ,outref=&outref
+    ,schema=&schema
+    ,outds=&ds
+    ,flavour=&flavour
+    ,maxobs=&maxobs
+  )
+%end;
+
+%mend mp_lib2inserts;

build.py

@@ -84,7 +84,7 @@ options noquotelenmax;
 """
 f = open('all.sas', "w")             # r / r+ / rb / rb+ / w / wb
 f.write(header)
-folders=['base','meta','metax','viya','lua']
+folders=['base','meta','metax','viya','lua','fcmp']
 for folder in folders:
     filenames = [fn for fn in Path('./' + folder).iterdir() if fn.match("*.sas")]
     filenames.sort()

fcmp/mcf_stpsrv_header.sas

@@ -0,0 +1,94 @@
+/**
+  @file
+  @brief Provides a replacement for the stpsrv_header function
+  @details The stpsrv_header is normally a built-in function, used to set the
+  headers for SAS 9 Stored Processes as documented here:
+  https://go.documentation.sas.com/doc/en/itechcdc/9.4/stpug/srvhead.htm
+
+  The purpose of this custom function is to provide a replacement when running
+  similar code as a web service against
+  [sasjs/server](https://github.com/sasjs/server).  It operates by creating a
+  text file with the headers.  The location of this text file is determined by
+  a macro variable (`sasjs_stpsrv_header_loc`) which needs to be injected into
+  each service by the calling process, eg:
+
+      %let sasjs_stpsrv_header_loc = C:/temp/some_uuid/stpsrv_header.txt;
+
+  Note - the function works by appending headers to the file.  If multiple same-
+  named headers are provided, they will all be appended - the calling process
+  needs to pick up the last one.  This will mean removing the attribute if the
+  final record has an empty value.
+
+  The function takes the following (positional) parameters:
+
+  | PARAMETER | DESCRIPTION |
+  |------------|-------------|
+  | name $  | name of the header attribute to create|
+  | value  $  | value of the header attribute|
+
+  It returns 0 if successful, or -1 if an error occured.
+
+  Usage:
+
+      %let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/stpsrv_header.txt;
+
+      %mcf_stpsrv_header(wrap=YES, insert_cmplib=YES)
+
+      data _null_;
+        rc=stpsrv_header('Content-type','application/text');
+        rc=stpsrv_header('Content-disposition',"attachment; filename=file.txt");
+      run;
+
+      data _null_;
+        infile "&sasjs_stpsrv_header_loc";
+        input;
+        putlog _infile_;
+      run;
+
+
+  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
+  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
+    CMPLIB reference.
+  @param [out] lib= (work) The output library in which to create the catalog.
+  @param [out] cat= (sasjs) The output catalog in which to create the package.
+  @param [out] pkg= (utils) The output package in which to create the function.
+    Uses a 3 part format:  libref.catalog.package
+
+**/
+
+%macro mcf_stpsrv_header(wrap=NO
+  ,insert_cmplib=NO
+  ,lib=WORK
+  ,cat=SASJS
+  ,pkg=UTILS
+)/*/STORE SOURCE*/;
+
+%if &wrap=YES  %then %do;
+  proc fcmp outcat=&lib..&cat..&pkg;
+%end;
+
+function stpsrv_header(name $, value $);
+  length loc $128 val $512;
+  loc=symget('sasjs_stpsrv_header_loc');
+  val=trim(name)!!': '!!value;
+  length fref $8;
+  rc=filename(fref,loc);
+  if (rc ne 0) then return( -1 );
+  fid = fopen(fref,'a');
+  if (fid = 0) then return( -1 );
+  rc=fput(fid, val);
+  rc=fwrite(fid);
+  rc=fclose(fid);
+  rc=filename(fref);
+  return(0);
+endsub;
+
+%if &wrap=YES %then %do;
+  quit;
+%end;
+
+%if &insert_cmplib=YES %then %do;
+  options insert=(CMPLIB=(&lib..&cat));
+%end;
+
+%mend mcf_stpsrv_header;

fcmp/mcf_string2file.sas

@@ -0,0 +1,79 @@
+/**
+  @file
+  @brief Adds a string to a file
+  @details Creates an fcmp function for appending a string to an external file.
+  If the file does not exist, it is created.
+
+  The function itself takes the following (positional) parameters:
+
+  | PARAMETER | DESCRIPTION |
+  |------------|-------------|
+  | filepath $  | full path to the file|
+  | string  $  | string to add to the file |
+  | mode $     | mode of the output - either APPEND (default) or CREATE |
+
+  It returns 0 if successful, or -1 if an error occured.
+
+  Usage:
+
+      %mcf_string2file(wrap=YES, insert_cmplib=YES)
+
+      data _null_;
+        rc=mcf_string2file(
+          "%sysfunc(pathname(work))/newfile.txt"
+          , "This is a test"
+          , "CREATE");
+      run;
+
+      data _null_;
+        infile "%sysfunc(pathname(work))/newfile.txt";
+        input;
+        putlog _infile_;
+      run;
+
+  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
+  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
+    CMPLIB reference.
+  @param [out] lib= (work) The output library in which to create the catalog.
+  @param [out] cat= (sasjs) The output catalog in which to create the package.
+  @param [out] pkg= (utils) The output package in which to create the function.
+    Uses a 3 part format:  libref.catalog.package
+
+**/
+
+%macro mcf_string2file(wrap=NO
+  ,insert_cmplib=NO
+  ,lib=WORK
+  ,cat=SASJS
+  ,pkg=UTILS
+)/*/STORE SOURCE*/;
+
+%if &wrap=YES  %then %do;
+  proc fcmp outcat=&lib..&cat..&pkg;
+%end;
+
+function mcf_string2file(filepath $, string $, mode $);
+  if mode='APPEND' then fmode='a';
+  else fmode='o';
+  length fref $8;
+  rc=filename(fref,filepath);
+  if (rc ne 0) then return( -1 );
+  fid = fopen(fref,fmode);
+  if (fid = 0) then return( -1 );
+  rc=fput(fid, string);
+  rc=fwrite(fid);
+  rc=fclose(fid);
+  rc=filename(fref);
+  return(0);
+endsub;
+
+
+%if &wrap=YES %then %do;
+  quit;
+%end;
+
+%if &insert_cmplib=YES %then %do;
+  options insert=(CMPLIB=(&lib..&cat));
+%end;
+
+%mend mcf_string2file;

main.dox

@@ -20,6 +20,19 @@
 
  */
 
+/*! \dir fcmp
+ *  \brief Macros for generating FCMP functions
+ *  \details These macros have the following attributes:
+
+        * Macro name matches compiled function / subroutine name
+        * Prefixes:  _mcf_, _mcs_
+
+    The macro part is just a wrapper for the underlying function / subroutine,
+    and has switches for including the proc fcmp / quit statements and whether
+    to insert the package into the CMPLIB option.
+
+ */
+
 /*! \dir meta
  *  \brief Metadata Aware Macros
  *  \details These macros have the following attributes:

meta/mm_createfolder.sas

@@ -14,10 +14,10 @@
 
   usage:
 
-    %mm_createfolder(path=/some/meta/folder)
+      %mm_createfolder(path=/some/meta/folder)
 
-  @param path= Name of the folder to create.
-  @param mdebug= set DBG to 1 to disable DEBUG messages
+  @param [in] path= Name of the folder to create.
+  @param [in] mdebug= set DBG to 1 to disable DEBUG messages
 
   @version 9.4
   @author Allan Bowe

meta/mm_gettables.sas

@@ -1,18 +1,20 @@
 /**
   @file
   @brief Creates a dataset with all metadata tables for a particular library
-  @details Will only show the tables to which a user has the requisite
-    metadata access.
+  @details Will only show the tables for which the executing user has the
+    requisite metadata access.
 
   usage:
 
-    %mm_gettables(uri=A5X8AHW1.B40001S5)
+      %mm_gettables(uri=A5X8AHW1.B40001S5)
 
-  @param outds the dataset to create that contains the list of tables
-  @param uri the uri of the library for which to return tables
-  @param getauth= YES|NO - fetch the authdomain used in database connections.
-  Set to NO to improve runtimes in larger environments, as there can be a
-  performance hit on the `metadata_getattr(domainuri, "Name", AuthDomain)` call.
+  @param [in] uri= the uri of the library for which to return tables
+  @param [out] outds= (work.mm_gettables) the dataset to contain the list of
+    tables
+  @param [in] getauth= (YES) Fetch the authdomain used in database connections.
+    Set to NO to improve runtimes in larger environments, as there can be a
+    performance hit on the `metadata_getattr(domainuri, "Name", AuthDomain)`
+    call.
 
   @returns outds  dataset containing all groups in a column named "metagroup"
     (defaults to work.mm_getlibs). The following columns are provided:
@@ -40,8 +42,8 @@ data &outds;
     libdesc $200 libref engine $8 IsDBMSLibname $1
     tablename $50 /* metadata table names can be longer than $32 */
     ;
-  keep libname libdesc libref engine ServerContext path_schema AuthDomain tableuri
-    tablename IsPreassigned IsDBMSLibname id;
+  keep libname libdesc libref engine ServerContext path_schema AuthDomain
+    tableuri tablename IsPreassigned IsDBMSLibname id;
   call missing (of _all_);
 
   uri=symget('uri');

package-lock.json

@@ -5,47 +5,45 @@
   "packages": {
     "": {
       "name": "@sasjs/core",
-      "hasInstallScript": true,
       "license": "MIT",
       "devDependencies": {
-        "@sasjs/cli": "^2.27.0"
+        "@sasjs/cli": "^2.37.2"
       }
     },
     "node_modules/@sasjs/adapter": {
-      "version": "2.6.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.6.3.tgz",
-      "integrity": "sha512-Mg1AIDd0JDa0/UjvvmrGxjkpYmGEpqmD1dOYmS55EJG5loAENaVKJg9Ht8bU2XFyVaMbFpgB2yqDd0VWPP4eBA==",
+      "version": "2.10.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.10.0.tgz",
+      "integrity": "sha512-GbvyIgbODnAJaBaz/Tz8/IwcujNOsZcwzbIuKQtG5y13BzgVPtx/e7b2TAhdoBqd+8uVh0CdrSDfOV/SuvPurg==",
       "dev": true,
-      "hasInstallScript": true,
       "dependencies": {
-        "@sasjs/utils": "^2.18.0",
+        "@sasjs/utils": "^2.27.1",
         "axios": "^0.21.1",
         "axios-cookiejar-support": "^1.0.1",
         "form-data": "^4.0.0",
         "https": "^1.0.0",
-        "tough-cookie": "^4.0.0",
-        "url": "^0.11.0"
+        "tough-cookie": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=15"
       }
     },
     "node_modules/@sasjs/cli": {
-      "version": "2.27.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.27.0.tgz",
-      "integrity": "sha512-jdQBNFbqvkidCL2+V6kuGk66YGH9BshHetPIzYSUCJPYrsKdWYRk0+OP+dZZ4qcNk8yLbcwZH+Z6Y1xh9yroFg==",
+      "version": "2.37.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.37.2.tgz",
+      "integrity": "sha512-Bcm8UFN9Y/ZON4T31Gu9mf1REn1pZoStHFVrix/yp7mchFt5rrrY5RbIqO/AI1jzhShSVg9P7oU4VPJrqki+SA==",
       "dev": true,
       "dependencies": {
-        "@sasjs/adapter": "2.6.3",
-        "@sasjs/core": "2.34.5",
-        "@sasjs/lint": "1.11.1",
-        "@sasjs/utils": "2.21.0",
-        "@types/url-parse": "1.4.3",
-        "btoa": "1.2.1",
+        "@sasjs/adapter": "2.10.0",
+        "@sasjs/core": "2.35.4",
+        "@sasjs/lint": "1.11.2",
+        "@sasjs/utils": "2.27.1",
         "chalk": "4.1.1",
         "csv-stringify": "5.6.2",
         "dotenv": "10.0.0",
         "esm": "3.2.25",
         "find": "0.3.0",
-        "fs-extra": "10.0.0",
         "get-installed-path": "4.0.8",
+        "js-base64": "3.6.1",
         "jsdom": "16.6.0",
         "jwt-decode": "3.1.2",
         "lodash.groupby": "4.6.0",
@@ -61,36 +59,39 @@
       }
     },
     "node_modules/@sasjs/core": {
-      "version": "2.34.5",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.34.5.tgz",
-      "integrity": "sha512-aq84ce9zmyAnkhKlSQ2SGZSQyJ5+EQAaUwktWfPhVdV9B6FJJSrQL3s1K/Dw9IuMczJcZcBlS4HO/w+goQBgyw==",
-      "dev": true,
-      "hasInstallScript": true
+      "version": "2.35.4",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.35.4.tgz",
+      "integrity": "sha512-Exr3+yRdvacKvbXQoi1RfGHi5NtZUkc7RwFNkemHTFXLYaIzPI8CGSCaQmKkwM1UteOJly2e2pw2YT6kHNY1NA==",
+      "dev": true
     },
     "node_modules/@sasjs/lint": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.1.tgz",
-      "integrity": "sha512-nb0xFwXSDrE1qQLYV8whUoaXX2lh4nN3jbA/dOlLpjmF+8uvpgZo9GHy5vz5cpIhtguxiFd7lOGN/zyrzKO8yg==",
+      "version": "1.11.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.2.tgz",
+      "integrity": "sha512-zEonhvha9kwrD+hxhG0hEhtfqpXwffH4vRDIr6eDiXkC7S8M3yImpjyFBvX/THJO5+8iuY8TYkOXKl7+nK/wAg==",
       "dev": true,
-      "hasInstallScript": true,
       "dependencies": {
         "@sasjs/utils": "^2.19.0"
       }
     },
     "node_modules/@sasjs/utils": {
-      "version": "2.21.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.21.0.tgz",
-      "integrity": "sha512-ehRF1VHBiF8FNg2w4VocjBlb+ZkyMasUZOvUn19bsdqpF0otJVYu66gFOJF0rIpykh/vbpLi6dhqYOxgCRAYIw==",
+      "version": "2.27.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.27.1.tgz",
+      "integrity": "sha512-CYTQwEj89cc7H3tGiQQcyDkZYaWRc1HZJpOF8o2RHYS37fIAOy0SyyJdq6mcQ74Nb1u5AmFXPFIvnRCMEcTYeQ==",
       "dev": true,
-      "hasInstallScript": true,
       "dependencies": {
+        "@types/fs-extra": "^9.0.11",
         "@types/prompts": "^2.0.13",
         "chalk": "^4.1.1",
         "cli-table": "^0.3.6",
         "consola": "^2.15.0",
         "fs-extra": "^10.0.0",
+        "jwt-decode": "^3.1.2",
         "prompts": "^2.4.1",
+        "rimraf": "^3.0.2",
         "valid-url": "^1.0.9"
+      },
+      "engines": {
+        "node": ">=15"
       }
     },
     "node_modules/@tootallnate/once": {
@@ -102,34 +103,37 @@
         "node": ">= 6"
       }
     },
+    "node_modules/@types/fs-extra": {
+      "version": "9.0.12",
+      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.12.tgz",
+      "integrity": "sha512-I+bsBr67CurCGnSenZZ7v94gd3tc3+Aj2taxMT4yu4ABLuOgOjeFxX3dokG24ztSRg5tnT00sL8BszO7gSMoIw==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/node": {
-      "version": "15.12.4",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.4.tgz",
-      "integrity": "sha512-zrNj1+yqYF4WskCMOHwN+w9iuD12+dGm0rQ35HLl9/Ouuq52cEtd0CH9qMgrdNmi5ejC1/V7vKEXYubB+65DkA==",
+      "version": "16.3.1",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.3.1.tgz",
+      "integrity": "sha512-N87VuQi7HEeRJkhzovao/JviiqKjDKMVKxKMfUvSKw+MbkbW8R0nA3fi/MQhhlxV2fQ+2ReM+/Nt4efdrJx3zA==",
       "dev": true
     },
     "node_modules/@types/prompts": {
-      "version": "2.0.13",
-      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.13.tgz",
-      "integrity": "sha512-jwMOIGy49VruR/gYehhJYgpVzB+EVpEE7t7j9m1oTo4HMpOe7KmsyqdBuoxAzA5B4caUgx0cKrWr7wUEqMXJ7Q==",
+      "version": "2.0.14",
+      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.14.tgz",
+      "integrity": "sha512-HZBd99fKxRWpYCErtm2/yxUZv6/PBI9J7N4TNFffl5JbrYMHBwF25DjQGTW3b3jmXq+9P6/8fCIb2ee57BFfYA==",
       "dev": true,
       "dependencies": {
         "@types/node": "*"
       }
     },
     "node_modules/@types/tough-cookie": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.0.tgz",
-      "integrity": "sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A==",
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.1.tgz",
+      "integrity": "sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==",
       "dev": true,
       "peer": true
     },
-    "node_modules/@types/url-parse": {
-      "version": "1.4.3",
-      "resolved": "https://registry.npmjs.org/@types/url-parse/-/url-parse-1.4.3.tgz",
-      "integrity": "sha512-4kHAkbV/OfW2kb5BLVUuUMoumB3CP8rHqlw48aHvFy5tf9ER0AfOonBlX29l/DD68G70DmyhRlSYfQPSYpC5Vw==",
-      "dev": true
-    },
     "node_modules/abab": {
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.5.tgz",
@@ -301,18 +305,6 @@
       "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==",
       "dev": true
     },
-    "node_modules/btoa": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz",
-      "integrity": "sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g==",
-      "dev": true,
-      "bin": {
-        "btoa": "bin/btoa.js"
-      },
-      "engines": {
-        "node": ">= 0.4.0"
-      }
-    },
     "node_modules/buffer": {
       "version": "5.7.1",
       "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
@@ -660,9 +652,9 @@
       }
     },
     "node_modules/follow-redirects": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz",
-      "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg==",
+      "version": "1.14.2",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.2.tgz",
+      "integrity": "sha512-yLR6WaE2lbF0x4K2qE2p9PEXKLDjUjnR/xmjS3wHAYxtlsI9MLLBJUZirAHKzUZDGLxje7w/cXR49WOUo4rbsA==",
       "dev": true,
       "funding": [
         {
@@ -993,6 +985,12 @@
       "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
       "dev": true
     },
+    "node_modules/js-base64": {
+      "version": "3.6.1",
+      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.6.1.tgz",
+      "integrity": "sha512-Frdq2+tRRGLQUIQOgsIGSCd1VePCS2fsddTG5dTCqR0JHgltXWfsxnY0gIXPoMeRmdom6Oyq+UMOFg5suduOjQ==",
+      "dev": true
+    },
     "node_modules/jsdom": {
       "version": "16.6.0",
       "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.6.0.tgz",
@@ -1276,9 +1274,9 @@
       }
     },
     "node_modules/path-parse": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz",
-      "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==",
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
       "dev": true
     },
     "node_modules/pify": {
@@ -1330,15 +1328,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/querystring": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
-      "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=",
-      "dev": true,
-      "engines": {
-        "node": ">=0.4.x"
-      }
-    },
     "node_modules/readable-stream": {
       "version": "3.6.0",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@@ -1597,22 +1586,6 @@
         "node": ">= 10.0.0"
       }
     },
-    "node_modules/url": {
-      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz",
-      "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=",
-      "dev": true,
-      "dependencies": {
-        "punycode": "1.3.2",
-        "querystring": "0.2.0"
-      }
-    },
-    "node_modules/url/node_modules/punycode": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
-      "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=",
-      "dev": true
-    },
     "node_modules/util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
@@ -1762,39 +1735,36 @@
   },
   "dependencies": {
     "@sasjs/adapter": {
-      "version": "2.6.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.6.3.tgz",
-      "integrity": "sha512-Mg1AIDd0JDa0/UjvvmrGxjkpYmGEpqmD1dOYmS55EJG5loAENaVKJg9Ht8bU2XFyVaMbFpgB2yqDd0VWPP4eBA==",
+      "version": "2.10.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.10.0.tgz",
+      "integrity": "sha512-GbvyIgbODnAJaBaz/Tz8/IwcujNOsZcwzbIuKQtG5y13BzgVPtx/e7b2TAhdoBqd+8uVh0CdrSDfOV/SuvPurg==",
       "dev": true,
       "requires": {
-        "@sasjs/utils": "^2.18.0",
+        "@sasjs/utils": "^2.27.1",
         "axios": "^0.21.1",
         "axios-cookiejar-support": "^1.0.1",
         "form-data": "^4.0.0",
         "https": "^1.0.0",
-        "tough-cookie": "^4.0.0",
-        "url": "^0.11.0"
+        "tough-cookie": "^4.0.0"
       }
     },
     "@sasjs/cli": {
-      "version": "2.27.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.27.0.tgz",
-      "integrity": "sha512-jdQBNFbqvkidCL2+V6kuGk66YGH9BshHetPIzYSUCJPYrsKdWYRk0+OP+dZZ4qcNk8yLbcwZH+Z6Y1xh9yroFg==",
+      "version": "2.37.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.37.2.tgz",
+      "integrity": "sha512-Bcm8UFN9Y/ZON4T31Gu9mf1REn1pZoStHFVrix/yp7mchFt5rrrY5RbIqO/AI1jzhShSVg9P7oU4VPJrqki+SA==",
       "dev": true,
       "requires": {
-        "@sasjs/adapter": "2.6.3",
-        "@sasjs/core": "2.34.5",
-        "@sasjs/lint": "1.11.1",
-        "@sasjs/utils": "2.21.0",
-        "@types/url-parse": "1.4.3",
-        "btoa": "1.2.1",
+        "@sasjs/adapter": "2.10.0",
+        "@sasjs/core": "2.35.4",
+        "@sasjs/lint": "1.11.2",
+        "@sasjs/utils": "2.27.1",
         "chalk": "4.1.1",
         "csv-stringify": "5.6.2",
         "dotenv": "10.0.0",
         "esm": "3.2.25",
         "find": "0.3.0",
-        "fs-extra": "10.0.0",
         "get-installed-path": "4.0.8",
+        "js-base64": "3.6.1",
         "jsdom": "16.6.0",
         "jwt-decode": "3.1.2",
         "lodash.groupby": "4.6.0",
@@ -1807,32 +1777,35 @@
       }
     },
     "@sasjs/core": {
-      "version": "2.34.5",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.34.5.tgz",
-      "integrity": "sha512-aq84ce9zmyAnkhKlSQ2SGZSQyJ5+EQAaUwktWfPhVdV9B6FJJSrQL3s1K/Dw9IuMczJcZcBlS4HO/w+goQBgyw==",
+      "version": "2.35.4",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.35.4.tgz",
+      "integrity": "sha512-Exr3+yRdvacKvbXQoi1RfGHi5NtZUkc7RwFNkemHTFXLYaIzPI8CGSCaQmKkwM1UteOJly2e2pw2YT6kHNY1NA==",
       "dev": true
     },
     "@sasjs/lint": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.1.tgz",
-      "integrity": "sha512-nb0xFwXSDrE1qQLYV8whUoaXX2lh4nN3jbA/dOlLpjmF+8uvpgZo9GHy5vz5cpIhtguxiFd7lOGN/zyrzKO8yg==",
+      "version": "1.11.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.2.tgz",
+      "integrity": "sha512-zEonhvha9kwrD+hxhG0hEhtfqpXwffH4vRDIr6eDiXkC7S8M3yImpjyFBvX/THJO5+8iuY8TYkOXKl7+nK/wAg==",
       "dev": true,
       "requires": {
         "@sasjs/utils": "^2.19.0"
       }
     },
     "@sasjs/utils": {
-      "version": "2.21.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.21.0.tgz",
-      "integrity": "sha512-ehRF1VHBiF8FNg2w4VocjBlb+ZkyMasUZOvUn19bsdqpF0otJVYu66gFOJF0rIpykh/vbpLi6dhqYOxgCRAYIw==",
+      "version": "2.27.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.27.1.tgz",
+      "integrity": "sha512-CYTQwEj89cc7H3tGiQQcyDkZYaWRc1HZJpOF8o2RHYS37fIAOy0SyyJdq6mcQ74Nb1u5AmFXPFIvnRCMEcTYeQ==",
       "dev": true,
       "requires": {
+        "@types/fs-extra": "^9.0.11",
         "@types/prompts": "^2.0.13",
         "chalk": "^4.1.1",
         "cli-table": "^0.3.6",
         "consola": "^2.15.0",
         "fs-extra": "^10.0.0",
+        "jwt-decode": "^3.1.2",
         "prompts": "^2.4.1",
+        "rimraf": "^3.0.2",
         "valid-url": "^1.0.9"
       }
     },
@@ -1842,34 +1815,37 @@
       "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==",
       "dev": true
     },
+    "@types/fs-extra": {
+      "version": "9.0.12",
+      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.12.tgz",
+      "integrity": "sha512-I+bsBr67CurCGnSenZZ7v94gd3tc3+Aj2taxMT4yu4ABLuOgOjeFxX3dokG24ztSRg5tnT00sL8BszO7gSMoIw==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/node": {
-      "version": "15.12.4",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.4.tgz",
-      "integrity": "sha512-zrNj1+yqYF4WskCMOHwN+w9iuD12+dGm0rQ35HLl9/Ouuq52cEtd0CH9qMgrdNmi5ejC1/V7vKEXYubB+65DkA==",
+      "version": "16.3.1",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.3.1.tgz",
+      "integrity": "sha512-N87VuQi7HEeRJkhzovao/JviiqKjDKMVKxKMfUvSKw+MbkbW8R0nA3fi/MQhhlxV2fQ+2ReM+/Nt4efdrJx3zA==",
       "dev": true
     },
     "@types/prompts": {
-      "version": "2.0.13",
-      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.13.tgz",
-      "integrity": "sha512-jwMOIGy49VruR/gYehhJYgpVzB+EVpEE7t7j9m1oTo4HMpOe7KmsyqdBuoxAzA5B4caUgx0cKrWr7wUEqMXJ7Q==",
+      "version": "2.0.14",
+      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.14.tgz",
+      "integrity": "sha512-HZBd99fKxRWpYCErtm2/yxUZv6/PBI9J7N4TNFffl5JbrYMHBwF25DjQGTW3b3jmXq+9P6/8fCIb2ee57BFfYA==",
       "dev": true,
       "requires": {
         "@types/node": "*"
       }
     },
     "@types/tough-cookie": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.0.tgz",
-      "integrity": "sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A==",
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.1.tgz",
+      "integrity": "sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==",
       "dev": true,
       "peer": true
     },
-    "@types/url-parse": {
-      "version": "1.4.3",
-      "resolved": "https://registry.npmjs.org/@types/url-parse/-/url-parse-1.4.3.tgz",
-      "integrity": "sha512-4kHAkbV/OfW2kb5BLVUuUMoumB3CP8rHqlw48aHvFy5tf9ER0AfOonBlX29l/DD68G70DmyhRlSYfQPSYpC5Vw==",
-      "dev": true
-    },
     "abab": {
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.5.tgz",
@@ -1994,12 +1970,6 @@
       "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==",
       "dev": true
     },
-    "btoa": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz",
-      "integrity": "sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g==",
-      "dev": true
-    },
     "buffer": {
       "version": "5.7.1",
       "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
@@ -2253,9 +2223,9 @@
       }
     },
     "follow-redirects": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz",
-      "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg==",
+      "version": "1.14.2",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.2.tgz",
+      "integrity": "sha512-yLR6WaE2lbF0x4K2qE2p9PEXKLDjUjnR/xmjS3wHAYxtlsI9MLLBJUZirAHKzUZDGLxje7w/cXR49WOUo4rbsA==",
       "dev": true
     },
     "form-data": {
@@ -2493,6 +2463,12 @@
       "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
       "dev": true
     },
+    "js-base64": {
+      "version": "3.6.1",
+      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.6.1.tgz",
+      "integrity": "sha512-Frdq2+tRRGLQUIQOgsIGSCd1VePCS2fsddTG5dTCqR0JHgltXWfsxnY0gIXPoMeRmdom6Oyq+UMOFg5suduOjQ==",
+      "dev": true
+    },
     "jsdom": {
       "version": "16.6.0",
       "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.6.0.tgz",
@@ -2717,9 +2693,9 @@
       "dev": true
     },
     "path-parse": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz",
-      "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==",
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
       "dev": true
     },
     "pify": {
@@ -2756,12 +2732,6 @@
       "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
       "dev": true
     },
-    "querystring": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
-      "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=",
-      "dev": true
-    },
     "readable-stream": {
       "version": "3.6.0",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@@ -2954,24 +2924,6 @@
       "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==",
       "dev": true
     },
-    "url": {
-      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz",
-      "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=",
-      "dev": true,
-      "requires": {
-        "punycode": "1.3.2",
-        "querystring": "0.2.0"
-      },
-      "dependencies": {
-        "punycode": {
-          "version": "1.3.2",
-          "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
-          "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=",
-          "dev": true
-        }
-      }
-    },
     "util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",

package.json

@@ -27,12 +27,12 @@
   "main": "index.js",
   "scripts": {
     "build": "sasjs cbd -t viya",
-    "docs": "sasjs doc && ./sasjs/utils/build.sh",
+    "docs": "sasjs doc -t docsonly && ./sasjs/utils/build.sh",
     "test": "sasjs test -t viya",
     "lint": "sasjs lint",
-    "prepare": "[ -d .git ] && git config core.hooksPath ./.git-hooks || true"
+    "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "2.27.0"
+    "@sasjs/cli": "^2.37.2"
   }
 }

sasjs/sasjsconfig.json

@@ -2,11 +2,12 @@
   "$schema": "https://cli.sasjs.io/sasjsconfig-schema.json",
   "macroFolders": [
     "base",
+    "fcmp",
     "meta",
     "metax",
     "viya",
     "lua",
-    "tests/base"
+    "tests/crossplatform"
   ],
   "docConfig": {
     "displayMacroCore": false,
@@ -33,7 +34,7 @@
       "allowInsecureRequests": false,
       "appLoc": "/Public/temp/macrocore",
       "macroFolders": [
-        "tests/viya"
+        "tests/viyaonly"
       ],
       "programFolders": [],
       "deployConfig": {
@@ -48,7 +49,15 @@
       "serverType": "SAS9",
       "appLoc": "/Shared Data/temp/macrocore",
       "macroFolders": [
-        "tests/meta"
+        "tests/sas9only"
+      ]
+    },
+    {
+      "name": "docsonly",
+      "serverType": "SAS9",
+      "macroFolders": [
+        "tests/sas9only",
+        "tests/viyaonly"
       ]
     }
   ]

tests/crossplatform/mcf_stpsrv_header.test.sas

@@ -0,0 +1,39 @@
+/**
+  @file
+  @brief Testing mcf_stpsrv_header macro
+
+  <h4> SAS Macros </h4>
+  @li mcf_stpsrv_header.sas
+  @li mp_assert.sas
+
+**/
+
+%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/stpsrv_header.txt;
+
+%mcf_stpsrv_header(wrap=YES, insert_cmplib=YES)
+
+data _null_;
+  rc=stpsrv_header('Content-type','application/text');
+  rc=stpsrv_header('Content-disposition',"attachment; filename=file.txt");
+run;
+
+%let test1=FAIL;
+%let test2=FAIL;
+
+data _null_;
+  infile "&sasjs_stpsrv_header_loc";
+  input;
+  if _n_=1 and _infile_='Content-type: application/text'
+  then call symputx('test1','PASS');
+  else if _n_=2 & _infile_='Content-disposition: attachment; filename=file.txt'
+  then call symputx('test2','PASS');
+run;
+
+%mp_assert(
+  iftrue=(%str(&test1)=%str(PASS)),
+  desc=Check first header line
+)
+%mp_assert(
+  iftrue=(%str(&test2)=%str(PASS)),
+  desc=Check second header line
+)

tests/crossplatform/mcf_string2file.test.sas

@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief Testing mcf_string2file macro
+
+  <h4> SAS Macros </h4>
+  @li mcf_string2file.sas
+  @li mp_assert.sas
+
+**/
+
+
+%mcf_string2file(wrap=YES, insert_cmplib=YES)
+
+data _null_;
+  rc=mcf_string2file(
+    "%sysfunc(pathname(work))/newfile.txt"
+    , "line1"
+    , "APPEND");
+  rc=mcf_string2file(
+    "%sysfunc(pathname(work))/newfile.txt"
+    , "line2"
+    , "APPEND");
+run;
+
+data _null_;
+  infile "%sysfunc(pathname(work))/newfile.txt";
+  input;
+  if _n_=2 then call symputx('val',_infile_);
+run;
+
+%mp_assert(
+  iftrue=(%str(&val)=%str(line2)),
+  desc=Check if APPEND works
+)
+
+data _null_;
+  rc=mcf_string2file(
+    "%sysfunc(pathname(work))/newfile.txt"
+    , "creating"
+    , "CREATE");
+run;
+
+data _null_;
+  infile "%sysfunc(pathname(work))/newfile.txt";
+  input;
+  if _n_=1 then call symputx('val2',_infile_);
+run;
+
+%mp_assert(
+  iftrue=(%str(&val2)=%str(creating)),
+  desc=Check if CREATE works
+)

tests/crossplatform/mf_existfileref.test.sas

@@ -0,0 +1,35 @@
+/**
+  @file
+  @brief Testing mf_existfileref macro
+
+  <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mp_assert.sas
+
+**/
+
+filename ref1 temp;
+filename ref2 temp;
+
+data _null_;
+  file ref1;
+  put 'exists';
+run;
+
+%mp_assert(
+  iftrue=(%mf_existfileref(ref1)=1),
+  desc=Checking fileref WITH target file exists,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_existfileref(ref2)=1),
+  desc=Checking fileref WITHOUT target file exists,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_existfileref(ref3)=0),
+  desc=Checking non-existant fref does not exist,
+  outds=work.test_results
+)

tests/crossplatform/mf_existfunction.test.sas

@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Testing mf_existfunction macro
+
+  <h4> SAS Macros </h4>
+  @li mf_existfunction.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(%mf_existfunction(CAT)=1),
+  desc=Checking if CAT function exists,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_existfunction(DOG)=0),
+  desc=Checking DOG function does not exist,
+  outds=work.test_results
+)
+

tests/crossplatform/mf_getapploc.test.sas

@@ -0,0 +1,49 @@
+/**
+  @file
+  @brief Testing mf_getapploc macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getapploc.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/loc/tests/services/x/service)"="/some/loc"
+  ),
+  desc=Checking test appLoc matches,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/loc/tests/services/tests/service)"="/some/loc"
+  ),
+  desc=Checking nested services appLoc matches,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/services/admin/service)"="/some/area"
+  ),
+  desc=Checking services appLoc matches,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/jobs/jobs/job)"="/some/area"
+  ),
+  desc=Checking jobs appLoc matches,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/tests/macros/somemacro.sas)"="/some/area"
+  ),
+  desc=Checking tests/macros appLoc matches (which has no subfolder),
+  outds=work.test_results
+)

tests/crossplatform/mp_base64copy.test.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Testing mp_base64copy.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_base64copy.sas
+  @li mp_assert.sas
+
+**/
+
+
+/* TEST 1 - regular base64 decode */
+
+%let string1=base ik ally;
+filename tmp temp;
+data _null_;
+  file tmp;
+  put "&string1";
+run;
+%mp_base64copy(inref=tmp, outref=myref, action=ENCODE)
+
+data _null_;
+  infile myref;
+  input;
+  put _infile_;
+run;
+%mp_base64copy(inref=myref, outref=mynewref, action=DECODE)
+data _null_;
+  infile mynewref lrecl=5000;
+  input;
+  put _infile_;
+  call symputx('string1_check',_infile_);
+  stop;
+run;
+%mp_assert(
+  iftrue=("&string1"="&string1_check"),
+  desc=Basic String Compare,
+  outds=work.test_results
+)
+
+
+/* multibyte string check */
+
+filename tmp2 temp;
+data _null_;
+  file tmp2;
+  put "'╤', '╔', '╗', '═', '╧', '╚', '╝', '║', '╟', '─', '┼', '║', '╢', '│'";
+run;
+%mp_base64copy(inref=tmp2, outref=myref2, action=ENCODE)
+
+%mp_base64copy(inref=myref2, outref=newref2, action=DECODE)
+data _null_;
+  infile newref2 lrecl=5000;
+  input;
+  list;
+  /* do not print the string to the log else viya 3.5 throws exception */
+  if trim(_infile_)=
+    "'╤', '╔', '╗', '═', '╧', '╚', '╝', '║', '╟', '─', '┼', '║', '╢', '│'"
+  then call symputx('check2',1);
+  else call symputx('check2',0);
+  stop;
+run;
+%mp_assert(
+  iftrue=("&check2"="1"),
+  desc=Double Byte String Compare,
+  outds=work.test_results
+)

tests/crossplatform/mp_binarycopy.test.sas

@@ -0,0 +1,99 @@
+/**
+  @file
+  @brief Testing mp_binarycopy.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_binarycopy.sas
+  @li mp_assert.sas
+
+**/
+
+
+/* TEST 1 - regular file copy */
+%let string1=test1;
+filename tmp temp;
+filename myref temp;
+data _null_;
+  file tmp;
+  put "&string1";
+run;
+%mp_binarycopy(inref=tmp, outref=myref)
+data _null_;
+  infile myref;
+  input;
+  put _infile_;
+  call symputx('string1_check',_infile_);
+  stop;
+run;
+%mp_assert(
+  iftrue=("&string1"="&string1_check"),
+  desc=Basic String Compare,
+  outds=work.test_results
+)
+
+
+/* TEST 2 - File append */
+%let string2=test2;
+%let path2=%sysfunc(pathname(work))/somefile.txt;
+data _null_;
+  file "&path2";
+  put "&string2";
+run;
+%mp_binarycopy(inloc="&path2", outref=myref, mode=APPEND)
+data _null_;
+  infile myref;
+  input;
+  put _infile_;
+  if _n_=2 then call symputx('string2_check',_infile_);
+run;
+%mp_assert(
+  iftrue=("&string2"="&string2_check"),
+  desc=Append Check (file to ref),
+  outds=work.test_results
+)
+
+/* TEST 3 - File create (ref to existing file) */
+%let string3=test3;
+%let path3=%sysfunc(pathname(work))/somefile3.txt;
+filename tmp3 temp;
+data _null_;
+  file tmp3;
+  put "&string3";
+run;
+data _null_;
+  file "&path3";
+  put "this should not be returned";
+run;
+%mp_binarycopy(inref=tmp3, outloc="&path3")
+data _null_;
+  infile "&path3";
+  input;
+  put _infile_;
+  if _n_=1 then call symputx('string3_check',_infile_);
+run;
+%mp_assert(
+  iftrue=("&string3"="&string3_check"),
+  desc=Append Check (ref to existing file),
+  outds=work.test_results
+)
+
+/* TEST 4 - File append (ref to file) */
+%let string4=test4;
+%let string4_check=;
+filename tmp4 temp;
+data _null_;
+  file tmp4;
+  put "&string4";
+run;
+%mp_binarycopy(inref=tmp4, outloc="&path3",mode=APPEND)
+data _null_;
+  infile "&path3";
+  input;
+  put _infile_;
+  if _n_=2 then call symputx('string4_check',_infile_);
+run;
+%mp_assert(
+  iftrue=("&string4"="&string4_check"),
+  desc=Append Check (ref to file),
+  outds=work.test_results
+)

tests/crossplatform/mp_ds2inserts.test.sas

@@ -0,0 +1,31 @@
+/**
+  @file
+  @brief Testing mp_ds2inserts.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2inserts.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * test 1 - rebuild an existing dataset
+  * Cars is a great dataset - it contains leading spaces, and formatted numerics
+  */
+
+%mp_ds2inserts(sashelp.cars,outref=testref,schema=work,outds=test)
+
+data work.test;
+  set sashelp.cars;
+  stop;
+proc sql;
+%inc testref;
+
+proc compare base=sashelp.cars compare=work.test;
+quit;
+
+%mp_assert(
+  iftrue=(&sysinfo=1),
+  desc=sashelp.cars is identical except for ds label,
+  outds=work.test_results
+)

tests/crossplatform/mp_getddl.test.sas

@@ -0,0 +1,23 @@
+/**
+  @file
+  @brief Testing mp_getddl.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_getddl.sas
+  @li mp_assert.sas
+
+**/
+
+data test(index=(pk=(x y)/unique /nomiss));
+  x=1;
+  y='blah';
+  label x='blah';
+run;
+proc sql; describe table &syslast;
+%mp_getddl(work,test,flavour=tsql,showlog=YES)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=mp_getddl runs without errors,
+  outds=work.test_results
+)

tests/crossplatform/mp_lib2inserts.test.sas

@@ -0,0 +1,42 @@
+/**
+  @file
+  @brief Testing mp_ds2inserts.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mp_getddl.sas
+  @li mp_lib2inserts.sas
+  @li mp_assert.sas
+
+**/
+
+/* grab 20 datasets from SASHELP */
+%let path=%sysfunc(pathname(work));
+%mf_mkdir(&path)
+libname sashlp "&path";
+proc sql noprint;
+create table members as
+  select distinct lowcase(memname) as memname
+  from dictionary.tables
+  where upcase(libname)="SASHELP"
+    and memtype='DATA'; /* exclude views */
+data _null_;
+  set work.members;
+  call execute(cats('data sashlp.',memname,';set sashelp.',memname,';run;'));
+  if _n_>20 then stop;
+run;
+
+/* export DDL and inserts */
+%mp_getddl(sashlp, schema=work, fref=tempref)
+%mp_lib2inserts(sashlp, schema=work, outref=tempref,maxobs=50)
+
+/* check if it actually runs */
+options source2;
+%inc tempref;
+
+/* without errors.. */
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Able to export 20 tables from sashelp using mp_lib2inserts,
+  outds=work.test_results
+)

tests/viyaonly/mv_getfoldermembers.test.sas

@@ -0,0 +1,24 @@
+/**
+  @file
+  @brief Testing mv_getfoldermembers macro
+  @details Testing is performed by ensuring that the tests/macros folder
+  contains more than 20 results (which also means the default was increased)
+
+  <h4> SAS Macros </h4>
+  @li mf_getapploc.sas
+  @li mp_assertdsobs.sas
+  @li mv_getfoldermembers.sas
+
+**/
+options mprint;
+
+%mv_getfoldermembers(
+  root=%mf_getapploc()/tests/macros,
+  outds=work.results
+)
+
+%mp_assertdsobs(work.results,
+  desc=%str(Ensuring over 20 results found in %mf_getapploc()/tests/macros),
+  test=ATLEAST 21,
+  outds=work.test_results
+)

viya/mv_getfoldermembers.sas

@@ -1,22 +1,25 @@
 /**
-  @file mv_getfoldermembers.sas
-  @brief Gets a list of folders (and ids) for a given root
-  @details Works for both root level and below, oauth or password. Default is
-    oauth, and the token is expected in a global ACCESS_TOKEN variable.
+  @file
+  @brief Gets a list of folder members (and ids) for a given root
+  @details Returns all members for a particular Viya folder.  Works at both root
+  level and below, and results are created in an output dataset.
 
-        %mv_getfoldermembers(root=/Public)
+        %mv_getfoldermembers(root=/Public, outds=work.mymembers)
 
 
-  @param root= The path for which to return the list of folders
-  @param outds= The output dataset to create (default is work.mv_getfolders).  Format:
+  @param [in] root= (/) The path for which to return the list of folders
+  @param [out] outds= (work.mv_getfolders) The output dataset to create. Format:
   |ordinal_root|ordinal_items|creationTimeStamp| modifiedTimeStamp|createdBy|modifiedBy|id| uri|added| type|name|description|
   |---|---|---|---|---|---|---|---|---|---|---|---|
   |1|1|2021-05-25T11:15:04.204Z|2021-05-25T11:15:04.204Z|allbow|allbow|4f1e3945-9655-462b-90f2-c31534b3ca47|/folders/folders/ed701ff3-77e8-468d-a4f5-8c43dec0fd9e|2021-05-25T11:15:04.212Z|child|my_folder_name|My folder Description|
 
-  @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values are "password" or "authorization_code" (unquoted).
-    The default is authorization_code.
-
+  @param [in] access_token_var= (ACCESS_TOKEN) The global macro variable to
+    contain the access token
+  @param [in] grant_type= (sas_services) Valid values are:
+    @li password
+    @li authorization_code
+    @li detect
+    @li sas_services
 
   @version VIYA V.03.04
   @author Allan Bowe, source: https://github.com/sasjs/core
@@ -28,6 +31,12 @@
   @li mf_getuniquelibref.sas
   @li mf_isblank.sas
 
+  <h4> Related Macros </h4>
+  @li mv_createfolder.sas
+  @li mv_deletefoldermember.sas
+  @li mv_deleteviyafolder.sas
+  @li mv_getfoldermembers.test.sas
+
 **/
 
 %macro mv_getfoldermembers(root=/
@@ -67,7 +76,7 @@ options noquotelenmax;
 %if "&root"="/" %then %do;
   /* if root just list root folders */
   proc http method='GET' out=&fname1 &oauth_bearer
-      url="&base_uri/folders/rootFolders";
+      url="&base_uri/folders/rootFolders?limit=1000";
   %if &grant_type=authorization_code %then %do;
       headers "Authorization"="Bearer &&&access_token_var";
   %end;
@@ -88,13 +97,17 @@ options noquotelenmax;
   /*data _null_;infile &fname1;input;putlog _infile_;run;*/
   libname &libref1 JSON fileref=&fname1;
   /* now get the followon link to list members */
-  %local href;
-  %let href=0;
+  %local href cnt;
+  %let cnt=0;
   data _null_;
     set &libref1..links;
-    if rel='members' then call symputx('href',quote("&base_uri"!!trim(href)),'l');
+    if rel='members' then do;
+      url=cats("'","&base_uri",href,"?limit=10000'");
+      call symputx('href',url,'l');
+      call symputx('cnt',1,'l');
+    end;
   run;
-  %if &href=0 %then %do;
+  %if &cnt=0 %then %do;
     %put NOTE:;%put NOTE-  No members found in &root!!;%put NOTE-;
     %return;
   %end;