fix: quoting memsize in ms_webout

fix: correct target path. Closes #100

.all-contributorsrc

@@ -98,7 +98,27 @@
       "contributions": [
         "ideas"
       ]
+    },
+    {
+      "login": "VladislavParhomchik",
+      "name": "Vladislav Parhomchik",
+      "avatar_url": "https://avatars.githubusercontent.com/u/83717836?v=4",
+      "profile": "https://github.com/VladislavParhomchik",
+      "contributions": [
+        "test",
+        "review"
+      ]
+    },
+    {
+      "login": "vznesh",
+      "name": "Vignesh T.",
+      "avatar_url": "https://avatars.githubusercontent.com/u/28916792?v=4",
+      "profile": "https://github.com/vznesh",
+      "contributions": [
+        "bug"
+      ]
     }
   ],
-  "contributorsPerLine": 7
+  "contributorsPerLine": 7,
+  "skipCi": true
 }

.editorconfig

@@ -1,12 +0,0 @@
-# http://editorconfig.org
-root = true
-
-[*]
-charset = utf-8
-indent_style = space
-indent_size = 2
-end_of_line = lf
-insert_final_newline = false
-trim_trailing_whitespace = true
-
-

.github/workflows/run-tests.yml

@@ -12,12 +12,12 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x]
+        node-version: [lts/fermium]
 
     steps:
       - uses: actions/checkout@v2
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node-version }}
 

.gitpod.yml

@@ -1,8 +1,25 @@
 tasks:
-  - init: nvm install --latest-npm && npm i -g @sasjs/cli
+  - init: nvm install --lts && npm i -g @sasjs/cli
 
 image:
   file: .gitpod.dockerfile
 vscode:
   extensions:
     - sasjs.sasjs-for-vscode
+
+github:
+  prebuilds:
+    # enable for the master/default branch (defaults to true)
+    master: true
+    # enable for all branches in this repo (defaults to false)
+    branches: false
+    # enable for pull requests coming from this repo (defaults to true)
+    pullRequests: true
+    # enable for pull requests coming from forks (defaults to false)
+    pullRequestsFromForks: true
+    # add a "Review in Gitpod" button as a comment to pull requests (defaults to true)
+    addComment: true
+    # add a "Review in Gitpod" button to pull requests (defaults to false)
+    addBadge: false
+    # add a label once the prebuild is ready to pull requests (defaults to false)
+    addLabel: prebuilt-in-gitpod

.npmignore

@@ -9,3 +9,4 @@ sasjs/
 main.dox
 make_singlefile.sh
 *.md
+.all-contributorsrc

README.md

@@ -1,7 +1,6 @@
 # Macro Core
 [![npm package][npm-image]][npm-url]
 [![Github Workflow][githubworkflow-image]][githubworkflow-url]
-[![Dependency Status][dependency-image]][dependency-url]
 [![npm](https://img.shields.io/npm/dt/@sasjs/core)]()
 ![Snyk Vulnerabilities for npm package](https://img.shields.io/snyk/vulnerabilities/npm/@sasjs/core)
 [![License](https://img.shields.io/apm/l/atomic-design-ui.svg)](/LICENSE)
@@ -16,11 +15,9 @@
 [npm-url]:http://npmjs.org/package/@sasjs/core
 [githubworkflow-image]:https://github.com/sasjs/core/actions/workflows/main.yml/badge.svg
 [githubworkflow-url]:https://github.com/sasjs/core/blob/main/.github/workflows/main.yml
-[dependency-image]:https://david-dm.org/sasjs/core.svg
 [dependency-url]:https://github.com/sasjs/core/blob/main/package.json
 
 
-
 Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
 
 You can download and compile them all in just two lines of SAS code:
@@ -32,49 +29,61 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 
 Documentation: https://core.sasjs.io
 
-# Components
+## Components
 
-**base** library (SAS9/Viya)
+### **base** library (SAS9/Viya)
 
 - OS independent
 - Not metadata aware
 - No X command
 - Prefixes: _mf_, _mp_
 
-**fcmp** library (SAS9/Viya)
+#### **fcmp** library (SAS9/Viya)
 - Function and macro names are identical, except for special cases
 - Prefixes: _mcf_
 
 The fcmp macros are used to generate fcmp functions, and can be used with or
 without the `proc fcmp` wrapper.
 
-**meta** library (SAS9 only)
+### **meta** library (SAS9 only)
+
+Macros used in SAS EBI, which connect to the metadata server.
 
 - OS independent
 - Metadata aware
 - No X command
 - Prefixes: _mm_
 
-**viya** library (Viya only)
+### **server** library (@sasjs/server only)
+These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
+
+- OS independent
+- @sasjs/server aware
+- No X command
+- Prefixes: _ms_
+
+### **viya** library (Viya only)
+
+Macros used for interfacing with SAS Viya.
 
 - OS independent
 - No X command
-- Prefixes: _mv_
+- Prefixes: _mv_, _mvf_
 
-**metax** library (SAS9 only)
+### **metax** library (SAS9 only)
 
 - OS specific
 - Metadata aware
 - X command enabled
 - Prefixes: _mmw_,_mmu_,_mmx_
 
-**lua** library
+### **lua** library
 
 Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
 
 To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert your LUA into a data step with put statements, and create the macro wrapper with a `ml_` prefix.  You can then use your module in any program by running:
 
-```
+```sas
 /* compile the lua module */
 %ml_yourmodule()
 
@@ -89,7 +98,7 @@ run;
 - X command enabled
 - Prefixes: _mmw_,_mmu_,_mmx_
 
-# Installation
+## Installation
 
 First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available, eg:
 
@@ -107,9 +116,9 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 %inc mc;
 ```
 
-# Standards
+## Standards
 
-## File Properties
+### File Properties
 
 - filenames much match macro names
 - filenames must be lowercase, without spaces
@@ -117,19 +126,20 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 - one macro per file
 - prefixes:
   - _mf_ for macro functions (can be used in open code).
-  - _mp_ for macro procedures (which generate sas code)
+  - _ml_ for macros that are used to compile LUA modules
   - _mm_ for metadata macros (interface with the metadata server).
   - _mmx_ for macros that use metadata and are XCMD enabled
+  - _mp_ for macro procedures (which generate sas code)
+  - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
+  - _mv_ for macro procedures that will only work in Viya
   - _mx_ for macros that are XCMD enabled
-  - _ml_ for macros that are used to compile LUA modules
-  - _mv_ for macros that will only work in Viya
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
 - UTF-8
 
 
-## Header Properties
+### Header Properties
 
 The **Macro Core** documentation is created using [doxygen](http://www.doxygen.nl). A full list of attributes can be found [here](http://www.doxygen.nl/manual/commands.html) but the following are most relevant:
 
@@ -143,7 +153,7 @@ The **Macro Core** documentation is created using [doxygen](http://www.doxygen.n
 
 All macros must be commented in the doxygen format, to enable the [online documentation](https://core.sasjs.io).
 
-### Dependencies
+#### Dependencies
 SAS code can contain one of two types of dependency - SAS Macros, and SAS Includes.  When compiling projects using the [SASjs CLI](https://cli.sasjs.io) the doxygen header is scanned for `  @li` items under the following headers:
 
 ```sas
@@ -161,7 +171,7 @@ The CLI can then extract all the dependencies and insert as precode (SAS Macros)
 When contributing to this library, it is therefore important to ensure that all dependencies are listed in the header in this format.
 
 
-## Coding Standards
+### Coding Standards
 
 - Indentation = 2 spaces. No tabs!
 - no trailing white space
@@ -174,7 +184,7 @@ When contributing to this library, it is therefore important to ensure that all
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
 - The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
 
-# General Notes
+## General Notes
 
 - All macros should be compatible with SAS versions from support level B and above (so currently 9.2 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully (eg `%if %sysevalf(&sysver<9.3) %then %return`).
 
@@ -186,10 +196,9 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 
 
-
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-8-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-10-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -208,6 +217,8 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
   </tr>
   <tr>
     <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
+    <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
+    <td align="center"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
   </tr>
 </table>
 

base/mf_getvarformat.sas

@@ -51,7 +51,8 @@
     %end;
   %end;
   %else %do;
-    %put dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
     %return;
   %end;
 

base/mf_getvarlen.sas

@@ -43,7 +43,11 @@
       %let vlen = %str( );
     %end;
   %end;
-  %else %put dataset &libds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));

base/mf_getvarnum.sas

@@ -43,7 +43,11 @@ returns:
       %let vnum = %str( );
     %end;
   %end;
-  %else %put dataset &ds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));

base/mf_getvartype.sas

@@ -39,7 +39,11 @@ Usage:
       %let vtype = %str( );
     %end;
   %end;
-  %else %put dataset &libds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));

base/mp_abort.sas

@@ -158,7 +158,7 @@
     /* send response in SASjs JSON format */
     data _null_;
       file _webout mod lrecl=32000 encoding='utf-8';
-      length msg $32767 debug $8;
+      length msg $32767 ;
       sasdatetime=datetime();
       msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
       /* escape the quotes */
@@ -169,7 +169,7 @@
       msg=cats('"',msg,'"');
       if symexist('_debug') then debug=quote(trim(symget('_debug')));
       else debug='""';
-      if debug ge '"131"' then put '>>weboutBEGIN<<';
+      put '>>weboutBEGIN<<';
       put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
       put ',"sasjsAbort" : [{';
       put ' "MSG":' msg ;
@@ -189,23 +189,26 @@
       _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
       put ',"_PROGRAM" : ' _PROGRAM ;
       put ",""SYSCC"" : ""&syscc"" ";
-      put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+      syserrortext=quote(trim(symget('syserrortext')));
+      put ",""SYSERRORTEXT"" : " syserrortext;
       put ",""SYSHOSTNAME"" : ""&syshostname"" ";
       put ",""SYSJOBID"" : ""&sysjobid"" ";
+      put ",""SYSSCPL"" : ""&sysscpl"" ";
       put ",""SYSSITE"" : ""&syssite"" ";
       sysvlong=quote(trim(symget('sysvlong')));
       put ',"SYSVLONG" : ' sysvlong;
-      put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+      syswarningtext=quote(trim(symget('syswarningtext')));
+      put ",""SYSWARNINGTEXT"" : " syswarningtext;
       put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
       put "}" @;
-      if debug ge '"131"' then put '>>weboutEND<<';
+      put '>>weboutEND<<';
     run;
 
     %put _all_;
 
     %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
       data _null_;
-        putlog 'stpsrvset program error and syscc';
+        putlog 'stpsrvset program err and syscc';
         rc=stpsrvset('program error', 0);
         call symputx("syscc",0,"g");
       run;

base/mp_appendfile.sas

@@ -0,0 +1,57 @@
+/**
+  @file
+  @brief Append (concatenate) two or more files.
+  @details Will append one more more `appendrefs` (filerefs) to a `baseref`.
+  Uses a binary mechanism, so will work with any file type.  For that reason -
+  use with care!   And supply your own trailing carriage returns in each file..
+
+  Usage:
+
+        filename tmp1 temp;
+        filename tmp2 temp;
+        filename tmp3 temp;
+        data _null_; file tmp1; put 'base file';
+        data _null_; file tmp2; put 'append1';
+        data _null_; file tmp3; put 'append2';
+        run;
+        %mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3)
+
+
+  @param [in] baseref= Fileref of the base file (should exist)
+  @param [in] appendrefs= One or more filerefs to be appended to the base
+    fileref.  Space separated.
+
+  @version 9.2
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+
+
+**/
+
+%macro mp_appendfile(
+  baseref=0,
+  appendrefs=0
+)/*/STORE SOURCE*/;
+
+%mp_abort(iftrue= (&baseref=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Baseref NOT specified!)
+)
+%mp_abort(iftrue= (&appendrefs=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Appendrefs NOT specified!)
+)
+
+%local i;
+%do i=1 %to %sysfunc(countw(&appendrefs));
+  %mp_abort(iftrue= (&syscc>0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc)
+  )
+  %mp_binarycopy(inref=%scan(&appendrefs,&i), outref=&baseref, mode=APPEND)
+%end;
+
+%mend mp_appendfile;

base/mp_assertcolvals.sas

@@ -30,6 +30,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
+  @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
 
@@ -115,6 +116,26 @@
   select count(*) into: orig from &lib..&ds;
   quit;
 
+  %local notfound tmp1 tmp2;
+  %let tmp1=%mf_getuniquename();
+  %let tmp2=%mf_getuniquename();
+
+  /* this is a bit convoluted - but using sql outobs=10 throws warnings */
+  proc sql noprint;
+  create view &tmp1 as
+    select distinct &col
+    from &lib..&ds
+    where &col not in (
+      select &ccol from &clib..&cds
+    );
+  data &tmp2;
+    set &tmp1;
+    if _n_>10 then stop;
+  run;
+  proc sql;
+  select distinct &col  into: notfound separated by ' ' from &tmp2;
+
+
   %mp_abort(iftrue= (&syscc ne 0)
     ,mac=&sysmacroname
     ,msg=%str(syscc=&syscc after macro query)
@@ -125,7 +146,7 @@
     test_description=symget('desc');
     test_result='FAIL';
     test_comments="&sysmacroname: &lib..&ds..&col has &result values "
-      !!"not in &clib..&cds..&ccol ";
+      !!"not in &clib..&cds..&ccol.. First 10 vals:"!!symget('notfound');
   %if &test=ANYVAL %then %do;
     if &result < &orig then test_result='PASS';
   %end;

base/mp_copyfolder.sas

@@ -0,0 +1,79 @@
+/**
+  @file
+  @brief A macro to recursively copy a directory
+  @details Performs a recursive directory listing then works from top to bottom
+    copying files and creating subdirectories.
+
+  Usage:
+
+      %let rootdir=%sysfunc(pathname(work))/demo;
+      %let copydir=%sysfunc(pathname(work))/demo_copy;
+      %mf_mkdir(&rootdir)
+      %mf_mkdir(&rootdir/subdir)
+      %mf_mkdir(&rootdir/subdir/subsubdir)
+      data "&rootdir/subdir/example.sas7bdat";
+      run;
+
+      %mp_copyfolder(&rootdir,&copydir)
+
+  @param source Unquoted path to the folder to copy from.
+  @param target Unquoted path to the folder to copy to.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_isdir.sas
+  @li mf_mkdir.sas
+  @li mp_abort.sas
+  @li mp_dirlist.sas
+
+  <h4> Related Macros </h4>
+  @li mp_copyfolder.test.sas
+
+**/
+
+%macro mp_copyfolder(source,target);
+
+  %mp_abort(iftrue=(%mf_isdir(&source)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Source dir does not exist (&source))
+  )
+
+  %mf_mkdir(&target)
+
+  %mp_abort(iftrue=(%mf_isdir(&target)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Target dir could not be created (&target))
+  )
+
+  /* prep temp table */
+  %local tempds;
+  %let tempds=%mf_getuniquename();
+
+  /* recursive directory listing */
+  %mp_dirlist(path=&source,outds=work.&tempds, maxdepth=MAX)
+
+  /* create folders and copy content */
+  data _null_;
+    set work.&tempds;
+    if _n_ = 1 then dpos+sum(length(directory),2);
+    filepath2="&target/"!!substr(filepath,dpos);
+    if file_or_folder='folder' then call execute('%mf_mkdir('!!filepath2!!')');
+    else do;
+      length fref1 fref2 $8;
+      rc1=filename(fref1,filepath,'disk','recfm=n');
+      rc2=filename(fref2,filepath2,'disk','recfm=n');
+      if fcopy(fref1,fref2) ne 0 then do;
+        sysmsg=sysmsg();
+        putlog "%str(ERR)OR: Unable to copy " filepath " to " filepath2;
+        putlog sysmg=;
+      end;
+    end;
+    rc=filename(fref1);
+    rc=filename(fref2);
+  run;
+
+  /* tidy up */
+  proc sql;
+  drop table work.&tempds;
+
+%mend mp_copyfolder;

base/mp_deletefolder.sas

@@ -0,0 +1,69 @@
+/**
+  @file
+  @brief A macro to delete a directory
+  @details Will delete all folder content (including subfolder content) and
+    finally, the folder itself.
+
+  Usage:
+
+      %let rootdir=%sysfunc(pathname(work))/demo;
+      %mf_mkdir(&rootdir)
+      %mf_mkdir(&rootdir/subdir)
+      %mf_mkdir(&rootdir/subdir/subsubdir)
+      data "&rootdir/subdir/example.sas7bdat";
+      run;
+
+      %mp_deletefolder(&rootdir)
+
+  @param path Unquoted path to the folder to delete.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_isdir.sas
+  @li mp_dirlist.sas
+
+  <h4> Related Macros </h4>
+  @li mp_deletefolder.test.sas
+
+**/
+
+%macro mp_deletefolder(folder);
+  /* proceed if valid directory */
+  %if %mf_isdir(&folder)=1 %then %do;
+
+    /* prep temp table */
+    %local tempds;
+    %let tempds=%mf_getuniquename();
+
+    /* recursive directory listing */
+    %mp_dirlist(path=&folder,outds=work.&tempds, maxdepth=MAX)
+
+    /* sort descending level so can delete folder contents before folders */
+    proc sort data=work.&tempds;
+      by descending level;
+    run;
+
+    /* ensure top level folder is removed at the end */
+    proc sql;
+    insert into work.&tempds set filepath="&folder";
+
+    /* delete everything */
+    data _null_;
+      set work.&tempds end=last;
+      length fref $8;
+      rc=filename(fref,filepath);
+      rc=fdelete(fref);
+      if rc then do;
+        msg=sysmsg();
+        put "&sysmacroname:" / rc= / msg= / filepath=;
+      end;
+      rc=filename(fref);
+    run;
+
+    /* tidy up */
+    proc sql;
+    drop table work.&tempds;
+
+  %end;
+  %else %put &sysmacroname: &folder: is not a valid / accessible folder. ;
+%mend mp_deletefolder;

base/mp_dirlist.sas

@@ -3,20 +3,13 @@
   @brief Returns all files and subdirectories within a specified parent
   @details When used with getattrs=NO, is not OS specific (uses dopen / dread).
 
-  If getattrs=YES then the doptname / foptname functions are used to scan all
-  properties - any characters that are not valid in a SAS name (v7) are simply
-  stripped, and the table is transposed so theat each property is a column
-  and there is one file per row.  An attempt is made to get all properties
-  whether a file or folder, but some files/folders cannot be accessed, and so
-  not all properties can / will be populated.
-
   Credit for the rename approach:
   https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003
 
 
   usage:
 
-      %mp_dirlist(path=/some/location,outds=myTable)
+      %mp_dirlist(path=/some/location, outds=myTable, maxdepth=MAX)
 
       %mp_dirlist(outds=cwdfileprops, getattrs=YES)
 
@@ -30,11 +23,19 @@
   X CMD) do please raise an issue!
 
 
-  @param path= for which to return contents
-  @param fref= Provide a DISK engine fileref as an alternative to PATH
-  @param outds= the output dataset to create
-  @param getattrs= YES/NO (default=NO).  Uses doptname and foptname to return
-  all attributes for each file / folder.
+  @param [in] path= for which to return contents
+  @param [in] fref= Provide a DISK engine fileref as an alternative to PATH
+  @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
+    subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
+  @param [out] outds= the output dataset to create
+  @param [out] getattrs= (NO)  If getattrs=YES then the doptname / foptname
+    functions are used to scan all properties - any characters that are not
+    valid in a SAS name (v7) are simply stripped, and the table is transposed
+    so theat each property is a column and there is one file per row.  An
+    attempt is made to get all properties whether a file or folder, but some
+    files/folders cannot be accessed, and so not all properties can / will be
+    populated.
 
 
   @returns outds contains the following variables:
@@ -44,8 +45,15 @@
     - filename (just the file name)
     - ext (.extension)
     - msg (system message if any issues)
+    - level (depth of folder)
     - OS SPECIFIC variables, if <code>getattrs=</code> is used.
 
+  <h4> SAS Macros </h4>
+  @li mp_dropmembers.sas
+
+  <h4> Related Macros </h4>
+  @li mp_dirlist.test.sas
+
   @version 9.2
   @author Allan Bowe
 **/
@@ -54,14 +62,27 @@
     , fref=0
     , outds=work.mp_dirlist
     , getattrs=NO
+    , maxdepth=0
+    , level=0 /* The level of recursion to perform.  For internal use only. */
 )/*/STORE SOURCE*/;
 %let getattrs=%upcase(&getattrs)XX;
 
-data &outds(compress=no
-    keep=file_or_folder filepath filename ext msg directory
+/* temp table */
+%local out_ds;
+data;run;
+%let out_ds=%str(&syslast);
+
+/* drop main (top) table if it exists */
+%if &level=0 %then %do;
+  %mp_dropmembers(%scan(&outds,-1,.), libref=WORK)
+%end;
+
+data &out_ds(compress=no
+    keep=file_or_folder filepath filename ext msg directory level
   );
   length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
     ext $20 msg $200;
+  retain level &level;
   %if &fref=0 %then %do;
     rc = filename(fref, "&path");
   %end;
@@ -119,8 +140,8 @@ data &outds(compress=no
 run;
 
 %if %substr(&getattrs,1,1)=Y %then %do;
-  data &outds;
-    set &outds;
+  data &out_ds;
+    set &out_ds;
     length infoname infoval $60 fref $8;
     rc=filename(fref,filepath);
     drop rc infoname fid i close fref;
@@ -161,10 +182,36 @@ run;
   run;
   proc sort;
     by filepath sasname;
-  proc transpose data=&outds out=&outds(drop=_:);
+  proc transpose data=&out_ds out=&out_ds(drop=_:);
     id sasname;
     var infoval;
     by filepath file_or_folder filename ext ;
   run;
 %end;
+
+data &out_ds;
+  set &out_ds(where=(filepath ne ''));
+run;
+
+/* update main table */
+proc append base=&outds data=&out_ds;
+run;
+
+/* recursive call */
+%if &maxdepth>&level or &maxdepth=MAX %then %do;
+  data _null_;
+    set &out_ds;
+    where file_or_folder='folder';
+    length code $10000;
+    code=cats('%nrstr(%mp_dirlist(path=',filepath,",outds=&outds"
+      ,",getattrs=&getattrs,level=%eval(&level+1),maxdepth=&maxdepth))");
+    put code=;
+    call execute(code);
+  run;
+%end;
+
+/* tidy up */
+proc sql;
+drop table &out_ds;
+
 %mend mp_dirlist;

base/mp_ds2cards.sas

@@ -2,12 +2,21 @@
   @file
   @brief Create a CARDS file from a SAS dataset.
   @details Uses dataset attributes to convert all data into datalines.
-    Running the generated file will rebuild the original dataset.
+    Running the generated file will rebuild the original dataset.  Includes
+    support for large decimals, binary data, PROCESSED_DTTM columns, and
+    alternative encoding.  If the input dataset is empty, the cards file will
+    still be created.
+
+    Additional support to generate a random sample and max rows.
+
   Usage:
 
       %mp_ds2cards(base_ds=sashelp.class
+        , tgt_ds=work.class
         , cards_file= "C:\temp\class.sas"
-        , maxobs=5)
+        , showlog=NO
+        , maxobs=5
+      )
 
   TODO:
     - labelling the dataset
@@ -18,15 +27,24 @@
                   that is converted to a cards file.
   @param [in] tgt_ds= Table that the generated cards file would create.
     Optional - if omitted, will be same as BASE_DS.
-  @param [out] cards_file= Location in which to write the (.sas) cards file
-  @param [in] maxobs= to limit output to the first <code>maxobs</code>
-    observations
-  @param [in] showlog= whether to show generated cards file in the SAS log
-    (YES/NO)
-  @param [in] outencoding= provide encoding value for file statement (eg utf-8)
-  @param [in] append= If NO then will rebuild the cards file if it already
+  @param [out] cards_file= ("%sysfunc(pathname(work))/cardgen.sas") Location in
+    which to write the (.sas) cards file
+  @param [in] maxobs= (max) To limit output to the first <code>maxobs</code>
+    observations, enter an integer here.
+  @param [in] random_sample= (NO) Set to YES to generate a random sample of
+    data.  Can be quite slow.
+  @param [in] showlog= (YES) Whether to show generated cards file in the SAS
+    log.  Valid values:
+    @li YES
+    @li NO
+  @param [in] outencoding= Provide encoding value for file statement (eg utf-8)
+  @param [in] append= (NO) If NO then will rebuild the cards file if it already
     exists, otherwise will append to it.  Used by the mp_lib2cards.sas macro.
 
+  <h4> Related Macros </h4>
+  @li mp_lib2cards.sas
+  @li mp_ds2inserts.sas
+  @li mp_mdtablewrite.sas
 
   @version 9.2
   @author Allan Bowe
@@ -51,15 +69,15 @@
 %if (&tgt_ds = ) %then %let tgt_ds=&base_ds;
 %if %index(&tgt_ds,.)=0 %then %let tgt_ds=WORK.%scan(&base_ds,2,.);
 %if ("&outencoding" ne "") %then %let outencoding=encoding="&outencoding";
-%if ("&append" = "") %then %let append=;
+%if ("&append" = "" or "&append" = "NO") %then %let append=;
 %else %let append=mod;
 
 /* get varcount */
 %let nvars=0;
 proc sql noprint;
 select count(*) into: nvars from dictionary.columns
-  where libname="%scan(%upcase(&base_ds),1)"
-    and memname="%scan(%upcase(&base_ds),2)";
+  where upcase(libname)="%scan(%upcase(&base_ds),1)"
+    and upcase(memname)="%scan(%upcase(&base_ds),2)";
 %if &nvars=0 %then %do;
   %put %str(WARN)ING: Dataset &base_ds has no variables, will not be converted.;
   %return;
@@ -115,8 +133,8 @@ proc sql
 reset outobs=max;
 create table datalines1 as
   select name,type,length,varnum,format,label from dictionary.columns
-  where libname="%upcase(%scan(&base_ds,1))"
-    and memname="%upcase(%scan(&base_ds,2))";
+  where upcase(libname)="%upcase(%scan(&base_ds,1))"
+    and upcase(memname)="%upcase(%scan(&base_ds,2))";
 
 /**
   Due to long decimals cannot use best. format
@@ -137,7 +155,18 @@ data datalines_2;
       ,put(',name,',best32.-l)
       ,substrn(put(',name,',bestd32.-l),1
       ,findc(put(',name,',bestd32.-l),"0","TBK")))');
-  else dataline=name;
+  /**
+    * binary data must be converted, to store in text format.  It is identified
+    * by the presence of the $HEX keyword in the format.
+    */
+  else if upcase(format)=:'$HEX' then
+    dataline=cats('put(trim(',name,'),',format,')');
+  /**
+    * There is no easy way to store line breaks in a cards file.
+    * To discuss this, use: https://github.com/sasjs/core/issues/80
+    * Removing all nonprintables with kw (keep writeable)
+    */
+  else dataline=cats('compress(',name,', ,"kw")');
 run;
 
 proc sql noprint;
@@ -162,7 +191,8 @@ data _null_;
 
 
 /* Build input statement */
-  if type='char' then type3=':$char.';
+  if upcase(format)=:'$HEX' then type3=':'!!format;
+  else if type='char' then type3=':$char.';
   str2=put(name,$33.)||type3;
 
 
@@ -184,11 +214,12 @@ data _null_;
   file &cards_file. &outencoding lrecl=32767 termstr=nl &append;
   length __attrib $32767;
   if _n_=1 then do;
-    put '/*******************************************************************';
-    put " Datalines for %upcase(%scan(&base_ds,2)) dataset ";
-    put " Generated by %nrstr(%%)mp_ds2cards()";
-    put " Available on github.com/sasjs/core";
-    put '********************************************************************/';
+    put '/**';
+    put '  @file';
+    put "  @brief Datalines for %upcase(%scan(&base_ds,2)) dataset";
+    put "  @details Generated by %nrstr(%%)mp_ds2cards()";
+    put "  Available on github.com/sasjs/core";
+    put '**/';
     put "data &tgt_ds &indexes;";
     put "attrib ";
     %do i = 1 %to &nvars;
@@ -212,11 +243,11 @@ data _null_;
       put 'run;';
     end;
     else do;
-      put "infile cards dsd delimiter=',';";
+      put "infile cards dsd;";
       put "input ";
       %do i = 1 %to &nvars.;
         %if(%length(&&input_stmt_&i..)) %then
-          put "   &&input_stmt_&i..";
+          put "  &&input_stmt_&i..";
         ;
       %end;
       put ";";

base/mp_getcols.sas

@@ -0,0 +1,65 @@
+/**
+  @file
+  @brief Creates a dataset with column metadata.
+  @details This macro takes the `proc contents` output and "tidies it up" in the
+  following ways:
+
+    @li Blank labels are filled in with column names
+    @li Formats are reconstructed with default values
+    @li Types such as DATE / TIME / DATETIME are inferred from the formats
+
+  Example usage:
+
+      %mp_getcols(sashelp.airline,outds=work.myds)
+
+  @param ds The dataset from which to obtain column metadata
+  @param outds= (work.cols) The output dataset to create. Sample data:
+  |NAME $|LENGTH 8|VARNUM 8|LABEL $|FORMAT $49|TYPE $1 |DDTYPE $|
+  |---|---|---|---|---|---|---|
+  |AIR|8|2|international airline travel (thousands)|8.|N|NUMERIC|
+  |DATE|8|1|DATE|MONYY.|N|DATE|
+  |REGION|3|3|REGION|$3.|C|CHARACTER|
+
+  <h4> Related Macros </h4>
+  @li mf_getvarlist.sas
+  @li mm_getcols.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_getcols(ds, outds=work.cols);
+
+proc contents noprint data=&ds
+  out=_data_ (keep=name type length label varnum format:);
+run;
+data &outds(keep=name type length varnum format label ddtype);
+  set &syslast(rename=(format=format2 type=type2));
+  name=upcase(name);
+  if type2=2 then do;
+    length format $49.;
+    if format2='' then format=cats('$',length,'.');
+    else if formatl=0 then format=cats(format2,'.');
+    else format=cats(format2,formatl,'.');
+    type='C';
+    ddtype='CHARACTER';
+  end;
+  else do;
+    if format2='' then format=cats(length,'.');
+    else if formatl=0 then format=cats(format2,'.');
+    else if formatd=0 then format=cats(format2,formatl,'.');
+    else format=cats(format2,formatl,'.',formatd);
+    type='N';
+    if format=:'DATETIME' then ddtype='DATETIME';
+    else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
+      or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
+      or format=:'MONYY'
+    then ddtype='DATE';
+    else if format=:'TIME' then ddtype='TIME';
+    else ddtype='NUMERIC';
+  end;
+  if label='' then label=name;
+run;
+
+%mend mp_getcols;

base/mp_getconstraints.sas

@@ -39,21 +39,24 @@
 /* must use SQL as proc datasets does not support length changes */
 proc sql noprint;
 create table &outds as
-  select a.TABLE_CATALOG as libref
-    ,a.TABLE_NAME
+  select upcase(a.TABLE_CATALOG) as libref
+    ,upcase(a.TABLE_NAME) as TABLE_NAME
     ,a.constraint_type
     ,a.constraint_name
     ,b.column_name
   from dictionary.TABLE_CONSTRAINTS a
   left join dictionary.constraint_column_usage  b
-  on a.TABLE_CATALOG=b.TABLE_CATALOG
-    and a.TABLE_NAME=b.TABLE_NAME
+  on upcase(a.TABLE_CATALOG)=upcase(b.TABLE_CATALOG)
+    and upcase(a.TABLE_NAME)=upcase(b.TABLE_NAME)
     and a.constraint_name=b.constraint_name
-  where a.TABLE_CATALOG="&lib"
-    and b.TABLE_CATALOG="&lib"
+/**
+  * We cannot apply this clause to the underlying dictionary table.  See:
+  * https://communities.sas.com/t5/SAS-Programming/Unexpected-Where-Clause-behaviour-in-dictionary-TABLE/m-p/771554#M244867
+  */
+  where calculated libref="&lib"
   %if "&ds" ne "" %then %do;
-    and a.TABLE_NAME="&ds"
-    and b.TABLE_NAME="&ds"
+    and upcase(a.TABLE_NAME)="&ds"
+    and upcase(b.TABLE_NAME)="&ds"
   %end;
   ;
 

base/mp_getddl.sas

@@ -211,7 +211,7 @@ run;
   proc sql noprint;
   select sysvalue into: schemaactual
     from dictionary.libnames
-    where libname="&libref" and engine='SQLSVR';
+    where upcase(libname)="&libref" and engine='SQLSVR';
   %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
 
   %do x=1 %to %sysfunc(countw(&dsnlist));
@@ -304,7 +304,7 @@ run;
   proc sql noprint;
   select sysvalue into: schemaactual
     from dictionary.libnames
-    where libname="&libref" and engine='POSTGRES';
+    where upcase(libname)="&libref" and engine='POSTGRES';
   %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
   data _null_;
     file &fref mod;

base/mp_gsubfile.sas

@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Performs a text substitution on a file
+  @details Makes use of the GSUB function in LUA to perform a text substitution
+  in a file - either in-place, or writing to a new location.  The benefit of
+  using LUA is that the entire file can be loaded into a single variable,
+  thereby side stepping the 32767 character limit in a data step.
+
+  Usage:
+
+      %let file=%sysfunc(pathname(work))/file.txt;
+      %let str=replace/me;
+      %let rep=with/this;
+      data _null_;
+        file "&file";
+        put "&str";
+      run;
+      %mp_gsubfile(file=&file, patternvar=str, replacevar=rep)
+      data _null_;
+        infile "&file";
+        input;
+        list;
+      run;
+
+  @param file= (0) The file to perform the substitution on
+  @param patternvar= A macro variable containing the Lua
+    [pattern](https://www.lua.org/pil/20.2.html) to search for.  Due to the use
+    of special (magic) characters in Lua patterns, it is safer to pass the NAME
+    of the macro variable containing the string, rather than the value itself.
+  @param replacevar= The name of the macro variable containing the replacement
+    _string_.
+  @param outfile= (0) The file to write the output to. If zero, then the file
+    is overwritten in-place.
+
+  <h4> SAS Macros </h4>
+  @li ml_gsubfile.sas
+
+  <h4> Related Macros </h4>
+  @li mp_gsubfile.test.sas
+
+  @version 9.4
+  @author Allan Bowe
+**/
+
+%macro mp_gsubfile(file=0,
+  patternvar=,
+  replacevar=,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+  %ml_gsubfile()
+
+%mend mp_gsubfile;

base/mp_lockanytable.sas

@@ -0,0 +1,255 @@
+/**
+  @file
+  @brief Mechanism for locking tables to prevent parallel modifications
+  @details Uses a control table to enable ANY table to be locked for updates.
+  Only useful if every update uses the macro!   Used heavily within
+  [Data Controller for SAS](https://datacontroller.io).
+
+  The underlying table is structured as per the MAKETABLE action.
+
+  @param [in] action The action to be performed.  Valid values:
+    @li LOCK - Sets the lock flag, also confirms if a SAS lock is available
+    @li UNLOCK - Unlocks the table
+    @li MAKETABLE - creates the control table (ctl_ds)
+  @param [in] lib= (WORK) The libref of the table to lock.  Should already be
+    assigned.
+  @param [in] ds= The dataset to lock
+  @param [in] ref= A meaningful reference to enable the lock to be traced. Max
+    length is 200 characters.
+  @param [out] ctl_ds= (0) The control table which controls the actual locking.
+    Should already be assigned and available.
+  @param [in] loops= (25) Number of times to check for a lock.
+  @param [in] loop_secs= (1) Seconds to wait between each lock attempt
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_lockfilecheck.sas
+  @li mf_getuser.sas
+
+  <h4> Related Macros </h4>
+  @li mp_lockanytable.test.sas
+
+  @version 9.2
+
+**/
+
+%macro mp_lockanytable(
+  action
+  ,lib= WORK
+  ,ds=0
+  ,ref=
+  ,ctl_ds=0
+  ,loops=25
+  ,loop_secs=1
+  );
+data _null_;
+  if _n_=1 then putlog "&sysmacroname entry vars:";
+  set sashelp.vmacro;
+  where scope="&sysmacroname";
+  put name '=' value;
+run;
+
+%mp_abort(iftrue= (&ds=0 and &action ne MAKETABLE)
+  ,mac=&sysmacroname
+  ,msg=%str(dataset was not provided)
+)
+%mp_abort(iftrue= (&ctl_ds=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Control dataset was not provided)
+)
+
+/* set up lib & mac vars */
+%let lib=%upcase(&lib);
+%let ds=%upcase(&ds);
+%let action=%upcase(&action);
+%local user x trans msg abortme;
+%let user=%mf_getuser();
+%let abortme=0;
+
+%mp_abort(iftrue= (&action ne LOCK & &action ne UNLOCK & &action ne MAKETABLE)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid action (&action) provided)
+)
+
+/* if an err condition exists, exit before we even begin */
+%mp_abort(iftrue= (&syscc>0 and &action=LOCK)
+  ,mac=&sysmacroname
+  ,msg=%str(aborting due to syscc=&syscc on LOCK entry)
+)
+
+/* do not bother locking work tables (else may affect all WORK libraries) */
+%if (%upcase(&lib)=WORK or %str(&lib)=%str()) & &action ne MAKETABLE %then %do;
+  %put NOTE: WORK libraries will not be registered in the locking system.;
+  %return;
+%end;
+
+/* do not proceed if no observations can be processed */
+%mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)
+  ,mac=&sysmacroname
+  ,msg=%str(options obs = 0. syserrortext=&syserrortext)
+)
+
+%if &ACTION=LOCK %then %do;
+
+  /* abort if a SAS lock is already in place, or cannot be applied */
+  %mp_lockfilecheck(&lib..&ds)
+
+  /* next, check there is a record for this table */
+  %local record_exists_check;
+  proc sql noprint;
+  select count(*) into: record_exists_check from &ctl_ds
+    where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+  quit;
+  %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
+  %if &record_exists_check=0 %then %do;
+    data _null_;
+      putlog "&sysmacroname: adding record to lock table..";
+    run;
+
+    data ;
+      if 0 then set &ctl_ds;
+      LOCK_LIB ="&lib";
+      LOCK_DS="&ds";
+      LOCK_STATUS_CD='LOCKED';
+      LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt;
+      LOCK_USER_NM="&user";
+      LOCK_PID="&sysjobid";
+      LOCK_REF="&ref";
+      output;stop;
+    run;
+    %let trans=&syslast;
+    proc append base=&ctl_ds data=&trans;
+    run;
+  %end;
+  /* if record does exist, perform lock attempts */
+  %else %do x=1 %to &loops;
+    data _null_;
+      putlog "&sysmacroname: attempting lock (iteration &x) "@;
+      putlog "at %sysfunc(datetime(),datetime19.) ..";
+    run;
+
+    proc sql;
+    update &ctl_ds
+      set LOCK_STATUS_CD='LOCKED'
+        , LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_USER_NM="&user"
+        , LOCK_PID="&sysjobid"
+        , LOCK_REF="&ref"
+      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+    quit;
+    /**
+      * NOTE - occasionally SQL server will return an err code (deadlocked
+      * transaction).  If so, ignore it, keep calm, and carry on..
+      */
+    %if &syscc>0 %then %do;
+      data _null_;
+        putlog 'NOTE-' / 'NOTE-';
+        putlog "NOTE- &sysmacroname: Update failed. "@;
+        putlog "Resetting err conditions and re-attempting.";
+        putlog "NOTE- syscc=&syscc syserr=&syserr sqlrc=&sqlrc";
+        putlog 'NOTE-' / 'NOTE-';
+      run;
+      %let syscc=0;
+      %let sqlrc=0;
+    %end;
+
+    /* now check if the record was successfully updated */
+    %local success_check;
+    proc sql noprint;
+    select count(*) into: success_check from &ctl_ds
+      where LOCK_LIB ="&lib" and LOCK_DS="&ds"
+        and LOCK_PID="&sysjobid" and LOCK_STATUS_CD='LOCKED';
+    quit;
+    %if &success_check=0 %then %do;
+      %if &x < &loops %then %do;
+        /* pause before next check */
+        data _null_;
+          putlog 'NOTE-' / 'NOTE-';
+          putlog "NOTE- &sysmacroname: table locked, waiting "@;
+          putlog "%sysfunc(sleep(&loop_inc)) seconds.. ";
+          putlog "NOTE- (iteration &x of &loops)";
+          putlog 'NOTE-' / 'NOTE-';
+        run;
+      %end;
+      %else %do;
+        %let msg=Unable to lock &lib..&ds via &ctl_ds after &loops attempts.\n
+            Please ask your administrator to investigate!;
+        %let abortme=1;
+      %end;
+    %end;
+    %else %do;
+      data _null_;
+        putlog 'NOTE-' / 'NOTE-';
+        putlog "NOTE- &sysmacroname: Table &lib..&ds locked at "@
+        putlog " %sysfunc(datetime(),datetime19.) (iteration &x)"@;
+        putlog 'NOTE-' / 'NOTE-';
+      run;
+      %if &syscc>0 %then %do;
+        %put setting syscc(&syscc) back to 0;
+        %let syscc=0;
+      %end;
+      %let x=&loops;  /* no more iterations needed */
+    %end;
+  %end;
+%end;
+%else %if &ACTION=UNLOCK %then %do;
+  %local status;
+  proc sql noprint;
+  select LOCK_STATUS_CD into: status from &ctl_ds
+    where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+  quit;
+  %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
+  %if &status=LOCKED %then %do;
+    data _null_;
+      putlog "&sysmacroname: unlocking &lib..&ds:";
+    run;
+    proc sql;
+    update &ctl_ds
+      set LOCK_STATUS_CD='UNLOCKED'
+        , LOCK_END_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_USER_NM="&user"
+        , LOCK_PID="&sysjobid"
+        , LOCK_REF="&ref"
+      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+    quit;
+  %end;
+  %else %if &status=UNLOCKED %then %do;
+    %put %str(WAR)NING: &lib..&ds is already unlocked!;
+  %end;
+  %else %do;
+    %put NOTE: Unrecognised STATUS_CD (&status) in &ctl_ds;
+    %let abortme=1;
+  %end;
+%end;
+%else %if &action=MAKETABLE %then %do;
+  proc sql;
+  create table &ctl_ds(
+      lock_lib char(8),
+      lock_ds char(32),
+      lock_status_cd char(10) not null,
+      lock_user_nm char(100) not null ,
+      lock_ref char(200),
+      lock_pid char(10),
+      lock_start_dttm num format=E8601DT26.6,
+      lock_end_dttm num format=E8601DT26.6,
+    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+%end;
+%else %do;
+  %let msg=lock_anytable given unsupported action (&action);
+  %let abortme=1;
+%end;
+
+/* catch errors - mp_abort must be called outside of a logic block */
+%mp_abort(iftrue=(&abortme=1),
+  msg=%superq(msg),
+  mac=&sysmacroname
+)
+
+%exit_macro:
+data _null_;
+  put "&sysmacroname: Exit vars: action=&action lib=&lib ds=&ds";
+  put " syscc=&syscc sqlrc=&sqlrc syserr=&syserr";
+run;
+%mend mp_lockanytable;
+
+

base/mp_lockfilecheck.sas

@@ -0,0 +1,97 @@
+/**
+  @file
+  @brief Aborts if a SAS lock file is in place, or if one cannot be applied.
+  @details Used in conjuction with the mp_lockanytable macro.
+  More info here: https://sasensei.com/flash/24
+
+  Usage:
+
+      data work.test; a=1;run;
+      %mp_lockfilecheck(work.test)
+
+  @param [in] libds The libref.dataset for which to check the lock status
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_getattrc.sas
+
+  <h4> Related Macros </h4>
+  @li mp_lockanytable.sas
+  @li mp_lockfilecheck.test.sas
+
+  @version 9.2
+**/
+
+%macro mp_lockfilecheck(
+  libds
+)/*/STORE SOURCE*/;
+
+data _null_;
+  if _n_=1 then putlog "&sysmacroname entry vars:";
+  set sashelp.vmacro;
+  where scope="&sysmacroname";
+  put name '=' value;
+run;
+
+%mp_abort(iftrue= (&syscc>0)
+  ,mac=checklock.sas
+  ,msg=Aborting with syscc=&syscc on entry.
+)
+%mp_abort(iftrue= (&libds=0)
+  ,mac=&sysmacroname
+  ,msg=%str(libds not provided)
+)
+
+%local msg lib ds;
+%let lib=%upcase(%scan(&libds,1,.));
+%let ds=%upcase(%scan(&libds,2,.));
+
+/* do not proceed if no observations can be processed */
+%let msg=options obs = 0. syserrortext=%superq(syserrortext);
+%mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)
+  ,mac=checklock.sas
+  ,msg=%superq(msg)
+)
+
+data _null_;
+  putlog "Checking engine & member type";
+run;
+%local engine memtype;
+%let memtype=%mf_getattrc(&libds,MTYPE);
+%let engine=%mf_getattrc(&libds,ENGINE);
+
+%if &engine ne V9 and &engine ne BASE %then %do;
+  data _null_;
+    putlog "Lib &lib  is not assigned using BASE engine - uses &engine instead";
+    putlog "SAS lock check will not be performed";
+  run;
+  %return;
+%end;
+%else %if &memtype ne DATA %then %do;
+  %put NOTE: Cannot lock a VIEW!! Memtype=&memtype;
+  %return;
+%end;
+
+data _null_;
+  putlog "Engine = &engine, memtype=&memtype";
+  putlog "Attempting lock statement";
+run;
+
+lock &libds;
+
+%local abortme;
+%let abortme=0;
+%if &syscc>0 or &SYSLCKRC ne 0 %then %do;
+  %let msg=Unable to apply lock on &libds (SYSLCKRC=&SYSLCKRC syscc=&syscc);
+  %put %str(ERR)OR: &sysmacroname: &msg;
+  %let abortme=1;
+%end;
+
+lock &libds clear;
+
+%mp_abort(iftrue= (&abortme=1)
+  ,mac=&sysmacroname
+  ,msg=%superq(msg)
+)
+
+%mend mp_lockfilecheck;

base/mp_mdtablewrite.sas

@@ -14,14 +14,14 @@
   We take the standard definition one step further by embedding the informat
   in the table header row, like so:
 
-      |var1:$|var2:best.|var3:date9.|
+      |var1:$32|var2:best.|var3:date9.|
       |---|---|---|
       |some text|42|01JAN1960|
       |blah|1|31DEC1999|
 
   Which resolves to:
 
-  |var1:$|var2:best.|var3:date9.|
+  |var1:$32|var2:best.|var3:date9.|
   |---|---|---|
   |some text|42|01JAN1960|
   |blah|1|31DEC1999|

base/mp_unzip.sas

@@ -2,10 +2,10 @@
   @file mp_unzip.sas
   @brief Unzips a zip file
   @details Opens the zip file and copies all the contents to another directory.
-    It is not possible to retain permissions / timestamps, also the BOF marker
-    is lost so it cannot extract binary files.
+  It is not possible to retain permissions / timestamps, also the BOF marker
+  is lost so it cannot extract binary files.
 
-    Usage:
+  Usage:
 
       filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
@@ -16,8 +16,9 @@
   @li mf_mkdir.sas
   @li mf_getuniquefileref.sas
 
-  @param ziploc= fileref or quoted full path to zip file ("/path/to/file.zip")
-  @param outdir= directory in which to write the outputs (created if non existant)
+  @param ziploc= Fileref or quoted full path to zip file ("/path/to/file.zip")
+  @param outdir= (%sysfunc(pathname(work))) Directory in which to write the
+    outputs (created if non existant)
 
   @version 9.4
   @author Allan Bowe
@@ -35,7 +36,8 @@
 %let fname2=%mf_getuniquefileref();
 %let fname3=%mf_getuniquefileref();
 
-filename &fname1 ZIP &ziploc; * Macro variable &datazip would be read from the file*;
+/* Macro variable &datazip would be read from the file */
+filename &fname1 ZIP &ziploc;
 
 /* Read the "members" (files) from the ZIP file */
 data _data_(keep=memname isFolder);

base/mp_wait4file.sas

@@ -0,0 +1,37 @@
+/**
+  @file
+  @brief Wait until a file arrives before continuing execution
+  @details Loops with a `sleep()` command until a file arrives or the max wait
+  period expires.
+
+  @example
+
+  Wait 3 minutes OR for /tmp/flag.txt to appear
+
+    %mp_wait4file(/tmp/flag.txt , maxwait=60*3)
+
+  @param [in] file The file to wait for.  Must be provided.
+  @param [in] maxwait= (0) Number of seconds to wait.  If set to zero, will
+    loop indefinitely (to a maximum of 46 days, per SAS [documentation](
+      https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a001418809.htm
+    )).  Otherwise, execution will proceed upon sleep expiry.
+  @param [in] interval= (1) The wait period between sleeps, in seconds
+
+
+**/
+
+%macro mp_wait4file(file, maxwait=0, interval=1);
+
+%if %str(&file)=%str() %then %do;
+  %put %str(ERR)OR: file not provided;
+%end;
+
+data _null_;
+  maxwait=&maxwait;
+  if maxwait=0 then maxwait=60*60*24*46;
+  do until (fileexist("&file") or slept>maxwait );
+    slept=sum(slept,sleep(&interval,1));
+  end;
+run;
+
+%mend mp_wait4file;

base/mp_webin.sas

@@ -51,7 +51,7 @@
 /* If Viya, create temporary fileref(s) */
 %local i;
 %if %mf_getplatform()=SASVIYA %then %do i=1 %to &_webin_file_count;
-  %let _webin_fileref&i=%mf_getuniquefileref(prefix=0);
+  %let _webin_fileref&i=%mf_getuniquefileref();
   filename &&_webin_fileref&i filesrvc "&&_webin_fileuri&i";
 %end;
 

base/mp_zip.sas

@@ -16,11 +16,18 @@
   @li mp_dirlist.sas
 
   @param in= unquoted filepath, dataset of files or directory to zip
-  @param type= FILE, DATASET, DIRECTORY. (FILE / DATASET not ready yet)
-  @param outname= output file to create, without .zip extension
-  @param outpath= location for output zip file
+  @param type= (FILE) Valid values:
+    @li FILE - /full/path/and/filename.extension to a particular file
+    @li DATASET - a dataset containing a list of files to zip (see `incol`)
+    @li DIRECTORY - a directory to zip
+  @param outname= (FILE) Output file to create, _without_ .zip extension
+  @param outpath= (%sysfunc(pathname(WORK))) Parent folder for output zip file
   @param incol= if DATASET input, say which column contains the filepath
 
+  <h4> Related Macros </h4>
+  @li mp_unzip.sas
+  @li mp_zip.test.sas
+
   @version 9.2
   @author Allan Bowe
   @source https://github.com/sasjs/core
@@ -51,9 +58,9 @@ ods package open nopf;
     set &ds;
     length __command $4000;
     if file_or_folder='file';
-    command=cats('ods package add file="',filepath
+    __command=cats('ods package add file="',filepath
       ,'" mimetype="application/x-compress";');
-    call execute(command);
+    call execute(__command);
   run;
   /* tidy up */
   %if &debug=NO %then %do;
@@ -64,11 +71,10 @@ ods package open nopf;
   data _null_;
     set &in;
     length __command $4000;
-    command=cats('ods package add file="',&incol
+    __command=cats('ods package add file="',&incol
       ,'" mimetype="application/x-compress";');
-    call execute(command);
+    call execute(__command);
   run;
-  ods package add file="&in" mimetype="application/x-compress";
 %end;
 
 

build.py

@@ -22,7 +22,7 @@ for file in files:
         for line in infile:
           ml.write("  put '" + line.rstrip().replace("'","''") + " ';\n")
     ml.write("run;\n\n")
-    ml.write("%inc \"%sysfunc(pathname(work))/" + name + ".lua\";\n\n")
+    ml.write("%inc \"%sysfunc(pathname(work))/" + name + ".lua\" /source2;\n\n")
     ml.write("%mend " + name + ";\n")
 
 ml.close()
@@ -84,7 +84,7 @@ options noquotelenmax;
 """
 f = open('all.sas', "w")             # r / r+ / rb / rb+ / w / wb
 f.write(header)
-folders=['base','meta','metax','viya','lua','fcmp']
+folders=['base','meta','metax','server','viya','lua','fcmp']
 for folder in folders:
     filenames = [fn for fn in Path('./' + folder).iterdir() if fn.match("*.sas")]
     filenames.sort()

fcmp/mcf_stpsrv_header.sas

@@ -54,6 +54,9 @@
   @param [out] pkg= (utils) The output package in which to create the function.
     Uses a 3 part format:  libref.catalog.package
 
+  <h4> SAS Macros </h4>
+  @li mf_existfunction.sas
+
 **/
 
 %macro mcf_stpsrv_header(wrap=NO
@@ -63,6 +66,8 @@
   ,pkg=UTILS
 )/*/STORE SOURCE*/;
 
+%if %mf_existfunction(stpsrv_header)=1 %then %return;
+
 %if &wrap=YES  %then %do;
   proc fcmp outcat=&lib..&cat..&pkg;
 %end;

lua/gsubfile.lua

@@ -0,0 +1,25 @@
+local fpath, outpath, file, fcontent
+
+-- configure in / out paths
+fpath = sas.symget("file")
+outpath = sas.symget("outfile")
+if ( outpath == 0 )
+then
+   outpath=fpath
+end
+
+-- open file and perform the substitution
+file = io.open(fpath,"r")
+fcontent = file:read("*all")
+file:close()
+fcontent = string.gsub(
+  fcontent,
+  sas.symget(sas.symget("patternvar")),
+  sas.symget(sas.symget("replacevar"))
+)
+
+-- write the file back out
+file = io.open(outpath, "w+")
+io.output(file)
+io.write(fcontent)
+io.close(file)

lua/ml_gsubfile.sas

@@ -0,0 +1,44 @@
+/**
+  @file ml_gsubfile.sas
+  @brief Compiles the gsubfile.lua lua file
+  @details Writes gsubfile.lua to the work directory
+  and then includes it.
+  Usage:
+
+      %ml_gsubfile()
+
+**/
+
+%macro ml_gsubfile();
+data _null_;
+  file "%sysfunc(pathname(work))/ml_gsubfile.lua";
+  put 'local fpath, outpath, file, fcontent ';
+  put ' ';
+  put '-- configure in / out paths ';
+  put 'fpath = sas.symget("file") ';
+  put 'outpath = sas.symget("outfile") ';
+  put 'if ( outpath == 0 ) ';
+  put 'then ';
+  put '   outpath=fpath ';
+  put 'end ';
+  put ' ';
+  put '-- open file and perform the substitution ';
+  put 'file = io.open(fpath,"r") ';
+  put 'fcontent = file:read("*all") ';
+  put 'file:close() ';
+  put 'fcontent = string.gsub( ';
+  put '  fcontent, ';
+  put '  sas.symget(sas.symget("patternvar")), ';
+  put '  sas.symget(sas.symget("replacevar")) ';
+  put ') ';
+  put ' ';
+  put '-- write the file back out ';
+  put 'file = io.open(outpath, "w+") ';
+  put 'io.output(file) ';
+  put 'io.write(fcontent) ';
+  put 'io.close(file) ';
+run;
+
+%inc "%sysfunc(pathname(work))/ml_gsubfile.lua" /source2;
+
+%mend ml_gsubfile;

lua/ml_json.sas

@@ -389,6 +389,6 @@ data _null_;
   put '-- JSON.LUA ENDS HERE ';
 run;
 
-%inc "%sysfunc(pathname(work))/ml_json.lua";
+%inc "%sysfunc(pathname(work))/ml_json.lua" /source2;
 
 %mend ml_json;

main.dox

@@ -55,7 +55,18 @@
 
  */
 
- /*! \dir Tests
+/*! \dir server
+ *  \brief Macros used with [sasjs/server](https://server.sasjs.io)
+ *  \details These macros have the following attributes:
+
+        * OS independent
+        * sasjs/server aware
+        * No X command
+        * Prefixes: _ms_
+
+ */
+
+/*! \dir tests
  *  \brief SASjs Tests
  *  \details These folders contain the macro tests.  They are first compiled
    and deployed (sasjs cbd) then executed (sasjs test).
@@ -72,7 +83,7 @@
 
  */
 
- /*! \dir lua
+/*! \dir lua
  *  \brief Lua macros
  *  \details These macros have the following attributes:
 

make_singlefile.sh

@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-
-# Concatenate all macros into a single file
-
-OUTFILE='./macrocore.sas'
-
-cat > $OUTFILE <<'EOL'
-/**
-  @file
-  @brief Auto-generated file
-  @details
-    This file contains all the macros in a single file - which means it can be
-    'included' in SAS with just 2 lines of code:
-
-      filename mc url
-        "https://raw.githubusercontent.com/sasjs/core/main/macrocore.sas";
-      %inc mc;
-
-    The `build.sh` file in the https://github.com/sasjs/core repo
-    is used to create this file.
-
-  @author Allan Bowe
-**/
-EOL
-
-cat base/* >> $OUTFILE
-cat meta/* >> $OUTFILE
-cat metax/* >> $OUTFILE
-cat viya/* >> $OUTFILE

meta/mm_createwebservice.sas

@@ -385,6 +385,7 @@ data _null_;
   put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
+  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';

meta/mm_webout.sas

@@ -155,6 +155,7 @@
     put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSJOBID"" : ""&sysjobid"" ";
+    put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;

package-lock.json

@@ -7,43 +7,42 @@
       "name": "@sasjs/core",
       "license": "MIT",
       "devDependencies": {
-        "@sasjs/cli": "^2.37.8"
+        "@sasjs/cli": "^2.39.0"
       }
     },
     "node_modules/@sasjs/adapter": {
-      "version": "2.10.4",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.10.4.tgz",
-      "integrity": "sha512-9b3WGZUzRgszZVyPfrabSx6TycL4JFFXQR/RQKsFCDDwT8UgLGfJ4JmgaGCSmGCcsqML/Q41QipdgWu1M/QA3g==",
+      "version": "2.12.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.12.0.tgz",
+      "integrity": "sha512-zzIuohhR8KUDl3DfIFOW38gv3LADPnOBCLOvLoKu4hH5R/UJDkjZ/Gdgc8B35vI7aOprYOLK/T5D/Z44OaTkqw==",
       "dev": true,
+      "hasInstallScript": true,
       "dependencies": {
-        "@sasjs/utils": "^2.27.1",
-        "axios": "^0.21.1",
+        "@sasjs/utils": "^2.32.0",
+        "axios": "^0.21.4",
         "axios-cookiejar-support": "^1.0.1",
         "form-data": "^4.0.0",
         "https": "^1.0.0",
         "tough-cookie": "^4.0.0"
-      },
-      "engines": {
-        "node": ">=15"
       }
     },
     "node_modules/@sasjs/cli": {
-      "version": "2.37.8",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.37.8.tgz",
-      "integrity": "sha512-V1FGK0ByS5v5+xOX5J1rlQDOXSA37YLL18etEUQOmWK5X9R5tbK4LPIx/pEjLyfZcCHddrm+0nGzRroC715Ilg==",
+      "version": "2.39.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.39.0.tgz",
+      "integrity": "sha512-n2LcU4n0QCEbUpXqZnBz/Ey5Td0nMJmgJpZRymMGfYEM0Y0x/CeXemd+kXHPjUvgQ+FX+SQzcvUQTEY/YlT4hA==",
       "dev": true,
+      "hasInstallScript": true,
       "dependencies": {
-        "@sasjs/adapter": "2.10.4",
-        "@sasjs/core": "2.42.0",
+        "@sasjs/adapter": "2.12.0",
+        "@sasjs/core": "2.45.2",
         "@sasjs/lint": "1.11.2",
-        "@sasjs/utils": "2.28.0",
+        "@sasjs/utils": "2.32.0",
         "chalk": "4.1.2",
-        "csv-stringify": "5.6.2",
+        "csv-stringify": "5.6.5",
         "dotenv": "10.0.0",
         "esm": "3.2.25",
         "find": "0.3.0",
         "get-installed-path": "4.0.8",
-        "js-base64": "3.6.1",
+        "js-base64": "3.7.2",
         "jsdom": "17.0.0",
         "jwt-decode": "3.1.2",
         "lodash.groupby": "4.6.0",
@@ -53,16 +52,16 @@
         "rimraf": "3.0.2",
         "shelljs": "0.8.4",
         "xml": "1.0.1",
-        "yargs": "17.1.1"
+        "yargs": "17.2.1"
       },
       "bin": {
         "sasjs": "build/index.js"
       }
     },
     "node_modules/@sasjs/core": {
-      "version": "2.42.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.42.0.tgz",
-      "integrity": "sha512-EISPPCEv+vB3Nqp03NUaIvMZwEnggzGJeEpVfm2Sb504ySN1I2xiJ8bOHXIzvvYP5N/V9X8bXe1raLYnnt8HGA==",
+      "version": "2.45.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.45.2.tgz",
+      "integrity": "sha512-tg+oZCD8GFMXsg+vDL66LMnyU+t151Hrqd7yl+pMXH2qwkA14N/j6QdkTBZOchskqOA/3PnpOlAZN/xxMW2gdg==",
       "dev": true
     },
     "node_modules/@sasjs/lint": {
@@ -75,24 +74,23 @@
       }
     },
     "node_modules/@sasjs/utils": {
-      "version": "2.28.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.28.0.tgz",
-      "integrity": "sha512-aZAbBDSjvW2TCY1lkuGqqiyWtR4W8GFpCNCUKa72VIkl4zwTE/pnSHQ+v8QilGDSHSPbPEsJGaBaldMGwoI12w==",
+      "version": "2.32.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.32.0.tgz",
+      "integrity": "sha512-xnvdEuI4PhTtulcdDEIMK7IxVj9bOMU1JTnxRuSEKWcsclY9P9Fw3cnMOOEgXCDffrOPn3f54DP7Wb1GXd+f8g==",
       "dev": true,
+      "hasInstallScript": true,
       "dependencies": {
         "@types/fs-extra": "^9.0.11",
         "@types/prompts": "^2.0.13",
         "chalk": "^4.1.1",
         "cli-table": "^0.3.6",
         "consola": "^2.15.0",
+        "csv-stringify": "^5.6.5",
         "fs-extra": "^10.0.0",
         "jwt-decode": "^3.1.2",
         "prompts": "^2.4.1",
         "rimraf": "^3.0.2",
         "valid-url": "^1.0.9"
-      },
-      "engines": {
-        "node": ">=15"
       }
     },
     "node_modules/@tootallnate/once": {
@@ -197,9 +195,9 @@
       }
     },
     "node_modules/ansi-regex": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
-      "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
       "dev": true,
       "engines": {
         "node": ">=8"
@@ -478,9 +476,9 @@
       "dev": true
     },
     "node_modules/csv-stringify": {
-      "version": "5.6.2",
-      "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-5.6.2.tgz",
-      "integrity": "sha512-n3rIVbX6ylm1YsX2NEug9IaPV8xRnT+9/NNZbrA/bcHgOSSeqtWla6XnI/xmyu57wIw+ASCAoX1oM6EZtqJV0A==",
+      "version": "5.6.5",
+      "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-5.6.5.tgz",
+      "integrity": "sha512-PjiQ659aQ+fUTQqSrd1XEDnOr52jh30RBurfzkscaE2tPaFsDH5wOAHJiw8XAHphRknCwMUE9KRayc4K/NbO8A==",
       "dev": true
     },
     "node_modules/data-urls": {
@@ -679,9 +677,9 @@
       }
     },
     "node_modules/follow-redirects": {
-      "version": "1.14.3",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.3.tgz",
-      "integrity": "sha512-3MkHxknWMUtb23apkgz/83fDoe+y+qr0TdgacGIA7bew+QLBo3vdgEN2xEsuXNivpFy4CyDhBBZnNZOtalmenw==",
+      "version": "1.14.5",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.5.tgz",
+      "integrity": "sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA==",
       "dev": true,
       "funding": [
         {
@@ -1031,9 +1029,9 @@
       "dev": true
     },
     "node_modules/js-base64": {
-      "version": "3.6.1",
-      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.6.1.tgz",
-      "integrity": "sha512-Frdq2+tRRGLQUIQOgsIGSCd1VePCS2fsddTG5dTCqR0JHgltXWfsxnY0gIXPoMeRmdom6Oyq+UMOFg5suduOjQ==",
+      "version": "3.7.2",
+      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.2.tgz",
+      "integrity": "sha512-NnRs6dsyqUXejqk/yv2aiXlAvOs56sLkX6nUdeaNezI5LFFLlsZjOThmwnrcwh5ZZRwZlCMnVAY3CvhIhoVEKQ==",
       "dev": true
     },
     "node_modules/jsdom": {
@@ -1806,9 +1804,9 @@
       }
     },
     "node_modules/yargs": {
-      "version": "17.1.1",
-      "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.1.1.tgz",
-      "integrity": "sha512-c2k48R0PwKIqKhPMWjeiF6y2xY/gPMUlro0sgxqXpbOIohWiLNXWslsootttv7E1e73QPAMQSg5FeySbVcpsPQ==",
+      "version": "17.2.1",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.2.1.tgz",
+      "integrity": "sha512-XfR8du6ua4K6uLGm5S6fA+FIJom/MdJcFNVY8geLlp2v8GYbOXD4EB1tPNZsRn4vBzKGMgb5DRZMeWuFc2GO8Q==",
       "dev": true,
       "dependencies": {
         "cliui": "^7.0.2",
@@ -1835,13 +1833,13 @@
   },
   "dependencies": {
     "@sasjs/adapter": {
-      "version": "2.10.4",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.10.4.tgz",
-      "integrity": "sha512-9b3WGZUzRgszZVyPfrabSx6TycL4JFFXQR/RQKsFCDDwT8UgLGfJ4JmgaGCSmGCcsqML/Q41QipdgWu1M/QA3g==",
+      "version": "2.12.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.12.0.tgz",
+      "integrity": "sha512-zzIuohhR8KUDl3DfIFOW38gv3LADPnOBCLOvLoKu4hH5R/UJDkjZ/Gdgc8B35vI7aOprYOLK/T5D/Z44OaTkqw==",
       "dev": true,
       "requires": {
-        "@sasjs/utils": "^2.27.1",
-        "axios": "^0.21.1",
+        "@sasjs/utils": "^2.32.0",
+        "axios": "^0.21.4",
         "axios-cookiejar-support": "^1.0.1",
         "form-data": "^4.0.0",
         "https": "^1.0.0",
@@ -1849,22 +1847,22 @@
       }
     },
     "@sasjs/cli": {
-      "version": "2.37.8",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.37.8.tgz",
-      "integrity": "sha512-V1FGK0ByS5v5+xOX5J1rlQDOXSA37YLL18etEUQOmWK5X9R5tbK4LPIx/pEjLyfZcCHddrm+0nGzRroC715Ilg==",
+      "version": "2.39.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.39.0.tgz",
+      "integrity": "sha512-n2LcU4n0QCEbUpXqZnBz/Ey5Td0nMJmgJpZRymMGfYEM0Y0x/CeXemd+kXHPjUvgQ+FX+SQzcvUQTEY/YlT4hA==",
       "dev": true,
       "requires": {
-        "@sasjs/adapter": "2.10.4",
-        "@sasjs/core": "2.42.0",
+        "@sasjs/adapter": "2.12.0",
+        "@sasjs/core": "2.45.2",
         "@sasjs/lint": "1.11.2",
-        "@sasjs/utils": "2.28.0",
+        "@sasjs/utils": "2.32.0",
         "chalk": "4.1.2",
-        "csv-stringify": "5.6.2",
+        "csv-stringify": "5.6.5",
         "dotenv": "10.0.0",
         "esm": "3.2.25",
         "find": "0.3.0",
         "get-installed-path": "4.0.8",
-        "js-base64": "3.6.1",
+        "js-base64": "3.7.2",
         "jsdom": "17.0.0",
         "jwt-decode": "3.1.2",
         "lodash.groupby": "4.6.0",
@@ -1874,13 +1872,13 @@
         "rimraf": "3.0.2",
         "shelljs": "0.8.4",
         "xml": "1.0.1",
-        "yargs": "17.1.1"
+        "yargs": "17.2.1"
       }
     },
     "@sasjs/core": {
-      "version": "2.42.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.42.0.tgz",
-      "integrity": "sha512-EISPPCEv+vB3Nqp03NUaIvMZwEnggzGJeEpVfm2Sb504ySN1I2xiJ8bOHXIzvvYP5N/V9X8bXe1raLYnnt8HGA==",
+      "version": "2.45.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.45.2.tgz",
+      "integrity": "sha512-tg+oZCD8GFMXsg+vDL66LMnyU+t151Hrqd7yl+pMXH2qwkA14N/j6QdkTBZOchskqOA/3PnpOlAZN/xxMW2gdg==",
       "dev": true
     },
     "@sasjs/lint": {
@@ -1893,9 +1891,9 @@
       }
     },
     "@sasjs/utils": {
-      "version": "2.28.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.28.0.tgz",
-      "integrity": "sha512-aZAbBDSjvW2TCY1lkuGqqiyWtR4W8GFpCNCUKa72VIkl4zwTE/pnSHQ+v8QilGDSHSPbPEsJGaBaldMGwoI12w==",
+      "version": "2.32.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.32.0.tgz",
+      "integrity": "sha512-xnvdEuI4PhTtulcdDEIMK7IxVj9bOMU1JTnxRuSEKWcsclY9P9Fw3cnMOOEgXCDffrOPn3f54DP7Wb1GXd+f8g==",
       "dev": true,
       "requires": {
         "@types/fs-extra": "^9.0.11",
@@ -1903,6 +1901,7 @@
         "chalk": "^4.1.1",
         "cli-table": "^0.3.6",
         "consola": "^2.15.0",
+        "csv-stringify": "^5.6.5",
         "fs-extra": "^10.0.0",
         "jwt-decode": "^3.1.2",
         "prompts": "^2.4.1",
@@ -1993,9 +1992,9 @@
       }
     },
     "ansi-regex": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
-      "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
       "dev": true
     },
     "ansi-styles": {
@@ -2198,9 +2197,9 @@
       }
     },
     "csv-stringify": {
-      "version": "5.6.2",
-      "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-5.6.2.tgz",
-      "integrity": "sha512-n3rIVbX6ylm1YsX2NEug9IaPV8xRnT+9/NNZbrA/bcHgOSSeqtWla6XnI/xmyu57wIw+ASCAoX1oM6EZtqJV0A==",
+      "version": "5.6.5",
+      "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-5.6.5.tgz",
+      "integrity": "sha512-PjiQ659aQ+fUTQqSrd1XEDnOr52jh30RBurfzkscaE2tPaFsDH5wOAHJiw8XAHphRknCwMUE9KRayc4K/NbO8A==",
       "dev": true
     },
     "data-urls": {
@@ -2347,9 +2346,9 @@
       }
     },
     "follow-redirects": {
-      "version": "1.14.3",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.3.tgz",
-      "integrity": "sha512-3MkHxknWMUtb23apkgz/83fDoe+y+qr0TdgacGIA7bew+QLBo3vdgEN2xEsuXNivpFy4CyDhBBZnNZOtalmenw==",
+      "version": "1.14.5",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.5.tgz",
+      "integrity": "sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA==",
       "dev": true
     },
     "form-data": {
@@ -2600,9 +2599,9 @@
       "dev": true
     },
     "js-base64": {
-      "version": "3.6.1",
-      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.6.1.tgz",
-      "integrity": "sha512-Frdq2+tRRGLQUIQOgsIGSCd1VePCS2fsddTG5dTCqR0JHgltXWfsxnY0gIXPoMeRmdom6Oyq+UMOFg5suduOjQ==",
+      "version": "3.7.2",
+      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.2.tgz",
+      "integrity": "sha512-NnRs6dsyqUXejqk/yv2aiXlAvOs56sLkX6nUdeaNezI5LFFLlsZjOThmwnrcwh5ZZRwZlCMnVAY3CvhIhoVEKQ==",
       "dev": true
     },
     "jsdom": {
@@ -3192,9 +3191,9 @@
       "dev": true
     },
     "yargs": {
-      "version": "17.1.1",
-      "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.1.1.tgz",
-      "integrity": "sha512-c2k48R0PwKIqKhPMWjeiF6y2xY/gPMUlro0sgxqXpbOIohWiLNXWslsootttv7E1e73QPAMQSg5FeySbVcpsPQ==",
+      "version": "17.2.1",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.2.1.tgz",
+      "integrity": "sha512-XfR8du6ua4K6uLGm5S6fA+FIJom/MdJcFNVY8geLlp2v8GYbOXD4EB1tPNZsRn4vBzKGMgb5DRZMeWuFc2GO8Q==",
       "dev": true,
       "requires": {
         "cliui": "^7.0.2",

package.json

@@ -33,6 +33,6 @@
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "^2.37.8"
+    "@sasjs/cli": "^2.39.0"
   }
 }

sasjs/sasjsconfig.json

@@ -5,6 +5,7 @@
     "fcmp",
     "meta",
     "metax",
+    "server",
     "viya",
     "lua",
     "tests/crossplatform"
@@ -50,7 +51,22 @@
       "appLoc": "/Shared Data/temp/macrocore",
       "macroFolders": [
         "tests/sas9only"
-      ]
+      ],
+      "deployConfig": {
+        "deployServicePack": true
+      }
+    },
+    {
+      "name": "server",
+      "serverUrl": "https://sas.analytium.co.uk:5001",
+      "serverType": "SASJS",
+      "appLoc": "/Shared Data/temp/macrocore",
+      "macroFolders": [
+        "tests/serveronly"
+      ],
+      "deployConfig": {
+        "deployServicePack": true
+      }
     },
     {
       "name": "docsonly",

server/ms_webout.sas

@@ -0,0 +1,171 @@
+/**
+  @file
+  @brief Send data to/from @sasjs/server
+  @details This macro should be added to the start of each web service,
+  **immediately** followed by a call to:
+
+        %ms_webout(FETCH)
+
+    This will read all the input data and create same-named SAS datasets in the
+    WORK library.  You can then insert your code, and send data back using the
+    following syntax:
+
+        data some datasets; * make some data ;
+          retain some columns;
+        run;
+
+        %ms_webout(OPEN)
+        %ms_webout(ARR,some)  * Array format, fast, suitable for large tables ;
+        %ms_webout(OBJ,datasets) * Object format, easier to work with ;
+        %ms_webout(CLOSE)
+
+
+  @param action Either FETCH, OPEN, ARR, OBJ or CLOSE
+  @param ds The dataset to send back to the frontend
+  @param dslabel= value to use instead of the real name for sending to JSON
+  @param fmt=(Y) Set to N to send back unformatted values
+  @param fref=(_webout) The fileref to which to write the JSON
+
+  <h4> SAS Macros </h4>
+  @li mp_jsonout.sas
+  @li mf_getuser.sas
+
+  <h4> Related Macros </h4>
+  @li mv_webout.sas
+  @li mm_webout.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y);
+%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
+  sasjs_tables;
+
+%local i tempds;
+%let action=%upcase(&action);
+
+%if &action=FETCH %then %do;
+  %if %str(&_debug) ge 131 %then %do;
+    options mprint notes mprintnest;
+  %end;
+  %let _webin_file_count=%eval(&_webin_file_count+0);
+  /* now read in the data */
+  %do i=1 %to &_webin_file_count;
+    %if &_webin_file_count=1 %then %do;
+      %let _webin_fileref1=&_webin_fileref;
+      %let _webin_name1=&_webin_name;
+    %end;
+    data _null_;
+      infile &&_webin_fileref&i termstr=crlf;
+      input;
+      call symputx('input_statement',_infile_);
+      putlog "&&_webin_name&i input statement: "  _infile_;
+      stop;
+    data &&_webin_name&i;
+      infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding='utf-8';
+      input &input_statement;
+      %if %str(&_debug) ge 131 %then %do;
+        if _n_<20 then putlog _infile_;
+      %end;
+    run;
+    %let sasjs_tables=&sasjs_tables &&_webin_name&i;
+  %end;
+%end;
+
+%else %if &action=OPEN %then %do;
+  /* fix encoding */
+  OPTIONS NOBOMFILE;
+
+  /* setup json */
+  data _null_;file &fref encoding='utf-8';
+  %if %str(&_debug) ge 131 %then %do;
+    put '>>weboutBEGIN<<';
+  %end;
+    put '{"SYSDATE" : "' "&SYSDATE" '"';
+    put ',"SYSTIME" : "' "&SYSTIME" '"';
+  run;
+
+%end;
+
+%else %if &action=ARR or &action=OBJ %then %do;
+  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
+    ,engine=DATASTEP,dbg=%str(&_debug)
+  )
+%end;
+%else %if &action=CLOSE %then %do;
+  %if %str(&_debug) ge 131 %then %do;
+    /* if debug mode, send back first 10 records of each work table also */
+    options obs=10;
+    data;run;%let tempds=%scan(&syslast,2,.);
+    ods output Members=&tempds;
+    proc datasets library=WORK memtype=data;
+    %local wtcnt;%let wtcnt=0;
+    data _null_;
+      set &tempds;
+      if not (upcase(name) =:"DATA"); /* ignore temp datasets */
+      i+1;
+      call symputx('wt'!!left(i),name,'l');
+      call symputx('wtcnt',i,'l');
+    data _null_; file &fref mod encoding='utf-8';
+      put ",""WORK"":{";
+    %do i=1 %to &wtcnt;
+      %let wt=&&wt&i;
+      proc contents noprint data=&wt
+        out=_data_ (keep=name type length format:);
+      run;%let tempds=%scan(&syslast,2,.);
+      data _null_; file &fref mod encoding='utf-8';
+        dsid=open("WORK.&wt",'is');
+        nlobs=attrn(dsid,'NLOBS');
+        nvars=attrn(dsid,'NVARS');
+        rc=close(dsid);
+        if &i>1 then put ','@;
+        put " ""&wt"" : {";
+        put '"nlobs":' nlobs;
+        put ',"nvars":' nvars;
+      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
+      data _null_; file &fref mod encoding='utf-8';
+        put "}";
+    %end;
+    data _null_; file &fref mod encoding='utf-8';
+      put "}";
+    run;
+  %end;
+  /* close off json */
+  data _null_;file &fref mod encoding='utf-8';
+    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
+    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
+    put ",""_DEBUG"" : ""&_debug"" ";
+    put ',"_PROGRAM" : ' _PROGRAM ;
+    put ",""SYSCC"" : ""&syscc"" ";
+    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    SYSHOSTINFOLONG=quote(trim(symget('SYSHOSTINFOLONG')));
+    put ',"SYSHOSTINFOLONG" : ' SYSHOSTINFOLONG;
+    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
+    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
+    put ",""SYSPROCESSNAME"" : ""&SYSPROCESSNAME"" ";
+    put ",""SYSJOBID"" : ""&sysjobid"" ";
+    put ",""SYSSCPL"" : ""&sysscpl"" ";
+    put ",""SYSSITE"" : ""&syssite"" ";
+    put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" ";
+    sysvlong=quote(trim(symget('sysvlong')));
+    put ',"SYSVLONG" : ' sysvlong;
+    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+    autoexec=quote(trim(getoption('autoexec')));
+    put ',"AUTOEXEC" : ' autoexec;
+    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
+    memsize=quote(cats(memsize));
+    put ',"MEMSIZE" : ' memsize;
+    put "}" @;
+  %if %str(&_debug) ge 131 %then %do;
+    put '>>weboutEND<<';
+  %end;
+  run;
+%end;
+
+%mend ms_webout;

tests/crossplatform/mp_appendfile.test.sas

@@ -0,0 +1,41 @@
+/**
+  @file
+  @brief Testing mp_appendfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_appendfile.sas
+  @li mp_assert.sas
+
+**/
+
+
+filename tmp1 temp;
+filename tmp2 temp;
+filename tmp3 temp;
+data _null_; file tmp1; put 'base file';
+data _null_; file tmp2; put 'append1';
+data _null_; file tmp3; put 'append2';
+run;
+%mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3)
+data _null_;
+  infile tmp1;
+  input;
+  put _infile_;
+  call symputx(cats('check',_n_),_infile_);
+run;
+%global check1 check2 check3;
+%mp_assert(
+  iftrue=("&check1"="base file"),
+  desc=Line 1 of file tmp1 is correct,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&check2"="append1"),
+  desc=Line 2 of file tmp1 is correct,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&check3"="append2"),
+  desc=Line 3 of file tmp1 is correct,
+  outds=work.test_results
+)

tests/crossplatform/mp_copyfolder.test.sas

@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief Testing mp_copyfolder.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_copyfolder.sas
+  @li mf_mkdir.sas
+  @li mf_nobs.sas
+  @li mp_assert.sas
+  @li mp_dirlist.sas
+
+**/
+
+/**
+  * make a directory structure
+  */
+
+%let root=%sysfunc(pathname(work))/top;
+%mf_mkdir(&root)
+%mf_mkdir(&root/a)
+%mf_mkdir(&root/b)
+%mf_mkdir(&root/a/d)
+%mf_mkdir(&root/a/e)
+%mf_mkdir(&root/a/e/f)
+data "&root/a/e/f/ds1.sas7bdat";x=1;
+data "&root/a/e/ds2.sas7bdat";x=1;
+data "&root/a/ds3.sas7bdat";x=1;
+run;
+
+%mp_dirlist(path=&root, outds=myTable, maxdepth=MAX)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable)=8),
+  desc=Temp data successfully created,
+  outds=work.test_results
+)
+
+/**
+  * copy it
+  */
+%let newfolder=%sysfunc(pathname(work))/new;
+%mp_copyfolder(&root,&newfolder)
+
+%mp_dirlist(path=&newfolder, outds=work.myTable2, maxdepth=MAX)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable2)=8),
+  desc=Folder successfully copied,
+  outds=work.test_results
+)
+
+

tests/crossplatform/mp_deletefolder.test.sas

@@ -0,0 +1,50 @@
+/**
+  @file
+  @brief Testing mp_deletefolder.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_deletefolder.sas
+  @li mf_mkdir.sas
+  @li mf_nobs.sas
+  @li mp_assert.sas
+  @li mp_dirlist.sas
+
+**/
+
+/**
+  * make a directory structure
+  */
+
+%let root=%sysfunc(pathname(work))/top;
+%mf_mkdir(&root)
+%mf_mkdir(&root/a)
+%mf_mkdir(&root/b)
+%mf_mkdir(&root/a/d)
+%mf_mkdir(&root/a/e)
+%mf_mkdir(&root/a/e/f)
+data "&root/a/e/f/ds1.sas7bdat";
+  x=1;
+run;
+
+%mp_dirlist(path=&root, outds=myTable, maxdepth=MAX)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable)=6),
+  desc=Temp data successfully created,
+  outds=work.test_results
+)
+
+%mp_deletefolder(&root/a)
+
+%mp_dirlist(path=&root, outds=work.myTable2, maxdepth=MAX)
+
+data _null_;
+  set work.mytable2;
+  putlog (_all_)(=);
+run;
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable2)=1),
+  desc=Subfolder and contents successfully deleted,
+  outds=work.test_results
+)

tests/crossplatform/mp_dirlist.test.sas

@@ -0,0 +1,50 @@
+/**
+  @file
+  @brief Testing mp_dirlist.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mf_mkdir.sas
+  @li mp_dirlist.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * make a directory structure
+  */
+
+%let root=%sysfunc(pathname(work))/top;
+%mf_mkdir(&root)
+%mf_mkdir(&root/a)
+%mf_mkdir(&root/b)
+%mf_mkdir(&root/a/d)
+%mf_mkdir(&root/a/e)
+%mf_mkdir(&root/a/e/f)
+data "&root/a/e/f/ds1.sas7bdat";
+  x=1;
+run;
+
+%mp_dirlist(path=&root, outds=myTable, maxdepth=MAX)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable)=6),
+  desc=All levels returned,
+  outds=work.test_results
+)
+
+%mp_dirlist(path=&root, outds=myTable2, maxdepth=2)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable2)=5),
+  desc=Top two levels returned,
+  outds=work.test_results
+)
+
+%mp_dirlist(path=&root, outds=work.myTable3, maxdepth=0)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable3)=2),
+  desc=Top level returned,
+  outds=work.test_results
+)

tests/crossplatform/mp_ds2cards.test.sas

@@ -0,0 +1,60 @@
+/**
+  @file
+  @brief Testing mp_ds2cards.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2cards.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * test 1 - rebuild an existing dataset
+  * Cars is a great dataset - it contains leading spaces, and formatted numerics
+  */
+
+%mp_ds2cards(base_ds=sashelp.cars
+  , tgt_ds=work.test
+  , cards_file= "%sysfunc(pathname(work))/cars.sas"
+  , showlog=NO
+)
+%inc "%sysfunc(pathname(work))/cars.sas"/source2;
+
+proc compare base=sashelp.cars compare=work.test;
+quit;
+
+%mp_assert(
+  iftrue=(&sysinfo=1),
+  desc=sashelp.cars is identical except for ds label,
+  outds=work.test_results
+)
+
+/**
+  * test 2 - binary data compare
+  */
+data work.binarybase;
+  format bin $hex500. z $hex.;
+  do x=1 to 250;
+    z=byte(x);
+    bin=trim(bin)!!z;
+    output;
+  end;
+run;
+
+%mp_ds2cards(base_ds=work.binarybase
+  , showlog=YES
+  , cards_file="%sysfunc(pathname(work))/c2.sas"
+  , tgt_ds=work.binarycompare
+  , append=
+)
+
+%inc "%sysfunc(pathname(work))/c2.sas"/source2;
+
+proc compare base=work.binarybase compare=work.binarycompare;
+run;
+
+%mp_assert(
+  iftrue=(&sysinfo=0),
+  desc=work.binarybase dataset is identical,
+  outds=work.test_results
+)

tests/crossplatform/mp_getcols.test.sas

@@ -0,0 +1,33 @@
+/**
+  @file
+  @brief Testing mp_getcols macro
+
+  <h4> SAS Macros </h4>
+  @li mp_getcols.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
+
+**/
+
+
+/* valid filter */
+%mp_getcols(sashelp.airline,outds=work.info)
+
+
+%mp_assertdsobs(work.info,
+  desc=Has 3 records,
+  test=EQUALS 3,
+  outds=work.test_results
+)
+
+data work.check;
+  length val $10;
+  do val='NUMERIC','DATE','CHARACTER';
+    output;
+  end;
+run;
+%mp_assertcolvals(work.info.ddtype,
+  checkvals=work.check.val,
+  desc=All values have a match,
+  test=ALLVALS
+)

tests/crossplatform/mp_getconstraints.test.sas

@@ -0,0 +1,29 @@
+/**
+  @file
+  @brief Testing mp_getconstraints.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_getconstraints.sas
+  @li mp_assert.sas
+
+**/
+
+proc sql;
+create table work.example(
+  TX_FROM float format=datetime19.,
+  DD_TYPE char(16),
+  DD_SOURCE char(2048),
+  DD_SHORTDESC char(256),
+  constraint pk primary key(tx_from, dd_type,dd_source),
+  constraint unq unique(tx_from, dd_type),
+  constraint nnn not null(DD_SHORTDESC)
+);
+
+%mp_getconstraints(lib=work,ds=example,outds=work.constraints)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.constraints)=6),
+  desc=Output table work.constraints created with correct number of records,
+  outds=work.test_results
+)

tests/crossplatform/mp_gsubfile.test.sas

@@ -0,0 +1,66 @@
+/**
+  @file
+  @brief Testing mp_gsubfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_gsubfile.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * test 1 - simple replace
+  */
+%global str1;
+%let file=%sysfunc(pathname(work))/file.txt;
+%let pat=replace/me;
+%let str=with/this;
+data _null_;
+  file "&file";
+  put "&pat";
+run;
+%mp_gsubfile(file=&file, patternvar=pat, replacevar=str)
+data _null_;
+  infile "&file";
+  input;
+  call symputx('str1',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&str1"="&str"),
+  desc=Check that simple replacement was successful,
+  outds=work.test_results
+)
+
+/**
+  * test 2 - replace from additional line
+  */
+%global str2 strcheck2 strcheck2b;
+%let file2=%sysfunc(pathname(work))/file2.txt;
+%let pat2=replace/me;
+%let str2=with/this;
+data _null_;
+  file "&file2";
+  put 'line1';output;
+  put "&pat2";output;
+  put "&pat2";output;
+run;
+%mp_gsubfile(file=&file2, patternvar=pat2, replacevar=str2)
+data _null_;
+  infile "&file2";
+  input;
+  if _n_=2 then call symputx('strcheck2',_infile_);
+  if _n_=3 then call symputx('strcheck2b',_infile_);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=("&strcheck2"="&str2"),
+  desc=Check that multi line replacement was successful (line2),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&strcheck2b"="&str2"),
+  desc=Check that multi line replacement was successful (line3),
+  outds=work.test_results
+)

tests/crossplatform/mp_lockanytable.test.sas

@@ -0,0 +1,62 @@
+/**
+  @file
+  @brief Testing mp_lockfilecheck macro
+
+  <h4> SAS Macros </h4>
+  @li mp_lockanytable.sas
+  @li mp_assertcols.sas
+  @li mp_assertcolvals.sas
+
+**/
+
+/* check create table */
+
+%mp_lockanytable(MAKETABLE, ctl_ds=work.controller)
+
+%mp_assertcols(work.controller,
+  cols=lock_status_cd lock_lib lock_ds lock_user_nm lock_ref lock_pid
+    lock_start_dttm lock_end_dttm,
+  test=ALL,
+  desc=check all control columns exist
+)
+
+/* check lock table */
+options dlcreatedir;
+libname tmp "%sysfunc(pathname(work))/tmp";
+data tmp.sometable;
+  x=1;
+run;
+
+%mp_lockanytable(LOCK,lib=tmp,ds=sometable,ref=This Ref, ctl_ds=work.controller)
+
+data work.checkds1;
+  checkval='SOMETABLE';
+run;
+%mp_assertcolvals(work.controller.lock_ds,
+  checkvals=work.checkds1.checkval,
+  desc=table is captured in lock,
+  test=ANYVAL
+)
+
+data work.checkds2;
+  checkval='LOCKED';
+run;
+%mp_assertcolvals(work.controller.lock_status_cd,
+  checkvals=work.checkds2.checkval,
+  desc=code is captured in lock,
+  test=ANYVAL
+)
+
+
+
+/* check for unsuccessful unlock */
+%mp_lockanytable(UNLOCK,lib=tmp,ds=sometable,ref=bye, ctl_ds=work.controller)
+
+data work.checkds3;
+  checkval='UNLOCKED';
+run;
+%mp_assertcolvals(work.controller.lock_status_cd,
+  checkvals=work.checkds3.checkval,
+  desc=Ref is captured in unlock,
+  test=ANYVAL
+)

tests/crossplatform/mp_lockfilecheck.test.sas

@@ -0,0 +1,36 @@
+/**
+  @file
+  @brief Testing mp_lockfilecheck macro
+
+  <h4> SAS Macros </h4>
+  @li mp_lockfilecheck.sas
+  @li mp_assert.sas
+
+**/
+
+
+/* check for regular lock */
+data work.test; a=1;run;
+%mp_lockfilecheck(work.test)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking regular table can be locked,
+  outds=work.test_results
+)
+
+
+/* check for unsuccessful lock */
+%global success abortme;
+%let success=0;
+%macro mp_abort(iftrue=,mac=,msg=);
+  %if &abortme=1 %then %let success=1;
+%mend mp_abort;
+
+%mp_lockfilecheck(sashelp.class)
+
+%mp_assert(
+  iftrue=(&success=1),
+  desc=Checking sashelp table cannot be locked,
+  outds=work.test_results
+)

tests/crossplatform/mp_zip.test.sas

@@ -0,0 +1,115 @@
+/**
+  @file
+  @brief Testing mp_zip macro
+
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mp_assert.sas
+  @li mp_zip.sas
+  @li mp_unzip.sas
+
+**/
+
+%let work=%sysfunc(pathname(work));
+%let root=&work/zipme;
+
+/* TEST 1 - zip a file */
+%mf_mkdir(&root)
+
+data _null_;
+  file "&root/test.txt";
+  put "houston, this is a test";
+run;
+
+%mp_zip(in=&root/test.txt
+  ,type=FILE
+  ,outpath=&work
+  ,outname=myFile
+)
+
+%mp_unzip(ziploc="&work/myFile.zip",outdir=&work)
+
+data _null_;
+  infile "&work/test.txt";
+  input;
+  call symputx('content1',_infile_);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(
+    %str(&content1)=%str(houston, this is a test)
+  ),
+  desc=Checking if file zip / unzip works,
+  outds=work.test_results
+)
+
+/* TEST 2 - zip a dataset of files */
+data _null_;
+  file "&root/test2.txt";
+  put "houston, this is test2";
+run;
+libname tmp "&root";
+data tmp.test;
+  filepath="&root/test2.txt";
+run;
+
+%mp_zip(in=tmp.test
+  ,incol=filepath
+  ,type=DATASET
+  ,outpath=&work
+  ,outname=myFile2
+)
+
+%mp_unzip(ziploc="&work/myFile2.zip",outdir=&work)
+
+data _null_;
+  infile "&work/test2.txt";
+  input;
+  call symputx('content2',_infile_);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(
+    %str(&content2)=%str(houston, this is test2)
+  ),
+  desc=Checking if file zip / unzip from a dataset works,
+  outds=work.test_results
+)
+
+/* TEST 3 - zip a dataset of files */
+%mf_mkdir(&work/out3)
+
+%mp_zip(in=&root
+  ,type=DIRECTORY
+  ,outpath=&work
+  ,outname=myFile3
+)
+
+%mp_unzip(ziploc="&work/myFile3.zip",outdir=&work/out3)
+
+data _null_;
+  infile "&work/out3/test.txt";
+  input;
+  call symputx('content3a',_infile_);
+  putlog _infile_;
+run;
+data _null_;
+  infile "&work/out3/test2.txt";
+  input;
+  call symputx('content3b',_infile_);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(
+    %str(&content3a)=%str(houston, this is a test)
+    and
+    %str(&content3b)=%str(houston, this is test2)
+  ),
+  desc=Checking if file zip / unzip from a directory works,
+  outds=work.test_results
+)
+
+

tests/serveronly/ms_webout.test.sas

@@ -0,0 +1,35 @@
+/**
+  @file
+  @brief Testing ms_webout macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li ms_webout.sas
+  @li mp_assert.sas
+
+**/
+
+
+%let fref=%mf_getuniquefileref();
+%global _metaperson;
+data some datasets;
+  x=1;
+run;
+%ms_webout(OPEN,fref=&fref)
+%ms_webout(ARR,some,fref=&fref)
+%ms_webout(OBJ,datasets,fref=&fref)
+%ms_webout(CLOSE,fref=&fref)
+
+libname test JSON (&fref);
+data root;
+  set test.root;
+  call symputx('checkval',sysvlong);
+run;
+data alldata;
+  set test.alldata;
+run;
+
+%mp_assert(
+  iftrue=(%str(&checkval)=%str(&sysvlong)),
+  desc=Check if the sysvlong value was created
+)

tests/testinit.sas

@@ -5,4 +5,12 @@
 **/
 
 /* location in metadata or SAS Drive for temporary files */
-%let mcTestAppLoc=/Public/temp/macrocore;
+%let mcTestAppLoc=/Public/temp/macrocore;
+
+%macro loglevel();
+  %if &_debug=2477 %then %do;
+    options mprint;
+  %end;
+%mend loglevel;
+
+%loglevel()

viya/mv_createwebservice.sas

@@ -590,6 +590,7 @@ data _null_;
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
   put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
+  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';

viya/mv_getfoldermembers.sas

@@ -122,6 +122,7 @@ options noquotelenmax;
   run;
   libname &libref2 JSON fileref=&fname2;
   data &outds;
+    length id $36 name $128 uri $64 type $32 description $256;
     set &libref2..items;
   run;
   filename &fname2 clear;

viya/mv_webout.sas

@@ -215,6 +215,7 @@
     put ",""SYSCC"" : ""&syscc"" ";
     put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;