Merge pull request #99 from sasjs/issue98

feat: adding ms_webout macro for server responses on sasjs/server.

.all-contributorsrc

@@ -0,0 +1,124 @@
+{
+  "projectName": "core",
+  "projectOwner": "sasjs",
+  "repoType": "github",
+  "repoHost": "https://github.com",
+  "files": [
+    "README.md"
+  ],
+  "imageSize": 100,
+  "commit": false,
+  "commitConvention": "angular",
+  "contributors": [
+    {
+      "login": "allanbowe",
+      "name": "Allan Bowe",
+      "avatar_url": "https://avatars.githubusercontent.com/u/4420615?v=4",
+      "profile": "https://github.com/allanbowe",
+      "contributions": [
+        "business",
+        "code",
+        "content",
+        "doc",
+        "infra",
+        "maintenance",
+        "mentoring",
+        "question",
+        "review",
+        "test"
+      ]
+    },
+    {
+      "login": "rafgag",
+      "name": "rafgag",
+      "avatar_url": "https://avatars.githubusercontent.com/u/69139928?v=4",
+      "profile": "https://github.com/rafgag",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "tmoody",
+      "name": "Trevor Moody",
+      "avatar_url": "https://avatars.githubusercontent.com/u/79837106?v=4",
+      "profile": "https://github.com/tmoody",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "krishna-acondy",
+      "name": "Krishna Acondy",
+      "avatar_url": "https://avatars.githubusercontent.com/u/2980428?v=4",
+      "profile": "https://krishna-acondy.io/",
+      "contributions": [
+        "code",
+        "infra",
+        "blog",
+        "content",
+        "ideas",
+        "video"
+      ]
+    },
+    {
+      "login": "saadjutt01",
+      "name": "Muhammad Saad ",
+      "avatar_url": "https://avatars.githubusercontent.com/u/8914650?v=4",
+      "profile": "https://github.com/saadjutt01",
+      "contributions": [
+        "code",
+        "ideas"
+      ]
+    },
+    {
+      "login": "YuryShkoda",
+      "name": "Yury Shkoda",
+      "avatar_url": "https://avatars.githubusercontent.com/u/25773492?v=4",
+      "profile": "https://www.erudicat.com/",
+      "contributions": [
+        "code",
+        "infra",
+        "video"
+      ]
+    },
+    {
+      "login": "medjedovicm",
+      "name": "Mihajlo Medjedovic",
+      "avatar_url": "https://avatars.githubusercontent.com/u/18329105?v=4",
+      "profile": "https://github.com/medjedovicm",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "kkchandok",
+      "name": "kkchandok",
+      "avatar_url": "https://avatars.githubusercontent.com/u/46090627?v=4",
+      "profile": "https://github.com/kkchandok",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "VladislavParhomchik",
+      "name": "Vladislav Parhomchik",
+      "avatar_url": "https://avatars.githubusercontent.com/u/83717836?v=4",
+      "profile": "https://github.com/VladislavParhomchik",
+      "contributions": [
+        "test",
+        "review"
+      ]
+    },
+    {
+      "login": "vznesh",
+      "name": "Vignesh T.",
+      "avatar_url": "https://avatars.githubusercontent.com/u/28916792?v=4",
+      "profile": "https://github.com/vznesh",
+      "contributions": [
+        "bug"
+      ]
+    }
+  ],
+  "contributorsPerLine": 7,
+  "skipCi": true
+}

.editorconfig

@@ -1,12 +0,0 @@
-# http://editorconfig.org
-root = true
-
-[*]
-charset = utf-8
-indent_style = space
-indent_size = 2
-end_of_line = lf
-insert_final_newline = false
-trim_trailing_whitespace = true
-
-

.github/workflows/run-tests.yml

@@ -12,12 +12,12 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x]
+        node-version: [lts/fermium]
 
     steps:
       - uses: actions/checkout@v2
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node-version }}
 
@@ -39,7 +39,7 @@ jobs:
           sudo apt install apt-transport-https
           sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
           sudo apt-key add openvpn-repo-pkg-key.pub
-          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-bionic.list
+          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-focal.list
           sudo apt update
           sudo apt install openvpn3
 

.gitpod.yml

@@ -1,8 +1,25 @@
 tasks:
-  - init: nvm install --latest-npm && npm i -g @sasjs/cli
+  - init: nvm install --lts && npm i -g @sasjs/cli
 
 image:
   file: .gitpod.dockerfile
 vscode:
   extensions:
     - sasjs.sasjs-for-vscode
+
+github:
+  prebuilds:
+    # enable for the master/default branch (defaults to true)
+    master: true
+    # enable for all branches in this repo (defaults to false)
+    branches: false
+    # enable for pull requests coming from this repo (defaults to true)
+    pullRequests: true
+    # enable for pull requests coming from forks (defaults to false)
+    pullRequestsFromForks: true
+    # add a "Review in Gitpod" button as a comment to pull requests (defaults to true)
+    addComment: true
+    # add a "Review in Gitpod" button to pull requests (defaults to false)
+    addBadge: false
+    # add a label once the prebuild is ready to pull requests (defaults to false)
+    addLabel: prebuilt-in-gitpod

.npmignore

@@ -9,3 +9,4 @@ sasjs/
 main.dox
 make_singlefile.sh
 *.md
+.all-contributorsrc

README.md

@@ -1,7 +1,6 @@
 # Macro Core
 [![npm package][npm-image]][npm-url]
 [![Github Workflow][githubworkflow-image]][githubworkflow-url]
-[![Dependency Status][dependency-image]][dependency-url]
 [![npm](https://img.shields.io/npm/dt/@sasjs/core)]()
 ![Snyk Vulnerabilities for npm package](https://img.shields.io/snyk/vulnerabilities/npm/@sasjs/core)
 [![License](https://img.shields.io/apm/l/atomic-design-ui.svg)](/LICENSE)
@@ -16,11 +15,9 @@
 [npm-url]:http://npmjs.org/package/@sasjs/core
 [githubworkflow-image]:https://github.com/sasjs/core/actions/workflows/main.yml/badge.svg
 [githubworkflow-url]:https://github.com/sasjs/core/blob/main/.github/workflows/main.yml
-[dependency-image]:https://david-dm.org/sasjs/core.svg
 [dependency-url]:https://github.com/sasjs/core/blob/main/package.json
 
 
-
 Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
 
 You can download and compile them all in just two lines of SAS code:
@@ -32,42 +29,61 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 
 Documentation: https://core.sasjs.io
 
-# Components
+## Components
 
-**base** library (SAS9/Viya)
+### **base** library (SAS9/Viya)
 
 - OS independent
 - Not metadata aware
 - No X command
 - Prefixes: _mf_, _mp_
 
-**meta** library (SAS9 only)
+#### **fcmp** library (SAS9/Viya)
+- Function and macro names are identical, except for special cases
+- Prefixes: _mcf_
+
+The fcmp macros are used to generate fcmp functions, and can be used with or
+without the `proc fcmp` wrapper.
+
+### **meta** library (SAS9 only)
+
+Macros used in SAS EBI, which connect to the metadata server.
 
 - OS independent
 - Metadata aware
 - No X command
 - Prefixes: _mm_
 
-**viya** library (Viya only)
+### **server** library (@sasjs/server only)
+These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
+
+- OS independent
+- @sasjs/server aware
+- No X command
+- Prefixes: _ms_
+
+### **viya** library (Viya only)
+
+Macros used for interfacing with SAS Viya.
 
 - OS independent
 - No X command
-- Prefixes: _mv_
+- Prefixes: _mv_, _mvf_
 
-**metax** library (SAS9 only)
+### **metax** library (SAS9 only)
 
 - OS specific
 - Metadata aware
 - X command enabled
 - Prefixes: _mmw_,_mmu_,_mmx_
 
-**lua** library
+### **lua** library
 
 Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
 
 To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert your LUA into a data step with put statements, and create the macro wrapper with a `ml_` prefix.  You can then use your module in any program by running:
 
-```
+```sas
 /* compile the lua module */
 %ml_yourmodule()
 
@@ -82,9 +98,9 @@ run;
 - X command enabled
 - Prefixes: _mmw_,_mmu_,_mmx_
 
-# Installation
+## Installation
 
-First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available,eg:
+First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available, eg:
 
 ```sas
 options insert=(sasautos="/your/path/macrocore/base");
@@ -100,9 +116,9 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 %inc mc;
 ```
 
-# Standards
+## Standards
 
-## File Properties
+### File Properties
 
 - filenames much match macro names
 - filenames must be lowercase, without spaces
@@ -110,19 +126,20 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 - one macro per file
 - prefixes:
   - _mf_ for macro functions (can be used in open code).
-  - _mp_ for macro procedures (which generate sas code)
+  - _ml_ for macros that are used to compile LUA modules
   - _mm_ for metadata macros (interface with the metadata server).
   - _mmx_ for macros that use metadata and are XCMD enabled
+  - _mp_ for macro procedures (which generate sas code)
+  - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
+  - _mv_ for macro procedures that will only work in Viya
   - _mx_ for macros that are XCMD enabled
-  - _ml_ for macros that are used to compile LUA modules
-  - _mv_ for macros that will only work in Viya
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
 - UTF-8
 
 
-## Header Properties
+### Header Properties
 
 The **Macro Core** documentation is created using [doxygen](http://www.doxygen.nl). A full list of attributes can be found [here](http://www.doxygen.nl/manual/commands.html) but the following are most relevant:
 
@@ -136,7 +153,7 @@ The **Macro Core** documentation is created using [doxygen](http://www.doxygen.n
 
 All macros must be commented in the doxygen format, to enable the [online documentation](https://core.sasjs.io).
 
-### Dependencies
+#### Dependencies
 SAS code can contain one of two types of dependency - SAS Macros, and SAS Includes.  When compiling projects using the [SASjs CLI](https://cli.sasjs.io) the doxygen header is scanned for `  @li` items under the following headers:
 
 ```sas
@@ -154,7 +171,7 @@ The CLI can then extract all the dependencies and insert as precode (SAS Macros)
 When contributing to this library, it is therefore important to ensure that all dependencies are listed in the header in this format.
 
 
-## Coding Standards
+### Coding Standards
 
 - Indentation = 2 spaces. No tabs!
 - no trailing white space
@@ -167,7 +184,7 @@ When contributing to this library, it is therefore important to ensure that all
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
 - The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
 
-# General Notes
+## General Notes
 
 - All macros should be compatible with SAS versions from support level B and above (so currently 9.2 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully (eg `%if %sysevalf(&sysver<9.3) %then %return`).
 
@@ -179,3 +196,35 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 
 
+## Contributors ✨
+<!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
+[![All Contributors](https://img.shields.io/badge/all_contributors-10-orange.svg?style=flat-square)](#contributors-)
+<!-- ALL-CONTRIBUTORS-BADGE:END -->
+Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
+
+<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-disable -->
+<table>
+  <tr>
+    <td align="center"><a href="https://github.com/allanbowe"><img src="https://avatars.githubusercontent.com/u/4420615?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Allan Bowe</b></sub></a><br /><a href="#business-allanbowe" title="Business development">💼</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Code">💻</a> <a href="#content-allanbowe" title="Content">🖋</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Documentation">📖</a> <a href="#infra-allanbowe" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#maintenance-allanbowe" title="Maintenance">🚧</a> <a href="#mentoring-allanbowe" title="Mentoring">🧑‍🏫</a> <a href="#question-allanbowe" title="Answering Questions">💬</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3Aallanbowe" title="Reviewed Pull Requests">👀</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Tests">⚠️</a></td>
+    <td align="center"><a href="https://github.com/rafgag"><img src="https://avatars.githubusercontent.com/u/69139928?v=4?s=100" width="100px;" alt=""/><br /><sub><b>rafgag</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rafgag" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/tmoody"><img src="https://avatars.githubusercontent.com/u/79837106?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Trevor Moody</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=tmoody" title="Code">💻</a></td>
+    <td align="center"><a href="https://krishna-acondy.io/"><img src="https://avatars.githubusercontent.com/u/2980428?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Krishna Acondy</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=krishna-acondy" title="Code">💻</a> <a href="#infra-krishna-acondy" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#blog-krishna-acondy" title="Blogposts">📝</a> <a href="#content-krishna-acondy" title="Content">🖋</a> <a href="#ideas-krishna-acondy" title="Ideas, Planning, & Feedback">🤔</a> <a href="#video-krishna-acondy" title="Videos">📹</a></td>
+    <td align="center"><a href="https://github.com/saadjutt01"><img src="https://avatars.githubusercontent.com/u/8914650?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Muhammad Saad </b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=saadjutt01" title="Code">💻</a> <a href="#ideas-saadjutt01" title="Ideas, Planning, & Feedback">🤔</a></td>
+    <td align="center"><a href="https://www.erudicat.com/"><img src="https://avatars.githubusercontent.com/u/25773492?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Yury Shkoda</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=YuryShkoda" title="Code">💻</a> <a href="#infra-YuryShkoda" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#video-YuryShkoda" title="Videos">📹</a></td>
+    <td align="center"><a href="https://github.com/medjedovicm"><img src="https://avatars.githubusercontent.com/u/18329105?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Mihajlo Medjedovic</b></sub></a><br /><a href="#infra-medjedovicm" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+  </tr>
+  <tr>
+    <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
+    <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
+    <td align="center"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
+  </tr>
+</table>
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+
+<!-- ALL-CONTRIBUTORS-LIST:END -->
+
+This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!

base/mf_existfileref.sas

@@ -17,7 +17,13 @@
 %macro mf_existfileref(fref
 )/*/STORE SOURCE*/;
 
-  %if %sysfunc(fileref(&fref))=0 %then %do;
+  %local rc;
+  %let rc=%sysfunc(fileref(&fref));
+  %if &rc=0 %then %do;
+    1
+  %end;
+  %else %if &rc<0 %then %do;
+    %put &sysmacroname: Fileref &fref exists but the underlying file does not;
     1
   %end;
   %else %do;

base/mf_existfunction.sas

@@ -0,0 +1,37 @@
+/**
+  @file
+  @brief Checks if a function exists
+  @details Returns 1 if the function exists, else 0.  Note that this function
+  can be slow as it needs to open the sashelp.vfuncs table.
+
+  Usage:
+
+      %put %mf_existfunction(CAT);
+      %put %mf_existfunction(DOG);
+
+  Full credit to [Bart](https://sasensei.com/user/305) for the vfunc pointer
+  and the tidy approach for pure macro data set filtering.
+  Check out his [SAS Packages](https://github.com/yabwon/SAS_PACKAGES)
+  framework!  Where you can find the same [function](
+https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionexists-macro
+  ).
+
+  @param [in] name (positional) - function name
+
+  @author Allan Bowe
+**/
+/** @cond */
+%macro mf_existfunction(name
+)/*/STORE SOURCE*/;
+
+  %local dsid rc exist;
+  %let dsid=%sysfunc(open(sashelp.vfunc(where=(fncname="%upcase(&name)"))));
+  %let exist=1;
+  %let exist=%sysfunc(fetch(&dsid, NOSET));
+  %let rc=%sysfunc(close(&dsid));
+
+  %sysevalf(0 = &exist)
+
+%mend mf_existfunction;
+
+/** @endcond */

base/mf_getapploc.sas

@@ -0,0 +1,73 @@
+/**
+  @file
+  @brief Returns the appLoc from the _program variable
+  @details When working with SASjs apps, web services / tests / jobs are always
+  deployed to a root (app) location in the SAS logical folder tree.
+
+  When building apps for use in other environments, you do not necessarily know
+  where the backend services will be deployed.  Therefore a function like this
+  is handy in order to dynamically figure out the appLoc, and enable other
+  services to be connected by a relative reference.
+
+  SASjs apps always have the same immediate substructure (one or more of the
+  following):
+
+  @li /data
+  @li /jobs
+  @li /services
+  @li /tests/jobs
+  @li /tests/services
+  @li /tests/macros
+
+  This function works by testing for the existence of any of the above in the
+  automatic _program variable, and returning the part to the left of it.
+
+  Usage:
+
+      %put %mf_getapploc(&_program)
+
+      %put %mf_getapploc(/some/location/services/admin/myservice);
+      %put %mf_getapploc(/some/location/jobs/extract/somejob/);
+      %put %mf_getapploc(/some/location/tests/jobs/somejob/);
+
+
+  @author Allan Bowe
+**/
+
+%macro mf_getapploc(pgm);
+%if "&pgm"="" %then %do;
+  %if %symexist(_program) %then %let pgm=&_program;
+  %else %do;
+    %put &sysmacroname: No value provided and no _program variable available;
+    %return;
+  %end;
+%end;
+%local root;
+
+/**
+  * First check we are not in the tests/macros folder (which has no subfolders)
+  */
+%if %index(&pgm,/tests/macros/) %then %do;
+  %let root=%substr(&pgm,1,%index(&pgm,/tests/macros)-1);
+  &root
+  %return;
+%end;
+
+/**
+  * Next, move up two levels to avoid matches on subfolder or service name
+  */
+%let root=%substr(&pgm,1,%length(&pgm)-%length(%scan(&pgm,-1,/))-1);
+%let root=%substr(&root,1,%length(&root)-%length(%scan(&root,-1,/))-1);
+
+%if %index(&root,/tests/) %then %do;
+  %let root=%substr(&root,1,%index(&root,/tests/)-1);
+%end;
+%else %if %index(&root,/services) %then %do;
+  %let root=%substr(&root,1,%index(&root,/services)-1);
+%end;
+%else %if %index(&root,/jobs) %then %do;
+  %let root=%substr(&root,1,%index(&root,/jobs)-1);
+%end;
+%else %put &sysmacroname: Could not find an app location from &pgm;
+  &root
+%mend mf_getapploc ;

base/mf_getuniquefileref.sas

@@ -1,37 +1,58 @@
 /**
   @file
   @brief Assigns and returns an unused fileref
-  @details
+  @details  Using the native approach for assigning filerefs fails as some
+  procedures (such as proc http) do not recognise the temporary names (starting
+  with a hash), returning a message such as:
+
+  > ERROR 22-322: Expecting a name.
+
+  This macro works by attempting a random fileref (with a prefix), seeing if it
+  is already assigned, and if not - returning the fileref.
+
+  If your process can accept filerefs with the hash (#) prefix, then set
+  `prefix=0` to revert to the native approach - which is significantly faster
+  when there are a lot of filerefs in a session.
+
   Use as follows:
 
       %let fileref1=%mf_getuniquefileref();
-      %let fileref2=%mf_getuniquefileref();
+      %let fileref2=%mf_getuniquefileref(prefix=0);
       %put &fileref1 &fileref2;
 
-  which returns:
+  which returns filerefs similar to:
 
-> mcref0 mcref1
+> _7432233 #LN00070
 
-  @param prefix= first part of fileref. Remember that filerefs can only be 8
-    characters, so a 7 letter prefix would mean that `maxtries` should be 10.
-  @param maxtries= the last part of the libref.  Provide an integer value.
+  @param [in] prefix= (_) first part of fileref. Remember that filerefs can only
+    be 8 characters, so a 7 letter prefix would mean `maxtries` should be 10.
+    if using zero (0) as the prefix, a native assignment is used.
+  @param [in] maxtries= (1000) the last part of the libref. Must be an integer.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-%macro mf_getuniquefileref(prefix=mcref,maxtries=1000);
-  %local x fname;
-  %let x=0;
-  %do x=0 %to &maxtries;
-  %if %sysfunc(fileref(&prefix&x)) > 0 %then %do;
-    %let fname=&prefix&x;
+%macro mf_getuniquefileref(prefix=_,maxtries=1000);
+  %local rc fname;
+  %if &prefix=0 %then %do;
     %let rc=%sysfunc(filename(fname,,temp));
     %if &rc %then %put %sysfunc(sysmsg());
-    &prefix&x
-    %*put &sysmacroname: Fileref &prefix&x was assigned and returned;
-    %return;
+    &fname
   %end;
+  %else %do;
+    %local x len;
+    %let len=%eval(8-%length(&prefix));
+    %let x=0;
+    %do x=0 %to &maxtries;
+      %let fname=&prefix%substr(%sysfunc(ranuni(0)),3,&len);
+      %if %sysfunc(fileref(&fname)) > 0 %then %do;
+        %let rc=%sysfunc(filename(fname,,temp));
+        %if &rc %then %put %sysfunc(sysmsg());
+        &fname
+        %return;
+      %end;
+    %end;
+    %put unable to find available fileref after &maxtries attempts;
   %end;
-  %put unable to find available fileref in range &prefix.0-&maxtries;
 %mend mf_getuniquefileref;

base/mf_getvarformat.sas

@@ -51,7 +51,8 @@
     %end;
   %end;
   %else %do;
-    %put dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
     %return;
   %end;
 

base/mf_getvarlen.sas

@@ -43,7 +43,11 @@
       %let vlen = %str( );
     %end;
   %end;
-  %else %put dataset &libds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));

base/mf_getvarnum.sas

@@ -43,7 +43,11 @@ returns:
       %let vnum = %str( );
     %end;
   %end;
-  %else %put dataset &ds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));

base/mf_getvartype.sas

@@ -39,7 +39,11 @@ Usage:
       %let vtype = %str( );
     %end;
   %end;
-  %else %put dataset &libds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));

base/mp_abort.sas

@@ -15,24 +15,47 @@
     recognise this and fetch the log of the parent session instead)
   @li STP environments must finish cleanly to avoid the log being sent to
     _webout.  To assist with this, we also run stpsrvset('program error', 0)
-    and set SYSCC=0.  For 9.4m3 we take a unique approach - we open a macro
-    but don't close it!  This provides a graceful abort, EXCEPT when called
-    called within a %include within a macro (and that macro contains additional
-    logic).  See mp_abort.test.nofix.sas for the example case.
-    If you know of another way to gracefully abort a 9.4m3 STP session, we'd
-    love to hear about it!
+    and set SYSCC=0.  We take a unique "soft abort" approach - we open a macro
+    but don't close it!  This works everywhere EXCEPT inside a \%include inside
+    a macro.  For that, we recommend you use mp_include.sas to perform the
+    include, and then call \%mp_abort(mode=INCLUDE) from the source program (ie,
+    OUTSIDE of the top-parent macro).
 
 
   @param mac= to contain the name of the calling macro
   @param msg= message to be returned
   @param iftrue= supply a condition under which the macro should be executed.
+  @param errds= (work.mp_abort_errds) There is no clean way to end a process
+    within a %include called within a macro.  Furthermore, there is no way to
+    test if a macro is called within a %include.  To handle this particular
+    scenario, the %include should be switched for the mp_include.sas macro.
+    This provides an indicator that we are running a macro within a \%include
+    (`_SYSINCLUDEFILEDEVICE`) and allows us to provide a dataset with the abort
+    values (msg, mac).
+    We can then run an abort cancel FILE to stop the include running, and pass
+    the dataset back to the calling program to run a regular \%mp_abort().
+    The dataset will contain the following fields:
+    @li iftrue (1=1)
+    @li msg (the message)
+    @li mac (the mac param)
 
-  @version 9.4M3
+  @param mode= (REGULAR) If mode=INCLUDE then the &errds dataset is checked for
+    an abort status.
+    Valid values:
+    @li REGULAR (default)
+    @li INCLUDE
+
+  <h4> Related Macros </h4>
+  @li mp_include.sas
+
+  @version 9.4
   @author Allan Bowe
   @cond
 **/
 
 %macro mp_abort(mac=mp_abort.sas, type=, msg=, iftrue=%str(1=1)
+  , errds=work.mp_abort_errds
+  , mode=REGULAR
 )/*/STORE SOURCE*/;
 
   %global sysprocessmode sysprocessname;
@@ -43,9 +66,44 @@
   %if %length(&mac)>0 %then %put NOTE- called by &mac;
   %put NOTE - &msg;
 
+  %if %symexist(_SYSINCLUDEFILEDEVICE) %then %do;
+    %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do;
+      data &errds;
+        iftrue='1=1';
+        length mac $100 msg $5000;
+        mac=symget('mac');
+        msg=symget('msg');
+      run;
+      data _null_;
+        abort cancel FILE;
+      run;
+      %return;
+    %end;
+  %end;
+
   /* Stored Process Server web app context */
-  %if %symexist(_metaperson) or "&SYSPROCESSNAME "="Compute Server " %then %do;
+  %if %symexist(_metaperson)
+    or "&SYSPROCESSNAME "="Compute Server "
+    or &mode=INCLUDE
+  %then %do;
     options obs=max replace nosyntaxcheck mprint;
+    %if &mode=INCLUDE %then %do;
+      %if %sysfunc(exist(&errds))=1 %then %do;
+        data _null_;
+          set &errds;
+          call symputx('iftrue',iftrue,'l');
+          call symputx('mac',mac,'l');
+          call symputx('msg',msg,'l');
+          putlog (_all_)(=);
+        run;
+        %if (&iftrue)=0 %then %return;
+      %end;
+      %else %do;
+        %put &sysmacroname: No include errors found;
+        %return;
+      %end;
+    %end;
+
     /* extract log errs / warns, if exist */
     %local logloc logline;
     %global logmsg; /* capture global messages */
@@ -100,7 +158,7 @@
     /* send response in SASjs JSON format */
     data _null_;
       file _webout mod lrecl=32000 encoding='utf-8';
-      length msg $32767 debug $8;
+      length msg $32767 ;
       sasdatetime=datetime();
       msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
       /* escape the quotes */
@@ -111,7 +169,7 @@
       msg=cats('"',msg,'"');
       if symexist('_debug') then debug=quote(trim(symget('_debug')));
       else debug='""';
-      if debug ge '"131"' then put '>>weboutBEGIN<<';
+      put '>>weboutBEGIN<<';
       put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
       put ',"sasjsAbort" : [{';
       put ' "MSG":' msg ;
@@ -131,29 +189,45 @@
       _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
       put ',"_PROGRAM" : ' _PROGRAM ;
       put ",""SYSCC"" : ""&syscc"" ";
-      put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+      syserrortext=quote(trim(symget('syserrortext')));
+      put ",""SYSERRORTEXT"" : " syserrortext;
+      put ",""SYSHOSTNAME"" : ""&syshostname"" ";
       put ",""SYSJOBID"" : ""&sysjobid"" ";
+      put ",""SYSSCPL"" : ""&sysscpl"" ";
+      put ",""SYSSITE"" : ""&syssite"" ";
       sysvlong=quote(trim(symget('sysvlong')));
       put ',"SYSVLONG" : ' sysvlong;
-      put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+      syswarningtext=quote(trim(symget('syswarningtext')));
+      put ",""SYSWARNINGTEXT"" : " syswarningtext;
       put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
       put "}" @;
-      if debug ge '"131"' then put '>>weboutEND<<';
+      put '>>weboutEND<<';
     run;
 
     %put _all_;
 
     %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
       data _null_;
-        putlog 'stpsrvset program error and syscc';
+        putlog 'stpsrvset program err and syscc';
         rc=stpsrvset('program error', 0);
         call symputx("syscc",0,"g");
       run;
-      %if "%substr(&sysvlong.xxxxxxxxx,1,9)" ne "9.04.01M3" %then %do;
-        %put NOTE: Ending SAS session due to:;
-        %put NOTE- &msg;
-        endsas;
-      %end;
+      /**
+        * endsas kills 9.4m3 deployments by orphaning multibridges.
+        * Abort variants are ungraceful (non zero return code)
+        * This approach lets SAS run silently until the end :-)
+        * Caution - fails when called within a %include within a macro
+        * Use mp_include() to handle this.
+        */
+      filename skip temp;
+      data _null_;
+        file skip;
+        put '%macro skip();';
+        comment '%mend skip; -> fix lint ';
+        put '%macro skippy();';
+        comment '%mend skippy; -> fix lint ';
+      run;
+      %inc skip;
     %end;
     %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
       /* endsas kills the session making it harder to fetch results */
@@ -165,27 +239,10 @@
         sysuserid=symget('sysuserid');
         iftrue=symget('iftrue');
         put (_all_)(/=);
+        call symputx('syscc',0);
         abort cancel nolist;
       run;
     %end;
-    %else %if "%substr(&sysvlong.xxxxxxxxx,1,9)" = "9.04.01M3" %then %do;
-      /**
-        * endsas kills 9.4m3 deployments by orphaning multibridges.
-        * Abort variants are ungraceful (non zero return code)
-        * This approach lets SAS run silently until the end :-)
-        * Caution - fails when called within a %include within a macro
-        * See tests/mp_abort.test.1 for an example case.
-        */
-      filename skip temp;
-      data _null_;
-        file skip;
-        put '%macro skip();';
-        comment '%mend skip; -> fix lint ';
-        put '%macro skippy();';
-        comment '%mend skippy; -> fix lint ';
-      run;
-      %inc skip;
-    %end;
     %else %do;
       %abort cancel;
     %end;

base/mp_appendfile.sas

@@ -0,0 +1,57 @@
+/**
+  @file
+  @brief Append (concatenate) two or more files.
+  @details Will append one more more `appendrefs` (filerefs) to a `baseref`.
+  Uses a binary mechanism, so will work with any file type.  For that reason -
+  use with care!   And supply your own trailing carriage returns in each file..
+
+  Usage:
+
+        filename tmp1 temp;
+        filename tmp2 temp;
+        filename tmp3 temp;
+        data _null_; file tmp1; put 'base file';
+        data _null_; file tmp2; put 'append1';
+        data _null_; file tmp3; put 'append2';
+        run;
+        %mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3)
+
+
+  @param [in] baseref= Fileref of the base file (should exist)
+  @param [in] appendrefs= One or more filerefs to be appended to the base
+    fileref.  Space separated.
+
+  @version 9.2
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+
+
+**/
+
+%macro mp_appendfile(
+  baseref=0,
+  appendrefs=0
+)/*/STORE SOURCE*/;
+
+%mp_abort(iftrue= (&baseref=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Baseref NOT specified!)
+)
+%mp_abort(iftrue= (&appendrefs=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Appendrefs NOT specified!)
+)
+
+%local i;
+%do i=1 %to %sysfunc(countw(&appendrefs));
+  %mp_abort(iftrue= (&syscc>0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc)
+  )
+  %mp_binarycopy(inref=%scan(&appendrefs,&i), outref=&baseref, mode=APPEND)
+%end;
+
+%mend mp_appendfile;

base/mp_assertcolvals.sas

@@ -30,6 +30,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
+  @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
 
@@ -115,6 +116,26 @@
   select count(*) into: orig from &lib..&ds;
   quit;
 
+  %local notfound tmp1 tmp2;
+  %let tmp1=%mf_getuniquename();
+  %let tmp2=%mf_getuniquename();
+
+  /* this is a bit convoluted - but using sql outobs=10 throws warnings */
+  proc sql noprint;
+  create view &tmp1 as
+    select distinct &col
+    from &lib..&ds
+    where &col not in (
+      select &ccol from &clib..&cds
+    );
+  data &tmp2;
+    set &tmp1;
+    if _n_>10 then stop;
+  run;
+  proc sql;
+  select distinct &col  into: notfound separated by ' ' from &tmp2;
+
+
   %mp_abort(iftrue= (&syscc ne 0)
     ,mac=&sysmacroname
     ,msg=%str(syscc=&syscc after macro query)
@@ -125,7 +146,7 @@
     test_description=symget('desc');
     test_result='FAIL';
     test_comments="&sysmacroname: &lib..&ds..&col has &result values "
-      !!"not in &clib..&cds..&ccol ";
+      !!"not in &clib..&cds..&ccol.. First 10 vals:"!!symget('notfound');
   %if &test=ANYVAL %then %do;
     if &result < &orig then test_result='PASS';
   %end;

base/mp_base64copy.sas

@@ -4,6 +4,7 @@
   @details Creates a new version of a file either encoded or decoded using
   Base64.  Inspired by this post by Michael Dixon:
   https://support.selerity.com.au/hc/en-us/articles/223345708-Tip-SAS-and-Base64
+
   Usage:
 
         filename tmp temp;
@@ -27,16 +28,18 @@
           put _infile_;
         run;
 
-  @param inref= Fileref of the input file (should exist)
-  @param outref= Output filref. If it does not exist, it is created.
-  @param action= (ENCODE) The action to take. Valid values:
-    @li ENCODE Convert the file to base64 format
-    @li DECODE Decode the file from base64 format
+  @param [in] inref= Fileref of the input file (should exist)
+  @param [out] outref= Output fileref. If it does not exist, it is created.
+  @param [in] action= (ENCODE) The action to take. Valid values:
+    @li ENCODE - Convert the file to base64 format
+    @li DECODE - Decode the file from base64 format
+
+  @version 9.2
+  @author Allan Bowe, source: https://github.com/sasjs/core
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas
 
-  @version 9.2
 
 **/
 
@@ -73,7 +76,7 @@ run;
 
 %if &outfound=0 %then %do;
   filename &outref temp lrecl=2097088;
-%%end;
+%end;
 
 %if &action=ENCODE %then %do;
   data _null_;
@@ -81,12 +84,12 @@ run;
     retain line "";
     infile &inref recfm=F lrecl= 1 end=eof;
     input @1 stream $char1.;
-    file &outref lrecl=76;
+    file &outref recfm=N;
     substr(line,(_N_-(CEIL(_N_/57)-1)*57),1) = byte(rank(stream));
     if mod(_N_,57)=0 or EOF then do;
       if eof then b64=put(trim(line),$base64X76.);
       else b64=put(line, $base64X76.);
-      put b64;
+      put b64 + (-1) @;
       line="";
     end;
   run;
@@ -98,19 +101,13 @@ run;
     fileout = fopen("&outref",'O',3,'B');
     char= '20'x;
     do while(fread(filein)=0);
-      raw="1234";
+      length raw $4;
       do i=1 to 4;
         rc=fget(filein,char,1);
         substr(raw,i,1)=char;
       end;
-      val="123";
-      val=input(raw,$base64X4.);
-      do i=1 to 3;
-        length byte $1;
-        byte=byte(rank(substr(val,i,1)));
-        rc = fput(fileout, byte);
-      end;
-      rc =fwrite(fileout);
+      rc = fput(fileout,input(raw,$base64X4.));
+      rc = fwrite(fileout);
     end;
     rc = fclose(filein);
     rc = fclose(fileout);

base/mp_binarycopy.sas

@@ -9,10 +9,29 @@
 
         %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
 
-  @param inloc full, quoted "path/and/filename.ext" of the object to be copied
-  @param outloc full, quoted "path/and/filename.ext" of object to be created
-  @param inref can override default input fileref to avoid naming clash
-  @param outref an override default output fileref to avoid naming clash
+  To append to a file, use the mode option, eg:
+
+      filename tmp1 temp;
+      filename tmp2 temp;
+      data _null_;
+        file tmp1;
+        put 'stacking';
+      run;
+
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+
+
+  @param [in] inloc quoted "path/and/filename.ext" of the file to be copied
+  @param [out] outloc quoted "path/and/filename.ext" of the file to be created
+  @param [in] inref (____in) If provided, this fileref will take precedence over
+    the `inloc` parameter
+  @param [out] outref (____in) If provided, this fileref will take precedence
+    over the `outloc` parameter.  It must already exist!
+  @param [in] mode (CREATE) Valid values:
+    @li CREATE - Create the file (even if it already exists)
+    @li APPEND - Append to the file (don't overwrite)
+
   @returns nothing
 
   @version 9.2
@@ -24,20 +43,29 @@
     ,outloc=          /* full path and filename of object to be created */
     ,inref=____in   /* override default to use own filerefs */
     ,outref=____out /* override default to use own filerefs */
+    ,mode=CREATE
 )/*/STORE SOURCE*/;
+  %local mod outmode;
+  %if &mode=APPEND %then %do;
+    %let mod=mod;
+    %let outmode='a';
+  %end;
+  %else %do;
+    %let outmode='o';
+  %end;
   /* these IN and OUT filerefs can point to anything */
   %if &inref = ____in %then %do;
     filename &inref &inloc lrecl=1048576 ;
   %end;
   %if &outref=____out %then %do;
-    filename &outref &outloc lrecl=1048576 ;
+    filename &outref &outloc lrecl=1048576 &mod;
   %end;
 
   /* copy the file byte-for-byte  */
   data _null_;
     length filein 8 fileid 8;
     filein = fopen("&inref",'I',1,'B');
-    fileid = fopen("&outref",'O',1,'B');
+    fileid = fopen("&outref",&outmode,1,'B');
     rec = '20'x;
     do while(fread(filein)=0);
       rc = fget(filein,rec,1);

base/mp_copyfolder.sas

@@ -0,0 +1,78 @@
+/**
+  @file
+  @brief A macro to recursively copy a directory
+  @details Performs a recursive directory listing then works from top to bottom
+    copying files and creating subdirectories.
+
+  Usage:
+
+      %let rootdir=%sysfunc(pathname(work))/demo;
+      %let copydir=%sysfunc(pathname(work))/demo_copy;
+      %mf_mkdir(&rootdir)
+      %mf_mkdir(&rootdir/subdir)
+      %mf_mkdir(&rootdir/subdir/subsubdir)
+      data "&rootdir/subdir/example.sas7bdat";
+      run;
+
+      %mp_copyfolder(&rootdir,&copydir)
+
+  @param source Unquoted path to the folder to copy from.
+  @param target Unquoted path to the folder to copy to.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_isdir.sas
+  @li mf_mkdir.sas
+  @li mp_abort.sas
+  @li mp_dirlist.sas
+
+  <h4> Related Macros </h4>
+  @li mp_copyfolder.test.sas
+
+**/
+
+%macro mp_copyfolder(source,target);
+
+  %mp_abort(iftrue=(%mf_isdir(&source)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Source dir does not exist (&source))
+  )
+
+  %mf_mkdir(&target)
+
+  %mp_abort(iftrue=(%mf_isdir(&target)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Target dir could not be created (&target))
+  )
+
+  /* prep temp table */
+  %local tempds;
+  %let tempds=%mf_getuniquename();
+
+  /* recursive directory listing */
+  %mp_dirlist(path=&source,outds=work.&tempds, maxdepth=MAX)
+
+  /* create folders and copy content */
+  data _null_;
+    set work.&tempds;
+    filepath2="&target/"!!substr(filepath,%length(&source)+2);
+    if file_or_folder='folder' then call execute('%mf_mkdir('!!filepath2!!')');
+    else do;
+      length fref1 fref2 $8;
+      rc1=filename(fref1,filepath,'disk','recfm=n');
+      rc2=filename(fref2,filepath2,'disk','recfm=n');
+      if fcopy(fref1,fref2) ne 0 then do;
+        sysmsg=sysmsg();
+        putlog "%str(ERR)OR: Unable to copy " filepath " to " filepath2;
+        putlog sysmg=;
+      end;
+    end;
+    rc=filename(fref1);
+    rc=filename(fref2);
+  run;
+
+  /* tidy up */
+  proc sql;
+  drop table work.&tempds;
+
+%mend mp_copyfolder;

base/mp_deletefolder.sas

@@ -0,0 +1,69 @@
+/**
+  @file
+  @brief A macro to delete a directory
+  @details Will delete all folder content (including subfolder content) and
+    finally, the folder itself.
+
+  Usage:
+
+      %let rootdir=%sysfunc(pathname(work))/demo;
+      %mf_mkdir(&rootdir)
+      %mf_mkdir(&rootdir/subdir)
+      %mf_mkdir(&rootdir/subdir/subsubdir)
+      data "&rootdir/subdir/example.sas7bdat";
+      run;
+
+      %mp_deletefolder(&rootdir)
+
+  @param path Unquoted path to the folder to delete.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_isdir.sas
+  @li mp_dirlist.sas
+
+  <h4> Related Macros </h4>
+  @li mp_deletefolder.test.sas
+
+**/
+
+%macro mp_deletefolder(folder);
+  /* proceed if valid directory */
+  %if %mf_isdir(&folder)=1 %then %do;
+
+    /* prep temp table */
+    %local tempds;
+    %let tempds=%mf_getuniquename();
+
+    /* recursive directory listing */
+    %mp_dirlist(path=&folder,outds=work.&tempds, maxdepth=MAX)
+
+    /* sort descending level so can delete folder contents before folders */
+    proc sort data=work.&tempds;
+      by descending level;
+    run;
+
+    /* ensure top level folder is removed at the end */
+    proc sql;
+    insert into work.&tempds set filepath="&folder";
+
+    /* delete everything */
+    data _null_;
+      set work.&tempds end=last;
+      length fref $8;
+      rc=filename(fref,filepath);
+      rc=fdelete(fref);
+      if rc then do;
+        msg=sysmsg();
+        put "&sysmacroname:" / rc= / msg= / filepath=;
+      end;
+      rc=filename(fref);
+    run;
+
+    /* tidy up */
+    proc sql;
+    drop table work.&tempds;
+
+  %end;
+  %else %put &sysmacroname: &folder: is not a valid / accessible folder. ;
+%mend mp_deletefolder;

base/mp_dirlist.sas

@@ -3,20 +3,13 @@
   @brief Returns all files and subdirectories within a specified parent
   @details When used with getattrs=NO, is not OS specific (uses dopen / dread).
 
-  If getattrs=YES then the doptname / foptname functions are used to scan all
-  properties - any characters that are not valid in a SAS name (v7) are simply
-  stripped, and the table is transposed so theat each property is a column
-  and there is one file per row.  An attempt is made to get all properties
-  whether a file or folder, but some files/folders cannot be accessed, and so
-  not all properties can / will be populated.
-
   Credit for the rename approach:
   https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003
 
 
   usage:
 
-      %mp_dirlist(path=/some/location,outds=myTable)
+      %mp_dirlist(path=/some/location, outds=myTable, maxdepth=MAX)
 
       %mp_dirlist(outds=cwdfileprops, getattrs=YES)
 
@@ -30,11 +23,19 @@
   X CMD) do please raise an issue!
 
 
-  @param path= for which to return contents
-  @param fref= Provide a DISK engine fileref as an alternative to PATH
-  @param outds= the output dataset to create
-  @param getattrs= YES/NO (default=NO).  Uses doptname and foptname to return
-  all attributes for each file / folder.
+  @param [in] path= for which to return contents
+  @param [in] fref= Provide a DISK engine fileref as an alternative to PATH
+  @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
+    subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
+  @param [out] outds= the output dataset to create
+  @param [out] getattrs= (NO)  If getattrs=YES then the doptname / foptname
+    functions are used to scan all properties - any characters that are not
+    valid in a SAS name (v7) are simply stripped, and the table is transposed
+    so theat each property is a column and there is one file per row.  An
+    attempt is made to get all properties whether a file or folder, but some
+    files/folders cannot be accessed, and so not all properties can / will be
+    populated.
 
 
   @returns outds contains the following variables:
@@ -44,8 +45,15 @@
     - filename (just the file name)
     - ext (.extension)
     - msg (system message if any issues)
+    - level (depth of folder)
     - OS SPECIFIC variables, if <code>getattrs=</code> is used.
 
+  <h4> SAS Macros </h4>
+  @li mp_dropmembers.sas
+
+  <h4> Related Macros </h4>
+  @li mp_dirlist.test.sas
+
   @version 9.2
   @author Allan Bowe
 **/
@@ -54,14 +62,27 @@
     , fref=0
     , outds=work.mp_dirlist
     , getattrs=NO
+    , maxdepth=0
+    , level=0 /* The level of recursion to perform.  For internal use only. */
 )/*/STORE SOURCE*/;
 %let getattrs=%upcase(&getattrs)XX;
 
-data &outds(compress=no
-    keep=file_or_folder filepath filename ext msg directory
+/* temp table */
+%local out_ds;
+data;run;
+%let out_ds=%str(&syslast);
+
+/* drop main (top) table if it exists */
+%if &level=0 %then %do;
+  %mp_dropmembers(%scan(&outds,-1,.), libref=WORK)
+%end;
+
+data &out_ds(compress=no
+    keep=file_or_folder filepath filename ext msg directory level
   );
   length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
     ext $20 msg $200;
+  retain level &level;
   %if &fref=0 %then %do;
     rc = filename(fref, "&path");
   %end;
@@ -119,8 +140,8 @@ data &outds(compress=no
 run;
 
 %if %substr(&getattrs,1,1)=Y %then %do;
-  data &outds;
-    set &outds;
+  data &out_ds;
+    set &out_ds;
     length infoname infoval $60 fref $8;
     rc=filename(fref,filepath);
     drop rc infoname fid i close fref;
@@ -161,10 +182,36 @@ run;
   run;
   proc sort;
     by filepath sasname;
-  proc transpose data=&outds out=&outds(drop=_:);
+  proc transpose data=&out_ds out=&out_ds(drop=_:);
     id sasname;
     var infoval;
     by filepath file_or_folder filename ext ;
   run;
 %end;
+
+data &out_ds;
+  set &out_ds(where=(filepath ne ''));
+run;
+
+/* update main table */
+proc append base=&outds data=&out_ds;
+run;
+
+/* recursive call */
+%if &maxdepth>&level or &maxdepth=MAX %then %do;
+  data _null_;
+    set &out_ds;
+    where file_or_folder='folder';
+    length code $10000;
+    code=cats('%nrstr(%mp_dirlist(path=',filepath,",outds=&outds"
+      ,",getattrs=&getattrs,level=%eval(&level+1),maxdepth=&maxdepth))");
+    put code=;
+    call execute(code);
+  run;
+%end;
+
+/* tidy up */
+proc sql;
+drop table &out_ds;
+
 %mend mp_dirlist;

base/mp_ds2cards.sas

@@ -2,12 +2,21 @@
   @file
   @brief Create a CARDS file from a SAS dataset.
   @details Uses dataset attributes to convert all data into datalines.
-    Running the generated file will rebuild the original dataset.
+    Running the generated file will rebuild the original dataset.  Includes
+    support for large decimals, binary data, PROCESSED_DTTM columns, and
+    alternative encoding.  If the input dataset is empty, the cards file will
+    still be created.
+
+    Additional support to generate a random sample and max rows.
+
   Usage:
 
       %mp_ds2cards(base_ds=sashelp.class
+        , tgt_ds=work.class
         , cards_file= "C:\temp\class.sas"
-        , maxobs=5)
+        , showlog=NO
+        , maxobs=5
+      )
 
   TODO:
     - labelling the dataset
@@ -18,15 +27,24 @@
                   that is converted to a cards file.
   @param [in] tgt_ds= Table that the generated cards file would create.
     Optional - if omitted, will be same as BASE_DS.
-  @param [out] cards_file= Location in which to write the (.sas) cards file
-  @param [in] maxobs= to limit output to the first <code>maxobs</code>
-    observations
-  @param [in] showlog= whether to show generated cards file in the SAS log
-    (YES/NO)
-  @param [in] outencoding= provide encoding value for file statement (eg utf-8)
-  @param [in] append= If NO then will rebuild the cards file if it already
+  @param [out] cards_file= ("%sysfunc(pathname(work))/cardgen.sas") Location in
+    which to write the (.sas) cards file
+  @param [in] maxobs= (max) To limit output to the first <code>maxobs</code>
+    observations, enter an integer here.
+  @param [in] random_sample= (NO) Set to YES to generate a random sample of
+    data.  Can be quite slow.
+  @param [in] showlog= (YES) Whether to show generated cards file in the SAS
+    log.  Valid values:
+    @li YES
+    @li NO
+  @param [in] outencoding= Provide encoding value for file statement (eg utf-8)
+  @param [in] append= (NO) If NO then will rebuild the cards file if it already
     exists, otherwise will append to it.  Used by the mp_lib2cards.sas macro.
 
+  <h4> Related Macros </h4>
+  @li mp_lib2cards.sas
+  @li mp_ds2inserts.sas
+  @li mp_mdtablewrite.sas
 
   @version 9.2
   @author Allan Bowe
@@ -51,15 +69,15 @@
 %if (&tgt_ds = ) %then %let tgt_ds=&base_ds;
 %if %index(&tgt_ds,.)=0 %then %let tgt_ds=WORK.%scan(&base_ds,2,.);
 %if ("&outencoding" ne "") %then %let outencoding=encoding="&outencoding";
-%if ("&append" = "") %then %let append=;
+%if ("&append" = "" or "&append" = "NO") %then %let append=;
 %else %let append=mod;
 
 /* get varcount */
 %let nvars=0;
 proc sql noprint;
 select count(*) into: nvars from dictionary.columns
-  where libname="%scan(%upcase(&base_ds),1)"
-    and memname="%scan(%upcase(&base_ds),2)";
+  where upcase(libname)="%scan(%upcase(&base_ds),1)"
+    and upcase(memname)="%scan(%upcase(&base_ds),2)";
 %if &nvars=0 %then %do;
   %put %str(WARN)ING: Dataset &base_ds has no variables, will not be converted.;
   %return;
@@ -115,8 +133,8 @@ proc sql
 reset outobs=max;
 create table datalines1 as
   select name,type,length,varnum,format,label from dictionary.columns
-  where libname="%upcase(%scan(&base_ds,1))"
-    and memname="%upcase(%scan(&base_ds,2))";
+  where upcase(libname)="%upcase(%scan(&base_ds,1))"
+    and upcase(memname)="%upcase(%scan(&base_ds,2))";
 
 /**
   Due to long decimals cannot use best. format
@@ -137,7 +155,18 @@ data datalines_2;
       ,put(',name,',best32.-l)
       ,substrn(put(',name,',bestd32.-l),1
       ,findc(put(',name,',bestd32.-l),"0","TBK")))');
-  else dataline=name;
+  /**
+    * binary data must be converted, to store in text format.  It is identified
+    * by the presence of the $HEX keyword in the format.
+    */
+  else if upcase(format)=:'$HEX' then
+    dataline=cats('put(trim(',name,'),',format,')');
+  /**
+    * There is no easy way to store line breaks in a cards file.
+    * To discuss this, use: https://github.com/sasjs/core/issues/80
+    * Removing all nonprintables with kw (keep writeable)
+    */
+  else dataline=cats('compress(',name,', ,"kw")');
 run;
 
 proc sql noprint;
@@ -162,7 +191,8 @@ data _null_;
 
 
 /* Build input statement */
-  if type='char' then type3=':$char.';
+  if upcase(format)=:'$HEX' then type3=':'!!format;
+  else if type='char' then type3=':$char.';
   str2=put(name,$33.)||type3;
 
 
@@ -184,11 +214,12 @@ data _null_;
   file &cards_file. &outencoding lrecl=32767 termstr=nl &append;
   length __attrib $32767;
   if _n_=1 then do;
-    put '/*******************************************************************';
-    put " Datalines for %upcase(%scan(&base_ds,2)) dataset ";
-    put " Generated by %nrstr(%%)mp_ds2cards()";
-    put " Available on github.com/sasjs/core";
-    put '********************************************************************/';
+    put '/**';
+    put '  @file';
+    put "  @brief Datalines for %upcase(%scan(&base_ds,2)) dataset";
+    put "  @details Generated by %nrstr(%%)mp_ds2cards()";
+    put "  Available on github.com/sasjs/core";
+    put '**/';
     put "data &tgt_ds &indexes;";
     put "attrib ";
     %do i = 1 %to &nvars;
@@ -212,11 +243,11 @@ data _null_;
       put 'run;';
     end;
     else do;
-      put "infile cards dsd delimiter=',';";
+      put "infile cards dsd;";
       put "input ";
       %do i = 1 %to &nvars.;
         %if(%length(&&input_stmt_&i..)) %then
-          put "   &&input_stmt_&i..";
+          put "  &&input_stmt_&i..";
         ;
       %end;
       put ";";

base/mp_ds2inserts.sas

@@ -0,0 +1,179 @@
+/**
+  @file
+  @brief Export a dataset to SQL insert statements
+  @details Converts dataset values to SQL insert statements for use across
+  multiple database types.
+
+  Usage:
+
+      %mp_ds2inserts(sashelp.class,outref=myref,outds=class)
+      data class;
+        set sashelp.class;
+        stop;
+      proc sql;
+      %inc myref;
+
+  @param [in] ds The dataset to be exported
+  @param [in] maxobs= (max) The max number of inserts to create
+  @param [out] outref= (0) The output fileref.  If it does not exist, it is
+    created. If it does exist, new records are APPENDED.
+  @param [out] schema= (0) The library (or schema) in which the target table is
+    located.  If not provided, is ignored.
+  @param [out] outds= (0) The output table to load.  If not provided, will
+    default to the table in the &ds parameter.
+  @param [in] flavour= (SAS) The SQL flavour to be applied to the output. Valid
+    options:
+    @li SAS (default) - suitable for regular proc sql
+    @li PGSQL - Used for Postgres databases
+  @param [in] applydttm= (YES) If YES, any columns using datetime formats will
+    be converted to native DB datetime literals
+
+  <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mf_getvarcount.sas
+  @li mf_getvarformat.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
+
+  @version 9.2
+  @author Allan Bowe (credit mjsq)
+**/
+
+%macro mp_ds2inserts(ds, outref=0,schema=0,outds=0,flavour=SAS,maxobs=max
+  ,applydttm=YES
+)/*/STORE SOURCE*/;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if %index(&ds,.)=0 %then %let ds=WORK.&ds;
+
+%let flavour=%upcase(&flavour);
+%if &flavour ne SAS and &flavour ne PGSQL %then %do;
+  %put %str(WAR)NING:  &flavour is not supported;
+  %return;
+%end;
+
+%if &outref=0 %then %do;
+  %put %str(WAR)NING:  Please provide a fileref;
+  %return;
+%end;
+%if %mf_existfileref(&outref)=0 %then %do;
+  filename &outref temp lrecl=66000;
+%end;
+
+%if &schema=0 %then %let schema=;
+%else %let schema=&schema..;
+
+%if &outds=0 %then %let outds=%scan(&ds,2,.);
+
+%local nobs;
+proc sql noprint;
+select count(*) into: nobs TRIMMED from &ds;
+%if &nobs=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No rows found in &ds */";
+  run;
+%end;
+
+%local vars;
+%let vars=%mf_getvarcount(&ds);
+%if &vars=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No columns found in &schema.&ds */";
+  run;
+  %return;
+%end;
+%else %if &vars>1600 and &flavour=PGSQL %then %do;
+  data _null_;
+    file &fref mod;
+    put "/* &schema.&ds contains &vars vars */";
+    put "/* Postgres cannot handle tables with over 1600 vars */";
+    put "/* No inserts will be generated for this table */";
+  run;
+  %return;
+%end;
+
+%local varlist varlistcomma;
+%let varlist=%mf_getvarlist(&ds);
+%let varlistcomma=%mf_getvarlist(&ds,dlm=%str(,),quote=double);
+
+/* next, export data */
+data _null_;
+  file &outref mod ;
+  if _n_=1 then put "/* &schema.&outds (&nobs rows, &vars columns) */";
+  set &ds;
+  %if &maxobs ne max %then %do;
+    if _n_>&maxobs then stop;
+  %end;
+  length _____str $32767;
+  format _numeric_ best.;
+  format _character_ ;
+  %local i comma var vtype vfmt;
+  %do i=1 %to %sysfunc(countw(&varlist));
+    %let var=%scan(&varlist,&i);
+    %let vtype=%mf_getvartype(&ds,&var);
+    %let vfmt=%upcase(%mf_getvarformat(&ds,&var,force=1));
+    %if &i=1 %then %do;
+      %if &flavour=SAS %then %do;
+        put "insert into &schema.&outds set ";
+        put "  &var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        _____str=cats(
+          "INSERT INTO &schema.&outds ("
+          ,symget('varlistcomma')
+          ,") VALUES ("
+        );
+        put _____str;
+        put "  "@;
+      %end;
+    %end;
+    %else %do;
+      %if &flavour=SAS %then %do;
+        put "  ,&var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        put "  ,"@;
+      %end;
+    %end;
+    %if &vtype=N %then %do;
+      %if &flavour=SAS %then %do;
+        put &var;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        if missing(&var) then put 'NULL';
+        %if &applydttm=YES and "%substr(&vfmt.xxxxxxxx,1,8)"="DATETIME"
+        %then %do;
+          else put "TIMESTAMP '" &var E8601DT25.6 "'";
+        %end;
+        %else %do;
+          else put &var;
+        %end;
+      %end;
+    %end;
+    %else %do;
+      _____str="'"!!trim(tranwrd(&var,"'","''"))!!"'";
+      put _____str;
+    %end;
+  %end;
+  %if &flavour=SAS %then %do;
+    put ';';
+  %end;
+  %else %if &flavour=PGSQL %then %do;
+    put ');';
+  %end;
+
+  if _n_=&nobs then put /;
+run;
+
+%mend mp_ds2inserts;

base/mp_filtervalidate.sas

@@ -78,7 +78,7 @@ run;
 
 data &outds;
   if &sqlrc or &syscc or &syserr then do;
-    REASON_CD='VALIDATION_ERROR: '!!
+    REASON_CD='VALIDATION_ERR'!!'OR: '!!
       coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
     output;
   end;

base/mp_getcols.sas

@@ -0,0 +1,65 @@
+/**
+  @file
+  @brief Creates a dataset with column metadata.
+  @details This macro takes the `proc contents` output and "tidies it up" in the
+  following ways:
+
+    @li Blank labels are filled in with column names
+    @li Formats are reconstructed with default values
+    @li Types such as DATE / TIME / DATETIME are inferred from the formats
+
+  Example usage:
+
+      %mp_getcols(sashelp.airline,outds=work.myds)
+
+  @param ds The dataset from which to obtain column metadata
+  @param outds= (work.cols) The output dataset to create. Sample data:
+  |NAME $|LENGTH 8|VARNUM 8|LABEL $|FORMAT $49|TYPE $1 |DDTYPE $|
+  |---|---|---|---|---|---|---|
+  |AIR|8|2|international airline travel (thousands)|8.|N|NUMERIC|
+  |DATE|8|1|DATE|MONYY.|N|DATE|
+  |REGION|3|3|REGION|$3.|C|CHARACTER|
+
+  <h4> Related Macros </h4>
+  @li mf_getvarlist.sas
+  @li mm_getcols.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_getcols(ds, outds=work.cols);
+
+proc contents noprint data=&ds
+  out=_data_ (keep=name type length label varnum format:);
+run;
+data &outds(keep=name type length varnum format label ddtype);
+  set &syslast(rename=(format=format2 type=type2));
+  name=upcase(name);
+  if type2=2 then do;
+    length format $49.;
+    if format2='' then format=cats('$',length,'.');
+    else if formatl=0 then format=cats(format2,'.');
+    else format=cats(format2,formatl,'.');
+    type='C';
+    ddtype='CHARACTER';
+  end;
+  else do;
+    if format2='' then format=cats(length,'.');
+    else if formatl=0 then format=cats(format2,'.');
+    else if formatd=0 then format=cats(format2,formatl,'.');
+    else format=cats(format2,formatl,'.',formatd);
+    type='N';
+    if format=:'DATETIME' then ddtype='DATETIME';
+    else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
+      or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
+      or format=:'MONYY'
+    then ddtype='DATE';
+    else if format=:'TIME' then ddtype='TIME';
+    else ddtype='NUMERIC';
+  end;
+  if label='' then label=name;
+run;
+
+%mend mp_getcols;

base/mp_getconstraints.sas

@@ -39,21 +39,24 @@
 /* must use SQL as proc datasets does not support length changes */
 proc sql noprint;
 create table &outds as
-  select a.TABLE_CATALOG as libref
-    ,a.TABLE_NAME
+  select upcase(a.TABLE_CATALOG) as libref
+    ,upcase(a.TABLE_NAME) as TABLE_NAME
     ,a.constraint_type
     ,a.constraint_name
     ,b.column_name
   from dictionary.TABLE_CONSTRAINTS a
   left join dictionary.constraint_column_usage  b
-  on a.TABLE_CATALOG=b.TABLE_CATALOG
-    and a.TABLE_NAME=b.TABLE_NAME
+  on upcase(a.TABLE_CATALOG)=upcase(b.TABLE_CATALOG)
+    and upcase(a.TABLE_NAME)=upcase(b.TABLE_NAME)
     and a.constraint_name=b.constraint_name
-  where a.TABLE_CATALOG="&lib"
-    and b.TABLE_CATALOG="&lib"
+/**
+  * We cannot apply this clause to the underlying dictionary table.  See:
+  * https://communities.sas.com/t5/SAS-Programming/Unexpected-Where-Clause-behaviour-in-dictionary-TABLE/m-p/771554#M244867
+  */
+  where calculated libref="&lib"
   %if "&ds" ne "" %then %do;
-    and a.TABLE_NAME="&ds"
-    and b.TABLE_NAME="&ds"
+    and upcase(a.TABLE_NAME)="&ds"
+    and upcase(b.TABLE_NAME)="&ds"
   %end;
   ;
 

base/mp_getddl.sas

@@ -5,6 +5,8 @@
     to create tables in SAS or a database.  The macro can be used at table or
     library level.  The default behaviour is to create DDL in SAS format.
 
+    Note - views are not currently supported.
+
   Usage:
 
       data test(index=(pk=(x y)/unique /nomiss));
@@ -16,12 +18,14 @@
       %mp_getddl(work,test,flavour=tsql,showlog=YES)
 
   <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mf_getvarcount.sas
   @li mp_getconstraints.sas
 
   @param lib libref of the library to create DDL for.  Should be assigned.
   @param ds dataset to create ddl for (optional)
-  @param fref= the fileref to which to write the DDL.  If not preassigned, will
-    be assigned to TEMP.
+  @param fref= the fileref to which to _append_ the DDL.  If it does not exist,
+    it will be created.
   @param flavour= The type of DDL to create (default=SAS). Supported=TSQL
   @param showlog= Set to YES to show the DDL in the log
   @param schema= Choose a preferred schema name (default is to use actual schema
@@ -37,9 +41,10 @@
 )/*/STORE SOURCE*/;
 
 /* check fileref is assigned */
-%if %sysfunc(fileref(&fref)) > 0 %then %do;
-  filename &fref temp;
+%if %mf_existfileref(&fref)=0 %then %do;
+  filename &fref temp ;
 %end;
+
 %if %length(&libref)=0 %then %let libref=WORK;
 %let flavour=%upcase(&flavour);
 
@@ -47,6 +52,7 @@ proc sql noprint;
 create table _data_ as
   select * from dictionary.tables
   where upcase(libname)="%upcase(&libref)"
+    and memtype='DATA' /* views not currently supported */
   %if %length(&ds)>0 %then %do;
     and upcase(memname)="%upcase(&ds)"
   %end;
@@ -118,7 +124,7 @@ create table _data_ as
 %mend addConst;
 
 data _null_;
-  file &fref;
+  file &fref mod;
   put "/* DDL generated by &sysuserid on %sysfunc(datetime(),datetime19.) */";
 run;
 
@@ -141,13 +147,15 @@ run;
           put "create table &libref..&curds(";
         end;
         else do;
+          /* just a placeholder - we filter out views at the top */
           put "create view &libref..&curds(";
         end;
         put "    "@@;
       end;
       else put "   ,"@@;
       if length(format)>1 then fmt=" format="!!cats(format);
-      if length(label)>1 then lab=" label="!!quote(trim(label));
+      if length(label)>1 then
+        lab=" label="!!cats("'",tranwrd(label,"'","''"),"'");
       if notnull='yes' then notnul=' not null';
       if type='char' then typ=cats('char(',length,')');
       else if length ne 8 then typ='num length='!!left(length);
@@ -203,7 +211,7 @@ run;
   proc sql noprint;
   select sysvalue into: schemaactual
     from dictionary.libnames
-    where libname="&libref" and engine='SQLSVR';
+    where upcase(libname)="&libref" and engine='SQLSVR';
   %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
 
   %do x=1 %to %sysfunc(countw(&dsnlist));
@@ -219,6 +227,7 @@ run;
           put "create table [&schema].[&curds](";
         end;
         else do;
+          /* just a placeholder - we filter out views at the top */
           put "create view [&schema].[&curds](";
         end;
         put "    "@@;
@@ -295,78 +304,88 @@ run;
   proc sql noprint;
   select sysvalue into: schemaactual
     from dictionary.libnames
-    where libname="&libref" and engine='POSTGRES';
+    where upcase(libname)="&libref" and engine='POSTGRES';
   %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
   data _null_;
     file &fref mod;
     put "CREATE SCHEMA &schema;";
   %do x=1 %to %sysfunc(countw(&dsnlist));
     %let curds=%scan(&dsnlist,&x);
-    data _null_;
-      file &fref mod;
-      put "/* Postgres Flavour DDL for &schema..&curds */";
-    data _null_;
-      file &fref mod;
-      set &colinfo (where=(upcase(memname)="&curds")) end=last;
-      length fmt $32;
-      if _n_=1 then do;
-        if memtype='DATA' then do;
-          put "CREATE TABLE &schema..&curds (";
+    %local curdsvarcount;
+    %let curdsvarcount=%mf_getvarcount(&libref..&curds);
+    %if &curdsvarcount>1600 %then %do;
+      data _null_;
+        file &fref mod;
+        put "/* &libref..&curds contains &curdsvarcount vars */";
+        put "/* Postgres cannot create tables with over 1600 vars */";
+        put "/* No DDL will be generated for this table";
+      run;
+    %end;
+    %else %do;
+      data _null_;
+        file &fref mod;
+        put "/* Postgres Flavour DDL for &schema..&curds */";
+      data _null_;
+        file &fref mod;
+        set &colinfo (where=(upcase(memname)="&curds")) end=last;
+        length fmt $32;
+        if _n_=1 then do;
+          if memtype='DATA' then do;
+            put "CREATE TABLE &schema..&curds (";
+          end;
+          else do;
+            /* just a placeholder - we filter out views at the top */
+            put "CREATE VIEW &schema..&curds (";
+          end;
+          put "    "@@;
         end;
-        else do;
-          put "CREATE VIEW &schema..&curds (";
-        end;
-        put "    "@@;
-      end;
-      else put "   ,"@@;
-      format=upcase(format);
-      if 1=0 then; /* dummy if */
-      %if &applydttm=YES %then %do;
-        else if format=:'DATETIME' then fmt=' TIMESTAMP ';
-      %end;
-      else if type='num' then fmt=' DOUBLE PRECISION';
-      else fmt='VARCHAR('!!cats(length)!!')';
-      if notnull='yes' then notnul=' NOT NULL';
-      /* quote column names in case they represent reserved words */
-      name2=quote(trim(name));
-      put name2 fmt notnul;
-    run;
+        else put "   ,"@@;
+        format=upcase(format);
+        if 1=0 then; /* dummy if */
+        %if &applydttm=YES %then %do;
+          else if format=:'DATETIME' then fmt=' TIMESTAMP ';
+        %end;
+        else if type='num' then fmt=' DOUBLE PRECISION';
+        else fmt='VARCHAR('!!cats(length)!!')';
+        if notnull='yes' then notnul=' NOT NULL';
+        /* quote column names in case they represent reserved words */
+        name2=quote(trim(name));
+        put name2 fmt notnul;
+      run;
 
-    /* Extra step for data constraints */
-    %addConst()
+      /* Extra step for data constraints */
+      %addConst()
 
-    data _null_;
-      file &fref mod;
-      put ');';
-    run;
+      data _null_;
+        file &fref mod;
+        put ');';
+      run;
 
-    /* Create Unique Indexes, but only if they were not already defined within
-      the Constraints section. */
-    data _null_;
-      *length ds $128;
-      set &idxinfo(
-        where=(
-          memname="&curds"
-          and unique='yes'
-          and indxname not in (
-            %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+      /* Create Unique Indexes, but only if they were not already defined within
+        the Constraints section. */
+      data _null_;
+        *length ds $128;
+        set &idxinfo(
+          where=(
+            memname="&curds"
+            and unique='yes'
+            and indxname not in (
+              %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+            )
           )
-        )
-      );
-      file &fref mod;
-      by idxusage indxname;
-/*       ds=cats(libname,'.',memname); */
-      if first.indxname then do;
+        );
+        file &fref mod;
+        by idxusage indxname;
+        if first.indxname then do;
           put 'CREATE UNIQUE INDEX "' indxname +(-1) '" ' "ON &schema..&curds(";
           put '  "' name +(-1) '"' ;
-      end;
-      else put '  ,"' name +(-1) '"';
-      *else put '    ,' name ;
-      if last.indxname then do;
-        put ');';
-      end;
-    run;
-
+        end;
+        else put '  ,"' name +(-1) '"';
+        if last.indxname then do;
+          put ');';
+        end;
+      run;
+    %end;
   %end;
 %end;
 %if %upcase(&showlog)=YES %then %do;

base/mp_gsubfile.sas

@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Performs a text substitution on a file
+  @details Makes use of the GSUB function in LUA to perform a text substitution
+  in a file - either in-place, or writing to a new location.  The benefit of
+  using LUA is that the entire file can be loaded into a single variable,
+  thereby side stepping the 32767 character limit in a data step.
+
+  Usage:
+
+      %let file=%sysfunc(pathname(work))/file.txt;
+      %let str=replace/me;
+      %let rep=with/this;
+      data _null_;
+        file "&file";
+        put "&str";
+      run;
+      %mp_gsubfile(file=&file, patternvar=str, replacevar=rep)
+      data _null_;
+        infile "&file";
+        input;
+        list;
+      run;
+
+  @param file= (0) The file to perform the substitution on
+  @param patternvar= A macro variable containing the Lua
+    [pattern](https://www.lua.org/pil/20.2.html) to search for.  Due to the use
+    of special (magic) characters in Lua patterns, it is safer to pass the NAME
+    of the macro variable containing the string, rather than the value itself.
+  @param replacevar= The name of the macro variable containing the replacement
+    _string_.
+  @param outfile= (0) The file to write the output to. If zero, then the file
+    is overwritten in-place.
+
+  <h4> SAS Macros </h4>
+  @li ml_gsubfile.sas
+
+  <h4> Related Macros </h4>
+  @li mp_gsubfile.test.sas
+
+  @version 9.4
+  @author Allan Bowe
+**/
+
+%macro mp_gsubfile(file=0,
+  patternvar=,
+  replacevar=,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+  %ml_gsubfile()
+
+%mend mp_gsubfile;

base/mp_hashdataset.sas

@@ -20,6 +20,7 @@
   @li mf_getvartype.sas
 
   @param [in] libds dataset to hash
+  @param [in] salt= Provide a salt (could be, for instance, the dataset name)
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
   will contain one column (hashkey) with one observation (a hex32.
   representation of the input hash)
@@ -33,7 +34,8 @@
 
 %macro mp_hashdataset(
   libds,
-  outds=
+  outds=,
+  salt=
 )/*/STORE SOURCE*/;
   %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
     %put %str(WARN)ING: Dataset &libds is empty;, or is not a dataset;
@@ -53,7 +55,7 @@
     %let varlist=%mf_getvarlist(&libds);
     data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
       length &prevkeyvar &keyvar $32;
-      retain &prevkeyvar;
+      retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
       set &libds end=&lastvar;
       /* hash should include previous row */
       &keyvar=put(md5(&prevkeyvar

base/mp_include.sas

@@ -0,0 +1,104 @@
+/**
+  @file
+  @brief Performs a wrapped \%include
+  @details This macro wrapper is necessary if you need your included code to
+  know that it is being \%included.
+
+  If you are using %include in a regular program, you could make use of the
+  following macro variables:
+
+  @li SYSINCLUDEFILEDEVICE
+  @li SYSINCLUDEFILEDIR
+  @li SYSINCLUDEFILEFILEREF
+  @li SYSINCLUDEFILENAME
+
+  However these variables are NOT available inside a macro, as documented here:
+https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/mcrolref/n1j5tcc0n2xczyn1kg1o0606gsv9.htm
+
+  This macro can be used in place of the %include statement, and will insert
+  the following (equivalent) global variables:
+
+  @li _SYSINCLUDEFILEDEVICE
+  @li _SYSINCLUDEFILEDIR
+  @li _SYSINCLUDEFILEFILEREF
+  @li _SYSINCLUDEFILENAME
+
+  These can be used whenever testing _within a macro_.  Outside of the macro,
+  the regular automatic variables will still be available (thanks to a
+  concatenated file list in the include statement).
+
+  Example usage:
+
+      filename example temp;
+      data _null_;
+        file example;
+        put '%macro test();';
+        put '%put &=_SYSINCLUDEFILEFILEREF;';
+        put '%put &=SYSINCLUDEFILEFILEREF;';
+        put '%mend; %test()';
+        put '%put &=SYSINCLUDEFILEFILEREF;';
+      run;
+      %mp_include(example)
+
+  @param [in] fileref The fileref of the file to be included. Must be provided.
+  @param [in] prefix= (_) The prefix to apply to the global variables.
+  @param [in] opts= (SOURCE2) The options to apply to the %inc statement
+  @param [in] errds= (work.mp_abort_errds) There is no clean way to end a
+    process within a %include called within a macro.  Furthermore, there is no
+    way to test if a macro is called within a %include.  To handle this
+    particular scenario, the %mp_abort() macro will test for the existence of
+    the `_SYSINCLUDEFILEDEVICE` variable and return the outputs (msg,mac) inside
+    this dataset.
+    It will then run an abort cancel FILE to stop the include running, and pass
+    the dataset back.
+    NOTE - it is NOT possible to read this dataset as part of _this_ macro -
+    when running abort cancel FILE, ALL macros are closed, so instead it is
+    necessary to invoke "%mp_abort(mode=INCLUDE)" OUTSIDE of any macro wrappers.
+
+
+  @version 9.4
+  @author Allan Bowe
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_abort.sas
+
+**/
+
+%macro mp_include(fileref
+  ,prefix=_
+  ,opts=SOURCE2
+  ,errds=work.mp_abort_errds
+)/*/STORE SOURCE*/;
+
+/* prepare precode */
+%local tempref;
+%let tempref=%mf_getuniquefileref();
+data _null_;
+  file &tempref;
+  set sashelp.vextfl(where=(fileref="%upcase(&fileref)"));
+  put '%let _SYSINCLUDEFILEDEVICE=' xengine ';';
+  name=scan(xpath,-1,'/\');
+  put '%let _SYSINCLUDEFILENAME=' name ';';
+  path=subpad(xpath,1,length(xpath)-length(name)-1);
+  put '%let _SYSINCLUDEFILEDIR=' path ';';
+  put '%let _SYSINCLUDEFILEFILEREF=' "&fileref;";
+run;
+
+/* prepare the errds */
+data &errds;
+  length msg mac $1000;
+  iftrue='1=0';
+run;
+
+/* include the include */
+%inc &tempref &fileref/&opts;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=%str(&_SYSINCLUDEFILEDIR/&_SYSINCLUDEFILENAME)
+  ,msg=%str(syscc=&syscc after executing &_SYSINCLUDEFILENAME)
+)
+
+filename &tempref clear;
+
+%mend mp_include;

base/mp_jsonout.sas

@@ -4,7 +4,7 @@
   @details PROC JSON is faster but will produce errs like the ones below if
   special chars are encountered.
 
-  > ERROR: Some code points did not transcode.
+  > (ERR)OR: Some code points did not transcode.
 
   > An object or array close is not valid at this point in the JSON text.
 

base/mp_lib2inserts.sas

@@ -0,0 +1,77 @@
+/**
+  @file
+  @brief Convert all data in a library to SQL insert statements
+  @details Gets list of members then calls the <code>%mp_ds2inserts()</code>
+  macro.
+  Usage:
+
+      %mp_getddl(sashelp, schema=work, fref=tempref)
+
+      %mp_lib2inserts(sashelp, schema=work, outref=tempref)
+
+      %inc tempref;
+
+
+  The output will be one file in the outref fileref.
+
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2inserts.sas
+
+
+  @param [in] lib Library in which to convert all datasets to inserts
+  @param [in] flavour= (SAS) The SQL flavour to be applied to the output. Valid
+    options:
+    @li SAS (default) - suitable for regular proc sql
+    @li PGSQL - Used for Postgres databases
+  @param [in] maxobs= (max) The max number of observations (per table) to create
+  @param [out] outref= Output fileref in which to create the insert statements.
+    If it exists, it will be appended to, otherwise it will be created.
+  @param [out] schema= (0) The schema of the target database, or the libref.
+  @param [in] applydttm= (YES) If YES, any columns using datetime formats will
+    be converted to native DB datetime literals
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_lib2inserts(lib
+    ,flavour=SAS
+    ,outref=0
+    ,schema=0
+    ,maxobs=max
+    ,applydttm=YES
+)/*/STORE SOURCE*/;
+
+/* Find the tables */
+%local x ds memlist;
+proc sql noprint;
+select distinct lowcase(memname)
+  into: memlist
+  separated by ' '
+  from dictionary.tables
+  where upcase(libname)="%upcase(&lib)"
+    and memtype='DATA'; /* exclude views */
+
+
+%let flavour=%upcase(&flavour);
+%if &flavour ne SAS and &flavour ne PGSQL %then %do;
+  %put %str(WAR)NING:  &flavour is not supported;
+  %return;
+%end;
+
+
+/* create the inserts */
+%do x=1 %to %sysfunc(countw(&memlist));
+  %let ds=%scan(&memlist,&x);
+  %mp_ds2inserts(&lib..&ds
+    ,outref=&outref
+    ,schema=&schema
+    ,outds=&ds
+    ,flavour=&flavour
+    ,maxobs=&maxobs
+    ,applydttm=&applydttm
+  )
+%end;
+
+%mend mp_lib2inserts;

base/mp_lockanytable.sas

@@ -0,0 +1,255 @@
+/**
+  @file
+  @brief Mechanism for locking tables to prevent parallel modifications
+  @details Uses a control table to enable ANY table to be locked for updates.
+  Only useful if every update uses the macro!   Used heavily within
+  [Data Controller for SAS](https://datacontroller.io).
+
+  The underlying table is structured as per the MAKETABLE action.
+
+  @param [in] action The action to be performed.  Valid values:
+    @li LOCK - Sets the lock flag, also confirms if a SAS lock is available
+    @li UNLOCK - Unlocks the table
+    @li MAKETABLE - creates the control table (ctl_ds)
+  @param [in] lib= (WORK) The libref of the table to lock.  Should already be
+    assigned.
+  @param [in] ds= The dataset to lock
+  @param [in] ref= A meaningful reference to enable the lock to be traced. Max
+    length is 200 characters.
+  @param [out] ctl_ds= (0) The control table which controls the actual locking.
+    Should already be assigned and available.
+  @param [in] loops= (25) Number of times to check for a lock.
+  @param [in] loop_secs= (1) Seconds to wait between each lock attempt
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_lockfilecheck.sas
+  @li mf_getuser.sas
+
+  <h4> Related Macros </h4>
+  @li mp_lockanytable.test.sas
+
+  @version 9.2
+
+**/
+
+%macro mp_lockanytable(
+  action
+  ,lib= WORK
+  ,ds=0
+  ,ref=
+  ,ctl_ds=0
+  ,loops=25
+  ,loop_secs=1
+  );
+data _null_;
+  if _n_=1 then putlog "&sysmacroname entry vars:";
+  set sashelp.vmacro;
+  where scope="&sysmacroname";
+  put name '=' value;
+run;
+
+%mp_abort(iftrue= (&ds=0 and &action ne MAKETABLE)
+  ,mac=&sysmacroname
+  ,msg=%str(dataset was not provided)
+)
+%mp_abort(iftrue= (&ctl_ds=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Control dataset was not provided)
+)
+
+/* set up lib & mac vars */
+%let lib=%upcase(&lib);
+%let ds=%upcase(&ds);
+%let action=%upcase(&action);
+%local user x trans msg abortme;
+%let user=%mf_getuser();
+%let abortme=0;
+
+%mp_abort(iftrue= (&action ne LOCK & &action ne UNLOCK & &action ne MAKETABLE)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid action (&action) provided)
+)
+
+/* if an err condition exists, exit before we even begin */
+%mp_abort(iftrue= (&syscc>0 and &action=LOCK)
+  ,mac=&sysmacroname
+  ,msg=%str(aborting due to syscc=&syscc on LOCK entry)
+)
+
+/* do not bother locking work tables (else may affect all WORK libraries) */
+%if (%upcase(&lib)=WORK or %str(&lib)=%str()) & &action ne MAKETABLE %then %do;
+  %put NOTE: WORK libraries will not be registered in the locking system.;
+  %return;
+%end;
+
+/* do not proceed if no observations can be processed */
+%mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)
+  ,mac=&sysmacroname
+  ,msg=%str(options obs = 0. syserrortext=&syserrortext)
+)
+
+%if &ACTION=LOCK %then %do;
+
+  /* abort if a SAS lock is already in place, or cannot be applied */
+  %mp_lockfilecheck(&lib..&ds)
+
+  /* next, check there is a record for this table */
+  %local record_exists_check;
+  proc sql noprint;
+  select count(*) into: record_exists_check from &ctl_ds
+    where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+  quit;
+  %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
+  %if &record_exists_check=0 %then %do;
+    data _null_;
+      putlog "&sysmacroname: adding record to lock table..";
+    run;
+
+    data ;
+      if 0 then set &ctl_ds;
+      LOCK_LIB ="&lib";
+      LOCK_DS="&ds";
+      LOCK_STATUS_CD='LOCKED';
+      LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt;
+      LOCK_USER_NM="&user";
+      LOCK_PID="&sysjobid";
+      LOCK_REF="&ref";
+      output;stop;
+    run;
+    %let trans=&syslast;
+    proc append base=&ctl_ds data=&trans;
+    run;
+  %end;
+  /* if record does exist, perform lock attempts */
+  %else %do x=1 %to &loops;
+    data _null_;
+      putlog "&sysmacroname: attempting lock (iteration &x) "@;
+      putlog "at %sysfunc(datetime(),datetime19.) ..";
+    run;
+
+    proc sql;
+    update &ctl_ds
+      set LOCK_STATUS_CD='LOCKED'
+        , LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_USER_NM="&user"
+        , LOCK_PID="&sysjobid"
+        , LOCK_REF="&ref"
+      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+    quit;
+    /**
+      * NOTE - occasionally SQL server will return an err code (deadlocked
+      * transaction).  If so, ignore it, keep calm, and carry on..
+      */
+    %if &syscc>0 %then %do;
+      data _null_;
+        putlog 'NOTE-' / 'NOTE-';
+        putlog "NOTE- &sysmacroname: Update failed. "@;
+        putlog "Resetting err conditions and re-attempting.";
+        putlog "NOTE- syscc=&syscc syserr=&syserr sqlrc=&sqlrc";
+        putlog 'NOTE-' / 'NOTE-';
+      run;
+      %let syscc=0;
+      %let sqlrc=0;
+    %end;
+
+    /* now check if the record was successfully updated */
+    %local success_check;
+    proc sql noprint;
+    select count(*) into: success_check from &ctl_ds
+      where LOCK_LIB ="&lib" and LOCK_DS="&ds"
+        and LOCK_PID="&sysjobid" and LOCK_STATUS_CD='LOCKED';
+    quit;
+    %if &success_check=0 %then %do;
+      %if &x < &loops %then %do;
+        /* pause before next check */
+        data _null_;
+          putlog 'NOTE-' / 'NOTE-';
+          putlog "NOTE- &sysmacroname: table locked, waiting "@;
+          putlog "%sysfunc(sleep(&loop_inc)) seconds.. ";
+          putlog "NOTE- (iteration &x of &loops)";
+          putlog 'NOTE-' / 'NOTE-';
+        run;
+      %end;
+      %else %do;
+        %let msg=Unable to lock &lib..&ds via &ctl_ds after &loops attempts.\n
+            Please ask your administrator to investigate!;
+        %let abortme=1;
+      %end;
+    %end;
+    %else %do;
+      data _null_;
+        putlog 'NOTE-' / 'NOTE-';
+        putlog "NOTE- &sysmacroname: Table &lib..&ds locked at "@
+        putlog " %sysfunc(datetime(),datetime19.) (iteration &x)"@;
+        putlog 'NOTE-' / 'NOTE-';
+      run;
+      %if &syscc>0 %then %do;
+        %put setting syscc(&syscc) back to 0;
+        %let syscc=0;
+      %end;
+      %let x=&loops;  /* no more iterations needed */
+    %end;
+  %end;
+%end;
+%else %if &ACTION=UNLOCK %then %do;
+  %local status;
+  proc sql noprint;
+  select LOCK_STATUS_CD into: status from &ctl_ds
+    where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+  quit;
+  %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
+  %if &status=LOCKED %then %do;
+    data _null_;
+      putlog "&sysmacroname: unlocking &lib..&ds:";
+    run;
+    proc sql;
+    update &ctl_ds
+      set LOCK_STATUS_CD='UNLOCKED'
+        , LOCK_END_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_USER_NM="&user"
+        , LOCK_PID="&sysjobid"
+        , LOCK_REF="&ref"
+      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+    quit;
+  %end;
+  %else %if &status=UNLOCKED %then %do;
+    %put %str(WAR)NING: &lib..&ds is already unlocked!;
+  %end;
+  %else %do;
+    %put NOTE: Unrecognised STATUS_CD (&status) in &ctl_ds;
+    %let abortme=1;
+  %end;
+%end;
+%else %if &action=MAKETABLE %then %do;
+  proc sql;
+  create table &ctl_ds(
+      lock_lib char(8),
+      lock_ds char(32),
+      lock_status_cd char(10) not null,
+      lock_user_nm char(100) not null ,
+      lock_ref char(200),
+      lock_pid char(10),
+      lock_start_dttm num format=E8601DT26.6,
+      lock_end_dttm num format=E8601DT26.6,
+    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+%end;
+%else %do;
+  %let msg=lock_anytable given unsupported action (&action);
+  %let abortme=1;
+%end;
+
+/* catch errors - mp_abort must be called outside of a logic block */
+%mp_abort(iftrue=(&abortme=1),
+  msg=%superq(msg),
+  mac=&sysmacroname
+)
+
+%exit_macro:
+data _null_;
+  put "&sysmacroname: Exit vars: action=&action lib=&lib ds=&ds";
+  put " syscc=&syscc sqlrc=&sqlrc syserr=&syserr";
+run;
+%mend mp_lockanytable;
+
+

base/mp_lockfilecheck.sas

@@ -0,0 +1,97 @@
+/**
+  @file
+  @brief Aborts if a SAS lock file is in place, or if one cannot be applied.
+  @details Used in conjuction with the mp_lockanytable macro.
+  More info here: https://sasensei.com/flash/24
+
+  Usage:
+
+      data work.test; a=1;run;
+      %mp_lockfilecheck(work.test)
+
+  @param [in] libds The libref.dataset for which to check the lock status
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_getattrc.sas
+
+  <h4> Related Macros </h4>
+  @li mp_lockanytable.sas
+  @li mp_lockfilecheck.test.sas
+
+  @version 9.2
+**/
+
+%macro mp_lockfilecheck(
+  libds
+)/*/STORE SOURCE*/;
+
+data _null_;
+  if _n_=1 then putlog "&sysmacroname entry vars:";
+  set sashelp.vmacro;
+  where scope="&sysmacroname";
+  put name '=' value;
+run;
+
+%mp_abort(iftrue= (&syscc>0)
+  ,mac=checklock.sas
+  ,msg=Aborting with syscc=&syscc on entry.
+)
+%mp_abort(iftrue= (&libds=0)
+  ,mac=&sysmacroname
+  ,msg=%str(libds not provided)
+)
+
+%local msg lib ds;
+%let lib=%upcase(%scan(&libds,1,.));
+%let ds=%upcase(%scan(&libds,2,.));
+
+/* do not proceed if no observations can be processed */
+%let msg=options obs = 0. syserrortext=%superq(syserrortext);
+%mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)
+  ,mac=checklock.sas
+  ,msg=%superq(msg)
+)
+
+data _null_;
+  putlog "Checking engine & member type";
+run;
+%local engine memtype;
+%let memtype=%mf_getattrc(&libds,MTYPE);
+%let engine=%mf_getattrc(&libds,ENGINE);
+
+%if &engine ne V9 and &engine ne BASE %then %do;
+  data _null_;
+    putlog "Lib &lib  is not assigned using BASE engine - uses &engine instead";
+    putlog "SAS lock check will not be performed";
+  run;
+  %return;
+%end;
+%else %if &memtype ne DATA %then %do;
+  %put NOTE: Cannot lock a VIEW!! Memtype=&memtype;
+  %return;
+%end;
+
+data _null_;
+  putlog "Engine = &engine, memtype=&memtype";
+  putlog "Attempting lock statement";
+run;
+
+lock &libds;
+
+%local abortme;
+%let abortme=0;
+%if &syscc>0 or &SYSLCKRC ne 0 %then %do;
+  %let msg=Unable to apply lock on &libds (SYSLCKRC=&SYSLCKRC syscc=&syscc);
+  %put %str(ERR)OR: &sysmacroname: &msg;
+  %let abortme=1;
+%end;
+
+lock &libds clear;
+
+%mp_abort(iftrue= (&abortme=1)
+  ,mac=&sysmacroname
+  ,msg=%superq(msg)
+)
+
+%mend mp_lockfilecheck;

base/mp_mdtablewrite.sas

@@ -14,14 +14,14 @@
   We take the standard definition one step further by embedding the informat
   in the table header row, like so:
 
-      |var1:$|var2:best.|var3:date9.|
+      |var1:$32|var2:best.|var3:date9.|
       |---|---|---|
       |some text|42|01JAN1960|
       |blah|1|31DEC1999|
 
   Which resolves to:
 
-  |var1:$|var2:best.|var3:date9.|
+  |var1:$32|var2:best.|var3:date9.|
   |---|---|---|
   |some text|42|01JAN1960|
   |blah|1|31DEC1999|

base/mp_testservice.sas

@@ -22,8 +22,11 @@
     |mustbevalidname|can be anything, oops, %abort!!|
 
   @param [in] debug= (log) Provide the _debug value
-  @param [in] viyaresult=(WEBOUT_JSON) The Viya result type to return.  For
+  @param [in] mdebug= (0) Set to 1 to provide macro debugging
+  @param [in] viyaresult= (WEBOUT_JSON) The Viya result type to return.  For
     more info, see mv_getjobresult.sas
+  @param [in] viyacontext= (SAS Job Execution compute context) The Viya compute
+    context on which to run the service
   @param [out] outlib= (0) Output libref to contain the final tables.  Set to
     0 if the service output is not in JSON format.
   @param [out] outref= (0) Output fileref to create, to contain the full _webout
@@ -47,17 +50,18 @@
   inputfiles=0,
   inputparams=0,
   debug=log,
+  mdebug=0,
   outlib=0,
   outref=0,
-  viyaresult=WEBOUT_JSON
+  viyaresult=WEBOUT_JSON,
+  viyacontext=SAS Job Execution compute context
 )/*/STORE SOURCE*/;
-%local mdebug;
-%if &debug ne 0 %then %do;
-  %let mdebug=1;
+%local dbg;
+%if &mdebug=1 %then %do;
   %put &sysmacroname entry vars:;
   %put _local_;
 %end;
-%else %let mdebug=0;
+%else %let dbg=*;
 
 /* sanitise inputparams */
 %local pcnt;
@@ -212,6 +216,7 @@
 
   data &ds1;
     retain _program "&program";
+    retain _contextname "&viyacontext";
     set &ds1;
     putlog "&sysmacroname inputparams:";
     putlog (_all_)(=);

base/mp_unzip.sas

@@ -2,10 +2,10 @@
   @file mp_unzip.sas
   @brief Unzips a zip file
   @details Opens the zip file and copies all the contents to another directory.
-    It is not possible to retain permissions / timestamps, also the BOF marker
-    is lost so it cannot extract binary files.
+  It is not possible to retain permissions / timestamps, also the BOF marker
+  is lost so it cannot extract binary files.
 
-    Usage:
+  Usage:
 
       filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
@@ -16,8 +16,9 @@
   @li mf_mkdir.sas
   @li mf_getuniquefileref.sas
 
-  @param ziploc= fileref or quoted full path to zip file ("/path/to/file.zip")
-  @param outdir= directory in which to write the outputs (created if non existant)
+  @param ziploc= Fileref or quoted full path to zip file ("/path/to/file.zip")
+  @param outdir= (%sysfunc(pathname(work))) Directory in which to write the
+    outputs (created if non existant)
 
   @version 9.4
   @author Allan Bowe
@@ -35,7 +36,8 @@
 %let fname2=%mf_getuniquefileref();
 %let fname3=%mf_getuniquefileref();
 
-filename &fname1 ZIP &ziploc; * Macro variable &datazip would be read from the file*;
+/* Macro variable &datazip would be read from the file */
+filename &fname1 ZIP &ziploc;
 
 /* Read the "members" (files) from the ZIP file */
 data _data_(keep=memname isFolder);

base/mp_wait4file.sas

@@ -0,0 +1,37 @@
+/**
+  @file
+  @brief Wait until a file arrives before continuing execution
+  @details Loops with a `sleep()` command until a file arrives or the max wait
+  period expires.
+
+  @example
+
+  Wait 3 minutes OR for /tmp/flag.txt to appear
+
+    %mp_wait4file(/tmp/flag.txt , maxwait=60*3)
+
+  @param [in] file The file to wait for.  Must be provided.
+  @param [in] maxwait= (0) Number of seconds to wait.  If set to zero, will
+    loop indefinitely (to a maximum of 46 days, per SAS [documentation](
+      https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a001418809.htm
+    )).  Otherwise, execution will proceed upon sleep expiry.
+  @param [in] interval= (1) The wait period between sleeps, in seconds
+
+
+**/
+
+%macro mp_wait4file(file, maxwait=0, interval=1);
+
+%if %str(&file)=%str() %then %do;
+  %put %str(ERR)OR: file not provided;
+%end;
+
+data _null_;
+  maxwait=&maxwait;
+  if maxwait=0 then maxwait=60*60*24*46;
+  do until (fileexist("&file") or slept>maxwait );
+    slept=sum(slept,sleep(&interval,1));
+  end;
+run;
+
+%mend mp_wait4file;

base/mp_webin.sas

@@ -0,0 +1,59 @@
+/**
+  @file
+  @brief Fix the `_WEBIN` variables provided to SAS web services
+  @details When uploading files to SAS Stored Processes or Viya Jobs a number
+  of global macro variables are automatically created - however there are some
+  differences in behaviour both between SAS 9 and Viya, and also between a
+  single file upload and a multi-file upload.
+
+  This macro "straightens" up the global macro variables to make it easier /
+  simpler to write code that works in both environments and with a variable
+  number of file inputs.
+
+  After running this macro, the following global variables will *always* exist:
+  @li `_WEBIN_FILE_COUNT`
+  @li `_WEBIN_FILENAME1`
+  @li `_WEBIN_FILEREF1`
+  @li `_WEBIN_NAME1`
+
+  Usage:
+
+    %mp_webin()
+
+  This was created as a macro procedure (over a macro function) as it will also
+  use the filename statement in Viya environments (where `_webin_fileuri` is
+  provided).
+
+  <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+
+**/
+
+%macro mp_webin();
+
+/* prepare global variables */
+%global _webin_file_count
+  _webin_filename _webin_filename1
+  _webin_fileref _webin_fileref1
+  _webin_fileuri _webin_fileuri1
+  _webin_name _webin_name1
+  ;
+
+/* create initial versions */
+%let _webin_file_count=%eval(&_webin_file_count+0);
+%let _webin_filename1=%sysfunc(coalescec(&_webin_filename1,&_webin_filename));
+%let _webin_fileref1=%sysfunc(coalescec(&_webin_fileref1,&_webin_fileref));
+%let _webin_fileuri1=%sysfunc(coalescec(&_webin_fileuri1,&_webin_fileuri));
+%let _webin_name1=%sysfunc(coalescec(&_webin_name1,&_webin_name));
+
+
+/* If Viya, create temporary fileref(s) */
+%local i;
+%if %mf_getplatform()=SASVIYA %then %do i=1 %to &_webin_file_count;
+  %let _webin_fileref&i=%mf_getuniquefileref();
+  filename &&_webin_fileref&i filesrvc "&&_webin_fileuri&i";
+%end;
+
+
+%mend mp_webin;

base/mp_zip.sas

@@ -16,11 +16,18 @@
   @li mp_dirlist.sas
 
   @param in= unquoted filepath, dataset of files or directory to zip
-  @param type= FILE, DATASET, DIRECTORY. (FILE / DATASET not ready yet)
-  @param outname= output file to create, without .zip extension
-  @param outpath= location for output zip file
+  @param type= (FILE) Valid values:
+    @li FILE - /full/path/and/filename.extension to a particular file
+    @li DATASET - a dataset containing a list of files to zip (see `incol`)
+    @li DIRECTORY - a directory to zip
+  @param outname= (FILE) Output file to create, _without_ .zip extension
+  @param outpath= (%sysfunc(pathname(WORK))) Parent folder for output zip file
   @param incol= if DATASET input, say which column contains the filepath
 
+  <h4> Related Macros </h4>
+  @li mp_unzip.sas
+  @li mp_zip.test.sas
+
   @version 9.2
   @author Allan Bowe
   @source https://github.com/sasjs/core
@@ -51,9 +58,9 @@ ods package open nopf;
     set &ds;
     length __command $4000;
     if file_or_folder='file';
-    command=cats('ods package add file="',filepath
+    __command=cats('ods package add file="',filepath
       ,'" mimetype="application/x-compress";');
-    call execute(command);
+    call execute(__command);
   run;
   /* tidy up */
   %if &debug=NO %then %do;
@@ -64,11 +71,10 @@ ods package open nopf;
   data _null_;
     set &in;
     length __command $4000;
-    command=cats('ods package add file="',&incol
+    __command=cats('ods package add file="',&incol
       ,'" mimetype="application/x-compress";');
-    call execute(command);
+    call execute(__command);
   run;
-  ods package add file="&in" mimetype="application/x-compress";
 %end;
 
 

build.py

@@ -22,7 +22,7 @@ for file in files:
         for line in infile:
           ml.write("  put '" + line.rstrip().replace("'","''") + " ';\n")
     ml.write("run;\n\n")
-    ml.write("%inc \"%sysfunc(pathname(work))/" + name + ".lua\";\n\n")
+    ml.write("%inc \"%sysfunc(pathname(work))/" + name + ".lua\" /source2;\n\n")
     ml.write("%mend " + name + ";\n")
 
 ml.close()
@@ -84,7 +84,7 @@ options noquotelenmax;
 """
 f = open('all.sas', "w")             # r / r+ / rb / rb+ / w / wb
 f.write(header)
-folders=['base','meta','metax','viya','lua']
+folders=['base','meta','metax','server','viya','lua','fcmp']
 for folder in folders:
     filenames = [fn for fn in Path('./' + folder).iterdir() if fn.match("*.sas")]
     filenames.sort()

fcmp/mcf_stpsrv_header.sas

@@ -0,0 +1,94 @@
+/**
+  @file
+  @brief Provides a replacement for the stpsrv_header function
+  @details The stpsrv_header is normally a built-in function, used to set the
+  headers for SAS 9 Stored Processes as documented here:
+  https://go.documentation.sas.com/doc/en/itechcdc/9.4/stpug/srvhead.htm
+
+  The purpose of this custom function is to provide a replacement when running
+  similar code as a web service against
+  [sasjs/server](https://github.com/sasjs/server).  It operates by creating a
+  text file with the headers.  The location of this text file is determined by
+  a macro variable (`sasjs_stpsrv_header_loc`) which needs to be injected into
+  each service by the calling process, eg:
+
+      %let sasjs_stpsrv_header_loc = C:/temp/some_uuid/stpsrv_header.txt;
+
+  Note - the function works by appending headers to the file.  If multiple same-
+  named headers are provided, they will all be appended - the calling process
+  needs to pick up the last one.  This will mean removing the attribute if the
+  final record has an empty value.
+
+  The function takes the following (positional) parameters:
+
+  | PARAMETER | DESCRIPTION |
+  |------------|-------------|
+  | name $  | name of the header attribute to create|
+  | value  $  | value of the header attribute|
+
+  It returns 0 if successful, or -1 if an error occured.
+
+  Usage:
+
+      %let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/stpsrv_header.txt;
+
+      %mcf_stpsrv_header(wrap=YES, insert_cmplib=YES)
+
+      data _null_;
+        rc=stpsrv_header('Content-type','application/text');
+        rc=stpsrv_header('Content-disposition',"attachment; filename=file.txt");
+      run;
+
+      data _null_;
+        infile "&sasjs_stpsrv_header_loc";
+        input;
+        putlog _infile_;
+      run;
+
+
+  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
+  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
+    CMPLIB reference.
+  @param [out] lib= (work) The output library in which to create the catalog.
+  @param [out] cat= (sasjs) The output catalog in which to create the package.
+  @param [out] pkg= (utils) The output package in which to create the function.
+    Uses a 3 part format:  libref.catalog.package
+
+**/
+
+%macro mcf_stpsrv_header(wrap=NO
+  ,insert_cmplib=NO
+  ,lib=WORK
+  ,cat=SASJS
+  ,pkg=UTILS
+)/*/STORE SOURCE*/;
+
+%if &wrap=YES  %then %do;
+  proc fcmp outcat=&lib..&cat..&pkg;
+%end;
+
+function stpsrv_header(name $, value $);
+  length loc $128 val $512;
+  loc=symget('sasjs_stpsrv_header_loc');
+  val=trim(name)!!': '!!value;
+  length fref $8;
+  rc=filename(fref,loc);
+  if (rc ne 0) then return( -1 );
+  fid = fopen(fref,'a');
+  if (fid = 0) then return( -1 );
+  rc=fput(fid, val);
+  rc=fwrite(fid);
+  rc=fclose(fid);
+  rc=filename(fref);
+  return(0);
+endsub;
+
+%if &wrap=YES %then %do;
+  quit;
+%end;
+
+%if &insert_cmplib=YES %then %do;
+  options insert=(CMPLIB=(&lib..&cat));
+%end;
+
+%mend mcf_stpsrv_header;

fcmp/mcf_string2file.sas

@@ -0,0 +1,79 @@
+/**
+  @file
+  @brief Adds a string to a file
+  @details Creates an fcmp function for appending a string to an external file.
+  If the file does not exist, it is created.
+
+  The function itself takes the following (positional) parameters:
+
+  | PARAMETER | DESCRIPTION |
+  |------------|-------------|
+  | filepath $  | full path to the file|
+  | string  $  | string to add to the file |
+  | mode $     | mode of the output - either APPEND (default) or CREATE |
+
+  It returns 0 if successful, or -1 if an error occured.
+
+  Usage:
+
+      %mcf_string2file(wrap=YES, insert_cmplib=YES)
+
+      data _null_;
+        rc=mcf_string2file(
+          "%sysfunc(pathname(work))/newfile.txt"
+          , "This is a test"
+          , "CREATE");
+      run;
+
+      data _null_;
+        infile "%sysfunc(pathname(work))/newfile.txt";
+        input;
+        putlog _infile_;
+      run;
+
+  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
+  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
+    CMPLIB reference.
+  @param [out] lib= (work) The output library in which to create the catalog.
+  @param [out] cat= (sasjs) The output catalog in which to create the package.
+  @param [out] pkg= (utils) The output package in which to create the function.
+    Uses a 3 part format:  libref.catalog.package
+
+**/
+
+%macro mcf_string2file(wrap=NO
+  ,insert_cmplib=NO
+  ,lib=WORK
+  ,cat=SASJS
+  ,pkg=UTILS
+)/*/STORE SOURCE*/;
+
+%if &wrap=YES  %then %do;
+  proc fcmp outcat=&lib..&cat..&pkg;
+%end;
+
+function mcf_string2file(filepath $, string $, mode $);
+  if mode='APPEND' then fmode='a';
+  else fmode='o';
+  length fref $8;
+  rc=filename(fref,filepath);
+  if (rc ne 0) then return( -1 );
+  fid = fopen(fref,fmode);
+  if (fid = 0) then return( -1 );
+  rc=fput(fid, string);
+  rc=fwrite(fid);
+  rc=fclose(fid);
+  rc=filename(fref);
+  return(0);
+endsub;
+
+
+%if &wrap=YES %then %do;
+  quit;
+%end;
+
+%if &insert_cmplib=YES %then %do;
+  options insert=(CMPLIB=(&lib..&cat));
+%end;
+
+%mend mcf_string2file;

lua/gsubfile.lua

@@ -0,0 +1,25 @@
+local fpath, outpath, file, fcontent
+
+-- configure in / out paths
+fpath = sas.symget("file")
+outpath = sas.symget("outfile")
+if ( outpath == 0 )
+then
+   outpath=fpath
+end
+
+-- open file and perform the substitution
+file = io.open(fpath,"r")
+fcontent = file:read("*all")
+file:close()
+fcontent = string.gsub(
+  fcontent,
+  sas.symget(sas.symget("patternvar")),
+  sas.symget(sas.symget("replacevar"))
+)
+
+-- write the file back out
+file = io.open(outpath, "w+")
+io.output(file)
+io.write(fcontent)
+io.close(file)

lua/ml_gsubfile.sas

@@ -0,0 +1,44 @@
+/**
+  @file ml_gsubfile.sas
+  @brief Compiles the gsubfile.lua lua file
+  @details Writes gsubfile.lua to the work directory
+  and then includes it.
+  Usage:
+
+      %ml_gsubfile()
+
+**/
+
+%macro ml_gsubfile();
+data _null_;
+  file "%sysfunc(pathname(work))/ml_gsubfile.lua";
+  put 'local fpath, outpath, file, fcontent ';
+  put ' ';
+  put '-- configure in / out paths ';
+  put 'fpath = sas.symget("file") ';
+  put 'outpath = sas.symget("outfile") ';
+  put 'if ( outpath == 0 ) ';
+  put 'then ';
+  put '   outpath=fpath ';
+  put 'end ';
+  put ' ';
+  put '-- open file and perform the substitution ';
+  put 'file = io.open(fpath,"r") ';
+  put 'fcontent = file:read("*all") ';
+  put 'file:close() ';
+  put 'fcontent = string.gsub( ';
+  put '  fcontent, ';
+  put '  sas.symget(sas.symget("patternvar")), ';
+  put '  sas.symget(sas.symget("replacevar")) ';
+  put ') ';
+  put ' ';
+  put '-- write the file back out ';
+  put 'file = io.open(outpath, "w+") ';
+  put 'io.output(file) ';
+  put 'io.write(fcontent) ';
+  put 'io.close(file) ';
+run;
+
+%inc "%sysfunc(pathname(work))/ml_gsubfile.lua" /source2;
+
+%mend ml_gsubfile;

lua/ml_json.sas

@@ -389,6 +389,6 @@ data _null_;
   put '-- JSON.LUA ENDS HERE ';
 run;
 
-%inc "%sysfunc(pathname(work))/ml_json.lua";
+%inc "%sysfunc(pathname(work))/ml_json.lua" /source2;
 
 %mend ml_json;

main.dox

@@ -20,6 +20,19 @@
 
  */
 
+/*! \dir fcmp
+ *  \brief Macros for generating FCMP functions
+ *  \details These macros have the following attributes:
+
+        * Macro name matches compiled function / subroutine name
+        * Prefixes:  _mcf_, _mcs_
+
+    The macro part is just a wrapper for the underlying function / subroutine,
+    and has switches for including the proc fcmp / quit statements and whether
+    to insert the package into the CMPLIB option.
+
+ */
+
 /*! \dir meta
  *  \brief Metadata Aware Macros
  *  \details These macros have the following attributes:
@@ -42,7 +55,18 @@
 
  */
 
- /*! \dir Tests
+/*! \dir server
+ *  \brief Macros used with [sasjs/server](https://server.sasjs.io)
+ *  \details These macros have the following attributes:
+
+        * OS independent
+        * sasjs/server aware
+        * No X command
+        * Prefixes: _ms_
+
+ */
+
+/*! \dir tests
  *  \brief SASjs Tests
  *  \details These folders contain the macro tests.  They are first compiled
    and deployed (sasjs cbd) then executed (sasjs test).
@@ -59,7 +83,7 @@
 
  */
 
- /*! \dir lua
+/*! \dir lua
  *  \brief Lua macros
  *  \details These macros have the following attributes:
 

make_singlefile.sh

@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-
-# Concatenate all macros into a single file
-
-OUTFILE='./macrocore.sas'
-
-cat > $OUTFILE <<'EOL'
-/**
-  @file
-  @brief Auto-generated file
-  @details
-    This file contains all the macros in a single file - which means it can be
-    'included' in SAS with just 2 lines of code:
-
-      filename mc url
-        "https://raw.githubusercontent.com/sasjs/core/main/macrocore.sas";
-      %inc mc;
-
-    The `build.sh` file in the https://github.com/sasjs/core repo
-    is used to create this file.
-
-  @author Allan Bowe
-**/
-EOL
-
-cat base/* >> $OUTFILE
-cat meta/* >> $OUTFILE
-cat metax/* >> $OUTFILE
-cat viya/* >> $OUTFILE

meta/mm_createfolder.sas

@@ -14,10 +14,10 @@
 
   usage:
 
-    %mm_createfolder(path=/some/meta/folder)
+      %mm_createfolder(path=/some/meta/folder)
 
-  @param path= Name of the folder to create.
-  @param mdebug= set DBG to 1 to disable DEBUG messages
+  @param [in] path= Name of the folder to create.
+  @param [in] mdebug= set DBG to 1 to disable DEBUG messages
 
   @version 9.4
   @author Allan Bowe

meta/mm_createwebservice.sas

@@ -385,6 +385,7 @@ data _null_;
   put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
+  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';

meta/mm_gettables.sas

@@ -1,18 +1,20 @@
 /**
   @file
   @brief Creates a dataset with all metadata tables for a particular library
-  @details Will only show the tables to which a user has the requisite
-    metadata access.
+  @details Will only show the tables for which the executing user has the
+    requisite metadata access.
 
   usage:
 
-    %mm_gettables(uri=A5X8AHW1.B40001S5)
+      %mm_gettables(uri=A5X8AHW1.B40001S5)
 
-  @param outds the dataset to create that contains the list of tables
-  @param uri the uri of the library for which to return tables
-  @param getauth= YES|NO - fetch the authdomain used in database connections.
-  Set to NO to improve runtimes in larger environments, as there can be a
-  performance hit on the `metadata_getattr(domainuri, "Name", AuthDomain)` call.
+  @param [in] uri= the uri of the library for which to return tables
+  @param [out] outds= (work.mm_gettables) the dataset to contain the list of
+    tables
+  @param [in] getauth= (YES) Fetch the authdomain used in database connections.
+    Set to NO to improve runtimes in larger environments, as there can be a
+    performance hit on the `metadata_getattr(domainuri, "Name", AuthDomain)`
+    call.
 
   @returns outds  dataset containing all groups in a column named "metagroup"
     (defaults to work.mm_getlibs). The following columns are provided:
@@ -40,8 +42,8 @@ data &outds;
     libdesc $200 libref engine $8 IsDBMSLibname $1
     tablename $50 /* metadata table names can be longer than $32 */
     ;
-  keep libname libdesc libref engine ServerContext path_schema AuthDomain tableuri
-    tablename IsPreassigned IsDBMSLibname id;
+  keep libname libdesc libref engine ServerContext path_schema AuthDomain
+    tableuri tablename IsPreassigned IsDBMSLibname id;
   call missing (of _all_);
 
   uri=symget('uri');

meta/mm_webout.sas

@@ -155,6 +155,7 @@
     put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSJOBID"" : ""&sysjobid"" ";
+    put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;

package.json

@@ -27,12 +27,12 @@
   "main": "index.js",
   "scripts": {
     "build": "sasjs cbd -t viya",
-    "docs": "sasjs doc && ./sasjs/utils/build.sh",
+    "docs": "sasjs doc -t docsonly && ./sasjs/utils/build.sh",
     "test": "sasjs test -t viya",
     "lint": "sasjs lint",
-    "postinstall": "[ -d .git ] && git config core.hooksPath ./.git-hooks || true"
+    "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "^2.27.0"
+    "@sasjs/cli": "^2.39.0"
   }
 }

sasjs/sasjsconfig.json

@@ -2,11 +2,13 @@
   "$schema": "https://cli.sasjs.io/sasjsconfig-schema.json",
   "macroFolders": [
     "base",
+    "fcmp",
     "meta",
     "metax",
+    "server",
     "viya",
     "lua",
-    "tests/base"
+    "tests/crossplatform"
   ],
   "docConfig": {
     "displayMacroCore": false,
@@ -33,7 +35,7 @@
       "allowInsecureRequests": false,
       "appLoc": "/Public/temp/macrocore",
       "macroFolders": [
-        "tests/viya"
+        "tests/viyaonly"
       ],
       "programFolders": [],
       "deployConfig": {
@@ -48,7 +50,30 @@
       "serverType": "SAS9",
       "appLoc": "/Shared Data/temp/macrocore",
       "macroFolders": [
-        "tests/meta"
+        "tests/sas9only"
+      ],
+      "deployConfig": {
+        "deployServicePack": true
+      }
+    },
+    {
+      "name": "server",
+      "serverUrl": "https://sas.analytium.co.uk:5001",
+      "serverType": "SASJS",
+      "appLoc": "/Shared Data/temp/macrocore",
+      "macroFolders": [
+        "tests/serveronly"
+      ],
+      "deployConfig": {
+        "deployServicePack": true
+      }
+    },
+    {
+      "name": "docsonly",
+      "serverType": "SAS9",
+      "macroFolders": [
+        "tests/sas9only",
+        "tests/viyaonly"
       ]
     }
   ]

server/ms_webout.sas

@@ -0,0 +1,170 @@
+/**
+  @file
+  @brief Send data to/from @sasjs/server
+  @details This macro should be added to the start of each web service,
+  **immediately** followed by a call to:
+
+        %ms_webout(FETCH)
+
+    This will read all the input data and create same-named SAS datasets in the
+    WORK library.  You can then insert your code, and send data back using the
+    following syntax:
+
+        data some datasets; * make some data ;
+          retain some columns;
+        run;
+
+        %ms_webout(OPEN)
+        %ms_webout(ARR,some)  * Array format, fast, suitable for large tables ;
+        %ms_webout(OBJ,datasets) * Object format, easier to work with ;
+        %ms_webout(CLOSE)
+
+
+  @param action Either FETCH, OPEN, ARR, OBJ or CLOSE
+  @param ds The dataset to send back to the frontend
+  @param dslabel= value to use instead of the real name for sending to JSON
+  @param fmt=(Y) Set to N to send back unformatted values
+  @param fref=(_webout) The fileref to which to write the JSON
+
+  <h4> SAS Macros </h4>
+  @li mp_jsonout.sas
+  @li mf_getuser.sas
+
+  <h4> Related Macros </h4>
+  @li mv_webout.sas
+  @li mm_webout.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y);
+%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
+  sasjs_tables;
+
+%local i tempds;
+%let action=%upcase(&action);
+
+%if &action=FETCH %then %do;
+  %if %str(&_debug) ge 131 %then %do;
+    options mprint notes mprintnest;
+  %end;
+  %let _webin_file_count=%eval(&_webin_file_count+0);
+  /* now read in the data */
+  %do i=1 %to &_webin_file_count;
+    %if &_webin_file_count=1 %then %do;
+      %let _webin_fileref1=&_webin_fileref;
+      %let _webin_name1=&_webin_name;
+    %end;
+    data _null_;
+      infile &&_webin_fileref&i termstr=crlf;
+      input;
+      call symputx('input_statement',_infile_);
+      putlog "&&_webin_name&i input statement: "  _infile_;
+      stop;
+    data &&_webin_name&i;
+      infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding='utf-8';
+      input &input_statement;
+      %if %str(&_debug) ge 131 %then %do;
+        if _n_<20 then putlog _infile_;
+      %end;
+    run;
+    %let sasjs_tables=&sasjs_tables &&_webin_name&i;
+  %end;
+%end;
+
+%else %if &action=OPEN %then %do;
+  /* fix encoding */
+  OPTIONS NOBOMFILE;
+
+  /* setup json */
+  data _null_;file &fref encoding='utf-8';
+  %if %str(&_debug) ge 131 %then %do;
+    put '>>weboutBEGIN<<';
+  %end;
+    put '{"SYSDATE" : "' "&SYSDATE" '"';
+    put ',"SYSTIME" : "' "&SYSTIME" '"';
+  run;
+
+%end;
+
+%else %if &action=ARR or &action=OBJ %then %do;
+  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
+    ,engine=&jsonengine,dbg=%str(&_debug)
+  )
+%end;
+%else %if &action=CLOSE %then %do;
+  %if %str(&_debug) ge 131 %then %do;
+    /* if debug mode, send back first 10 records of each work table also */
+    options obs=10;
+    data;run;%let tempds=%scan(&syslast,2,.);
+    ods output Members=&tempds;
+    proc datasets library=WORK memtype=data;
+    %local wtcnt;%let wtcnt=0;
+    data _null_;
+      set &tempds;
+      if not (upcase(name) =:"DATA"); /* ignore temp datasets */
+      i+1;
+      call symputx('wt'!!left(i),name,'l');
+      call symputx('wtcnt',i,'l');
+    data _null_; file &fref mod encoding='utf-8';
+      put ",""WORK"":{";
+    %do i=1 %to &wtcnt;
+      %let wt=&&wt&i;
+      proc contents noprint data=&wt
+        out=_data_ (keep=name type length format:);
+      run;%let tempds=%scan(&syslast,2,.);
+      data _null_; file &fref mod encoding='utf-8';
+        dsid=open("WORK.&wt",'is');
+        nlobs=attrn(dsid,'NLOBS');
+        nvars=attrn(dsid,'NVARS');
+        rc=close(dsid);
+        if &i>1 then put ','@;
+        put " ""&wt"" : {";
+        put '"nlobs":' nlobs;
+        put ',"nvars":' nvars;
+      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=&jsonengine)
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=&jsonengine)
+      data _null_; file &fref mod encoding='utf-8';
+        put "}";
+    %end;
+    data _null_; file &fref mod encoding='utf-8';
+      put "}";
+    run;
+  %end;
+  /* close off json */
+  data _null_;file &fref mod encoding='utf-8';
+    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
+    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
+    put ",""_DEBUG"" : ""&_debug"" ";
+    put ',"_PROGRAM" : ' _PROGRAM ;
+    put ",""SYSCC"" : ""&syscc"" ";
+    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    SYSHOSTINFOLONG=quote(trim(symget('SYSHOSTINFOLONG')));
+    put ',"SYSHOSTINFOLONG" : ' SYSHOSTINFOLONG;
+    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
+    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
+    put ",""SYSPROCESSNAME"" : ""&SYSPROCESSNAME"" ";
+    put ",""SYSJOBID"" : ""&sysjobid"" ";
+    put ",""SYSSCPL"" : ""&sysscpl"" ";
+    put ",""SYSSITE"" : ""&syssite"" ";
+    put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" ";
+    sysvlong=quote(trim(symget('sysvlong')));
+    put ',"SYSVLONG" : ' sysvlong;
+    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+    autoexec=quote(trim(getoption('autoexec')));
+    put ',"AUTOEXEC" : ' autoexec;
+    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
+    put ',"MEMSIZE" : ' memsize;
+    put "}" @;
+  %if %str(&_debug) ge 131 %then %do;
+    put '>>weboutEND<<';
+  %end;
+  run;
+%end;
+
+%mend ms_webout;

tests/crossplatform/mcf_stpsrv_header.test.sas

@@ -0,0 +1,39 @@
+/**
+  @file
+  @brief Testing mcf_stpsrv_header macro
+
+  <h4> SAS Macros </h4>
+  @li mcf_stpsrv_header.sas
+  @li mp_assert.sas
+
+**/
+
+%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/stpsrv_header.txt;
+
+%mcf_stpsrv_header(wrap=YES, insert_cmplib=YES)
+
+data _null_;
+  rc=stpsrv_header('Content-type','application/text');
+  rc=stpsrv_header('Content-disposition',"attachment; filename=file.txt");
+run;
+
+%let test1=FAIL;
+%let test2=FAIL;
+
+data _null_;
+  infile "&sasjs_stpsrv_header_loc";
+  input;
+  if _n_=1 and _infile_='Content-type: application/text'
+  then call symputx('test1','PASS');
+  else if _n_=2 & _infile_='Content-disposition: attachment; filename=file.txt'
+  then call symputx('test2','PASS');
+run;
+
+%mp_assert(
+  iftrue=(%str(&test1)=%str(PASS)),
+  desc=Check first header line
+)
+%mp_assert(
+  iftrue=(%str(&test2)=%str(PASS)),
+  desc=Check second header line
+)

tests/crossplatform/mcf_string2file.test.sas

@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief Testing mcf_string2file macro
+
+  <h4> SAS Macros </h4>
+  @li mcf_string2file.sas
+  @li mp_assert.sas
+
+**/
+
+
+%mcf_string2file(wrap=YES, insert_cmplib=YES)
+
+data _null_;
+  rc=mcf_string2file(
+    "%sysfunc(pathname(work))/newfile.txt"
+    , "line1"
+    , "APPEND");
+  rc=mcf_string2file(
+    "%sysfunc(pathname(work))/newfile.txt"
+    , "line2"
+    , "APPEND");
+run;
+
+data _null_;
+  infile "%sysfunc(pathname(work))/newfile.txt";
+  input;
+  if _n_=2 then call symputx('val',_infile_);
+run;
+
+%mp_assert(
+  iftrue=(%str(&val)=%str(line2)),
+  desc=Check if APPEND works
+)
+
+data _null_;
+  rc=mcf_string2file(
+    "%sysfunc(pathname(work))/newfile.txt"
+    , "creating"
+    , "CREATE");
+run;
+
+data _null_;
+  infile "%sysfunc(pathname(work))/newfile.txt";
+  input;
+  if _n_=1 then call symputx('val2',_infile_);
+run;
+
+%mp_assert(
+  iftrue=(%str(&val2)=%str(creating)),
+  desc=Check if CREATE works
+)

tests/crossplatform/mf_existfileref.test.sas

@@ -0,0 +1,35 @@
+/**
+  @file
+  @brief Testing mf_existfileref macro
+
+  <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mp_assert.sas
+
+**/
+
+filename ref1 temp;
+filename ref2 temp;
+
+data _null_;
+  file ref1;
+  put 'exists';
+run;
+
+%mp_assert(
+  iftrue=(%mf_existfileref(ref1)=1),
+  desc=Checking fileref WITH target file exists,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_existfileref(ref2)=1),
+  desc=Checking fileref WITHOUT target file exists,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_existfileref(ref3)=0),
+  desc=Checking non-existant fref does not exist,
+  outds=work.test_results
+)

tests/crossplatform/mf_existfunction.test.sas

@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Testing mf_existfunction macro
+
+  <h4> SAS Macros </h4>
+  @li mf_existfunction.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(%mf_existfunction(CAT)=1),
+  desc=Checking if CAT function exists,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_existfunction(DOG)=0),
+  desc=Checking DOG function does not exist,
+  outds=work.test_results
+)
+

tests/crossplatform/mf_getapploc.test.sas

@@ -0,0 +1,49 @@
+/**
+  @file
+  @brief Testing mf_getapploc macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getapploc.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/loc/tests/services/x/service)"="/some/loc"
+  ),
+  desc=Checking test appLoc matches,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/loc/tests/services/tests/service)"="/some/loc"
+  ),
+  desc=Checking nested services appLoc matches,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/services/admin/service)"="/some/area"
+  ),
+  desc=Checking services appLoc matches,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/jobs/jobs/job)"="/some/area"
+  ),
+  desc=Checking jobs appLoc matches,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/tests/macros/somemacro.sas)"="/some/area"
+  ),
+  desc=Checking tests/macros appLoc matches (which has no subfolder),
+  outds=work.test_results
+)

tests/crossplatform/mf_getuniquefileref.test.sas

@@ -0,0 +1,36 @@
+/**
+  @file
+  @brief Testing mf_getuniquefileref macro
+  @details To test performance you can also use the following macro:
+
+      %macro x(prefix);
+      %let now=%sysfunc(datetime());
+      %do x=1 %to 1000;
+        %let rc=%mf_getuniquefileref(prefix=&prefix);
+      %end;
+      %put %sysevalf(%sysfunc(datetime())-&now);
+      %mend;
+      %x(_)
+      %x(0)
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(
+    "%substr(%mf_getuniquefileref(prefix=0),1,1)"="#"
+  ),
+  desc=Checking for a natively assigned fileref,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%substr(%mf_getuniquefileref(),1,1)"="_"
+  ),
+  desc=Checking for a default fileref,
+  outds=work.test_results
+)

tests/crossplatform/mp_appendfile.test.sas

@@ -0,0 +1,41 @@
+/**
+  @file
+  @brief Testing mp_appendfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_appendfile.sas
+  @li mp_assert.sas
+
+**/
+
+
+filename tmp1 temp;
+filename tmp2 temp;
+filename tmp3 temp;
+data _null_; file tmp1; put 'base file';
+data _null_; file tmp2; put 'append1';
+data _null_; file tmp3; put 'append2';
+run;
+%mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3)
+data _null_;
+  infile tmp1;
+  input;
+  put _infile_;
+  call symputx(cats('check',_n_),_infile_);
+run;
+%global check1 check2 check3;
+%mp_assert(
+  iftrue=("&check1"="base file"),
+  desc=Line 1 of file tmp1 is correct,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&check2"="append1"),
+  desc=Line 2 of file tmp1 is correct,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&check3"="append2"),
+  desc=Line 3 of file tmp1 is correct,
+  outds=work.test_results
+)

tests/crossplatform/mp_base64copy.test.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Testing mp_base64copy.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_base64copy.sas
+  @li mp_assert.sas
+
+**/
+
+
+/* TEST 1 - regular base64 decode */
+
+%let string1=base ik ally;
+filename tmp temp;
+data _null_;
+  file tmp;
+  put "&string1";
+run;
+%mp_base64copy(inref=tmp, outref=myref, action=ENCODE)
+
+data _null_;
+  infile myref;
+  input;
+  put _infile_;
+run;
+%mp_base64copy(inref=myref, outref=mynewref, action=DECODE)
+data _null_;
+  infile mynewref lrecl=5000;
+  input;
+  put _infile_;
+  call symputx('string1_check',_infile_);
+  stop;
+run;
+%mp_assert(
+  iftrue=("&string1"="&string1_check"),
+  desc=Basic String Compare,
+  outds=work.test_results
+)
+
+
+/* multibyte string check */
+
+filename tmp2 temp;
+data _null_;
+  file tmp2;
+  put "'╤', '╔', '╗', '═', '╧', '╚', '╝', '║', '╟', '─', '┼', '║', '╢', '│'";
+run;
+%mp_base64copy(inref=tmp2, outref=myref2, action=ENCODE)
+
+%mp_base64copy(inref=myref2, outref=newref2, action=DECODE)
+data _null_;
+  infile newref2 lrecl=5000;
+  input;
+  list;
+  /* do not print the string to the log else viya 3.5 throws exception */
+  if trim(_infile_)=
+    "'╤', '╔', '╗', '═', '╧', '╚', '╝', '║', '╟', '─', '┼', '║', '╢', '│'"
+  then call symputx('check2',1);
+  else call symputx('check2',0);
+  stop;
+run;
+%mp_assert(
+  iftrue=("&check2"="1"),
+  desc=Double Byte String Compare,
+  outds=work.test_results
+)

tests/crossplatform/mp_binarycopy.test.sas

@@ -0,0 +1,99 @@
+/**
+  @file
+  @brief Testing mp_binarycopy.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_binarycopy.sas
+  @li mp_assert.sas
+
+**/
+
+
+/* TEST 1 - regular file copy */
+%let string1=test1;
+filename tmp temp;
+filename myref temp;
+data _null_;
+  file tmp;
+  put "&string1";
+run;
+%mp_binarycopy(inref=tmp, outref=myref)
+data _null_;
+  infile myref;
+  input;
+  put _infile_;
+  call symputx('string1_check',_infile_);
+  stop;
+run;
+%mp_assert(
+  iftrue=("&string1"="&string1_check"),
+  desc=Basic String Compare,
+  outds=work.test_results
+)
+
+
+/* TEST 2 - File append */
+%let string2=test2;
+%let path2=%sysfunc(pathname(work))/somefile.txt;
+data _null_;
+  file "&path2";
+  put "&string2";
+run;
+%mp_binarycopy(inloc="&path2", outref=myref, mode=APPEND)
+data _null_;
+  infile myref;
+  input;
+  put _infile_;
+  if _n_=2 then call symputx('string2_check',_infile_);
+run;
+%mp_assert(
+  iftrue=("&string2"="&string2_check"),
+  desc=Append Check (file to ref),
+  outds=work.test_results
+)
+
+/* TEST 3 - File create (ref to existing file) */
+%let string3=test3;
+%let path3=%sysfunc(pathname(work))/somefile3.txt;
+filename tmp3 temp;
+data _null_;
+  file tmp3;
+  put "&string3";
+run;
+data _null_;
+  file "&path3";
+  put "this should not be returned";
+run;
+%mp_binarycopy(inref=tmp3, outloc="&path3")
+data _null_;
+  infile "&path3";
+  input;
+  put _infile_;
+  if _n_=1 then call symputx('string3_check',_infile_);
+run;
+%mp_assert(
+  iftrue=("&string3"="&string3_check"),
+  desc=Append Check (ref to existing file),
+  outds=work.test_results
+)
+
+/* TEST 4 - File append (ref to file) */
+%let string4=test4;
+%let string4_check=;
+filename tmp4 temp;
+data _null_;
+  file tmp4;
+  put "&string4";
+run;
+%mp_binarycopy(inref=tmp4, outloc="&path3",mode=APPEND)
+data _null_;
+  infile "&path3";
+  input;
+  put _infile_;
+  if _n_=2 then call symputx('string4_check',_infile_);
+run;
+%mp_assert(
+  iftrue=("&string4"="&string4_check"),
+  desc=Append Check (ref to file),
+  outds=work.test_results
+)

tests/crossplatform/mp_copyfolder.test.sas

@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief Testing mp_copyfolder.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_copyfolder.sas
+  @li mf_mkdir.sas
+  @li mf_nobs.sas
+  @li mp_assert.sas
+  @li mp_dirlist.sas
+
+**/
+
+/**
+  * make a directory structure
+  */
+
+%let root=%sysfunc(pathname(work))/top;
+%mf_mkdir(&root)
+%mf_mkdir(&root/a)
+%mf_mkdir(&root/b)
+%mf_mkdir(&root/a/d)
+%mf_mkdir(&root/a/e)
+%mf_mkdir(&root/a/e/f)
+data "&root/a/e/f/ds1.sas7bdat";x=1;
+data "&root/a/e/ds2.sas7bdat";x=1;
+data "&root/a/ds3.sas7bdat";x=1;
+run;
+
+%mp_dirlist(path=&root, outds=myTable, maxdepth=MAX)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable)=8),
+  desc=Temp data successfully created,
+  outds=work.test_results
+)
+
+/**
+  * copy it
+  */
+%let newfolder=%sysfunc(pathname(work))/new;
+%mp_copyfolder(&root,&newfolder)
+
+%mp_dirlist(path=&newfolder, outds=work.myTable2, maxdepth=MAX)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable2)=8),
+  desc=Folder successfully copied,
+  outds=work.test_results
+)
+
+

tests/crossplatform/mp_deletefolder.test.sas

@@ -0,0 +1,50 @@
+/**
+  @file
+  @brief Testing mp_deletefolder.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_deletefolder.sas
+  @li mf_mkdir.sas
+  @li mf_nobs.sas
+  @li mp_assert.sas
+  @li mp_dirlist.sas
+
+**/
+
+/**
+  * make a directory structure
+  */
+
+%let root=%sysfunc(pathname(work))/top;
+%mf_mkdir(&root)
+%mf_mkdir(&root/a)
+%mf_mkdir(&root/b)
+%mf_mkdir(&root/a/d)
+%mf_mkdir(&root/a/e)
+%mf_mkdir(&root/a/e/f)
+data "&root/a/e/f/ds1.sas7bdat";
+  x=1;
+run;
+
+%mp_dirlist(path=&root, outds=myTable, maxdepth=MAX)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable)=6),
+  desc=Temp data successfully created,
+  outds=work.test_results
+)
+
+%mp_deletefolder(&root/a)
+
+%mp_dirlist(path=&root, outds=work.myTable2, maxdepth=MAX)
+
+data _null_;
+  set work.mytable2;
+  putlog (_all_)(=);
+run;
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable2)=1),
+  desc=Subfolder and contents successfully deleted,
+  outds=work.test_results
+)

tests/crossplatform/mp_dirlist.test.sas

@@ -0,0 +1,50 @@
+/**
+  @file
+  @brief Testing mp_dirlist.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mf_mkdir.sas
+  @li mp_dirlist.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * make a directory structure
+  */
+
+%let root=%sysfunc(pathname(work))/top;
+%mf_mkdir(&root)
+%mf_mkdir(&root/a)
+%mf_mkdir(&root/b)
+%mf_mkdir(&root/a/d)
+%mf_mkdir(&root/a/e)
+%mf_mkdir(&root/a/e/f)
+data "&root/a/e/f/ds1.sas7bdat";
+  x=1;
+run;
+
+%mp_dirlist(path=&root, outds=myTable, maxdepth=MAX)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable)=6),
+  desc=All levels returned,
+  outds=work.test_results
+)
+
+%mp_dirlist(path=&root, outds=myTable2, maxdepth=2)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable2)=5),
+  desc=Top two levels returned,
+  outds=work.test_results
+)
+
+%mp_dirlist(path=&root, outds=work.myTable3, maxdepth=0)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable3)=2),
+  desc=Top level returned,
+  outds=work.test_results
+)

tests/crossplatform/mp_ds2cards.test.sas

@@ -0,0 +1,60 @@
+/**
+  @file
+  @brief Testing mp_ds2cards.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2cards.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * test 1 - rebuild an existing dataset
+  * Cars is a great dataset - it contains leading spaces, and formatted numerics
+  */
+
+%mp_ds2cards(base_ds=sashelp.cars
+  , tgt_ds=work.test
+  , cards_file= "%sysfunc(pathname(work))/cars.sas"
+  , showlog=NO
+)
+%inc "%sysfunc(pathname(work))/cars.sas"/source2;
+
+proc compare base=sashelp.cars compare=work.test;
+quit;
+
+%mp_assert(
+  iftrue=(&sysinfo=1),
+  desc=sashelp.cars is identical except for ds label,
+  outds=work.test_results
+)
+
+/**
+  * test 2 - binary data compare
+  */
+data work.binarybase;
+  format bin $hex500. z $hex.;
+  do x=1 to 250;
+    z=byte(x);
+    bin=trim(bin)!!z;
+    output;
+  end;
+run;
+
+%mp_ds2cards(base_ds=work.binarybase
+  , showlog=YES
+  , cards_file="%sysfunc(pathname(work))/c2.sas"
+  , tgt_ds=work.binarycompare
+  , append=
+)
+
+%inc "%sysfunc(pathname(work))/c2.sas"/source2;
+
+proc compare base=work.binarybase compare=work.binarycompare;
+run;
+
+%mp_assert(
+  iftrue=(&sysinfo=0),
+  desc=work.binarybase dataset is identical,
+  outds=work.test_results
+)

tests/crossplatform/mp_ds2inserts.test.sas

@@ -0,0 +1,31 @@
+/**
+  @file
+  @brief Testing mp_ds2inserts.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2inserts.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * test 1 - rebuild an existing dataset
+  * Cars is a great dataset - it contains leading spaces, and formatted numerics
+  */
+
+%mp_ds2inserts(sashelp.cars,outref=testref,schema=work,outds=test)
+
+data work.test;
+  set sashelp.cars;
+  stop;
+proc sql;
+%inc testref;
+
+proc compare base=sashelp.cars compare=work.test;
+quit;
+
+%mp_assert(
+  iftrue=(&sysinfo=1),
+  desc=sashelp.cars is identical except for ds label,
+  outds=work.test_results
+)

tests/crossplatform/mp_getcols.test.sas

@@ -0,0 +1,33 @@
+/**
+  @file
+  @brief Testing mp_getcols macro
+
+  <h4> SAS Macros </h4>
+  @li mp_getcols.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
+
+**/
+
+
+/* valid filter */
+%mp_getcols(sashelp.airline,outds=work.info)
+
+
+%mp_assertdsobs(work.info,
+  desc=Has 3 records,
+  test=EQUALS 3,
+  outds=work.test_results
+)
+
+data work.check;
+  length val $10;
+  do val='NUMERIC','DATE','CHARACTER';
+    output;
+  end;
+run;
+%mp_assertcolvals(work.info.ddtype,
+  checkvals=work.check.val,
+  desc=All values have a match,
+  test=ALLVALS
+)

tests/crossplatform/mp_getconstraints.test.sas

@@ -0,0 +1,29 @@
+/**
+  @file
+  @brief Testing mp_getconstraints.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_getconstraints.sas
+  @li mp_assert.sas
+
+**/
+
+proc sql;
+create table work.example(
+  TX_FROM float format=datetime19.,
+  DD_TYPE char(16),
+  DD_SOURCE char(2048),
+  DD_SHORTDESC char(256),
+  constraint pk primary key(tx_from, dd_type,dd_source),
+  constraint unq unique(tx_from, dd_type),
+  constraint nnn not null(DD_SHORTDESC)
+);
+
+%mp_getconstraints(lib=work,ds=example,outds=work.constraints)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.constraints)=6),
+  desc=Output table work.constraints created with correct number of records,
+  outds=work.test_results
+)

tests/crossplatform/mp_getddl.test.sas

@@ -0,0 +1,23 @@
+/**
+  @file
+  @brief Testing mp_getddl.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_getddl.sas
+  @li mp_assert.sas
+
+**/
+
+data test(index=(pk=(x y)/unique /nomiss));
+  x=1;
+  y='blah';
+  label x='blah';
+run;
+proc sql; describe table &syslast;
+%mp_getddl(work,test,flavour=tsql,showlog=YES)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=mp_getddl runs without errors,
+  outds=work.test_results
+)

tests/crossplatform/mp_gsubfile.test.sas

@@ -0,0 +1,66 @@
+/**
+  @file
+  @brief Testing mp_gsubfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_gsubfile.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * test 1 - simple replace
+  */
+%global str1;
+%let file=%sysfunc(pathname(work))/file.txt;
+%let pat=replace/me;
+%let str=with/this;
+data _null_;
+  file "&file";
+  put "&pat";
+run;
+%mp_gsubfile(file=&file, patternvar=pat, replacevar=str)
+data _null_;
+  infile "&file";
+  input;
+  call symputx('str1',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&str1"="&str"),
+  desc=Check that simple replacement was successful,
+  outds=work.test_results
+)
+
+/**
+  * test 2 - replace from additional line
+  */
+%global str2 strcheck2 strcheck2b;
+%let file2=%sysfunc(pathname(work))/file2.txt;
+%let pat2=replace/me;
+%let str2=with/this;
+data _null_;
+  file "&file2";
+  put 'line1';output;
+  put "&pat2";output;
+  put "&pat2";output;
+run;
+%mp_gsubfile(file=&file2, patternvar=pat2, replacevar=str2)
+data _null_;
+  infile "&file2";
+  input;
+  if _n_=2 then call symputx('strcheck2',_infile_);
+  if _n_=3 then call symputx('strcheck2b',_infile_);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=("&strcheck2"="&str2"),
+  desc=Check that multi line replacement was successful (line2),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&strcheck2b"="&str2"),
+  desc=Check that multi line replacement was successful (line3),
+  outds=work.test_results
+)

tests/crossplatform/mp_lib2inserts.test.sas

@@ -0,0 +1,42 @@
+/**
+  @file
+  @brief Testing mp_ds2inserts.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mp_getddl.sas
+  @li mp_lib2inserts.sas
+  @li mp_assert.sas
+
+**/
+
+/* grab 20 datasets from SASHELP */
+%let path=%sysfunc(pathname(work));
+%mf_mkdir(&path)
+libname sashlp "&path";
+proc sql noprint;
+create table members as
+  select distinct lowcase(memname) as memname
+  from dictionary.tables
+  where upcase(libname)="SASHELP"
+    and memtype='DATA'; /* exclude views */
+data _null_;
+  set work.members;
+  call execute(cats('data sashlp.',memname,';set sashelp.',memname,';run;'));
+  if _n_>20 then stop;
+run;
+
+/* export DDL and inserts */
+%mp_getddl(sashlp, schema=work, fref=tempref)
+%mp_lib2inserts(sashlp, schema=work, outref=tempref,maxobs=50)
+
+/* check if it actually runs */
+options source2;
+%inc tempref;
+
+/* without errors.. */
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Able to export 20 tables from sashelp using mp_lib2inserts,
+  outds=work.test_results
+)

tests/crossplatform/mp_lockanytable.test.sas

@@ -0,0 +1,62 @@
+/**
+  @file
+  @brief Testing mp_lockfilecheck macro
+
+  <h4> SAS Macros </h4>
+  @li mp_lockanytable.sas
+  @li mp_assertcols.sas
+  @li mp_assertcolvals.sas
+
+**/
+
+/* check create table */
+
+%mp_lockanytable(MAKETABLE, ctl_ds=work.controller)
+
+%mp_assertcols(work.controller,
+  cols=lock_status_cd lock_lib lock_ds lock_user_nm lock_ref lock_pid
+    lock_start_dttm lock_end_dttm,
+  test=ALL,
+  desc=check all control columns exist
+)
+
+/* check lock table */
+options dlcreatedir;
+libname tmp "%sysfunc(pathname(work))/tmp";
+data tmp.sometable;
+  x=1;
+run;
+
+%mp_lockanytable(LOCK,lib=tmp,ds=sometable,ref=This Ref, ctl_ds=work.controller)
+
+data work.checkds1;
+  checkval='SOMETABLE';
+run;
+%mp_assertcolvals(work.controller.lock_ds,
+  checkvals=work.checkds1.checkval,
+  desc=table is captured in lock,
+  test=ANYVAL
+)
+
+data work.checkds2;
+  checkval='LOCKED';
+run;
+%mp_assertcolvals(work.controller.lock_status_cd,
+  checkvals=work.checkds2.checkval,
+  desc=code is captured in lock,
+  test=ANYVAL
+)
+
+
+
+/* check for unsuccessful unlock */
+%mp_lockanytable(UNLOCK,lib=tmp,ds=sometable,ref=bye, ctl_ds=work.controller)
+
+data work.checkds3;
+  checkval='UNLOCKED';
+run;
+%mp_assertcolvals(work.controller.lock_status_cd,
+  checkvals=work.checkds3.checkval,
+  desc=Ref is captured in unlock,
+  test=ANYVAL
+)

tests/crossplatform/mp_lockfilecheck.test.sas

@@ -0,0 +1,36 @@
+/**
+  @file
+  @brief Testing mp_lockfilecheck macro
+
+  <h4> SAS Macros </h4>
+  @li mp_lockfilecheck.sas
+  @li mp_assert.sas
+
+**/
+
+
+/* check for regular lock */
+data work.test; a=1;run;
+%mp_lockfilecheck(work.test)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking regular table can be locked,
+  outds=work.test_results
+)
+
+
+/* check for unsuccessful lock */
+%global success abortme;
+%let success=0;
+%macro mp_abort(iftrue=,mac=,msg=);
+  %if &abortme=1 %then %let success=1;
+%mend mp_abort;
+
+%mp_lockfilecheck(sashelp.class)
+
+%mp_assert(
+  iftrue=(&success=1),
+  desc=Checking sashelp table cannot be locked,
+  outds=work.test_results
+)

tests/crossplatform/mp_webin.test.sas

@@ -0,0 +1,39 @@
+/**
+  @file
+  @brief Testing mp_webin macro
+
+  <h4> SAS Macros </h4>
+  @li mp_webin.sas
+  @li mp_assert.sas
+
+**/
+
+/* force SAS9 tests as we don't have a valid URI available */
+%macro mf_getplatform();
+  SAS9
+%mend mf_getplatform;
+
+/* TEST 1 */
+%let _webin_file_count=1;
+%let _webin_filename=test;
+%mp_webin()
+
+%mp_assert(
+  iftrue=(
+    %symexist(_WEBIN_FILEREF1)
+  ),
+  desc=Checking if the macvar exists,
+  outds=work.test_results
+)
+
+/* TEST 2 */
+%global _WEBIN_FILENAME1;
+%mp_assert(
+  iftrue=(
+    %str(&_WEBIN_FILENAME1)=%str(test)
+  ),
+  desc=Checking if the macvar exists,
+  outds=work.test_results
+)
+
+

tests/crossplatform/mp_zip.test.sas

@@ -0,0 +1,115 @@
+/**
+  @file
+  @brief Testing mp_zip macro
+
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mp_assert.sas
+  @li mp_zip.sas
+  @li mp_unzip.sas
+
+**/
+
+%let work=%sysfunc(pathname(work));
+%let root=&work/zipme;
+
+/* TEST 1 - zip a file */
+%mf_mkdir(&root)
+
+data _null_;
+  file "&root/test.txt";
+  put "houston, this is a test";
+run;
+
+%mp_zip(in=&root/test.txt
+  ,type=FILE
+  ,outpath=&work
+  ,outname=myFile
+)
+
+%mp_unzip(ziploc="&work/myFile.zip",outdir=&work)
+
+data _null_;
+  infile "&work/test.txt";
+  input;
+  call symputx('content1',_infile_);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(
+    %str(&content1)=%str(houston, this is a test)
+  ),
+  desc=Checking if file zip / unzip works,
+  outds=work.test_results
+)
+
+/* TEST 2 - zip a dataset of files */
+data _null_;
+  file "&root/test2.txt";
+  put "houston, this is test2";
+run;
+libname tmp "&root";
+data tmp.test;
+  filepath="&root/test2.txt";
+run;
+
+%mp_zip(in=tmp.test
+  ,incol=filepath
+  ,type=DATASET
+  ,outpath=&work
+  ,outname=myFile2
+)
+
+%mp_unzip(ziploc="&work/myFile2.zip",outdir=&work)
+
+data _null_;
+  infile "&work/test2.txt";
+  input;
+  call symputx('content2',_infile_);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(
+    %str(&content2)=%str(houston, this is test2)
+  ),
+  desc=Checking if file zip / unzip from a dataset works,
+  outds=work.test_results
+)
+
+/* TEST 3 - zip a dataset of files */
+%mf_mkdir(&work/out3)
+
+%mp_zip(in=&root
+  ,type=DIRECTORY
+  ,outpath=&work
+  ,outname=myFile3
+)
+
+%mp_unzip(ziploc="&work/myFile3.zip",outdir=&work/out3)
+
+data _null_;
+  infile "&work/out3/test.txt";
+  input;
+  call symputx('content3a',_infile_);
+  putlog _infile_;
+run;
+data _null_;
+  infile "&work/out3/test2.txt";
+  input;
+  call symputx('content3b',_infile_);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(
+    %str(&content3a)=%str(houston, this is a test)
+    and
+    %str(&content3b)=%str(houston, this is test2)
+  ),
+  desc=Checking if file zip / unzip from a directory works,
+  outds=work.test_results
+)
+
+

tests/serveronly/ms_webout.test.sas

@@ -0,0 +1,35 @@
+/**
+  @file
+  @brief Testing ms_webout macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li ms_webout.sas
+  @li mp_assert.sas
+
+**/
+
+
+%let fref=%mf_getuniquefileref();
+%global _metaperson;
+data some datasets;
+  x=1;
+run;
+%ms_webout(OPEN,fref=&fref)
+%ms_webout(ARR,some,fref=&fref)
+%ms_webout(OBJ,datasets,fref=&fref)
+%ms_webout(CLOSE,fref=&fref)
+
+libname test JSON (&fref);
+data root;
+  set test.root;
+  call symputx('checkval',sysvlong);
+run;
+data alldata;
+  set test.alldata;
+run;
+
+%mp_assert(
+  iftrue=(%str(&checkval)=%str(&sysvlong)),
+  desc=Check if the sysvlong value was created
+)

tests/testinit.sas

@@ -5,4 +5,12 @@
 **/
 
 /* location in metadata or SAS Drive for temporary files */
-%let mcTestAppLoc=/Public/temp/macrocore;
+%let mcTestAppLoc=/Public/temp/macrocore;
+
+%macro loglevel();
+  %if &_debug=2477 %then %do;
+    options mprint;
+  %end;
+%mend loglevel;
+
+%loglevel()