fix: hard abort in mm_getstpcode when the stp does not exist

fix: enable reserved names in mp_jsonout.  Closes #249

.all-contributorsrc

@@ -117,6 +117,24 @@
       "contributions": [
         "bug"
       ]
+    },
+    {
+      "login": "yabwon",
+      "name": "Bart Jablonski",
+      "avatar_url": "https://avatars.githubusercontent.com/u/9314894?v=4",
+      "profile": "https://github.com/yabwon",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "eltociear",
+      "name": "Ikko Ashimine",
+      "avatar_url": "https://avatars.githubusercontent.com/u/22633385?v=4",
+      "profile": "https://bandism.net/",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.devcontainer/Dockerfile

@@ -0,0 +1,8 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/javascript-node/.devcontainer/base.Dockerfile
+
+# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
+ARG VARIANT="18-bullseye"
+FROM mcr.microsoft.com/vscode/devcontainers/javascript-node:0-${VARIANT}
+
+RUN apt-get update \
+    && apt-get install -y doxygen

.devcontainer/devcontainer.json

@@ -0,0 +1,26 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/typescript-node
+{
+	"name": "Node.js & TypeScript",
+	"build": {
+		"dockerfile": "Dockerfile",
+		// Update 'VARIANT' to pick a Node version: 18, 16, 14.
+		// Append -bullseye or -buster to pin to an OS version.
+		// Use -bullseye variants on local on arm64/Apple Silicon.
+		"args": {
+			"VARIANT": "16-bullseye"
+		}
+	},
+	// Set *default* container specific settings.json values on container create.
+	"settings": {},
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"SASjs.sasjs-for-vscode"
+	],
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "npm i && npm i -g @sasjs/cli",
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "node"
+}

.git-hooks/pre-commit

@@ -1,5 +1,12 @@
 #!/bin/bash
-sasjs lint
+
+# Ensure lint is passing
+LINT=`sasjs lint`
+if [[ "$LINT" != "✔ All matched files use @sasjs/lint code style!" ]]; then
+  echo "$LINT"
+  echo "To commit in spite of these warnings, use the -n parameter."
+  exit 1
+fi
 
 # Avoid commits to the master branch
 BRANCH=`git rev-parse --abbrev-ref HEAD`

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [sasjs]

.gitpod.yml

@@ -1,7 +1,7 @@
 tasks:
-  - init: |
-      nvm install --lts
-      npm i -g @sasjs/cli
+  - init: npm install -g npm
+  - command: npm i
+  - command: npm i -g @sasjs/cli
 
 image:
   file: .gitpod.dockerfile
@@ -24,4 +24,4 @@ github:
     # add a "Review in Gitpod" button to pull requests (defaults to false)
     addBadge: false
     # add a label once the prebuild is ready to pull requests (defaults to false)
-    addLabel: prebuilt-in-gitpod
+    addLabel: prebuilt-in-gitpod

.npmignore

@@ -6,7 +6,6 @@ sasjs/
 .github/
 .git-hooks/
 .vscode/
-main.dox
 make_singlefile.sh
 *.md
 .all-contributorsrc

README.md

@@ -18,7 +18,7 @@
 [dependency-url]:https://github.com/sasjs/core/blob/main/package.json
 
 
-Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
+Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/.github/CONTRIBUTING.md) are welcome.
 
 You can download and compile them all in just two lines of SAS code:
 
@@ -31,14 +31,14 @@ Documentation: https://core.sasjs.io
 
 ## Components
 
-### BASE library (All Platforms)
+### BASE folder (All Platforms)
 
 - OS independent
 - Works on all SAS Platforms
 - No X command
 - Prefixes: _mf_, _mp_
 
-### DDL library (All Platforms)
+### DDL folder (All Platforms)
 
 - OS independent
 - Works on all SAS Platforms
@@ -47,46 +47,14 @@ Documentation: https://core.sasjs.io
 
 This library will not be used for storing data entries (such as formats or datalines).  Where this becomes necessary in the future, a new repo will be created, in order to keep the NPM bundle size down (for the benefit of those looking to embed purely macros in their applications).
 
-#### FCMP library (All Platforms)
+### FCMP folder (All Platforms)
 
 - Function and macro names are identical, except for special cases
 - Prefixes: _mcf_
 
 The fcmp macros are used to generate fcmp functions, and can be used with or without the `proc fcmp` wrapper.
 
-### META library (SAS9 only)
-
-Macros used in SAS EBI, which connect to the metadata server.
-
-- OS independent
-- Metadata aware
-- No X command
-- Prefixes: _mm_
-
-### SERVER library (@sasjs/server only)
-These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
-
-- OS independent
-- @sasjs/server aware
-- No X command
-- Prefixes: _ms_
-
-### VIYA library (Viya only)
-
-Macros used for interfacing with SAS Viya.
-
-- OS independent
-- No X command
-- Prefixes: _mv_, _mvf_
-
-### METAX library (SAS9 only)
-
-- OS specific
-- Metadata aware
-- X command enabled
-- Prefixes: _mmw_,_mmu_,_mmx_
-
-### LUA library
+### LUA folder
 
 Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
 
@@ -106,13 +74,61 @@ run;
 
 - Prefixes: _ml_
 
+### META folder (SAS9 only)
+
+Macros used in SAS EBI, which connect to the metadata server.
+
+- OS independent
+- Metadata aware
+- No X command
+- Prefixes: _mm_
+
+### METAX folder (SAS9 only)
+
+- OS specific
+- Metadata aware
+- X command enabled
+- Prefixes: _mmw_,_mmu_,_mmx_
+
+### SERVER folder (@sasjs/server only)
+These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
+
+- OS independent
+- @sasjs/server aware
+- No X command
+- Prefixes: _ms_
+
+### VIYA folder (Viya only)
+
+Macros used for interfacing with SAS Viya.
+
+- OS independent
+- No X command
+- Prefixes: _mv_, _mvf_
+
+### XPLATFORM folder (Viya, Meta, and Server)
+
+Sometimes it is helpful to use a macro that can be used interchangeably regardless of the server type on which is is running (SASVIYA, SAS9, SASJS).
+
+- OS independent
+- No X command
+- Prefixes: _mx_
+
 ## Installation
 
 First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available, eg:
 
 ```sas
-options insert=(sasautos="/your/path/macrocore/base");
-options insert=(sasautos="/your/path/macrocore/meta");
+%let repoloc=/your/path/core;
+options insert=(sasautos="&repoloc/base");
+options insert=(sasautos="&repoloc/ddl");
+options insert=(sasautos="&repoloc/fcmp");
+options insert=(sasautos="&repoloc/lua");
+options insert=(sasautos="&repoloc/meta");
+options insert=(sasautos="&repoloc/metax");
+options insert=(sasautos="&repoloc/server");
+options insert=(sasautos="&repoloc/viya");
+options insert=(sasautos="&repoloc/xplatform");
 ```
 
 The above can be done directly in your sas program, via an autoexec, or an initialisation program.
@@ -142,7 +158,7 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
   - _mp_ for macro procedures (which generate sas code)
   - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
   - _mv_ for macro procedures that will only work in Viya
-  - _mx_ for macros that are XCMD enabled (working on both windows and unix)
+  - _mx_ for macros that work on Viya, SAS 9 EBI and SASjs Server
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
@@ -204,6 +220,7 @@ When contributing to this library, it is therefore important to ensure that all
 
 We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major/breaking release (v5) becomes necessary:
 
+* mf_getuniquelibref.sas to have the deprecated maxtried parameter removed (no longer needed)
 * mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
 * `insert_cmplib` option of mcf_xxx macros will be deprecated (the option is now checked automatically with value inserted only if needed)
 * mcf_xxx macros to have `wrap=` option defaulted to YES for convenience.  Set this option explicitly to avoid issues.
@@ -216,11 +233,20 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 ![](https://starchart.cc/sasjs/core.svg)
 
+## Other SAS Repositories
 
+The following repositories are also worth checking out:
+
+* [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
+* [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)
+* [KatjaGlassConsulting/SMILE-SmartSASMacros](https://github.com/KatjaGlassConsulting/SMILE-SmartSASMacros)
+* [rogerjdeangelis](https://github.com/rogerjdeangelis)
+* [scottbass/sas](https://github.com/scottbass/SAS)
+* [yabwon/sas_packages](https://github.com/yabwon/SAS_PACKAGES)
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-10-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-12-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -241,6 +267,8 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
     <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
     <td align="center"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
+    <td align="center"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
+    <td align="center"><a href="https://bandism.net/"><img src="https://avatars.githubusercontent.com/u/22633385?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Ikko Ashimine</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=eltociear" title="Code">💻</a></td>
   </tr>
 </table>
 

base/mf_deletefile.sas

@@ -0,0 +1,31 @@
+/**
+  @file
+  @brief Deletes a physical file, if it exists
+  @details Usage:
+
+      %mf_writefile(&sasjswork/myfile.txt,l1=some content)
+
+      %mf_deletefile(&sasjswork/myfile.txt)
+
+      %mf_deletefile(&sasjswork/myfile.txt)
+
+
+  @param filepath Full path to the target file
+
+  @returns The return code from the fdelete() invocation
+
+  <h4> Related Macros </h4>
+  @li mf_deletefile.test.sas
+  @li mf_writefile.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_deletefile(file
+)/*/STORE SOURCE*/;
+  %local rc fref;
+  %let rc= %sysfunc(filename(fref,&file));
+  %if %sysfunc(fdelete(&fref)) ne 0 %then %put %sysfunc(sysmsg());
+  %let rc= %sysfunc(filename(fref));
+%mend mf_deletefile;

base/mf_existfeature.sas

@@ -9,19 +9,17 @@
 
         %put %mf_existfeature(PROCLUA);
 
-  @param feature the feature to detect.  Leave blank to list all in log.
+  @param [in] feature The feature to detect.
 
   @return output returns 1 or 0 (or -1 if not found)
 
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
 
-
   @version 8
   @author Allan Bowe
 **/
 /** @cond */
-
 %macro mf_existfeature(feature
 )/*/STORE SOURCE*/;
   %let feature=%upcase(&feature);
@@ -29,7 +27,11 @@
   %let platform=%mf_getplatform();
 
   %if &feature= %then %do;
-    %put Supported features:  PROCLUA;
+    %put No feature was requested for detection;
+  %end;
+  %else %if &feature=COLCONSTRAINTS %then %do;
+    %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 0;
+    %else 1;
   %end;
   %else %if &feature=PROCLUA %then %do;
     /* https://blogs.sas.com/content/sasdummy/2015/08/03/using-lua-within-your-sas-programs */
@@ -38,10 +40,15 @@
     %else %if "&SYSVLONG" < "9.04.01M3" %then 0;
     %else 1;
   %end;
+  %else %if &feature=EXPORTXLS %then %do;
+    /* is it possible to PROC EXPORT an excel file? */
+    %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 1;
+    %else %if %sysfunc(sysprod(SAS/ACCESS Interface to PC Files)) = 1 %then 1;
+    %else 0;
+  %end;
   %else %do;
     -1
     %put &sysmacroname: &feature not found;
   %end;
 %mend mf_existfeature;
-
-/** @endcond */
+/** @endcond */

base/mf_fmtdttm.sas

@@ -0,0 +1,42 @@
+/**
+  @file
+  @brief Returns E8601DT26.6 if compatible else DATETIME19.3
+  @details From our experience in [Data Controller for SAS]
+  (https://datacontroller.io) deployments, the E8601DT26.6 datetime format has
+  the widest support when it comes to pass-through SQL queries.
+
+  However, it is not supported in WPS or early versions of SAS 9 (M3 and below)
+  when used as a datetime literal, eg:
+
+      data _null_;
+        demo="%sysfunc(datetime(),E8601DT26.6)"dt;
+        demo=;
+      run;
+
+  This macro will therefore return DATEITME19.3 as an alternative format
+  based on the runtime environment so that it can be used in such cases, eg:
+
+      data _null_;
+        demo="%sysfunc(datetime(),%mf_fmtdttm())"dt;
+        demo=;
+      run;
+
+  <h4> Related Macros </h4>
+  @li mf_fmtdttm.test.sas
+
+  @author Allan Bowe
+**/
+
+%macro mf_fmtdttm(
+)/*/STORE SOURCE*/;
+
+%if "&sysver"="9.2" or "&sysver"="9.3"
+  or ("&sysver"="9.4" and "%substr(&SYSVLONG,9,1)" le "3")
+  or "%substr(&sysver,1,1)"="4"
+  or "%substr(&sysver,1,1)"="5"
+%then %do;DATETIME19.3%end;
+%else %do;E8601DT26.6%end;
+
+%mend mf_fmtdttm;
+
+

base/mf_getplatform.sas

@@ -5,8 +5,12 @@
 
       %put %mf_getplatform();
 
-    returns:
-      SASMETA  (or SASVIYA)
+  returns one of:
+
+  @li SASMETA
+  @li SASVIYA
+  @li SASJS
+  @li BASESAS
 
   @param switch the param for which to return a platform specific variable
 
@@ -22,6 +26,12 @@
 )/*/STORE SOURCE*/;
 %local a b c;
 %if &switch.NONE=NONE %then %do;
+  %if %symexist(sasjsprocessmode) %then %do;
+    %if &sasjsprocessmode=Stored Program %then %do;
+      SASJS
+      %return;
+    %end;
+  %end;
   %if %symexist(sysprocessmode) %then %do;
     %if "&sysprocessmode"="SAS Object Server"
     or "&sysprocessmode"= "SAS Compute Server" %then %do;
@@ -62,4 +72,4 @@
 %else %if &switch=VIYARESTAPI %then %do;
   %mf_trimstr(%sysfunc(getoption(servicesbaseurl)),/)
 %end;
-%mend mf_getplatform;
+%mend mf_getplatform;

base/mf_getuniquefileref.sas

@@ -28,15 +28,17 @@
     be 8 characters, so a 7 letter prefix would mean `maxtries` should be 10.
     if using zero (0) as the prefix, a native assignment is used.
   @param [in] maxtries= (1000) the last part of the libref. Must be an integer.
+  @param [in] lrecl= (32767) Provide a default lrecl with which to initialise
+    the generated fileref.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-%macro mf_getuniquefileref(prefix=_,maxtries=1000);
+%macro mf_getuniquefileref(prefix=_,maxtries=1000,lrecl=32767);
   %local rc fname;
   %if &prefix=0 %then %do;
-    %let rc=%sysfunc(filename(fname,,temp));
+    %let rc=%sysfunc(filename(fname,,temp,lrecl=&lrecl));
     %if &rc %then %put %sysfunc(sysmsg());
     &fname
   %end;
@@ -47,7 +49,7 @@
     %do x=0 %to &maxtries;
       %let fname=&prefix%substr(%sysfunc(ranuni(0)),3,&len);
       %if %sysfunc(fileref(&fname)) > 0 %then %do;
-        %let rc=%sysfunc(filename(fname,,temp));
+        %let rc=%sysfunc(filename(fname,,temp,lrecl=&lrecl));
         %if &rc %then %put %sysfunc(sysmsg());
         &fname
         %return;

base/mf_getuniquelibref.sas

@@ -3,38 +3,55 @@
   @brief Returns an unused libref
   @details Use as follows:
 
-    libname mclib0 (work);
-    libname mclib1 (work);
-    libname mclib2 (work);
+      libname mclib0 (work);
+      libname mclib1 (work);
+      libname mclib2 (work);
 
-    %let libref=%mf_getuniquelibref();
-    %put &=libref;
+      %let libref=%mf_getuniquelibref();
+      %put &=libref;
 
   which returns:
 
 > mclib3
 
-  @param prefix= first part of libref.  Remember that librefs can only be 8 characters,
-    so a 7 letter prefix would mean that maxtries should be 10.
-  @param maxtries= the last part of the libref.  Provide an integer value.
+  A blank value is returned if no usable libname is determined.
+
+  @param [in] prefix= (mclib) first part of the returned libref. As librefs can
+    be as long as 8 characters, a maximum length of 7 characters is premitted
+    for this prefix.
+  @param [in] maxtries= Deprecated parameter. Remains here to ensure a
+    non-breaking change.  Will be removed in v5.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-
 %macro mf_getuniquelibref(prefix=mclib,maxtries=1000);
-  %local x libref;
-  %let x=0;
-  %do x=0 %to &maxtries;
-  %if %sysfunc(libref(&prefix&x)) ne 0 %then %do;
-    %let libref=&prefix&x;
-    %let rc=%sysfunc(libname(&libref,%sysfunc(pathname(work))));
-    %if &rc %then %put %sysfunc(sysmsg());
-    &prefix&x
-    %*put &sysmacroname: Libref &libref assigned as WORK and returned;
+  %local x;
+
+  %if ( %length(&prefix) gt 7 ) %then %do;
+    %put %str(ERR)OR: The prefix parameter cannot exceed 7 characters.;
+    0
     %return;
   %end;
+  %else %if (%sysfunc(NVALID(&prefix,v7))=0) %then %do;
+    %put %str(ERR)OR: Invalid prefix (&prefix);
+    0
+    %return;
   %end;
-  %put unable to find available libref in range &prefix.0-&maxtries;
+
+  /* Set maxtries equal to '10 to the power of [# unused characters] - 1' */
+  %let maxtries=%eval(10**(8-%length(&prefix))-1);
+
+  %do x = 0 %to &maxtries;
+    %if %sysfunc(libref(&prefix&x)) ne 0 %then %do;
+      &prefix&x
+      %return;
+    %end;
+    %let x = %eval(&x + 1);
+  %end;
+
+  %put %str(ERR)OR: No usable libref in range &prefix.0-&maxtries;
+  %put %str(ERR)OR- Try reducing the prefix or deleting some libraries!;
+  0
 %mend mf_getuniquelibref;

base/mf_getuser.sas

@@ -33,7 +33,8 @@
   %else %if %symexist(&metavar) %then %do;
     %if %length(&&&metavar)=0 %then %let user=&sysuserid;
     /* sometimes SAS will add @domain extension - remove for consistency */
-    %else %let user=%scan(&&&metavar,1,@);
+    /* but be sure to quote in case of usernames with commas */
+    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@));
   %end;
   %else %let user=&sysuserid;
 

base/mf_getvarcount.sas

@@ -2,31 +2,48 @@
   @file
   @brief Returns number of variables in a dataset
   @details Useful to identify those renagade datasets that have no columns!
+  Can also be used to count for numeric, or character columns
 
-        %put Number of Variables=%mf_getvarcount(sashelp.class);
+      %put Number of Variables=%mf_getvarcount(sashelp.class);
+      %put Character Variables=%mf_getvarcount(sashelp.class,typefilter=C);
+      %put Numeric Variables = %mf_getvarcount(sashelp.class,typefilter=N);
 
   returns:
   > Number of Variables=4
 
-  @param libds Two part dataset (or view) reference.
+
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] typefilter= (A) Filter for certain types of column.  Valid values:
+    @li A Count All columns
+    @li C Count Character columns only
+    @li N Count Numeric columns only
 
   @version 9.2
   @author Allan Bowe
 
 **/
 
-%macro mf_getvarcount(libds
+%macro mf_getvarcount(libds,typefilter=A
 )/*/STORE SOURCE*/;
-  %local dsid nvars rc ;
+  %local dsid nvars rc outcnt x;
   %let dsid=%sysfunc(open(&libds));
   %let nvars=.;
+  %let outcnt=0;
+  %let typefilter=%upcase(&typefilter);
   %if &dsid %then %do;
     %let nvars=%sysfunc(attrn(&dsid,NVARS));
+    %if &typefilter=A %then %let outcnt=&nvars;
+    %else %if &nvars>0 %then %do x=1 %to &nvars;
+      /* increment based on variable type */
+      %if %sysfunc(vartype(&dsid,&x))=&typefilter %then %do;
+        %let outcnt=%eval(&outcnt+1);
+      %end;
+    %end;
     %let rc=%sysfunc(close(&dsid));
   %end;
   %else %do;
     %put unable to open &libds (rc=&dsid);
     %let rc=%sysfunc(close(&dsid));
   %end;
-  &nvars
+  &outcnt
 %mend mf_getvarcount;

base/mf_isint.sas

@@ -20,8 +20,11 @@
 
 %macro mf_isint(arg
 )/*/STORE SOURCE*/;
-  /* remove minus sign if exists */
 
+  /* blank val is not an integer */
+  %if "&arg"="" %then %do;0%return;%end;
+
+  /* remove minus sign if exists */
   %local val;
   %if "%substr(%str(&arg),1,1)"="-" %then %let val=%substr(%str(&arg),2);
   %else %let val=&arg;
@@ -30,4 +33,4 @@
   %if %sysfunc(findc(%str(&val),,kd)) %then %do;0%end;
   %else %do;1%end;
 
-%mend mf_isint;
+%mend mf_isint;

base/mf_verifymacvars.sas

@@ -59,7 +59,7 @@
 
   %goto exit_success;
   %exit_err:
-    %put %str(ERR)OR: &abortmsg;
+    %put &abortmsg;
     %mf_abort(iftrue=(&mabort ne SOFT),
       mac=mf_verifymacvars,
       msg=%str(&abortmsg)

base/mp_abort.sas

@@ -84,12 +84,17 @@
     %end;
   %end;
 
-  /* Stored Process Server web app context */
-  %if %symexist(_metaperson)
+  /* Web App Context */
+  %if %symexist(_PROGRAM)
     or "&SYSPROCESSNAME "="Compute Server "
     or &mode=INCLUDE
   %then %do;
-    options obs=max replace nosyntaxcheck mprint;
+    options obs=max replace mprint;
+    %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5"
+    %then %do;
+      options nosyntaxcheck;
+    %end;
+
     %if &mode=INCLUDE %then %do;
       %if %sysfunc(exist(&errds))=1 %then %do;
         data _null_;
@@ -161,12 +166,14 @@
     /* send response in SASjs JSON format */
     data _null_;
       file _webout mod lrecl=32000 encoding='utf-8';
-      length msg $32767 ;
+      length msg syswarningtext syserrortext $32767 ;
       sasdatetime=datetime();
       msg=symget('msg');
     %if &logline>0 %then %do;
       msg=cats(msg,'\n\nLog Extract:\n',symget('logmsg'));
     %end;
+      /* escape the escapes */
+      msg=tranwrd(msg,'\','\\');
       /* escape the quotes */
       msg=tranwrd(msg,'"','\"');
       /* ditch the CRLFs as chrome complains */
@@ -260,4 +267,4 @@
   %end;
 %mend mp_abort;
 
-/** @endcond */
+/** @endcond */

base/mp_assertscope.sas

@@ -74,7 +74,8 @@
   outds=work.test_results
 )/*/STORE SOURCE*/;
 %local ds test_result test_comments del add mod ilist;
-%let ilist=%upcase(&sasjs_prefix._FUNCTIONS &ignorelist);
+%let ilist=%upcase(&sasjs_prefix._FUNCTIONS SYS_PROCHTTP_STATUS_CODE
+  SYS_PROCHTTP_STATUS_CODE SYS_PROCHTTP_STATUS_PHRASE &ignorelist);
 
 /**
   * this sets up the global vars, it will also enter STRICT mode.  If this
@@ -89,7 +90,7 @@
   create table &scopeds as
     select name,offset,value
     from dictionary.macros
-    where scope="&scope" and name not in (%mf_getquotedstr(&ilist))
+    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
     order by name,offset;
 %end;
 %else %if &action=COMPARE %then %do;
@@ -98,12 +99,14 @@
   create table _data_ as
     select name,offset,value
     from dictionary.macros
-    where scope="&scope" and name not in (%mf_getquotedstr(&ilist))
+    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
     order by name,offset;
 
   %let ds=&syslast;
 
-  proc compare base=&scopeds compare=&ds;
+  proc compare
+    base=&scopeds(where=(upcase(name) not in (%mf_getquotedstr(&ilist))))
+    compare=&ds noprint;
   run;
 
   %if &sysinfo=0 %then %do;
@@ -141,4 +144,4 @@
   drop table &ds;
 %end;
 
-%mend mp_assertscope;
+%mend mp_assertscope;

base/mp_chop.sas

@@ -0,0 +1,194 @@
+/**
+  @file
+  @brief Splits a file of ANY SIZE by reference to a search string.
+  @details Provide a fileref and a search string to chop off part of a file.
+
+  Works by reading in the file byte by byte, then marking the beginning and end
+  of each matched string, before finally doing the chop.
+
+  Choose whether to keep the FIRST or the LAST section of the file.  Optionally,
+  use an OFFSET to fix the precise chop point.
+
+  Usage:
+
+      %let src="%sysfunc(pathname(work))/file.txt";
+      %let str=Chop here!;
+      %let out1="%sysfunc(pathname(work))/file1.txt";
+      %let out2="%sysfunc(pathname(work))/file2.txt";
+      %let out3="%sysfunc(pathname(work))/file3.txt";
+      %let out4="%sysfunc(pathname(work))/file4.txt";
+
+      data _null_;
+        file &src;
+        put "startsection&str.endsection";
+      run;
+
+      %mp_chop(&src, matchvar=str, keep=FIRST, outfile=&out1)
+      %mp_chop(&src, matchvar=str, keep=LAST, outfile=&out2)
+      %mp_chop(&src, matchvar=str, keep=FIRST, matchpoint=END, outfile=&out3)
+      %mp_chop(&src, matchvar=str, keep=LAST, matchpoint=END, outfile=&out4)
+
+      filename results (&out1 &out2 &out3 &out4);
+      data _null_;
+        infile results;
+        input;
+        list;
+      run;
+
+  Results:
+    @li `startsection`
+    @li `Chop here!endsection`
+    @li `startsectionChop here!`
+    @li `endsection`
+
+  For more examples, see mp_chop.test.sas
+
+  @param [in] infile The QUOTED path to the file on which to perform the chop
+  @param [in] matchvar= Macro variable NAME containing the string to split by
+  @param [in] matchpoint= (START) Valid values:
+    @li START - chop at the beginning of the string in `matchvar`.
+    @li END - chop at the end of the string in `matchvar`.
+  @param [in] offset= (0) An adjustment to the precise chop location, by
+    by reference to the `matchpoint`. Should be a positive or negative integer.
+  @param [in] keep= (FIRST) Valid values:
+    @li FIRST - keep the section of the file before the chop
+    @li LAST - keep the section of the file after the chop
+  @param [in] mdebug= (0) Set to 1 to provide macro debugging
+  @param outfile= (0) Optional QUOTED path to the adjusted output file (avoids
+    overwriting the first file).
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mp_abort.sas
+  @li mp_gsubfile.sas
+  @li mp_replace.sas
+  @li mp_chop.test.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mp_chop(infile,
+  matchvar=,
+  matchpoint=START,
+  keep=FIRST,
+  offset=0,
+  mdebug=0,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+%local fref0 dttm ds1 outref;
+%let fref0=%mf_getuniquefileref();
+%let ds1=%mf_getuniquename(prefix=allchars);
+%let ds2=%mf_getuniquename(prefix=startmark);
+
+%if &outfile=0 %then %let outfile=&infile;
+
+%mp_abort(iftrue= (%length(%superq(&matchvar))=0)
+  ,mac=mp_chop.sas
+  ,msg=%str(&matchvar is an empty variable)
+)
+
+/* START */
+%let dttm=%sysfunc(datetime());
+
+filename &fref0 &infile lrecl=1 recfm=n;
+
+/* create dataset with one char per row */
+data &ds1;
+  infile &fref0;
+  input sourcechar $char1. @@;
+  format sourcechar hex2.;
+run;
+
+/* get start & stop position of first matchvar string (one row, two vars) */
+data &ds2;
+  /* set find string to length in bytes to cover trailing spaces */
+  length string $ %length(%superq(&matchvar));
+  string =symget("&matchvar");
+  drop string;
+
+  firstchar=char(string,1);
+  findlen=lengthm(string); /* <- for trailing bytes */
+
+  do _N_=1 to nobs;
+    set &ds1 nobs=nobs point=_N_;
+    if sourcechar=firstchar then do;
+      pos=1;
+      s=0;
+      do point=_N_ to min(_N_ + findlen -1,nobs);
+        set &ds1 point=point;
+        if sourcechar=char(string, pos) then s + 1;
+        else goto _leave_;
+        pos+1;
+      end;
+      _leave_:
+      if s=findlen then do;
+        START =_N_;
+        _N_ =_N_+ s - 1;
+        STOP =_N_;
+        output;
+        /* matched! */
+        stop;
+      end;
+    end;
+  end;
+  stop;
+  keep START STOP;
+run;
+
+%local split;
+%let split=0;
+data _null_;
+  set &ds2;
+  if "&matchpoint"='START' then do;
+    if "&keep"='FIRST' then mp=start;
+    else if "&keep"='LAST' then mp=start-1;
+  end;
+  else if "&matchpoint"='END' then do;
+    if "&keep"='FIRST' then mp=stop+1;
+    else if "&keep"='LAST' then mp=stop;
+  end;
+  split=mp+&offset;
+  call symputx('split',split,'l');
+%if &mdebug=1 %then %do;
+  put (_all_)(=);
+  %put &=offset;
+%end;
+run;
+%if &split=0 %then %do;
+  %put &sysmacroname: No match found in &infile for string %superq(&matchvar);
+  %return;
+%end;
+
+data _null_;
+  file &outfile recfm=n;
+  set &ds1;
+%if &keep=FIRST %then %do;
+  if _n_ ge &split then stop;
+%end;
+%else %do;
+  if _n_ gt &split;
+%end;
+  put sourcechar char1.;
+run;
+
+%if &mdebug=0 %then %do;
+  filename &fref0 clear;
+%end;
+%else %do;
+  data _null_;
+    infile &outfile lrecl=32767;
+    input;
+    list;
+    if _n_>50 then stop;
+  run;
+%end;
+/* END */
+%put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) seconds to run;
+
+%mend mp_chop;

base/mp_cntlout.sas

@@ -0,0 +1,85 @@
+/**
+  @file mp_cntlout.sas
+  @brief Creates a cntlout dataset in a consistent format
+  @details The dataset produced by proc format in the cntlout option will vary
+  according to its contents.
+
+  When dealing with formats from an ETL perspective (eg in [Data Controller for
+  SAS](https://datacontroller.io)), it is important that the output dataset
+  has a consistent model (and compariable values).
+
+  This macro makes use of mddl_sas_cntlout.sas to provide the consistent model,
+  and will left-align the start and end values when dealing with numeric ranges
+  to enable consistency when checking for differences.
+
+  usage:
+
+      %mp_cntlout(libcat=yourlib.cat,cntlout=work.formatexport)
+
+  @param [in] libcat The library.catalog reference
+  @param [in] fmtlist= (0) provide a space separated list of specific formats to
+    extract
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [out] cntlout= (work.fmtextract) Libds reference for the output dataset
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mf_getvarformat.sas
+  @li mp_getformats.sas
+  @li mp_loadformat.sas
+  @li mp_ds2fmtds.sas
+
+  @version 9.2
+  @author Allan Bowe
+  @cond
+**/
+
+%macro mp_cntlout(
+  iftrue=(1=1)
+  ,libcat=
+  ,cntlout=work.fmtextract
+  ,fmtlist=0
+)/*/STORE SOURCE*/;
+%local ddlds cntlds i;
+
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+%let ddlds=%mf_getuniquename();
+%let cntlds=%mf_getuniquename();
+
+%mddl_sas_cntlout(libds=&ddlds)
+
+%if %index(&libcat,-)>0 and %scan(&libcat,2,-)=FC %then %do;
+  %let libcat=%scan(&libcat,1,-);
+%end;
+
+proc format lib=&libcat cntlout=&cntlds;
+%if "&fmtlist" ne "0" %then %do;
+  select
+  %do i=1 %to %sysfunc(countw(&fmtlist));
+    %scan(&fmtlist,&i,%str( ))
+  %end;
+  ;
+%end;
+run;
+
+data &cntlout;
+  if 0 then set &ddlds;
+  set &cntlds;
+  if type="N" then do;
+    start=cats(start);
+    end=cats(end);
+  end;
+run;
+proc sort;
+  by fmtname start;
+run;
+
+proc sql;
+drop table &ddlds,&cntlds;
+
+%mend mp_cntlout;
+/** @endcond */

base/mp_copyfolder.sas

@@ -16,8 +16,11 @@
 
       %mp_copyfolder(&rootdir,&copydir)
 
-  @param source Unquoted path to the folder to copy from.
-  @param target Unquoted path to the folder to copy to.
+  @param [in] source Unquoted path to the folder to copy from.
+  @param [out] target Unquoted path to the folder to copy to.
+  @param [in] copymax=(MAX) Set to a positive integer to indicate the level of
+    subdirectory copy recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas
@@ -31,7 +34,7 @@
 
 **/
 
-%macro mp_copyfolder(source,target);
+%macro mp_copyfolder(source,target,copymax=MAX);
 
   %mp_abort(iftrue=(%mf_isdir(&source)=0)
     ,mac=&sysmacroname
@@ -50,7 +53,7 @@
   %let tempds=%mf_getuniquename();
 
   /* recursive directory listing */
-  %mp_dirlist(path=&source,outds=work.&tempds, maxdepth=MAX)
+  %mp_dirlist(path=&source,outds=work.&tempds,maxdepth=&copymax)
 
   /* create folders and copy content */
   data _null_;
@@ -78,4 +81,4 @@
   proc sql;
   drop table work.&tempds;
 
-%mend mp_copyfolder;
+%mend mp_copyfolder;

base/mp_coretable.sas

@@ -30,6 +30,7 @@
   @li mddl_dc_filtersummary.sas
   @li mddl_dc_locktable.sas
   @li mddl_dc_maxkeytable.sas
+  @li mf_getuniquename.sas
 
   <h4> Related Macros </h4>
   @li mp_filterstore.sas
@@ -46,7 +47,7 @@
 %macro mp_coretable(table_ref,libds=0
 )/*/STORE SOURCE*/;
 %local outds ;
-%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
+%let outds=%sysfunc(ifc(&libds=0,%mf_getuniquename(),&libds));
 proc sql;
 %if &table_ref=DIFFTABLE %then %do;
   %mddl_dc_difftable(libds=&outds)
@@ -65,7 +66,8 @@ proc sql;
 %end;
 
 %if &libds=0 %then %do;
+  proc sql;
   describe table &syslast;
   drop table &syslast;
 %end;
-%mend mp_coretable;
+%mend mp_coretable;

base/mp_createwebservice.sas

@@ -1,49 +1,13 @@
 /**
   @file mp_createwebservice.sas
-  @brief Create a web service in SAS 9 or Viya
-  @details Creates a SASJS ready Stored Process in SAS 9 or Job Execution
-  Service in SAS Viya
+  @brief Create a web service in SAS 9, Viya or SASjs Server
+  @details This is actually a wrapper for mx_createwebservice.sas, remaining
+  for legacy purposes.  For new apps, use mx_createwebservice.sas.
 
-Usage:
-
-    %* compile macros ;
-    filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
-    %inc mc;
-
-    %* write some code;
-    filename ft15f001 temp;
-    parmcards4;
-        %* fetch any data from frontend ;
-        %webout(FETCH)
-        data example1 example2;
-          set sashelp.class;
-        run;
-        %* send data back;
-        %webout(OPEN)
-        %webout(ARR,example1) * Array format, fast, suitable for large tables ;
-        %webout(OBJ,example2) * Object format, easier to work with ;
-        %webout(CLOSE)
-    ;;;;
-    %mp_createwebservice(path=/Public/app/common,name=appInit,replace=YES)
 
   <h4> SAS Macros </h4>
-  @li mf_getplatform.sas
-  @li mm_createwebservice.sas
-  @li mv_createwebservice.sas
+  @li mx_createwebservice.sas
 
-  @param [in,out] path= The full folder path where the service will be created
-  @param [in,out] name= Service name.  Avoid spaces.
-  @param [in] desc= The description of the service (optional)
-  @param [in] precode= Space separated list of filerefs, pointing to the code
-    that needs to be attached to the beginning of the service (optional)
-  @param [in] code= (ft15f001) Space seperated fileref(s) of the actual code to
-    be added
-  @param [in] replace= (YES) Select YES to replace any existing service in that
-    location
-
-
-  @version 9.2
-  @author Allan Bowe
 
 **/
 
@@ -53,33 +17,16 @@ Usage:
     ,code=ft15f001
     ,desc=This service was created by the mp_createwebservice macro
     ,replace=YES
+    ,mdebug=0
 )/*/STORE SOURCE*/;
 
-%if &syscc ge 4 %then %do;
-  %put syscc=&syscc - &sysmacroname will not execute in this state;
-  %return;
-%end;
-
-%local platform; %let platform=%mf_getplatform();
-%if &platform=SASVIYA %then %do;
-  %if "&path"="HOME" %then %let path=/Users/&sysuserid/My Folder;
-  %mv_createwebservice(path=&path
+  %mx_createwebservice(path=&path
     ,name=&name
-    ,code=&code
     ,precode=&precode
+    ,code=&code
     ,desc=&desc
     ,replace=&replace
+    ,mdebug=&mdebug
   )
-%end;
-%else %do;
-  %if "&path"="HOME" %then %let path=/User Folders/&sysuserid/My Folder;
-  %mm_createwebservice(path=&path
-    ,name=&name
-    ,code=&code
-    ,precode=&precode
-    ,desc=&desc
-    ,replace=&replace
-  )
-%end;
 
 %mend mp_createwebservice;

base/mp_dirlist.sas

@@ -94,6 +94,13 @@ data &out_ds(compress=no
   %end;
   if rc = 0 then do;
     did = dopen(fref);
+    if did=0 then do;
+      putlog "NOTE: This directory is empty, or does not exist - &path";
+      msg=sysmsg();
+      put msg;
+      put _all_;
+      stop;
+    end;
     /* attribute is OS-dependent - could be "Directory" or "Directory Name" */
     numopts=doptnum(did);
     do i=1 to numopts;
@@ -101,12 +108,6 @@ data &out_ds(compress=no
       if foption=:'Directory' then i=numopts;
     end;
     directory=dinfo(did,foption);
-    if did=0 then do;
-      putlog "NOTE: This directory is empty - " directory;
-      msg=sysmsg();
-      put _all_;
-      stop;
-    end;
     rc = filename(fref);
   end;
   else do;
@@ -151,6 +152,7 @@ run;
   data &out_ds;
     set &out_ds;
     length infoname infoval $60 fref $8;
+    if _n_=1 then call missing(fref);
     rc=filename(fref,filepath);
     drop rc infoname fid i close fref;
     if file_or_folder='file' then do;
@@ -242,4 +244,4 @@ run;
 proc sql;
 drop table &out_ds;
 
-%mend mp_dirlist;
+%mend mp_dirlist;

base/mp_ds2cards.sas

@@ -11,7 +11,7 @@
 
   Usage:
 
-      %mp_ds2cards(base_ds=sashelp.class
+      %mp_ds2cards(sashelp.class
         , tgt_ds=work.class
         , cards_file= "C:\temp\class.sas"
         , showlog=NO
@@ -23,7 +23,7 @@
     - explicity setting a unix LF
     - constraints / indexes etc
 
-  @param [in] base_ds= Should be two level - eg work.blah.  This is the table
+  @param [in] base_ds Should be two level - eg work.blah.  This is the table
                   that is converted to a cards file.
   @param [in] tgt_ds= Table that the generated cards file would create.
     Optional - if omitted, will be same as BASE_DS.
@@ -48,6 +48,7 @@
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
 %macro mp_ds2cards(base_ds=, tgt_ds=
@@ -219,7 +220,8 @@ data _null_;
     put '  @file';
     put "  @brief Datalines for %upcase(%scan(&base_ds,2)) dataset";
     put "  @details Generated by %nrstr(%%)mp_ds2cards()";
-    put "  Available on github.com/sasjs/core";
+    put "    Source: https://github.com/sasjs/core";
+    put '  @cond ';
     put '**/';
     put "data &tgt_ds &indexes;";
     put "attrib ";
@@ -264,6 +266,7 @@ data _null_;
   if __lastobs then do;
     put ';;;;';
     put 'run;';
+    put '/** @endcond **/';
     stop;
   end;
 run;
@@ -283,4 +286,5 @@ quit;
 %put NOTE-;%put NOTE-;
 %put NOTE- %sysfunc(dequote(&cards_file.));
 %put NOTE-;%put NOTE-;
-%mend mp_ds2cards;
+%mend mp_ds2cards;
+/** @endcond **/

base/mp_ds2md.sas

@@ -57,6 +57,11 @@
 %local vars;
 %let vars=%upcase(%mf_getvarlist(&libds));
 
+%if %trim(X&vars)=X %then %do;
+  %put &sysmacroname: Table &libds has no columns!!;
+  %return;
+%end;
+
 /* create the header row */
 data _null_;
   file &outref;

base/mp_filtercheck.sas

@@ -92,7 +92,36 @@ data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
-  length reason_cd $4032;
+  length reason_cd $4032 vtype $1 vnum dsid 8;
+
+  /* quick check to ensure column exists */
+  if upcase(VARIABLE_NM) not in
+  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
+  then do;
+    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
+    putlog REASON_CD= VARIABLE_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
+    output;
+    return;
+  end;
+
+  /* need to open the dataset to get the column type */
+  dsid=open("&targetds","i");
+  if dsid>0 then do;
+    vnum=varnum(dsid,VARIABLE_NM);
+    if vnum<1 then do;
+      /* should not happen as was also tested for above */
+      REASON_CD=cats("Variable (",VARIABLE_NM,") not found in &targetds");
+      putlog REASON_CD= dsid=;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
+      output;
+      return;
+    end;
+    /* now we can get the type */
+    else vtype=vartype(dsid,vnum);
+  end;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
@@ -116,17 +145,8 @@ data &outds;
     call symputx('nobs',_n_,'l');
     output;
   end;
-  if upcase(VARIABLE_NM) not in
-  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
-  then do;
-    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
-    putlog REASON_CD= VARIABLE_NM=;
-    call symputx('reason_cd',reason_cd,'l');
-    call symputx('nobs',_n_,'l');
-    output;
-  end;
   if OPERATOR_NM not in
-  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE')
+  ('=','>','<','<=','>=','NE','GE','LE','BETWEEN','IN','NOT IN','CONTAINS')
   then do;
     REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
@@ -135,6 +155,18 @@ data &outds;
     output;
   end;
 
+  /* special missing logic */
+  if vtype='N'
+    and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE')
+    and cats(upcase(raw_value)) in (
+      '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N'
+      '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._'
+    )
+  then do;
+    /* valid numeric - exit data step loop */
+    return;
+  end;
+
   /* special logic */
   if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ','');
   else if OPERATOR_NM in ('IN','NOT IN') then do;

base/mp_filtergenerate.sas

@@ -84,6 +84,9 @@ filename &outref temp;
   run;
 %end;
 %else %do;
+  proc sort data=&inds;
+    by SUBGROUP_ID;
+  run;
   data _null_;
     file &outref lrecl=32800;
     set &inds end=last;

base/mp_getconstraints.sas

@@ -40,6 +40,22 @@
 %let lib=%upcase(&lib);
 %let ds=%upcase(&ds);
 
+/**
+  * Cater for environments where sashelp.vcncolu is not available
+  */
+%if %sysfunc(exist(sashelp.vcncolu,view))=0 %then %do;
+  proc sql;
+  create table &outds(
+      libref char(8)
+    ,TABLE_NAME char(32)
+    ,constraint_type char(8)     label='Constraint Type'
+    ,constraint_name char(32)     label='Constraint Name'
+    ,column_name char(32)     label='Column'
+    ,constraint_order num
+  );
+  %return;
+%end;
+
 /**
   * Neither dictionary tables nor sashelp provides a constraint order column,
   * however they DO arrive in the correct order.  So, create the col.

base/mp_getformats.sas

@@ -113,10 +113,10 @@ create table &outsummary as
       select &&fmtname&i;
     run;
     data &tempds;
-      length label $256;
+      if 0 then set &outdetail;
       set &tempds;
     run;
-    proc append base=&outdetail data=&tempds;
+    proc append base=&outdetail data=&tempds ;
     run;
   %end;
 %end;

base/mp_getmaxvarlengths.sas

@@ -41,6 +41,7 @@
   <h4> SAS Macros </h4>
   @li mcf_length.sas
   @li mf_getuniquename.sas
+  @li mf_getvarcount.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
   @li mf_getvarformat.sas
@@ -60,7 +61,7 @@
   ,outds=work.mp_getmaxvarlengths
 )/*/STORE SOURCE*/;
 
-%local vars prefix x var fmt;
+%local vars prefix x var fmt srcds;
 %let vars=%mf_getvarlist(libds=&libds);
 %let prefix=%substr(%mf_getuniquename(),1,25);
 %let num2char=%upcase(&num2char);
@@ -70,6 +71,24 @@
   %mcf_length(wrap=YES, insert_cmplib=YES)
 %end;
 
+%if &num2char=NO
+  and ("%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5")
+  and %mf_getvarcount(&libds,typefilter=N) gt 0
+%then %do;
+  /* custom functions not supported in summary operations */
+  %let srcds=%mf_getuniquename();
+  data &srcds/view=&srcds;
+    set &libds;
+  %do x=1 %to %sysfunc(countw(&vars,%str( )));
+    %let var=%scan(&vars,&x);
+    %if %mf_getvartype(&libds,&var)=N %then %do;
+      &prefix.&x=mcf_length(&var);
+    %end;
+  %end;
+  run;
+%end;
+%else %let srcds=&libds;
+
 proc sql;
 create table &outds (rename=(
     %do x=1 %to %sysfunc(countw(&vars,%str( )));
@@ -94,10 +113,15 @@ create table &outds (rename=(
         %end;
       %end;
       %else %do;
-        max(mcf_length(&var)) as &prefix.&x
+        %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do;
+          max(&prefix.&x) as &prefix.&x
+        %end;
+        %else %do;
+          max(mcf_length(&var)) as &prefix.&x
+        %end;
       %end;
     %end;
-  from &libds;
+  from &srcds;
 
   proc transpose data=&outds
     out=&outds(rename=(_name_=NAME COL1=MAXLEN));

base/mp_getpk.sas

@@ -55,7 +55,8 @@
 )/*/STORE SOURCE*/;
 
 
-%local engine schema ds1 ds2 ds3 dsn tabs1 tabs2 sum pk4sure pkdefault finalpks;
+%local engine schema ds1 ds2 ds3 dsn tabs1 tabs2 sum pk4sure pkdefault finalpks
+  pkfromindex;
 
 %let lib=%upcase(&lib);
 %let ds=%upcase(&ds);
@@ -70,6 +71,7 @@
 %let sum=%mf_getuniquename(prefix=getpk_sum);
 %let pk4sure=%mf_getuniquename(prefix=getpk_pk4sure);
 %let pkdefault=%mf_getuniquename(prefix=getpk_pkdefault);
+%let pkfromindex=%mf_getuniquename(prefix=getpk_pkfromindex);
 %let finalpks=%mf_getuniquename(prefix=getpk_finalpks);
 
 %local dbg;
@@ -180,9 +182,23 @@ create table &ds1 as
       and a.constraint_name=b.constraint_name
     order by 1,2,3,4;
 
+  /* extract cols from the relevant unique INDEXES */
+  create table &pkfromindex as
+    select libname as libref
+      ,memname as table_name
+      ,indxname as constraint_name
+      ,indxpos as constraint_order
+      ,name
+    from dictionary.indexes
+    where nomiss='yes' and unique='yes' and upcase(libname)="&lib"
+  %if &ds ne 0 %then %do;
+      and upcase(memname)="&ds"
+  %end;
+    order by 1,2,3,4;
+
   /* create one table */
   data &finalpks;
-    set &pkdefault &pk4sure ;
+    set &pkdefault &pk4sure &pkfromindex;
     pk_ind=1;
     /* if there are multiple unique constraints, take the first */
     by libref table_name constraint_name;
@@ -256,4 +272,4 @@ create table &outds as
   iftrue=(&mdebug=0)
 )
 
-%mend mp_getpk;
+%mend mp_getpk;

base/mp_gsubfile.sas

@@ -48,6 +48,11 @@
   outfile=0
 )/*/STORE SOURCE*/;
 
+  %if "%substr(&sysver.XX,1,4)"="V.04" %then %do;
+    %put %str(ERR)OR: Viya 4 does not support the IO library in lua;
+    %return;
+  %end;
+
   %ml_gsubfile()
 
 %mend mp_gsubfile;

base/mp_hashdataset.sas

@@ -1,8 +1,8 @@
 /**
   @file
   @brief Returns a unique hash for a dataset
-  @details Ignores metadata attributes, used only to hash values. Compared
-  datasets must be in the same order.
+  @details Ignores metadata attributes, used only to hash values. If used to
+  compare datasets, they must have their columns and rows in the same order.
 
       %mp_hashdataset(sashelp.class,outds=myhash)
 
@@ -17,13 +17,16 @@
   @li mf_getattrn.sas
   @li mf_getuniquename.sas
   @li mf_getvarlist.sas
-  @li mf_getvartype.sas
+  @li mp_md5.sas
+
+  <h4> Related Files </h4>
+  @li mp_hashdataset.test.sas
 
   @param [in] libds dataset to hash
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
   @param [in] iftrue= A condition under which the macro should be executed.
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
-  will contain one column (hashkey) with one observation (a hex32.
+  will contain one column (hashkey) with one observation (a $hex32.
   representation of the input hash)
   |hashkey:$32.|
   |---|
@@ -35,48 +38,53 @@
 
 %macro mp_hashdataset(
   libds,
-  outds=,
+  outds=work._data_,
   salt=,
   iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
-  %if not(%eval(%unquote(&iftrue))) %then %return;
+%local keyvar /* roll up the md5 */
+  prevkeyvar /* retain prev record md5 */
+  lastvar /* last var in input ds */
+  cvars nvars;
 
-  %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
-    %put %str(WARN)ING: Dataset &libds is empty, or is not a dataset;
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+/* avoid naming conflict for hash key vars */
+%let keyvar=%mf_getuniquename();
+%let prevkeyvar=%mf_getuniquename();
+%let lastvar=%mf_getuniquename();
+
+%if %mf_getattrn(&libds,NLOBS)=0 %then %do;
+  data &outds;
+    length hashkey $32;
+    hashkey=put(md5("&salt"),$hex32.);
+    output;
+    stop;
+  run;
+  %put &sysmacroname: Dataset &libds is empty, or is not a dataset;
+  %put &sysmacroname: hashkey of &outds is based on salt (&salt) only;
+%end;
+%else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
+  %put %str(ERR)OR: Dataset &libds is not a dataset;
+%end;
+%else %do;
+  data &outds(rename=(&keyvar=hashkey) keep=&keyvar)
+  %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+    /nonote2err
   %end;
-  %else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
-    %put %str(ERR)OR: Dataset &libds is not a dataset;
-  %end;
-  %else %do;
-    %local keyvar /* roll up the md5 */
-      prevkeyvar /* retain prev record md5 */
-      lastvar /* last var in input ds */
-      varlist var i;
-    /* avoid naming conflict for hash key vars */
-    %let keyvar=%mf_getuniquename();
-    %let prevkeyvar=%mf_getuniquename();
-    %let lastvar=%mf_getuniquename();
-    %let varlist=%mf_getvarlist(&libds);
-    data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
-      length &prevkeyvar &keyvar $32;
-      retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
-      set &libds end=&lastvar;
-      /* hash should include previous row */
-      &keyvar=put(md5(&prevkeyvar
-      /* loop every column, hashing every individual value */
-    %do i=1 %to %sysfunc(countw(&varlist));
-      %let var=%scan(&varlist,&i,%str( ));
-      %if %mf_getvartype(&libds,&var)=C %then %do;
-          !!put(md5(trim(&var)),$hex32.)
-      %end;
-      %else %do;
-          !!put(md5(trim(put(&var*1,binary64.))),$hex32.)
-      %end;
-    %end;
-      ),$hex32.);
-      &prevkeyvar=&keyvar;
-      if &lastvar then output;
-    run;
-  %end;
-%mend mp_hashdataset;
+  ;
+    length &prevkeyvar &keyvar $32;
+    retain &prevkeyvar;
+    if _n_=1 then &prevkeyvar=put(md5("&salt"),$hex32.);
+    set &libds end=&lastvar;
+    /* hash should include previous row */
+    &keyvar=%mp_md5(
+      cvars=%mf_getvarlist(&libds,typefilter=C) &prevkeyvar,
+      nvars=%mf_getvarlist(&libds,typefilter=N)
+    );
+    &prevkeyvar=&keyvar;
+    if &lastvar then output;
+  run;
+%end;
+%mend mp_hashdataset;

base/mp_init.sas

@@ -67,10 +67,10 @@ options
   validvarname=V7         /* avoid special characters etc in variable names */
   varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
   varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
-%if %substr(&sysver,1,1) ne 4 %then %do;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
   noautocorrect           /* disallow misspelled procedure names            */
   dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
 %end;
 ;
 
-%mend mp_init;
+%mend mp_init;

base/mp_jsonout.sas

@@ -70,7 +70,7 @@
 
 %if &action=OPEN %then %do;
   options nobomfile;
-  data _null_;file &jref encoding='utf-8' ;
+  data _null_;file &jref encoding='utf-8' lrecl=200;
     put '{"PROCESSED_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '"';
   run;
 %end;
@@ -199,12 +199,23 @@
       format _numeric_ bart.;
     %do i=1 %to &numcols;
       %if &&typelong&i=char or &fmt=Y %then %do;
-        &&name&i='"'!!trim(prxchange('s/"/\"/',-1,
-                    prxchange('s/'!!'0A'x!!'/\n/',-1,
-                    prxchange('s/'!!'0D'x!!'/\r/',-1,
-                    prxchange('s/'!!'09'x!!'/\t/',-1,
-                    prxchange('s/\\/\\\\/',-1,&&name&i)
-        )))))!!'"';
+        if findc(&&name&i,'"\'!!'0A0D09000E0F01021011'x) then do;
+          &&name&i='"'!!trim(
+            prxchange('s/"/\\"/',-1,        /* double quote */
+            prxchange('s/\x0A/\n/',-1,      /* new line */
+            prxchange('s/\x0D/\r/',-1,      /* carriage return */
+            prxchange('s/\x09/\\t/',-1,     /* tab */
+            prxchange('s/\x00/\\u0000/',-1, /* NUL */
+            prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+            prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+            prxchange('s/\x01/\\u0001/',-1, /* SOH */
+            prxchange('s/\x02/\\u0002/',-1, /* STX */
+            prxchange('s/\x10/\\u0010/',-1, /* DLE */
+            prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+            prxchange('s/\\/\\\\/',-1,&&name&i)
+          ))))))))))))!!'"';
+        end;
+        else &&name&i=quote(cats(&&name&i));
       %end;
     %end;
     run;
@@ -220,7 +231,7 @@
       %do i=1 %to &numcols;
         %if &i>1 %then  "," ;
         %if &action=OBJ %then """&&name&i"":" ;
-        &&name&i
+        "&&name&i"n /* name literal for reserved variable names */
       %end;
       %if &action=ARR %then "]" ; %else "}" ; ;
     /* now write the long strings to _webout 1 byte at a time */

base/mp_loadformat.sas

@@ -0,0 +1,303 @@
+/**
+  @file
+  @brief Loads a format catalog from a staging dataset
+  @details When loading staged data, it is common to receive only the records
+  that have actually changed.  However, when loading a format catalog, if
+  records are missing they are presumed to be no longer required.
+
+  This macro will augment a staging dataset with other records from the same
+  format, to prevent loss of data - UNLESS the input dataset contains a marker
+  column, specifying that a particular row needs to be deleted (`delete_col=`).
+
+  This macro can also be used to identify which records would be (or were)
+  considered new, modified or deleted (`loadtarget=`) by creating the following
+  tables:
+
+  @li work.outds_add
+  @li work.outds_del
+  @li work.outds_mod
+
+  For example usage, see mp_loadformat.test.sas
+
+  @param [in] libcat The format catalog to be loaded
+  @param [in] libds The staging table to load
+  @param [in] loadtarget= (NO) Set to YES to actually load the target catalog
+  @param [in] delete_col= (_____DELETE__THIS__RECORD_____) The column used to
+    mark a record for deletion.  Values should be "Yes" or "No".
+  @param [out] auditlibds= (0) For change tracking, set to the libds of an audit
+    table as defined in mddl_dc_difftable.sas
+  @param [in] locklibds= (0) For multi-user (parallel) situations, set to the
+    libds of the DC lock table as defined in the mddl_dc_locktable.sas macro.
+  @param [out] outds_add= (0) Set a libds here to see the new records added
+  @param [out] outds_del= (0) Set a libds here to see the records deleted
+  @param [out] outds_mod= (0) Set a libds here to see the modified records
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_cntlout.sas
+  @li mp_lockanytable.sas
+  @li mp_storediffs.sas
+
+  <h4> Related Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mddl_dc_locktable.sas
+  @li mp_loadformat.test.sas
+  @li mp_lockanytable.sas
+  @li mp_stackdiffs.sas
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_loadformat(libcat,libds
+  ,loadtarget=NO
+  ,auditlibds=0
+  ,locklibds=0
+  ,delete_col=_____DELETE__THIS__RECORD_____
+  ,outds_add=0
+  ,outds_del=0
+  ,outds_mod=0
+  ,mdebug=0
+);
+/* set up local macro variables and temporary tables (with a prefix) */
+%local err msg prefix dslist i var fmtlist ibufsize;
+%let dslist=base_fmts template inlibds ds1 stagedata storediffs;
+%if &outds_add=0 %then %let dslist=&dslist outds_add;
+%if &outds_del=0 %then %let dslist=&dslist outds_del;
+%if &outds_mod=0 %then %let dslist=&dslist outds_mod;
+%let prefix=%substr(%mf_getuniquename(),1,21);
+%do i=1 %to %sysfunc(countw(&dslist));
+  %let var=%scan(&dslist,&i);
+  %local &var;
+  %let &var=%upcase(&prefix._&var);
+%end;
+
+/*
+format values can be up to 32767 wide.  SQL joins on such a wide column can
+cause buffer issues.  Update ibufsize and reset at the end.
+*/
+%let ibufsize=%sysfunc(getoption(ibufsize));
+options ibufsize=32767 ;
+
+/* in DC, format catalogs maybe specified in the libds with a -FC extension */
+%let libcat=%scan(&libcat,1,-);
+
+/* perform input validations */
+%let err=0;
+%let msg=0;
+data _null_;
+  if _n_=1 then putlog "&sysmacroname entry vars:";
+  set sashelp.vmacro;
+  where scope="&sysmacroname";
+  value=upcase(value);
+  if &mdebug=0 then put name '=' value;
+  if name=:'LOAD' and value not in ('YES','NO') then do;
+    call symputx('msg',"invalid value for "!!name!!":"!!value);
+    call symputx('err',1);
+    stop;
+  end;
+  else if name='LIBCAT' then do;
+    if exist(value,'CATALOG') le 0 then do;
+      call symputx('msg',"Unable to open catalog: "!!value);
+      call symputx('err',1);
+      stop;
+    end;
+  end;
+  else if name='LIBDS' then do;
+    if exist(value) le 0 then do;
+      call symputx('msg',"Unable to open staging table: "!!value);
+      call symputx('err',1);
+      stop;
+    end;
+  end;
+  else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
+  and missing(value) then do;
+    call symputx('msg',"missing value in var: "!!name);
+    call symputx('err',1);
+    stop;
+  end;
+run;
+
+%mp_abort(
+  iftrue=(&err ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(&msg)
+)
+
+/**
+  * First, extract only relevant formats from the catalog
+  */
+proc sql noprint;
+select distinct upcase(fmtname) into: fmtlist separated by ' ' from &libds;
+
+%mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
+
+
+/**
+  * Ensure input table and base_formats have consistent lengths and types
+  */
+%mddl_sas_cntlout(libds=&template)
+data &inlibds;
+  length &delete_col $3;
+  if 0 then set &template;
+  set &libds;
+  if &delete_col='' then &delete_col='No';
+  fmtname=upcase(fmtname);
+  if missing(type) then do;
+    if substr(fmtname,1,1)='$' then type='C';
+    else type='N';
+  end;
+  if type='N' then do;
+    start=cats(start);
+    end=cats(end);
+  end;
+run;
+
+/**
+  * Identify new records
+  */
+proc sql;
+create table &outds_add(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  left join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where b.fmtname is null
+    and upcase(a.&delete_col) ne "YES"
+  order by fmtname, start;;
+
+/**
+  * Identify deleted records
+  */
+create table &outds_del(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where upcase(a.&delete_col)="YES"
+  order by fmtname, start;
+
+/**
+  * Identify modified records
+  */
+create table &outds_mod (drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where upcase(a.&delete_col) ne "YES"
+  order by fmtname, start;
+
+options ibufsize=&ibufsize;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc prior to load prep)
+)
+
+%if &loadtarget=YES %then %do;
+  data &ds1;
+    merge &base_fmts(in=base)
+      &outds_mod(in=mod)
+      &outds_add(in=add)
+      &outds_del(in=del);
+    if not del and not mod;
+    by fmtname start;
+  run;
+  data &stagedata;
+    set &ds1 &outds_mod;
+  run;
+  proc sort;
+    by fmtname start;
+  run;
+%end;
+/* mp abort needs to run outside of conditional blocks */
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc prior to actual load)
+)
+%if &loadtarget=YES %then %do;
+  %if %mf_nobs(&stagedata)=0 %then %do;
+    %put There are no changes to load in &libcat!;
+    %return;
+  %end;
+  %if &locklibds ne 0 %then %do;
+    /* prevent parallel updates */
+    %mp_lockanytable(LOCK
+      ,lib=%scan(&libcat,1,.)
+      ,ds=%scan(&libcat,2,.)-FC
+      ,ref=MP_LOADFORMAT commencing format load
+      ,ctl_ds=&locklibds
+    )
+  %end;
+  /* do the actual load */
+  proc format lib=&libcat cntlin=&stagedata;
+  run;
+  %if &locklibds ne 0 %then %do;
+    /* unlock the table */
+    %mp_lockanytable(UNLOCK
+      ,lib=%scan(&libcat,1,.)
+      ,ds=%scan(&libcat,2,.)-FC
+      ,ref=MP_LOADFORMAT completed format load
+      ,ctl_ds=&locklibds
+    )
+  %end;
+  /* track the changes */
+  %if &auditlibds ne 0 %then %do;
+    %if &locklibds ne 0 %then %do;
+      %mp_lockanytable(LOCK
+        ,lib=%scan(&auditlibds,1,.)
+        ,ds=%scan(&auditlibds,2,.)
+        ,ref=MP_LOADFORMAT commencing audit table load
+        ,ctl_ds=&locklibds
+      )
+    %end;
+
+    %mp_storediffs(&libcat-FC
+      ,&base_fmts
+      ,FMTNAME START
+      ,delds=&outds_del
+      ,modds=&outds_mod
+      ,appds=&outds_add
+      ,outds=&storediffs
+      ,mdebug=&mdebug
+    )
+
+    proc append base=&auditlibds data=&storediffs;
+    run;
+
+    %if &locklibds ne 0 %then %do;
+      %mp_lockanytable(UNLOCK
+        ,lib=%scan(&auditlibds,1,.)
+        ,ds=%scan(&auditlibds,2,.)
+        ,ref=MP_LOADFORMAT commencing audit table load
+        ,ctl_ds=&locklibds
+      )
+    %end;
+  %end;
+%end;
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc after load)
+)
+
+%if &mdebug=0 %then %do;
+  proc datasets lib=work;
+    delete &prefix:;
+  run;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%mend mp_loadformat;

base/mp_lockanytable.sas

@@ -1,7 +1,8 @@
 /**
   @file
   @brief Mechanism for locking tables to prevent parallel modifications
-  @details Uses a control table to enable ANY table to be locked for updates.
+  @details Uses a control table to enable ANY table to be locked for updates
+  (not just SAS datasets).
   Only useful if every update uses the macro!   Used heavily within
   [Data Controller for SAS](https://datacontroller.io).
 
@@ -15,12 +16,13 @@
     length is 200 characters.
   @param [out] ctl_ds= (0) The control table which controls the actual locking.
     Should already be assigned and available.  The definition is available by
-    running mp_coretable.sas as follows:  `%mp_coretable(LOCKTABLE)`.
+    running the mddl_dc_locktable.sas macro.
 
   @param [in] loops= (25) Number of times to check for a lock.
   @param [in] loop_secs= (1) Seconds to wait between each lock attempt
 
   <h4> SAS Macros </h4>
+  @li mf_fmtdttm.sas
   @li mp_abort.sas
   @li mp_lockfilecheck.sas
   @li mf_getuser.sas
@@ -48,7 +50,7 @@ data _null_;
   put name '=' value;
 run;
 
-%mp_abort(iftrue= (&ds=0 and &action ne MAKETABLE)
+%mp_abort(iftrue= ("&ds"="0" and &action ne MAKETABLE)
   ,mac=&sysmacroname
   ,msg=%str(dataset was not provided)
 )
@@ -110,7 +112,7 @@ run;
       LOCK_LIB ="&lib";
       LOCK_DS="&ds";
       LOCK_STATUS_CD='LOCKED';
-      LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt;
+      LOCK_START_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt;
       LOCK_USER_NM="&user";
       LOCK_PID="&sysjobid";
       LOCK_REF="&ref";
@@ -130,7 +132,7 @@ run;
     proc sql;
     update &ctl_ds
       set LOCK_STATUS_CD='LOCKED'
-        , LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_START_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
         , LOCK_USER_NM="&user"
         , LOCK_PID="&sysjobid"
         , LOCK_REF="&ref"
@@ -205,7 +207,7 @@ run;
     proc sql;
     update &ctl_ds
       set LOCK_STATUS_CD='UNLOCKED'
-        , LOCK_END_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_END_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
         , LOCK_USER_NM="&user"
         , LOCK_PID="&sysjobid"
         , LOCK_REF="&ref"

base/mp_lockfilecheck.sas

@@ -37,7 +37,7 @@ run;
   ,mac=checklock.sas
   ,msg=Aborting with syscc=&syscc on entry.
 )
-%mp_abort(iftrue= (&libds=0)
+%mp_abort(iftrue= ("&libds"="0")
   ,mac=&sysmacroname
   ,msg=%str(libds not provided)
 )
@@ -46,6 +46,12 @@ run;
 %let lib=%upcase(%scan(&libds,1,.));
 %let ds=%upcase(%scan(&libds,2,.));
 
+/* in DC, format catalogs are passed with a -FC suffix. No saslock here! */
+%if %scan(&libds,2,-)=FC %then %do;
+  %put &sysmacroname: Format Catalog detected, no lockfile applied to &libds;
+  %return;
+%end;
+
 /* do not proceed if no observations can be processed */
 %let msg=options obs = 0. syserrortext=%superq(syserrortext);
 %mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)

base/mp_md5.sas

@@ -0,0 +1,58 @@
+/**
+  @file
+  @brief Generates an md5 expression for hashing a set of variables
+  @details This is the same algorithm used to hash records in
+  [Data Controller for SAS](https://datacontroller.io) (free for up
+  to 5 users).
+
+  It is not designed to be efficient - it is designed to be effective,
+  given the range of edge cases (large floating points, special missing
+  numerics, thousands of columns, very wide columns).
+
+  It can be used only in data step, eg as follows:
+
+      data _null_;
+        set sashelp.class;
+        hashvar=%mp_md5(cvars=name sex, nvars=age height weight);
+        put hashvar=;
+      run;
+
+  Unfortunately it will not run in SQL - it fails with the following message:
+
+  > The width value for HEX is out of bounds. It should be between 1 and 16
+
+  The macro will also cause errors if the data contains (non-special) missings
+  and the (undocumented) `options dsoptions=nonote2err;` is in effect.
+
+  This can be avoided in two ways:
+
+  @li Global option:  `options dsoptions=nonote2err;`
+  @li Data step option: `data YOURLIB.YOURDATASET /nonote2err;`
+
+  @param cvars= Space seperated list of character variables
+  @param nvars= Space seperated list of numeric variables
+
+  <h4> Related Programs </h4>
+  @li mp_init.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_md5(cvars=,nvars=);
+%local i var sep;
+put(md5(
+  %do i=1 %to %sysfunc(countw(&cvars));
+    %let var=%scan(&cvars,&i,%str( ));
+    &sep put(md5(trim(&var)),$hex32.)
+    %let sep=!!;
+  %end;
+  %do i=1 %to %sysfunc(countw(&nvars));
+    %let var=%scan(&nvars,&i,%str( ));
+    /* multiply by 1 to strip precision errors (eg 0 != 0) */
+    /* but ONLY if not missing, else will lose any special missing values */
+    &sep put(md5(trim(put(ifn(missing(&var),&var,&var*1),binary64.))),$hex32.)
+    %let sep=!!;
+  %end;
+),$hex32.)
+%mend mp_md5;

base/mp_replace.sas

@@ -0,0 +1,151 @@
+/**
+  @file
+  @brief Performs a text substitution on a file
+  @details Performs a find and replace on a file, either in place or to a new
+  file. Can be used on files where lines are longer than 32767.
+
+  Works by reading in the file byte by byte, then marking the beginning and end
+  of each matched string, before finally doing the replace.
+
+  Full credit for this highly efficient and syntactically satisfying SAS logic
+  goes to [Bartosz Jabłoński](https://www.linkedin.com/in/yabwon), founder of
+  the [SAS Packages](https://github.com/yabwon/SAS_PACKAGES) framework.
+
+  Usage:
+
+      %let file="%sysfunc(pathname(work))/file.txt";
+      %let str=replace/me;
+      %let rep=with/this;
+      data _null_;
+        file &file;
+        put 'blahblah';
+        put "blahblah&str.blah";
+        put 'blahblahblah';
+      run;
+      %mp_replace(&file, findvar=str, replacevar=rep)
+      data _null_;
+        infile &file;
+        input;
+        list;
+      run;
+
+  Note - if you are running a version of SAS that will allow the io package in
+  LUA, you can also use this macro: mp_gsubfile.sas
+
+  @param infile The QUOTED path to the file on which to perform the substitution
+  @param findvar= Macro variable NAME containing the string to search for
+  @param replacevar= Macro variable NAME containing the replacement string
+  @param outfile= (0) Optional QUOTED path to the adjusted output file (to
+    avoid overwriting the first file).
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mp_chop.sas
+  @li mp_gsubfile.sas
+  @li mp_replace.test.sas
+
+  @version 9.4
+  @author Bartosz Jabłoński
+  @author Allan Bowe
+**/
+
+%macro mp_replace(infile,
+  findvar=,
+  replacevar=,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+%local inref dttm ds1;
+%let inref=%mf_getuniquefileref();
+%let outref=%mf_getuniquefileref();
+%if &outfile=0 %then %let outfile=&infile;
+%let ds1=%mf_getuniquename(prefix=allchars);
+%let ds2=%mf_getuniquename(prefix=startmark);
+
+/* START */
+%let dttm=%sysfunc(datetime());
+
+filename &inref &infile lrecl=1 recfm=n;
+
+data &ds1;
+  infile &inref;
+  input sourcechar $char1. @@;
+  format sourcechar hex2.;
+run;
+
+data &ds2;
+  /* set find string to length in bytes to cover trailing spaces */
+  length string $ %length(%superq(&findvar));
+  string =symget("&findvar");
+  drop string;
+
+  firstchar=char(string,1);
+  findlen=lengthm(string); /* <- for trailing bytes */
+
+  do _N_=1 to nobs;
+    set &ds1 nobs=nobs point=_N_;
+    if sourcechar=firstchar then do;
+      pos=1;
+      s=0;
+      do point=_N_ to min(_N_ + findlen -1,nobs);
+        set &ds1 point=point;
+        if sourcechar=char(string, pos) then s + 1;
+        else goto _leave_;
+        pos+1;
+      end;
+      _leave_:
+      if s=findlen then do;
+        START =_N_;
+        _N_ =_N_+ s - 1;
+        STOP =_N_;
+        output;
+      end;
+    end;
+  end;
+  stop;
+  keep START STOP;
+run;
+
+data &ds1;
+  declare hash HS(dataset:"&ds2(keep=start)");
+  HS.defineKey("start");
+  HS.defineDone();
+  declare hash HE(dataset:"&ds2(keep=stop)");
+  HE.defineKey("stop");
+  HE.defineDone();
+  do until(eof);
+    set &ds1 end=eof curobs =n;
+    start = ^HS.check(key:n);
+    stop  = ^HE.check(key:n);
+    length strt $ 1;
+    strt =put(start,best. -L);
+    retain out 1;
+    if out   then output;
+    if start then out=0;
+    if stop  then out=1;
+  end;
+  stop;
+  keep sourcechar strt;
+run;
+
+filename &outref &outfile recfm=n;
+
+data _null_;
+  length replace $ %length(%superq(&replacevar));
+  replace=symget("&replacevar");
+  file &outref;
+  do until(eof);
+    set &ds1 end=eof;
+    if strt ="1" then put replace char.;
+    else put sourcechar char1.;
+  end;
+  stop;
+run;
+
+/* END */
+%put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) seconds to run;
+
+%mend mp_replace;

base/mp_resetoption.sas

@@ -21,15 +21,19 @@ https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-s
 %macro mp_resetoption(option /* the option to reset */
 )/*/STORE SOURCE*/;
 
-data _null_;
-  length code  $1500;
-  startup=getoption("&option",'startupvalue');
-  current=getoption("&option");
-  if startup ne current then do;
-    code =cat('OPTIONS ',getoption("&option",'keyword','startupvalue'),';');
-    putlog "NOTE: Resetting system option: " code ;
-    call execute(code );
-  end;
-run;
-
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  data _null_;
+    length code  $1500;
+    startup=getoption("&option",'startupvalue');
+    current=getoption("&option");
+    if startup ne current then do;
+      code =cat('OPTIONS ',getoption("&option",'keyword','startupvalue'),';');
+      putlog "NOTE: Resetting system option: " code ;
+      call execute(code );
+    end;
+  run;
+%end;
+%else %do;
+  %put &sysmacroname: reset option feature unavailable on &sysvlong;
+%end;
 %mend mp_resetoption;

base/mp_retainedkey.sas

@@ -58,6 +58,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_existvar.sas
+  @li mf_fmtdttm.sas
   @li mf_getquotedstr.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
@@ -217,12 +218,12 @@ quit;
     set keytable="&base_libds"
       ,keycolumn="&retained_key"
       ,max_key=%eval(&maxkey+&newkey_cnt)
-      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt;
+      ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt;
   %end;
   %else %do;
   update &maxkeytable
     set max_key=%eval(&maxkey+&newkey_cnt)
-      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt
+      ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt
     where keytable="&base_libds";
   %end;
   %mp_lockanytable(UNLOCK

base/mp_storediffs.sas

@@ -90,7 +90,7 @@
 %else %let dbg=*;
 
 /* set up unique and temporary vars */
-%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist;
+%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist vlist;
 %let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_ds1));
 %let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_ds2));
 %let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_ds3));
@@ -125,7 +125,7 @@
 data &ds1;
   set &dslist indsname=&inds_auto;
   &hashkey=put(md5(catx('|',%mf_getquotedstr(&key,quote=N))),$hex32.);
-  &inds_keep=&inds_auto;
+  &inds_keep=upcase(&inds_auto);
 proc sort;
   by &inds_keep &hashkey;
 run;
@@ -144,15 +144,24 @@ proc transpose data=&ds1
   by &inds_keep &hashkey;
   var _character_;
 run;
+
+%if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do;
+  /* this is a format catalog - cannot query cols directly */
+  %let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH"
+    ,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO"
+    ,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
+%end;
+%else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
+
 data &ds4;
   length &inds_keep $41 tgtvar_nm $32;
   set &ds2 &ds3 indsname=&inds_auto;
 
   tgtvar_nm=upcase(tgtvar_nm);
-  if tgtvar_nm in (%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE)));
+  if tgtvar_nm in (%upcase(&vlist));
 
-  if &inds_auto="&ds2" then tgtvar_type='N';
-  else if &inds_auto="&ds3" then tgtvar_type='C';
+  if upcase(&inds_auto)="&ds2" then tgtvar_type='N';
+  else if upcase(&inds_auto)="&ds3" then tgtvar_type='C';
   else do;
     putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
     call symputx('syscc',98);

base/mp_streamfile.sas

@@ -1,8 +1,9 @@
 /**
   @file
   @brief Streams a file to _webout according to content type
-  @details Will set headers using appropriate functions (SAS 9 vs Viya) and send
-  content as a binary stream.
+  @details Will set headers using appropriate functions per the server type
+  (Viya, EBI, [SASjs Server](https://github.com/sasjs/server)) and stream
+  content using mp_binarycopy().
 
   Usage:
 
@@ -12,7 +13,14 @@
 
       %mp_streamfile(contenttype=csv,inloc=/some/where.txt,outname=myfile.txt)
 
-  @param [in] contenttype= (TEXTS) Either TEXT, ZIP, CSV, EXCEL
+  @param [in] contenttype= (TEXT) Supported:
+    @li CSV
+    @li EXCEL
+    @li MARKDOWN
+    @li TEXT
+    @li ZIP
+    Feel free to submit PRs to support more mime types!  The official list is
+    here:  https://www.iana.org/assignments/media-types/media-types.xhtml
   @param [in] inloc= /path/to/file.ext to be sent
   @param [in] inref= fileref of file to be sent (if provided, overrides `inloc`)
   @param [in] iftrue= (1=1) Provide a condition under which to execute.
@@ -22,6 +30,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
+  @li mfs_httpheader.sas
   @li mp_binarycopy.sas
 
   @author Allan Bowe
@@ -55,9 +64,9 @@ data _null_;
 run;
 
 %if &contentype=CSV %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/csv');
+      rc=stpsrv_header('Content-Type','application/csv');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
@@ -66,12 +75,16 @@ run;
       contenttype='application/csv'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-Type,application/csv)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
 %else %if &contentype=EXCEL %then %do;
   /* suitable for XLS format */
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/vnd.ms-excel');
+      rc=stpsrv_header('Content-Type','application/vnd.ms-excel');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
@@ -80,28 +93,46 @@ run;
       contenttype='application/vnd.ms-excel'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-Type,application/vnd.ms-excel)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
 %else %if &contentype=GIF or &contentype=JPEG or &contentype=PNG %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type',"image/%lowcase(&contenttype)");
+      rc=stpsrv_header('Content-Type',"image/%lowcase(&contenttype)");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
     filename &outref filesrvc parenturi="&SYS_JES_JOB_URI"
       contenttype="image/%lowcase(&contenttype)";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-Type,image/%lowcase(&contenttype))
+  %end;
 %end;
-%else %if &contentype=HTML %then %do;
-  %if &platform=SASVIYA %then %do;
+%else %if &contentype=HTML or &contenttype=MARKDOWN %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-Type',"text/%lowcase(&contenttype)");
+      rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
     filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
-      contenttype="text/html";
+      contenttype="text/%lowcase(&contenttype)"
+      contentdisp="attachment; filename=&outname";
+  %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-Type,text/%lowcase(&contenttype))
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
   %end;
 %end;
 %else %if &contentype=TEXT %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/text');
+      rc=stpsrv_header('Content-Type','application/text');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
@@ -110,22 +141,29 @@ run;
       contenttype='application/text'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-Type,application/text)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
 %else %if &contentype=WOFF or &contentype=WOFF2 or &contentype=TTF %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type',"font/%lowcase(&contenttype)");
+      rc=stpsrv_header('Content-Type',"font/%lowcase(&contenttype)");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
     filename &outref filesrvc parenturi="&SYS_JES_JOB_URI"
       contenttype="font/%lowcase(&contenttype)";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-Type,font/%lowcase(&contenttype))
+  %end;
 %end;
 %else %if &contentype=XLSX %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type',
+      rc=stpsrv_header('Content-Type',
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
@@ -136,11 +174,17 @@ run;
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-Type
+      ,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+    )
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
 %else %if &contentype=ZIP %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/zip');
+      rc=stpsrv_header('Content-Type','application/zip');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
@@ -149,6 +193,10 @@ run;
       contenttype='application/zip'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-Type,application/zip)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
 %else %do;
   %put %str(ERR)OR: Content Type &contenttype NOT SUPPORTED by &sysmacroname!;

base/mp_testservice.sas

@@ -1,51 +1,11 @@
 /**
   @file
-  @brief Will execute a SASjs web service on SAS 9 or Viya
-  @details Prepares the input files and retrieves the resulting datasets from
-  the response JSON.
-
-
-
-  Note - the _webout fileref should NOT be assigned prior to running this macro.
-
-  @param [in] program The _PROGRAM endpoint to test
-  @param [in] inputfiles=(0) A list of space seperated fileref:filename pairs as
-    follows:
-        inputfiles=inref:filename inref2:filename2
-  @param [in] inputdatasets= (0) All datasets in this space seperated list are
-    converted into SASJS-formatted CSVs (see mp_ds2csv.sas) files and added to
-    the list of `inputfiles` for ingestion.  The dataset will be sent with the
-    same name (no need for a colon modifier).
-  @param [in] inputparams=(0) A dataset containing name/value pairs in the
-    following format:
-    |name:$32|value:$1000|
-    |---|---|
-    |stpmacname|some value|
-    |mustbevalidname|can be anything, oops, %abort!!|
-
-  @param [in] debug= (log) Provide the _debug value
-  @param [in] mdebug= (0) Set to 1 to provide macro debugging
-  @param [in] viyaresult= (WEBOUT_JSON) The Viya result type to return.  For
-    more info, see mv_getjobresult.sas
-  @param [in] viyacontext= (SAS Job Execution compute context) The Viya compute
-    context on which to run the service
-  @param [out] outlib= (0) Output libref to contain the final tables.  Set to
-    0 if the service output is not in JSON format.
-  @param [out] outref= (0) Output fileref to create, to contain the full _webout
-    response.
+  @brief To be deprecated.  Will execute a SASjs web service on SAS 9 or Viya
+  @details Use the mx_testservice.sas macro instead (documentation can be
+  found there)
 
   <h4> SAS Macros </h4>
-  @li mf_getplatform.sas
-  @li mf_getuniquefileref.sas
-  @li mf_getuniquename.sas
-  @li mp_abort.sas
-  @li mp_binarycopy.sas
-  @li mp_ds2csv.sas
-  @li mv_getjobresult.sas
-  @li mv_jobflow.sas
-
-  <h4> Related Programs </h4>
-  @li mp_testservice.test.sas
+  @li mx_testservice.sas
 
   @version 9.4
   @author Allan Bowe
@@ -63,216 +23,17 @@
   viyaresult=WEBOUT_JSON,
   viyacontext=SAS Job Execution compute context
 )/*/STORE SOURCE*/;
-%local dbg pcnt fref1 webref i webcount var platform;
-%if &mdebug=1 %then %do;
-  %put &sysmacroname entry vars:;
-  %put _local_;
-%end;
-%else %let dbg=*;
 
-/* sanitise inputparams */
-%let pcnt=0;
-%if &inputparams ne 0 %then %do;
-  data _null_;
-    set &inputparams;
-    if not nvalid(name,'v7') then putlog (_all_)(=);
-    else if name in (
-      'program','inputfiles','inputparams','debug','outlib','outref'
-    ) then putlog (_all_)(=);
-    else do;
-      x+1;
-      call symputx(name,quote(cats(value)),'l');
-      call symputx(cats('pval',x),name,'l');
-      call symputx('pcnt',x,'l');
-    end;
-  run;
-  %mp_abort(iftrue= (%mf_nobs(&inputparams) ne &pcnt)
-    ,mac=&sysmacroname
-    ,msg=%str(Invalid values in &inputparams)
-  )
-%end;
+%mx_testservice(&program,
+  inputfiles=&inputfiles,
+  inputdatasets=&inputdatasets,
+  inputparams=&inputparams,
+  debug=&debug,
+  mdebug=&mdebug,
+  outlib=&outlib,
+  outref=&outref,
+  viyaresult=&viyaresult,
+  viyacontext=&viyacontext
+)
 
-/* convert inputdatasets to filerefs */
-%if "&inputdatasets" ne "0" %then %do;
-  %if %quote(&inputfiles)=0 %then %let inputfiles=;
-  %do i=1 %to %sysfunc(countw(&inputdatasets,%str( )));
-    %let var=%scan(&inputdatasets,&i,%str( ));
-    %local dsref&i;
-    %let dsref&i=%mf_getuniquefileref();
-    %mp_ds2csv(&var,outref=&&dsref&i,headerformat=SASJS)
-    %let inputfiles=&inputfiles &&dsref&i:%scan(&var,-1,.);
-  %end;
-%end;
-
-
-%let fref1=%mf_getuniquefileref();
-%let webref=%mf_getuniquefileref();
-%let platform=%mf_getplatform();
-%if &platform=SASMETA %then %do;
-
-  /* parse the input files */
-  %if %quote(&inputfiles) ne 0 %then %do;
-    %let webcount=%sysfunc(countw(&inputfiles));
-    %put &=webcount;
-    %do i=1 %to &webcount;
-      %let var=%scan(&inputfiles,&i,%str( ));
-      %local webfref&i webname&i;
-      %let webref&i=%scan(&var,1,%str(:));
-      %let webname&i=%scan(&var,2,%str(:));
-      %put webref&i=&&webref&i;
-      %put webname&i=&&webname&i;
-    %end;
-  %end;
-  %else %let webcount=0;
-
-  proc stp program="&program";
-    inputparam _program="&program"
-  %do i=1 %to &webcount;
-    %if &webcount=1 %then %do;
-      _webin_fileref="&&webref&i"
-      _webin_name="&&webname&i"
-    %end;
-    %else %do;
-      _webin_fileref&i="&&webref&i"
-      _webin_name&i="&&webname&i"
-    %end;
-  %end;
-    _webin_file_count="&webcount"
-    _debug="&debug"
-  %do i=1 %to &pcnt;
-    /* resolve name only, proc stp fetches value */
-    &&pval&i=&&&&&&pval&i
-  %end;
-    ;
-  %do i=1 %to &webcount;
-    inputfile &&webref&i;
-  %end;
-    outputfile _webout=&webref;
-  run;
-
-  data _null_;
-    infile &webref;
-    file &fref1;
-    input;
-    length line $10000;
-    if index(_infile_,'>>weboutBEGIN<<') then do;
-        line=tranwrd(_infile_,'>>weboutBEGIN<<','');
-        put line;
-    end;
-    else if index(_infile_,'>>weboutEND<<') then do;
-        line=tranwrd(_infile_,'>>weboutEND<<','');
-        put line;
-        stop;
-    end;
-    else put _infile_;
-  run;
-  data _null_;
-    infile &fref1;
-    input;
-    put _infile_;
-  run;
-  %if &outlib ne 0 %then %do;
-    libname &outlib json (&fref1);
-  %end;
-  %if &outref ne 0 %then %do;
-    filename &outref temp;
-    %mp_binarycopy(inref=&webref,outref=&outref)
-  %end;
-
-%end;
-%else %if &platform=SASVIYA %then %do;
-
-  /* prepare inputparams */
-  %local ds1;
-  %let ds1=%mf_getuniquename();
-  %if "&inputparams" ne "0" %then %do;
-    proc transpose data=&inputparams out=&ds1;
-      id name;
-      var value;
-    run;
-  %end;
-  %else %do;
-    data &ds1;run;
-  %end;
-
-  /* parse the input files - convert to sasjs params */
-  %local webcount i var sasjs_tables;
-  %if %quote(&inputfiles) ne 0 %then %do;
-    %let webcount=%sysfunc(countw(&inputfiles));
-    %put &=webcount;
-    %do i=1 %to &webcount;
-      %let var=%scan(&inputfiles,&i,%str( ));
-      %local webfref&i webname&i sasjs&i.data;
-      %let webref&i=%scan(&var,1,%str(:));
-      %let webname&i=%scan(&var,2,%str(:));
-      %put webref&i=&&webref&i;
-      %put webname&i=&&webname&i;
-
-      %let sasjs_tables=&sasjs_tables &&webname&i;
-      data _null_;
-        infile &&webref&i lrecl=32767;
-        input;
-        if _n_=1 then call symputx("sasjs&i.data",_infile_);
-        else call symputx(
-          "sasjs&i.data",cats(symget("sasjs&i.data"),'0D0A'x,_infile_)
-        );
-        putlog "&sysmacroname infile: " _infile_;
-      run;
-      data &ds1;
-        set &ds1;
-        length sasjs&i.data $32767 sasjs_tables $1000;
-        sasjs&i.data=symget("sasjs&i.data");
-        sasjs_tables=symget("sasjs_tables");
-      run;
-    %end;
-  %end;
-  %else %let webcount=0;
-
-  data &ds1;
-    retain _program "&program";
-    retain _contextname "&viyacontext";
-    set &ds1;
-    putlog "&sysmacroname inputparams:";
-    putlog (_all_)(=);
-  run;
-
-  %mv_jobflow(inds=&ds1
-    ,maxconcurrency=1
-    ,outds=work.results
-    ,outref=&fref1
-    ,mdebug=&mdebug
-  )
-  /* show the log */
-  data _null_;
-    infile &fref1;
-    input;
-    putlog _infile_;
-  run;
-  /* get the uri to fetch results */
-  data _null_;
-    set work.results;
-    call symputx('uri',uri);
-    putlog "&sysmacroname: fetching results for " uri;
-  run;
-  /* fetch results from webout.json */
-  %mv_getjobresult(uri=&uri,
-    result=&viyaresult,
-    outref=&outref,
-    outlib=&outlib,
-    mdebug=&mdebug
-  )
-
-%end;
-%else %do;
-  %put %str(ERR)OR: Unrecognised platform:  &platform;
-%end;
-
-%if &mdebug=0 %then %do;
-  filename &webref clear;
-%end;
-%else %do;
-  %put &sysmacroname exit vars:;
-  %put _local_;
-%end;
-
-%mend mp_testservice;
+%mend mp_testservice;

base/mp_wait4file.sas

@@ -4,9 +4,7 @@
   @details Loops with a `sleep()` command until a file arrives or the max wait
   period expires.
 
-  @example
-
-  Wait 3 minutes OR for /tmp/flag.txt to appear
+  Example: Wait 3 minutes OR for /tmp/flag.txt to appear
 
     %mp_wait4file(/tmp/flag.txt , maxwait=60*3)
 

build.py

@@ -4,8 +4,8 @@ from pathlib import Path
 # Prepare Lua Macros
 files = [f for f in Path('lua').iterdir() if f.match("*.lua")]
 for file in files:
-    basename=os.path.basename(file)
-    name='ml_' + os.path.splitext(basename)[0]
+    basename = os.path.basename(file)
+    name = 'ml_' + os.path.splitext(basename)[0]
     ml = open('lua/' + name + '.sas', "w")
     ml.write("/**\n")
     ml.write("  @file " + name + '.sas\n')
@@ -20,50 +20,68 @@ for file in files:
     ml.write("  file \"%sysfunc(pathname(work))/" + name + ".lua\";\n")
     with open(file) as infile:
         for line in infile:
-          ml.write("  put '" + line.rstrip().replace("'","''") + " ';\n")
+            ml.write("  put '" + line.rstrip().replace("'", "''") + " ';\n")
     ml.write("run;\n\n")
-    ml.write("%inc \"%sysfunc(pathname(work))/" + name + ".lua\" /source2;\n\n")
+
+    ml.write("/* ensure big enough lrecl to avoid lua compilation issues */\n")
+    ml.write("%local optval;\n")
+    ml.write("%let optval=%sysfunc(getoption(lrecl));\n")
+    ml.write("options lrecl=1024;\n\n")
+    ml.write("/* execute the lua code by using a .lua extension */\n")
+    ml.write("%inc \"%sysfunc(pathname(work))/" +
+             name + ".lua\" /source2;\n\n")
+    ml.write("options lrecl=&optval;\n\n")
     ml.write("%mend " + name + ";\n")
 
 ml.close()
 
 # prepare web files
-files=['viya/mv_createwebservice.sas','meta/mm_createwebservice.sas']
+files = ['viya/mv_createwebservice.sas',
+         'meta/mm_createwebservice.sas', 'server/ms_createwebservice.sas']
 for file in files:
-    webout0=open('base/mp_jsonout.sas','r')
-    if file=='viya/mv_createwebservice.sas':
-        webout1=open('viya/mv_webout.sas',"r")
+    webout0 = open('base/mp_jsonout.sas', 'r')
+    webout1 = open('base/mf_getuser.sas', 'r')
+
+    if file == 'viya/mv_createwebservice.sas':
+        webout2 = open('viya/mv_webout.sas', "r")
+        weboutfiles = [webout0, webout1, webout2]
+    elif file == 'server/ms_createwebservice.sas':
+        webout2 = open('server/ms_webout.sas', "r")
+        webout3 = open('server/mfs_httpheader.sas', 'r')
+        weboutfiles = [webout0, webout1, webout2, webout3]
     else:
-        webout1=open('meta/mm_webout.sas','r')
-    webout2=open('base/mf_getuser.sas','r')
-    outfile=open(file + 'TEMP','w')
-    infile=open(file,'r')
-    delrow=0
+        webout2 = open('meta/mm_webout.sas', 'r')
+        weboutfiles = [webout0, webout1, webout2]
+    outfile = open(file + 'TEMP', 'w')
+    infile = open(file, 'r')
+    delrow = 0
     for line in infile:
-        if line=='/* WEBOUT BEGIN */\n':
-            delrow=1
+        if line == '/* WEBOUT BEGIN */\n':
+            delrow = 1
             outfile.write('/* WEBOUT BEGIN */\n')
-            weboutfiles=[webout0,webout1,webout2]
             for weboutfile in weboutfiles:
-                stripcomment=1
+                stripcomment = 1
                 for w in weboutfile:
-                    if w=='**/\n': stripcomment=0
-                    elif stripcomment==0:
-                        outfile.write("  put '" + w.rstrip().replace("'","''") + " ';\n")
-        elif delrow==1 and line=='/* WEBOUT END */\n':
-                delrow=0
-                outfile.write('/* WEBOUT END */\n')
-        elif delrow==0:
+                    if w == '**/\n':
+                        stripcomment = 0
+                    elif stripcomment == 0:
+                        outfile.write(
+                            "  put '" + w.rstrip().replace("'", "''") + " ';\n")
+        elif delrow == 1 and line == '/* WEBOUT END */\n':
+            delrow = 0
+            outfile.write('/* WEBOUT END */\n')
+        elif delrow == 0:
             outfile.write(line.rstrip() + "\n")
     webout0.close()
     webout1.close()
+    webout2.close()
     outfile.close()
     infile.close()
     os.remove(file)
-    os.rename(file + 'TEMP',file)
+    os.rename(file + 'TEMP', file)
 
 # Concatenate all macros into a single file
-header="""
+header = """
 /**
   @file
   @brief Auto-generated file
@@ -84,14 +102,15 @@ options noquotelenmax;
 """
 f = open('all.sas', "w")             # r / r+ / rb / rb+ / w / wb
 f.write(header)
-folders=['base','ddl','meta','metax','server','viya','lua','fcmp']
+folders = ['base', 'ddl', 'meta', 'metax', 'server', 'viya', 'lua', 'fcmp', 'xplatform']
 for folder in folders:
-    filenames = [fn for fn in Path('./' + folder).iterdir() if fn.match("*.sas")]
+    filenames = [fn for fn in Path(
+        './' + folder).iterdir() if fn.match("*.sas")]
     filenames.sort()
     with open('mc_' + folder + '.sas', 'w') as outfile:
         for fname in filenames:
             with open(fname) as infile:
                 outfile.write(infile.read())
-    with open('mc_' + folder + '.sas','r') as c:
+    with open('mc_' + folder + '.sas', 'r') as c:
         f.write(c.read())
 f.close()

ddl/mddl_dc_difftable.sas

@@ -26,9 +26,19 @@
       oldval_num num format=best32. label='Old (numeric) value',
       newval_num num format=best32. label='New (numeric) value',
       oldval_char char(32765) label='Old (character) value',
-      newval_char char(32765) label='New (character) value',
-    constraint pk_mpe_audit
-      primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
+      newval_char char(32765) label='New (character) value'
   );
 
+  %local lib;
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create
+      pk_mpe_audit=(load_ref libref dsn key_hash tgtvar_nm)
+      /nomiss unique;
+  quit;
+
 %mend mddl_dc_difftable;

ddl/mddl_dc_filterdetail.sas

@@ -9,19 +9,32 @@
 
 %macro mddl_dc_filterdetail(libds=WORK.FILTER_DETAIL);
 
+%local nn lib;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  %let nn=not null;
+%end;
+%else %let nn=;
+
   proc sql;
   create table &libds(
-      filter_hash char(32) not null,
-      filter_line num not null,
-      group_logic char(3) not null,
-      subgroup_logic char(3) not null,
-      subgroup_id num not null,
-      variable_nm varchar(32) not null,
-      operator_nm varchar(12) not null,
-      raw_value varchar(4000) not null,
-      processed_dttm num not null format=E8601DT26.6,
-    constraint pk_mpe_filteranytable
-      primary key(filter_hash,filter_line)
+      filter_hash char(32) &nn,
+      filter_line num &nn,
+      group_logic char(3) &nn,
+      subgroup_logic char(3) &nn,
+      subgroup_id num &nn,
+      variable_nm varchar(32) &nn,
+      operator_nm varchar(12) &nn,
+      raw_value varchar(4000) &nn,
+      processed_dttm num &nn format=E8601DT26.6
   );
 
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create pk_mpe_filterdetail=(filter_hash filter_line)/nomiss unique;
+  quit;
+
 %mend mddl_dc_filterdetail;

ddl/mddl_dc_filtersummary.sas

@@ -9,14 +9,27 @@
 
 %macro mddl_dc_filtersummary(libds=WORK.FILTER_SUMMARY);
 
+%local nn lib;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  %let nn=not null;
+%end;
+%else %let nn=;
+
   proc sql;
   create table &libds(
-      filter_rk num not null,
-      filter_hash char(32) not null,
-      filter_table char(41) not null,
-      processed_dttm num not null format=E8601DT26.6,
-    constraint pk_mpe_filteranytable
-      primary key(filter_rk)
+      filter_rk num &nn,
+      filter_hash char(32) &nn,
+      filter_table char(41) &nn,
+      processed_dttm num &nn format=E8601DT26.6
   );
 
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create filter_rk /nomiss unique;
+  quit;
+
 %mend mddl_dc_filtersummary;

ddl/mddl_dc_locktable.sas

@@ -9,17 +9,33 @@
 
 %macro mddl_dc_locktable(libds=WORK.LOCKTABLE);
 
+%local nn lib;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  %let nn=not null;
+%end;
+%else %let nn=;
+
   proc sql;
   create table &libds(
       lock_lib char(8),
       lock_ds char(32),
-      lock_status_cd char(10) not null,
-      lock_user_nm char(100) not null ,
+      lock_status_cd char(10) &nn,
+      lock_user_nm char(100) &nn ,
       lock_ref char(200),
       lock_pid char(10),
       lock_start_dttm num format=E8601DT26.6,
-      lock_end_dttm num format=E8601DT26.6,
-    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds)
+      lock_end_dttm num format=E8601DT26.6
   );
 
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create
+      pk_mp_lockanytable=(lock_lib lock_ds)
+      /nomiss unique;
+  quit;
+
 %mend mddl_dc_locktable;

ddl/mddl_dc_maxkeytable.sas

@@ -17,8 +17,17 @@
       max_key num label=
         'Integer representing current max RK or SK value in the KEYTABLE',
       processed_dttm num format=E8601DT26.6
-        label='Datetime this value was last updated',
-    constraint pk_mpe_maxkeyvalues
-        primary key(keytable));
+        label='Datetime this value was last updated'
+  );
+
+  %local lib;
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create keytable /nomiss unique;
+  quit;
 
 %mend mddl_dc_maxkeytable;

ddl/mddl_sas_cntlout.sas

@@ -12,27 +12,32 @@
 
 proc sql;
 create table &libds(
-    FMTNAME char(32)     label='Format name'
-    ,START char(16)     label='Starting value for format'
-    ,END char(16)     label='Ending value for format'
-    ,LABEL char(256)     label='Format value label'
+    FMTNAME char(32)      label='Format name'
+    /*
+      to accommodate larger START values, mp_loadformat.sas will need the
+      SQL dependency removed (proc sql needs to accommodate 3 index values in
+      a 32767 ibufsize limit)
+    */
+    ,START char(10000)    label='Starting value for format'
+    ,END char(32767)      label='Ending value for format'
+    ,LABEL char(32767)    label='Format value label'
     ,MIN num length=3     label='Minimum length'
     ,MAX num length=3     label='Maximum length'
-    ,DEFAULT num length=3     label='Default length'
-    ,LENGTH num length=3     label='Format length'
-    ,FUZZ num     label='Fuzz value'
-    ,PREFIX char(2)     label='Prefix characters'
-    ,MULT num     label='Multiplier'
-    ,FILL char(1)     label='Fill character'
-    ,NOEDIT num length=3     label='Is picture string noedit?'
-    ,TYPE char(1)     label='Type of format'
-    ,SEXCL char(1)     label='Start exclusion'
-    ,EEXCL char(1)     label='End exclusion'
-    ,HLO char(13)     label='Additional information'
-    ,DECSEP char(1)     label='Decimal separator'
-    ,DIG3SEP char(1)     label='Three-digit separator'
+    ,DEFAULT num length=3 label='Default length'
+    ,LENGTH num length=3  label='Format length'
+    ,FUZZ num             label='Fuzz value'
+    ,PREFIX char(2)       label='Prefix characters'
+    ,MULT num             label='Multiplier'
+    ,FILL char(1)         label='Fill character'
+    ,NOEDIT num length=3  label='Is picture string noedit?'
+    ,TYPE char(1)         label='Type of format'
+    ,SEXCL char(1)        label='Start exclusion'
+    ,EEXCL char(1)        label='End exclusion'
+    ,HLO char(13)         label='Additional information'
+    ,DECSEP char(1)       label='Decimal separator'
+    ,DIG3SEP char(1)      label='Three-digit separator'
     ,DATATYPE char(8)     label='Date/time/datetime?'
     ,LANGUAGE char(8)     label='Language for date strings'
 );
 
-%mend mddl_sas_cntlout;
+%mend mddl_sas_cntlout;

fcmp/mcf_stpsrv_header.sas

@@ -1,110 +0,0 @@
-/**
-  @file
-  @brief Provides a replacement for the stpsrv_header function
-  @details The stpsrv_header is normally a built-in function, used to set the
-  headers for SAS 9 Stored Processes as documented here:
-  https://go.documentation.sas.com/doc/en/itechcdc/9.4/stpug/srvhead.htm
-
-  The purpose of this custom function is to provide a replacement when running
-  similar code as a web service against
-  [sasjs/server](https://github.com/sasjs/server).  It operates by creating a
-  text file with the headers.  The location of this text file is determined by
-  a macro variable (`sasjs_stpsrv_header_loc`) which needs to be injected into
-  each service by the calling process, eg:
-
-      %let sasjs_stpsrv_header_loc = C:/temp/some_uuid/stpsrv_header.txt;
-
-  Note - the function works by appending headers to the file.  If multiple same-
-  named headers are provided, they will all be appended - the calling process
-  needs to pick up the last one.  This will mean removing the attribute if the
-  final record has an empty value.
-
-  The function takes the following (positional) parameters:
-
-  | PARAMETER | DESCRIPTION |
-  |------------|-------------|
-  | name $  | name of the header attribute to create|
-  | value  $  | value of the header attribute|
-
-  It returns 0 if successful, or -1 if an error occured.
-
-  Usage:
-
-      %let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/stpsrv_header.txt;
-
-      %mcf_stpsrv_header(wrap=YES, insert_cmplib=YES)
-
-      data _null_;
-        rc=stpsrv_header('Content-type','application/text');
-        rc=stpsrv_header('Content-disposition',"attachment; filename=file.txt");
-      run;
-
-      data _null_;
-        infile "&sasjs_stpsrv_header_loc";
-        input;
-        putlog _infile_;
-      run;
-
-
-  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
-  @param [out] lib= (work) The output library in which to create the catalog.
-  @param [out] cat= (sasjs) The output catalog in which to create the package.
-  @param [out] pkg= (utils) The output package in which to create the function.
-    Uses a 3 part format:  libref.catalog.package
-  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
-    values inserted only if needed.
-
-  <h4> SAS Macros </h4>
-  @li mcf_init.sas
-
-  <h4> Related Programs </h4>
-  @li mcf_stpsrv_header.test.sas
-  @li mp_init.sas
-
-**/
-
-%macro mcf_stpsrv_header(wrap=NO
-  ,insert_cmplib=DEPRECATED
-  ,lib=WORK
-  ,cat=SASJS
-  ,pkg=UTILS
-)/*/STORE SOURCE*/;
-%local i var cmpval found;
-%if %mcf_init(stpsrv_header)=1 %then %return;
-
-%if &wrap=YES  %then %do;
-  proc fcmp outlib=&lib..&cat..&pkg;
-%end;
-
-function stpsrv_header(name $, value $);
-  length loc $128 val $512;
-  loc=symget('sasjs_stpsrv_header_loc');
-  val=trim(name)!!': '!!value;
-  length fref $8;
-  rc=filename(fref,loc);
-  if (rc ne 0) then return( -1 );
-  fid = fopen(fref,'a');
-  if (fid = 0) then return( -1 );
-  rc=fput(fid, val);
-  rc=fwrite(fid);
-  rc=fclose(fid);
-  rc=filename(fref);
-  return(0);
-endsub;
-
-%if &wrap=YES %then %do;
-  quit;
-%end;
-
-/* insert the CMPLIB if not already there */
-%let cmpval=%sysfunc(getoption(cmplib));
-%let found=0;
-%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
-  %let var=%scan(&cmpval,&i,%str( %(%)));
-  %if &var=&lib..&cat %then %let found=1;
-%end;
-%if &found=0 %then %do;
-  options insert=(CMPLIB=(&lib..&cat));
-%end;
-
-%mend mcf_stpsrv_header;

lua/ml_gsubfile.sas

@@ -39,6 +39,14 @@ data _null_;
   put 'io.close(file) ';
 run;
 
+/* ensure big enough lrecl to avoid lua compilation issues */
+%local optval;
+%let optval=%sysfunc(getoption(lrecl));
+options lrecl=1024;
+
+/* execute the lua code by using a .lua extension */
 %inc "%sysfunc(pathname(work))/ml_gsubfile.lua" /source2;
 
+options lrecl=&optval;
+
 %mend ml_gsubfile;

lua/ml_json.sas

@@ -389,6 +389,14 @@ data _null_;
   put '-- JSON.LUA ENDS HERE ';
 run;
 
+/* ensure big enough lrecl to avoid lua compilation issues */
+%local optval;
+%let optval=%sysfunc(getoption(lrecl));
+options lrecl=1024;
+
+/* execute the lua code by using a .lua extension */
 %inc "%sysfunc(pathname(work))/ml_json.lua" /source2;
 
+options lrecl=&optval;
+
 %mend ml_json;

main.dox

@@ -18,7 +18,17 @@
     statements.  Those starting `mp_` are macro _procedures_, which generate
     SAS statements, and must therefore be applied accordingly.
 
- */
+*/
+
+/*! \dir ddl
+ *  \brief Data Definition Language files
+ *  \details Provides templates for commonly used tables in sasjs/core.
+    Attributes:
+
+        * OS independent
+        * No X command
+        * Prefixes: _mddl_
+*/
 
 /*! \dir fcmp
  *  \brief Macros for generating FCMP functions
@@ -92,4 +102,39 @@
         * Auto-generated from the plain source `.lua` files in the same directory
         * Prefixes: _ml_
 
- */
+*/
+
+/*! \dir tests/base
+ *  \brief Tests for Base macros
+ */
+
+/*! \dir tests/ddlonly
+ *  \brief Tests for DDL macros
+ */
+
+/*! \dir tests/sas9only
+ *  \brief Tests for SAS Metadata macros
+ */
+
+/*! \dir tests/serveronly
+ *  \brief Tests for SASjs Server macros
+ */
+
+/*! \dir tests/viyaonly
+ *  \brief Tests for Viya macros
+ */
+
+/*! \dir tests/x-platform
+ *  \brief Tests for cross-platform macros
+ */
+
+/*! \dir xplatform
+ *  \brief Cross Platform, works on all SAS servers (Viya, EBI, SASjs)
+ *  \details Useful when you need to run a single piece of code against Viya,
+     SAS 9 with metadata, or SASjs on Base SAS.
+
+        * OS independent
+        * No X command
+        * Prefixes: _mx_
+
+ */

meta/mm_assignlib.sas

@@ -13,8 +13,8 @@
   <h4> SAS Macros </h4>
   @li mp_abort.sas
 
-  @param libref the libref (not name) of the metadata library
-  @param mAbort= If not assigned, HARD will call %mp_abort(), SOFT will
+  @param [in] libref The libref (not name) of the metadata library
+  @param [in] mAbort= If not assigned, HARD will call %mp_abort(), SOFT will
     silently return
 
   @returns libname statement
@@ -28,11 +28,11 @@
     libref
     ,mAbort=HARD
 )/*/STORE SOURCE*/;
-
+%local mp_abort msg;
+%let mp_abort=0;
 %if %sysfunc(libref(&libref)) %then %do;
-  %local mp_abort msg; %let mp_abort=0;
   data _null_;
-    length liburi LibName $200;
+    length liburi LibName msg $200;
     call missing(of _all_);
     nobj=metadata_getnobj("omsobj:SASLibrary?@Libref='&libref'",1,liburi);
     if nobj=1 then do;
@@ -40,7 +40,30 @@
       /* now try and assign it */
       if libname("&libref",,'meta',cats('liburi="',liburi,'";')) ne 0 then do;
         putlog "&libref could not be assigned";
-        call symputx('msg',sysmsg(),'l');
+        putlog liburi=;
+        /**
+          * Fetch the system message for display in the abort modal.  This is
+          * not always helpful though.  One example, previously received:
+          * NOTE: Libref XX refers to the same library metadata as libref XX.
+          */
+        msg=sysmsg();
+        if msg=:'ERROR: Libref SAVE is not assigned.' then do;
+          msg=catx(" ",
+            "Could not assign %upcase(&libref).",
+            "Please check metadata permissions!  Libname:",libname,
+            "Liburi:",liburi
+          );
+        end;
+        else if msg="ERROR: User does not have appropriate authorization "!!
+          "level for library SAVE."
+        then do;
+          msg=catx(" ",
+            "ERROR: User does not have appropriate authorization level",
+            "for library %upcase(&libref), libname:",libname,
+            "Liburi:",liburi
+          );
+        end;
+        call symputx('msg',msg,'l');
         if "&mabort"='HARD' then call symputx('mp_abort',1,'l');
       end;
       else do;
@@ -59,20 +82,16 @@
     end;
   run;
 
-  %if &mp_abort=1 %then %do;
-    %mp_abort(iftrue= (&mp_abort=1)
-      ,mac=&sysmacroname
-      ,msg=&msg
-    )
-    %return;
-  %end;
-  %else %if %length(&msg)>2 %then %do;
-    %put NOTE: &msg;
-    %return;
-  %end;
+  %put NOTE: &msg;
 
 %end;
 %else %do;
   %put NOTE: Library &libref is already assigned;
 %end;
+
+%mp_abort(iftrue= (&mp_abort=1)
+  ,mac=mm_assignlib.sas
+  ,msg=%superq(msg)
+)
+
 %mend mm_assignlib;

meta/mm_createdataset.sas

@@ -1,22 +1,21 @@
 /**
-  @file mm_createdataset.sas
-  @brief Create a dataset from a metadata definition
+  @file
+  @brief Create an empty dataset from a metadata definition
   @details This macro was built to support viewing empty tables in
-    https://datacontroller.io - a free evaluation copy is available by
-    contacting the author (Allan Bowe).
+  https://datacontroller.io
 
-    The table can be retrieved using LIBRARY.DATASET reference, or directly
-    using the metadata URI.
+  The table can be retrieved using LIBRARY.DATASET reference, or directly
+  using the metadata URI.
 
-    The dataset is written to the WORK library.
+  The dataset is written to the WORK library.
 
-  usage:
+  Usage:
 
-    %mm_createdataset(libds=metlib.some_dataset)
+      %mm_createdataset(libds=metlib.some_dataset)
 
-    or
+  or
 
-    %mm_createdataset(tableuri=G5X8AFW1.BE00015Y)
+      %mm_createdataset(tableuri=G5X8AFW1.BE00015Y)
 
   <h4> SAS Macros </h4>
   @li mm_getlibs.sas
@@ -26,9 +25,9 @@
   @param libds= library.dataset metadata source.  Note - table names in metadata
     can be longer than 32 chars (just fyi, not an issue here)
   @param tableuri= Metadata URI of the table to be created
-  @param outds= The dataset to create, default is `work.mm_createdataset`.
-    The table name needs to be 32 chars or less as per SAS naming rules.
-  @param mdebug= set DBG to 1 to disable DEBUG messages
+  @param outds= (work.mm_createdataset) The dataset to create.  The table name
+    needs to be 32 chars or less as per SAS naming rules.
+  @param mdebug= (0) Set to 1 to enable DEBUG messages
 
   @version 9.4
   @author Allan Bowe
@@ -54,14 +53,23 @@
   %mm_gettables(uri=&liburi,outds=&tempds2)
   data _null_;
     set &tempds2;
-    if upcase(tablename)="%upcase(%scan(&libds,2,.))";
+    where upcase(tablename)="%upcase(%scan(&libds,2,.))";
+    &dbg putlog tableuri=;
     call symputx('tableuri',tableuri);
   run;
 %end;
 
-data;run;%let tempds3=&syslast;
+data;run;
+%let tempds3=&syslast;
 %mm_getcols(tableuri=&tableuri,outds=&tempds3)
 
+%if %mf_nobs(&tempds3)=0 %then %do;
+  %put &libds (&tableuri) has no columns defined!!;
+  data &outds;
+  run;
+  %return;
+%end;
+
 data _null_;
   set &tempds3 end=last;
   if _n_=1 then call execute('data &outds;');

meta/mm_createfolder.sas

@@ -12,13 +12,14 @@
     The macro is idempotent - if you run it twice, it will only create a folder
     once.
 
-  usage:
+  Usage:
 
       %mm_createfolder(path=/some/meta/folder)
 
   @param [in] path= Name of the folder to create.
   @param [in] mdebug= set DBG to 1 to disable DEBUG messages
 
+
   @version 9.4
   @author Allan Bowe
 

meta/mm_createstp.sas

@@ -12,7 +12,7 @@
     This macro is idempotent - if you run it twice, it will only create an STP
     once.
 
-  usage (type 1 STP):
+  Usage (type 1 STP):
 
       %mm_createstp(stpname=MyNewSTP
         ,filename=mySpecialProgram.sas
@@ -31,7 +31,8 @@
         putlog (_all_)(=);
       run;
 
-  usage (type 2 STP):
+  Usage (type 2 STP):
+
       %mm_createstp(stpname=MyNewType2STP
         ,filename=mySpecialProgram.sas
         ,directory=SASEnvironment/SASCode/STPs
@@ -74,8 +75,9 @@
   @li mf_verifymacvars.sas
   @li mm_getdirectories.sas
   @li mm_updatestpsourcecode.sas
-  @li mp_dropmembers.sas
   @li mm_getservercontexts.sas
+  @li mp_abort.sas
+  @li mp_dropmembers.sas
 
   <h4> Related Macros </h4>
   @li mm_createwebservice.sas

meta/mm_createwebservice.sas

@@ -103,7 +103,7 @@ data _null_;
   put ' ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
-  put '  data _null_;file &jref encoding=''utf-8'' ; ';
+  put '  data _null_;file &jref encoding=''utf-8'' lrecl=200; ';
   put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
   put '  run; ';
   put '%end; ';
@@ -232,12 +232,23 @@ data _null_;
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
-  put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
-  put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
-  put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
-  put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '        )))))!!''"''; ';
+  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; ';
+  put '          &&name&i=''"''!!trim( ';
+  put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '            prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '            prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '            prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '            prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '            prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '          ))))))))))))!!''"''; ';
+  put '        end; ';
+  put '        else &&name&i=quote(cats(&&name&i)); ';
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
@@ -253,7 +264,7 @@ data _null_;
   put '      %do i=1 %to &numcols; ';
   put '        %if &i>1 %then  "," ; ';
   put '        %if &action=OBJ %then """&&name&i"":" ; ';
-  put '        &&name&i ';
+  put '        "&&name&i"n /* name literal for reserved variable names */ ';
   put '      %end; ';
   put '      %if &action=ARR %then "]" ; %else "}" ; ; ';
   put '    /* now write the long strings to _webout 1 byte at a time */ ';
@@ -303,6 +314,25 @@ data _null_;
   put '  run; ';
   put '%end; ';
   put '%mend mp_jsonout; ';
+  put ' ';
+  put '%macro mf_getuser(type=META ';
+  put ')/*/STORE SOURCE*/; ';
+  put '  %local user metavar; ';
+  put '  %if &type=OS %then %let metavar=_secureusername; ';
+  put '  %else %let metavar=_metaperson; ';
+  put ' ';
+  put '  %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %let user=&SYS_COMPUTE_SESSION_OWNER; ';
+  put '  %else %if %symexist(&metavar) %then %do; ';
+  put '    %if %length(&&&metavar)=0 %then %let user=&sysuserid; ';
+  put '    /* sometimes SAS will add @domain extension - remove for consistency */ ';
+  put '    /* but be sure to quote in case of usernames with commas */ ';
+  put '    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@)); ';
+  put '  %end; ';
+  put '  %else %let user=&sysuserid; ';
+  put ' ';
+  put '  %quote(&user) ';
+  put ' ';
+  put '%mend mf_getuser; ';
   put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
   put '  ,showmeta=NO ';
   put '); ';
@@ -421,14 +451,16 @@ data _null_;
   put '    put '',"_METAPERSON": '' _METAPERSON; ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
+  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
+  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';
   put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
@@ -442,24 +474,6 @@ data _null_;
   put '%end; ';
   put ' ';
   put '%mend mm_webout; ';
-  put ' ';
-  put '%macro mf_getuser(type=META ';
-  put ')/*/STORE SOURCE*/; ';
-  put '  %local user metavar; ';
-  put '  %if &type=OS %then %let metavar=_secureusername; ';
-  put '  %else %let metavar=_metaperson; ';
-  put ' ';
-  put '  %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %let user=&SYS_COMPUTE_SESSION_OWNER; ';
-  put '  %else %if %symexist(&metavar) %then %do; ';
-  put '    %if %length(&&&metavar)=0 %then %let user=&sysuserid; ';
-  put '    /* sometimes SAS will add @domain extension - remove for consistency */ ';
-  put '    %else %let user=%scan(&&&metavar,1,@); ';
-  put '  %end; ';
-  put '  %else %let user=&sysuserid; ';
-  put ' ';
-  put '  %quote(&user) ';
-  put ' ';
-  put '%mend mf_getuser; ';
 /* WEBOUT END */
   put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO);';
   put '  %mm_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt,missing=&missing';

meta/mm_deletelibrary.sas

@@ -78,6 +78,7 @@ filename &fname2 clear;
 %local isgone;
 data _null_;
   length type uri $256;
+  call missing (of _all_);
   rc=metadata_resolve("omsobj:SASLibrary?@Id='&liburi'",type,uri);
   call symputx('isgone',type,'l');
 run;

meta/mm_getauthinfo.sas

@@ -1,16 +1,21 @@
 /**
   @file mm_getauthinfo.sas
-  @brief extracts authentication info
-  @details usage:
+  @brief Extracts authentication info for each user in metadata
+  @details
+  Usage:
 
-    %mm_getauthinfo(outds=auths)
+      %mm_getauthinfo(outds=auths)
 
-  @param outds= the ONE LEVEL work dataset to create
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
+  @param [out] outds= (mm_getauthinfo) The output dataset to create
 
   <h4> SAS Macros </h4>
-  @li mm_getobjects.sas
   @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
   @li mm_getdetails.sas
+  @li mm_getobjects.sas
+
 
   @version 9.4
   @author Allan Bowe
@@ -18,67 +23,69 @@
 **/
 
 %macro mm_getauthinfo(outds=mm_getauthinfo
+  ,mdebug=0
 )/*/STORE SOURCE*/;
+%local prefix fileref;
+%let prefix=%substr(%mf_getuniquename(),1,25);
 
-%if %length(&outds)>30 %then %do;
-  %put %str(ERR)OR: Temp tables are created with the &outds prefix, which
-    therefore needs to be 30 characters or less;
-  %return;
-%end;
-%if %index(&outds,'.')>0 %then %do;
-  %put %str(ERR)OR: Table &outds should be ONE LEVEL (no library);
-  %return;
-%end;
-
-%mm_getobjects(type=Login,outds=&outds.0)
+%mm_getobjects(type=Login,outds=&prefix.0)
 
 %local fileref;
 %let fileref=%mf_getuniquefileref();
 
 data _null_;
   file &fileref;
-  set &outds.0 end=last;
+  set &prefix.0 end=last;
   /* run macro */
-  str=cats('%mm_getdetails(uri=',id,",outattrs=&outds.d",_n_
-    ,",outassocs=&outds.a",_n_,")");
+  str=cats('%mm_getdetails(uri=',id,",outattrs=&prefix.d",_n_
+    ,",outassocs=&prefix.a",_n_,")");
   put str;
   /* transpose attributes */
-  str=cats("proc transpose data=&outds.d",_n_,"(drop=type) out=&outds.da"
+  str=cats("proc transpose data=&prefix.d",_n_,"(drop=type) out=&prefix.da"
     ,_n_,"(drop=_name_);var value;id name;run;");
   put str;
   /* add extra info to attributes */
-  str=cats("data &outds.da",_n_,";length login_id login_name $256; login_id="
-    ,quote(trim(id)),";set &outds.da",_n_
+  str=cats("data &prefix.da",_n_,";length login_id login_name $256; login_id="
+    ,quote(trim(id)),";set &prefix.da",_n_
     ,";login_name=trim(subpad(name,1,256));drop name;run;");
   put str;
   /* add extra info to associations */
-  str=cats("data &outds.a",_n_,";length login_id login_name $256; login_id="
+  str=cats("data &prefix.a",_n_,";length login_id login_name $256; login_id="
     ,quote(trim(id)),";login_name=",quote(trim(name))
-    ,";set &outds.a",_n_,";run;");
+    ,";set &prefix.a",_n_,";run;");
   put str;
   if last then do;
     /* collate attributes */
-    str=cats("data &outds._logat; set &outds.da1-&outds.da",_n_,";run;");
+    str=cats("data &prefix._logat; set &prefix.da1-&prefix.da",_n_,";run;");
     put str;
     /* collate associations */
-    str=cats("data &outds._logas; set &outds.a1-&outds.a",_n_,";run;");
+    str=cats("data &prefix._logas; set &prefix.a1-&prefix.a",_n_,";run;");
     put str;
     /* tidy up */
-    str=cats("proc delete data=&outds.da1-&outds.da",_n_,";run;");
+    str=cats("proc delete data=&prefix.da1-&prefix.da",_n_,";run;");
     put str;
-    str=cats("proc delete data=&outds.d1-&outds.d",_n_,";run;");
+    str=cats("proc delete data=&prefix.d1-&prefix.d",_n_,";run;");
     put str;
-    str=cats("proc delete data=&outds.a1-&outds.a",_n_,";run;");
+    str=cats("proc delete data=&prefix.a1-&prefix.a",_n_,";run;");
     put str;
   end;
 run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fileref;
+    if _n_=1 then putlog // "Now executing the following code:" //;
+    input; putlog _infile_;
+  run;
+%end;
 %inc &fileref;
+filename &fileref clear;
 
 /* get libraries */
-proc sort data=&outds._logas(where=(assoc='Libraries')) out=&outds._temp;
+proc sort data=&prefix._logas(where=(assoc='Libraries')) out=&prefix._temp;
   by login_id;
-data &outds._temp;
-  set &outds._temp;
+data &prefix._temp;
+  set &prefix._temp;
   by login_id;
   length library_list $32767;
   retain library_list;
@@ -86,31 +93,27 @@ data &outds._temp;
   else library_list=catx(' !! ',library_list,name);
 proc sql;
 /* get auth domain */
-create table &outds._dom as
+create table &prefix._dom as
   select login_id,name as domain
-  from &outds._logas
+  from &prefix._logas
   where assoc='Domain';
-create unique index login_id on &outds._dom(login_id);
+create unique index login_id on &prefix._dom(login_id);
 /* join it all together */
-create table &outds._logins as
+create table &outds as
   select a.*
     ,c.domain
     ,b.library_list
-  from &outds._logat (drop=ishidden lockedby usageversion publictype) a
-  left join &outds._temp b
+  from &prefix._logat (drop=ishidden lockedby usageversion publictype) a
+  left join &prefix._temp b
   on a.login_id=b.login_id
-  left join &outds._dom c
+  left join &prefix._dom c
   on a.login_id=c.login_id;
-drop table &outds._temp;
-drop table &outds._logat;
-drop table &outds._logas;
 
-data _null_;
-  infile &fileref;
-  if _n_=1 then putlog // "Now executing the following code:" //;
-  input; putlog _infile_;
-run;
+%if &mdebug=0 %then %do;
+  proc datasets lib=work;
+    delete &prefix:;
+  run;
+%end;
 
-filename &fileref clear;
 
 %mend mm_getauthinfo;

meta/mm_getstpcode.sas

@@ -21,6 +21,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
+  @li mp_abort.sas
 
   @author Allan Bowe
 
@@ -70,11 +71,10 @@ data _null_;
   else put (_all_)(=);
 run;
 
-%if &tsuri=stopifempty %then %do;
-  %put %str(WARN)ING:  &tree&name.(StoredProcess) not found!;
-  %return;
-%end;
-
+%mp_abort(iftrue= (&tsuri=stopifempty)
+  ,mac=mm_getstpcode
+  ,msg=%str(&tree&name.(StoredProcess) not found!)
+)
 
 /**
   * Now we can extract the textstore

meta/mm_spkexport.sas

@@ -49,20 +49,25 @@
 
 
   <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_isblank.sas
   @li mf_loc.sas
   @li mm_tree.sas
-  @li mf_getuniquefileref.sas
-  @li mf_isblank.sas
   @li mp_abort.sas
 
-  @param metaloc= the metadata folder to export
-  @param secureref= fileref containing the username / password (should point to
-    a file in a secure location).  Leave blank to substitute $bash type vars.
-  @param outref= fileref to which to write the command
-  @param cmdoutloc= the directory to which the command will write the SPK
-    (default=WORK)
-  @param cmdoutname= the name of the spk / log files to create (will be
-    identical just with .spk or .log extension)
+
+  @param [in] metaloc= the metadata folder to export
+  @param [in] secureref= fileref containing the username / password (should
+    point to a file in a secure location). Leave blank to substitute $bash vars.
+  @param [in] excludevars= (0) A space seperated list of macro variable names,
+    each of which contains a value that should be used to filter the output
+    objects.
+  @param [out] outref= fileref to which to write the command
+  @param [out] cmdoutloc= (%sysfunc(pathname(work))) The directory to which the
+    command will write the SPK
+  @param [out] cmdoutname= (mmxport) The name of the spk / log files to create
+    (will be identical just with .spk or .log extension)
 
   @version 9.4
   @author Allan Bowe
@@ -71,6 +76,7 @@
 
 %macro mm_spkexport(metaloc=
   ,secureref=
+  ,excludevars=0
   ,outref=
   ,cmdoutloc=%sysfunc(pathname(work))
   ,cmdoutname=mmxport
@@ -82,7 +88,7 @@
 %end;
 
 /* set creds */
-%local mmxuser mmxpath;
+%local mmxuser mmxpath i var;
 %let mmxuser=$1;
 %let mmxpass=$2;
 %if %mf_isblank(&secureref)=0 %then %do;
@@ -90,35 +96,51 @@
 %end;
 
 /* setup metadata connection options */
-%local host port platform_object_path connx_string;
+%local host port platform_object_path ds;
 %let host=%sysfunc(getoption(metaserver));
 %let port=%sysfunc(getoption(metaport));
 %let platform_object_path=%mf_loc(POF);
+%let ds=%mf_getuniquename(prefix=spkexportable);
 
-%let connx_string=%str(-host &host -port &port -user &mmxuser %trim(
-  )-password &mmxpass);
-
-%mm_tree(root=%str(&metaloc) ,types=EXPORTABLE ,outds=exportable)
+%mm_tree(root=%str(&metaloc),types=EXPORTABLE ,outds=&ds)
 
 %if %mf_isblank(&outref)=1 %then %let outref=%mf_getuniquefileref();
 
 data _null_;
-  set exportable end=last;
+  set &ds end=last;
   file &outref lrecl=32767;
   length str $32767;
   if _n_=1 then do;
+    put "# Script generated by &sysuserid on %sysfunc(datetime(),datetime19.)";
     put "cd ""&platform_object_path"" \";
-    put "; ./ExportPackage &connx_string -disableX11 \";
-    put " -package ""&cmdoutloc/&cmdoutname..spk"" \";
+    put "; ./ExportPackage -host &host -port &port -user &mmxuser \";
+    put "  -disableX11 -password &mmxpass \";
+    put "  -package ""&cmdoutloc/&cmdoutname..spk"" \";
   end;
+/* exclude particular patterns from the exported SPK */
+%if "&excludevars" ne "0" %then %do;
+  %do i=1 %to %sysfunc(countw(&excludevars));
+    %let var=%scan(&excludevars,&i);
+    if _n_=1 then do;
+      length excludestr&i $1000;
+      retain excludestr&i;
+      excludestr&i=symget("&var");
+      putlog excludestr&i=;
+      putlog path=;
+    end;
+    if index(path,cats(excludestr&i))=0 and index(name,cats(excludestr&i))=0;
+  %end;
+  /* ignore top level folder else all subcontent will be exported regardless */
+  if _n_>1;
+%end;
   str=' -objects '!!cats('"',path,'/',name,"(",publictype,')" \');
   put str;
   if last then put " -log ""&cmdoutloc/&cmdoutname..log"" 2>&1 ";
 run;
 
 %mp_abort(iftrue= (&syscc ne 0)
-  ,mac=&sysmacroname
+  ,mac=mm_spkexport
   ,msg=%str(syscc=&syscc)
 )
 
-%mend mm_spkexport;
+%mend mm_spkexport;

meta/mm_webout.sas

@@ -157,14 +157,16 @@
     put ',"_METAPERSON": ' _METAPERSON;
     put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";

package.json

@@ -26,14 +26,13 @@
   "homepage": "https://core.sasjs.io",
   "main": "index.js",
   "scripts": {
-    "build": "sasjs cbd -t viya",
+    "build": "sasjs cbd -t server",
     "docs": "sasjs doc -t docsonly && ./sasjs/utils/build.sh",
-    "test": "sasjs test -t viya",
+    "test": "sasjs test -t server",
     "lint": "sasjs lint",
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "3.6.0",
-    "@sasjs/core": "4.4.7"
+    "@sasjs/cli": "3.13.0"
   }
 }

sasjs/sasjsconfig.json

@@ -4,13 +4,12 @@
     "base",
     "ddl",
     "fcmp",
-    "meta",
-    "metax",
-    "server",
-    "viya",
     "lua",
-    "tests/crossplatform",
-    "tests/ddl"
+    "server",
+    "xplatform",
+    "tests/base",
+    "tests/ddlonly",
+    "tests/x-platform"
   ],
   "docConfig": {
     "displayMacroCore": false,
@@ -28,36 +27,33 @@
       "mcTestAppLoc": "/Public/temp/macrocore"
     }
   },
-  "defaultTarget": "viya",
+  "defaultTarget": "server",
   "targets": [
     {
       "name": "viya",
-      "serverUrl": "https://sas.analytium.co.uk",
+      "serverUrl": "",
       "serverType": "SASVIYA",
       "httpsAgentOptions": {
         "allowInsecureRequests": false
       },
-      "appLoc": "/Public/temp/macrocore",
+      "appLoc": "/Public/app/macrocore",
       "macroFolders": [
+        "viya",
         "tests/viyaonly"
       ],
-      "programFolders": [],
-      "binaryFolders": [],
-      "deployConfig": {
-        "deployServicePack": true,
-        "deployScripts": []
-      },
       "contextName": "SAS Job Execution compute context"
     },
     {
       "name": "sas9",
-      "serverUrl": "https://sas.analytium.co.uk:8343",
+      "serverUrl": "",
       "serverType": "SAS9",
       "httpsAgentOptions": {
         "allowInsecureRequests": false
       },
       "appLoc": "/Shared Data/temp/macrocore",
       "macroFolders": [
+        "meta",
+        "metax",
         "tests/sas9only"
       ],
       "programFolders": [],
@@ -71,23 +67,44 @@
     },
     {
       "name": "server",
-      "serverUrl": "https://sas.analytium.co.uk:5000",
+      "serverUrl": "https://sas.analytium.co.uk:5007",
       "serverType": "SASJS",
-      "appLoc": "/Shared Data/temp/macrocore",
+      "httpsAgentOptions": {
+        "allowInsecureRequests": false
+      },
+      "appLoc": "/sasjs/core",
       "macroFolders": [
+        "server",
         "tests/serveronly"
-      ],
-      "deployConfig": {
-        "deployServicePack": true
-      }
+      ]
     },
     {
       "name": "docsonly",
-      "serverType": "SAS9",
+      "serverType": "SASJS",
+      "appLoc": "dummy",
       "macroFolders": [
+        "meta",
+        "metax",
+        "server",
+        "viya",
         "tests/sas9only",
+        "tests/serveronly",
         "tests/viyaonly"
       ]
+    },
+    {
+      "name": "viya4",
+      "serverUrl": "https://azureuse011059.my-trials.sas.com",
+      "serverType": "SASVIYA",
+      "appLoc": "/Public/temp/macrocore",
+      "macroFolders": [
+        "viya",
+        "tests/viyaonly"
+      ],
+      "deployConfig": {
+        "deployServicePack": true
+      },
+      "contextName": "SAS Job Execution compute context"
     }
   ]
-}
+}

server/mfs_httpheader.sas

@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Sets the http headers in the SASjs/server response
+  @details For GET requests, SASjs server will use the file generated by this
+  macro for setting the appropriate http headers in the response.
+
+  It works by writing a file to the session directory, that is then ingested by
+  the server.
+
+  The location of this file is driven by the global variable
+  `sasjs_stpsrv_header_loc` which is made available in the autoexec.
+
+  Usage:
+
+      %mfs_httpheader(Content-Type,application/csv)
+
+  @param [in] header_name Name of the http header to set
+  @param [in] header_value Value of the http header to set
+
+  <h4> Related Macros </h4>
+  @li mcf_stpsrv_header.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mfs_httpheader(header_name
+  ,header_value
+)/*/STORE SOURCE*/;
+%global sasjs_stpsrv_header_loc;
+%local fref fid i;
+
+%if %sysfunc(filename(fref,&sasjs_stpsrv_header_loc)) ne 0 %then %do;
+  %put &=fref &=sasjs_stpsrv_header_loc;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let fid=%sysfunc(fopen(&fref,A));
+
+%if &fid=0 %then %do;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let rc=%sysfunc(fput(&fid,%str(&header_name): %str(&header_value)));
+%let rc=%sysfunc(fwrite(&fid));
+
+%let rc=%sysfunc(fclose(&fid));
+%let rc=%sysfunc(filename(&fref));
+
+%mend mfs_httpheader;

server/ms_createfile.sas

@@ -0,0 +1,112 @@
+/**
+  @file
+  @brief Creates a file on SASjs Drive
+  @details Creates a file on SASjs Drive. To use the file as a Stored Program,
+  it must have a ".sas" extension.
+
+  Example:
+
+      filename stpcode temp;
+      data _null_;
+        file stpcode;
+        put '%put hello world;';
+      run;
+      %ms_createfile(/some/stored/program.sas, inref=stpcode)
+
+  @param [in] driveloc The full path to the file in SASjs Drive
+  @param [in] inref= (0) The fileref containing the file to create.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+  @li ms_deletefile.sas
+
+**/
+
+%macro ms_createfile(driveloc
+    ,inref=0
+    ,mdebug=0
+  );
+
+/* first, delete in case it exists */
+%ms_deletefile(&driveloc,mdebug=&mdebug)
+
+%local fname0 fname1 fname2 boundary fname statcd msg optval;
+%let fname0=%mf_getuniquefileref();
+%let fname1=%mf_getuniquefileref();
+%let fname2=%mf_getuniquefileref();
+%let boundary=%mf_getuniquename();
+
+/* avoid sending bom marker to API */
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
+data _null_;
+  file &fname0 termstr=crlf lrecl=32767;
+  infile &inref end=eof lrecl=32767;
+  if _n_ = 1 then do;
+    put "--&boundary.";
+    put 'Content-Disposition: form-data; name="filePath"';
+    put ;
+    put "&driveloc";
+    put "--&boundary";
+    put 'Content-Disposition: form-data; name="file"; filename="ignore.sas"';
+    put "Content-Type: text/plain";
+    put ;
+  end;
+  input;
+  put _infile_; /* add the actual file to be sent */
+  if eof then do;
+    put ;
+    put "--&boundary--";
+  end;
+run;
+
+data _null_;
+  file &fname1 lrecl=1000;
+  infile "&_sasjs_tokenfile" lrecl=1000;
+  input;
+  if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary";
+  put _infile_;
+run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fname0 lrecl=32767;
+    input;
+    put _infile_;
+  data _null_;
+    infile &fname1 lrecl=32767;
+    input;
+    put _infile_;
+  run;
+%end;
+
+proc http method='POST' in=&fname0 headerin=&fname1 out=&fname2
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file";
+%if &mdebug=1 %then %do;
+  debug level=1;
+%end;
+run;
+
+%let statcd=0;
+data _null_;
+  infile &fname2;
+  input;
+  putlog _infile_;
+  if _infile_='{"status":"success"}' then call symputx('statcd',1,'l');
+  else call symputx('msg',_infile_,'l');
+run;
+
+%mp_abort(
+  iftrue=(&statcd=0)
+  ,mac=ms_createfile.sas
+  ,msg=%superq(msg)
+)
+
+/* reset options */
+options &optval;
+
+%mend ms_createfile;

server/ms_creategroup.sas

@@ -0,0 +1,149 @@
+/**
+  @file
+  @brief Creates a group on SASjs Server
+  @details Creates a group on SASjs Server with the following attributes:
+
+  @li name
+  @li description
+  @li isActive
+
+  Examples:
+
+      %ms_creategroup(mynewgroup)
+
+      %ms_creategroup(mynewergroup, desc=The group description)
+
+  @param [in] groupname The group name to create.  No spaces or special chars.
+  @param [in] desc= (0) If no description provided, group name will be used.
+  @param [in] isactive= (true) Set to false to create an inactive group.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outds= (work.ms_creategroup) This output dataset will contain the
+    values from the JSON response (such as the id of the new group)
+|DESCRIPTION:$1.|GROUPID:best.|ISACTIVE:best.|NAME:$11.|
+|---|---|---|---|
+|`The group description`|`2 `|`1 `|`mynewergroup `|
+
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mp_abort.sas
+
+  <h4> Related Files </h4>
+  @li ms_creategroup.test.sas
+  @li ms_getgroups.sas
+
+**/
+
+%macro ms_creategroup(groupname
+    ,desc=0
+    ,isactive=true
+    ,outds=work.ms_creategroup
+    ,mdebug=0
+  );
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_creategroup.sas
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+
+%local fref0 fref1 fref2 libref optval rc msg;
+%let fref0=%mf_getuniquefileref();
+%let fref1=%mf_getuniquefileref();
+%let fref2=%mf_getuniquefileref();
+%let libref=%mf_getuniquelibref();
+
+/* avoid sending bom marker to API */
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
+data _null_;
+  file &fref0 termstr=crlf;
+  name=quote(cats(symget('groupname')));
+  description=quote(cats(symget('desc')));
+  if cats(description)='"0"' then description=name;
+  isactive=symget('isactive');
+%if &mdebug=1 %then %do;
+  putlog _all_;
+%end;
+
+  put '{'@;
+  put '"name":' name @;
+  put ',"description":' description @;
+  put ',"isActive":' isactive @;
+  put '}';
+run;
+
+data _null_;
+  file &fref1 lrecl=1000;
+  infile "&_sasjs_tokenfile" lrecl=1000;
+  input;
+  if _n_=1 then do;
+    put "Content-Type: application/json";
+    put "accept: application/json";
+  end;
+  put _infile_;
+run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fref0;
+    input;
+    put _infile_;
+  data _null_;
+    infile &fref1;
+    input;
+    put _infile_;
+  run;
+%end;
+
+proc http method='POST' in=&fref0 headerin=&fref1 out=&fref2
+  url="&_sasjs_apiserverurl/SASjsApi/group";
+%if &mdebug=1 %then %do;
+  debug level=1;
+%end;
+run;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_creategroup.sas
+  ,msg=%str(Issue submitting query to SASjsApi/group)
+)
+
+libname &libref JSON fileref=&fref2;
+
+data &outds;
+  set &libref..root;
+  drop ordinal_root;
+%if &mdebug=1 %then %do;
+  putlog _all_;
+%end;
+run;
+
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_creategroup.sas
+  ,msg=%str(Issue reading response JSON)
+)
+
+/* reset options */
+options &optval;
+
+%if &mdebug=0 %then %do;
+  filename &fref0 clear;
+  filename &fref1 clear;
+  filename &fref2 clear;
+  libname &libref clear;
+%end;
+%else %do;
+  data _null_;
+    infile &fref2;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend ms_creategroup;

server/ms_createuser.sas

@@ -0,0 +1,146 @@
+/**
+  @file
+  @brief Creates a user on SASjs Server
+  @details Creates a user on SASjs Server with the following attributes:
+
+  @li UserName
+  @li Password
+  @li isAdmin
+  @li displayName
+
+  The userid is created by sasjs/server. All users are created with `isActive`
+  set to `true`.
+
+  Example:
+
+      %ms_createuser(newuser,secretpass,displayname=New User!)
+
+  @param [in] username The username to apply.  No spaces or special characters.
+  @param [in] password The initial password to set.
+  @param [in] isadmin= (false) Set to true to give the user admin rights
+  @param [in] displayName= (0) Set a friendly name (spaces & special characters
+    are ok).  If not provided, username will be used instead.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outds= (work.ms_createuser) This output dataset will contain the
+    values from the JSON response (such as the id of the new user)
+|ID:best.|DISPLAYNAME:$8.|USERNAME:$8.|ISACTIVE:best.|ISADMIN:best.|
+|---|---|---|---|---|
+|`6 `|`New User `|`newuser `|`1 `|`0 `|
+
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mp_abort.sas
+
+  <h4> Related Files </h4>
+  @li ms_createuser.test.sas
+  @li ms_getusers.sas
+
+**/
+
+%macro ms_createuser(username,password
+    ,isadmin=false
+    ,displayname=0
+    ,outds=work.ms_createuser
+    ,mdebug=0
+  );
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_createuser.sas
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+
+%local fref0 fref1 fref2 libref optval rc msg;
+%let fref0=%mf_getuniquefileref();
+%let fref1=%mf_getuniquefileref();
+%let fref2=%mf_getuniquefileref();
+%let libref=%mf_getuniquelibref();
+
+/* avoid sending bom marker to API */
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
+data _null_;
+  file &fref0 termstr=crlf;
+  username=quote(cats(symget('username')));
+  password=quote(cats(symget('password')));
+  isadmin=symget('isadmin');
+  displayname=quote(cats(symget('displayname')));
+  if displayname='"0"' then displayname=username;
+
+%if &mdebug=1 %then %do;
+  putlog _all_;
+%end;
+
+  put '{'@;
+  put '"displayName":' displayname @;
+  put ',"username":' username @;
+  put ',"password":' password @;
+  put ',"isAdmin":' isadmin @;
+  put ',"isActive": true }';
+run;
+
+data _null_;
+  file &fref1 lrecl=1000;
+  infile "&_sasjs_tokenfile" lrecl=1000;
+  input;
+  if _n_=1 then do;
+    put "Content-Type: application/json";
+    put "accept: application/json";
+  end;
+  put _infile_;
+run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fref0;
+    input;
+    put _infile_;
+  data _null_;
+    infile &fref1;
+    input;
+    put _infile_;
+  run;
+%end;
+
+proc http method='POST' in=&fref0 headerin=&fref1 out=&fref2
+  url="&_sasjs_apiserverurl/SASjsApi/user";
+%if &mdebug=1 %then %do;
+  debug level=1;
+%end;
+run;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_createuser.sas
+  ,msg=%str(Issue submitting query to SASjsApi/user)
+)
+
+libname &libref JSON fileref=&fref2;
+
+data &outds;
+  set &libref..root;
+  drop ordinal_root;
+run;
+
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_createuser.sas
+  ,msg=%str(Issue reading response JSON)
+)
+
+/* reset options */
+options &optval;
+
+%if &mdebug=1 %then %do;
+  filename &fref0 clear;
+  filename &fref1 clear;
+  filename &fref2 clear;
+  libname &libref clear;
+%end;
+
+%mend ms_createuser;

server/ms_createwebservice.sas

@@ -0,0 +1,537 @@
+/**
+  @file ms_createwebservice.sas
+  @brief Create a Web-Ready Stored Program
+  @details This macro creates a Stored Program along with the necessary precode
+  to enable the %webout() macro
+
+  Usage:
+  <code>
+    %* compile macros ;
+    filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+    %inc mc;
+
+    %* parmcards lets us write to a text file from open code ;
+    filename ft15f001 temp;
+    parmcards4;
+        %webout(FETCH)
+        %* do some sas, any inputs are now already WORK tables;
+        data example1 example2;
+          set sashelp.class;
+        run;
+        %* send data back;
+        %webout(OPEN)
+        %webout(ARR,example1) * Array format, fast, suitable for large tables ;
+        %webout(OBJ,example2) * Object format, easier to work with ;
+        %webout(CLOSE)
+    ;;;;
+    %ms_createwebservice(path=/Public/app/common,name=appInit,code=ft15f001)
+
+  </code>
+
+  For more examples of using these web services with the SASjs Adapter, see:
+  https://github.com/sasjs/adapter#readme
+
+  @param [in] path= (0) The full SASjs Drive path in which to create the service
+  @param [in] name= Stored Program name
+  @param [in] desc= The description of the service (not implemented yet)
+  @param [in] precode= Space separated list of filerefs, pointing to the code
+    that needs to be attached to the beginning of the service (optional)
+  @param [in] code= (ft15f001) Space seperated fileref(s) of the actual code to
+    be added
+  @param [in] mDebug= (0) set to 1 to show debug messages in the log
+
+  <h4> SAS Macros </h4>
+  @li ms_createfile.sas
+  @li mf_getuser.sas
+  @li mf_getuniquename.sas
+  @li mf_getuniquefileref.sas
+  @li mp_abort.sas
+
+  <h4> Related Files </h4>
+  @li ms_createwebservice.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro ms_createwebservice(path=0
+    ,name=initService
+    ,precode=
+    ,code=ft15f001
+    ,desc=Not currently used
+    ,mDebug=0
+)/*/STORE SOURCE*/;
+
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+%mp_abort(iftrue=(&syscc ge 4 )
+  ,mac=ms_createwebservice
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+%mp_abort(iftrue=("&path"="0")
+  ,mac=ms_createwebservice
+  ,msg=%str(Path not provided)
+)
+
+/* remove any trailing slash */
+%if "%substr(&path,%length(&path),1)" = "/" %then
+  %let path=%substr(&path,1,%length(&path)-1);
+
+/**
+  * Add webout macro
+  * These put statements are auto generated - to change the macro, change the
+  * source (ms_webout) and run `build.py`
+  */
+%local sasjsref;
+%let sasjsref=%mf_getuniquefileref();
+data _null_;
+  file &sasjsref termstr=crlf lrecl=512;
+  put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
+/* WEBOUT BEGIN */
+  put ' ';
+  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
+  put '  ,engine=DATASTEP ';
+  put '  ,missing=NULL ';
+  put '  ,showmeta=NO ';
+  put ')/*/STORE SOURCE*/; ';
+  put '%local tempds colinfo fmtds i numcols; ';
+  put '%let numcols=0; ';
+  put ' ';
+  put '%if &action=OPEN %then %do; ';
+  put '  options nobomfile; ';
+  put '  data _null_;file &jref encoding=''utf-8'' lrecl=200; ';
+  put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
+  put '  run; ';
+  put '%end; ';
+  put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
+  put '  options validvarname=upcase; ';
+  put '  data _null_; file &jref encoding=''utf-8'' mod; ';
+  put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
+  put ' ';
+  put '  /* grab col defs */ ';
+  put '  proc contents noprint data=&ds ';
+  put '    out=_data_(keep=name type length format formatl formatd varnum label); ';
+  put '  run; ';
+  put '  %let colinfo=%scan(&syslast,2,.); ';
+  put '  proc sort data=&colinfo; ';
+  put '    by varnum; ';
+  put '  run; ';
+  put '  /* move meta to mac vars */ ';
+  put '  data _null_; ';
+  put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
+  put '    set &colinfo end=last nobs=nobs; ';
+  put '    name=upcase(name); ';
+  put '    /* fix formats */ ';
+  put '    if type=2 or type=6 then do; ';
+  put '      typelong=''char''; ';
+  put '      length fmt $49.; ';
+  put '      if format='''' then fmt=cats(''$'',length,''.''); ';
+  put '      else if formatl=0 then fmt=cats(format,''.''); ';
+  put '      else fmt=cats(format,formatl,''.''); ';
+  put '      newlen=max(formatl,length); ';
+  put '    end; ';
+  put '    else do; ';
+  put '      typelong=''num''; ';
+  put '      if format='''' then fmt=''best.''; ';
+  put '      else if formatl=0 then fmt=cats(format,''.''); ';
+  put '      else if formatd=0 then fmt=cats(format,formatl,''.''); ';
+  put '      else fmt=cats(format,formatl,''.'',formatd); ';
+  put '      /* needs to be wide, for datetimes etc */ ';
+  put '      newlen=max(length,formatl,24); ';
+  put '    end; ';
+  put '    /* 32 char unique name */ ';
+  put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put ' ';
+  put '    call symputx(cats(''name'',_n_),name,''l''); ';
+  put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '    call symputx(cats(''len'',_n_),newlen,''l''); ';
+  put '    call symputx(cats(''length'',_n_),length,''l''); ';
+  put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
+  put '    call symputx(cats(''type'',_n_),type,''l''); ';
+  put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
+  put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
+  put '  run; ';
+  put ' ';
+  put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put ' ';
+  put '  %if &engine=PROCJSON %then %do; ';
+  put '    %if &missing=STRING %then %do; ';
+  put '      %put &sysmacroname: Special Missings not supported in proc json.; ';
+  put '      %put &sysmacroname: Switching to DATASTEP engine; ';
+  put '      %goto datastep; ';
+  put '    %end; ';
+  put '    data &tempds;set &ds; ';
+  put '    %if &fmt=N %then format _numeric_ best32.;; ';
+  put '    /* PRETTY is necessary to avoid line truncation in large files */ ';
+  put '    proc json out=&jref pretty ';
+  put '        %if &action=ARR %then nokeys ; ';
+  put '        ;export &tempds / nosastags fmtnumeric; ';
+  put '    run; ';
+  put '  %end; ';
+  put '  %else %if &engine=DATASTEP %then %do; ';
+  put '    %datastep: ';
+  put '    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 ';
+  put '    %then %do; ';
+  put '      %put &sysmacroname:  &ds NOT FOUND!!!; ';
+  put '      %return; ';
+  put '    %end; ';
+  put ' ';
+  put '    %if &fmt=Y %then %do; ';
+  put '      data _data_; ';
+  put '        /* rename on entry */ ';
+  put '        set &ds(rename=( ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&name&i=&&newname&i ';
+  put '      %end; ';
+  put '        )); ';
+  put '      %do i=1 %to &numcols; ';
+  put '        length &&name&i $&&len&i; ';
+  put '        %if &&typelong&i=num %then %do; ';
+  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '        %end; ';
+  put '        %else %do; ';
+  put '          &&name&i=put(&&newname&i,&&fmt&i); ';
+  put '        %end; ';
+  put '        drop &&newname&i; ';
+  put '      %end; ';
+  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '      run; ';
+  put '      %let fmtds=&syslast; ';
+  put '    %end; ';
+  put ' ';
+  put '    proc format; /* credit yabwon for special null removal */ ';
+  put '    value bart (default=40) ';
+  put '    %if &missing=NULL %then %do; ';
+  put '      ._ - .z = null ';
+  put '    %end; ';
+  put '    %else %do; ';
+  put '      ._ = [quote()] ';
+  put '      . = null ';
+  put '      .a - .z = [quote()] ';
+  put '    %end; ';
+  put '      other = [best.]; ';
+  put ' ';
+  put '    data &tempds; ';
+  put '      attrib _all_ label=''''; ';
+  put '      %do i=1 %to &numcols; ';
+  put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
+  put '          length &&name&i $32767; ';
+  put '          format &&name&i $32767.; ';
+  put '        %end; ';
+  put '      %end; ';
+  put '      %if &fmt=Y %then %do; ';
+  put '        set &fmtds; ';
+  put '      %end; ';
+  put '      %else %do; ';
+  put '        set &ds; ';
+  put '      %end; ';
+  put '      format _numeric_ bart.; ';
+  put '    %do i=1 %to &numcols; ';
+  put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
+  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; ';
+  put '          &&name&i=''"''!!trim( ';
+  put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '            prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '            prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '            prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '            prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '            prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '          ))))))))))))!!''"''; ';
+  put '        end; ';
+  put '        else &&name&i=quote(cats(&&name&i)); ';
+  put '      %end; ';
+  put '    %end; ';
+  put '    run; ';
+  put ' ';
+  put '    /* write to temp loc to avoid _webout truncation ';
+  put '      - https://support.sas.com/kb/49/325.html */ ';
+  put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
+  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
+  put '      if _n_=1 then put "["; ';
+  put '      set &tempds; ';
+  put '      if _n_>1 then put "," @; put ';
+  put '      %if &action=ARR %then "[" ; %else "{" ; ';
+  put '      %do i=1 %to &numcols; ';
+  put '        %if &i>1 %then  "," ; ';
+  put '        %if &action=OBJ %then """&&name&i"":" ; ';
+  put '        "&&name&i"n /* name literal for reserved variable names */ ';
+  put '      %end; ';
+  put '      %if &action=ARR %then "]" ; %else "}" ; ; ';
+  put '    /* now write the long strings to _webout 1 byte at a time */ ';
+  put '    data _null_; ';
+  put '      length filein 8 fileid 8; ';
+  put '      filein=fopen("_sjs",''I'',1,''B''); ';
+  put '      fileid=fopen("&jref",''A'',1,''B''); ';
+  put '      rec=''20''x; ';
+  put '      do while(fread(filein)=0); ';
+  put '        rc=fget(filein,rec,1); ';
+  put '        rc=fput(fileid, rec); ';
+  put '        rc=fwrite(fileid); ';
+  put '      end; ';
+  put '      /* close out the table */ ';
+  put '      rc=fput(fileid, "]"); ';
+  put '      rc=fwrite(fileid); ';
+  put '      rc=fclose(filein); ';
+  put '      rc=fclose(fileid); ';
+  put '    run; ';
+  put '    filename _sjs clear; ';
+  put '  %end; ';
+  put ' ';
+  put '  proc sql; ';
+  put '  drop table &colinfo, &tempds; ';
+  put ' ';
+  put '  %if &showmeta=YES %then %do; ';
+  put '    data _null_; file &jref encoding=''utf-8'' mod; ';
+  put '      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; ';
+  put '      do i=1 to &numcols; ';
+  put '        name=quote(trim(symget(cats(''name'',i)))); ';
+  put '        format=quote(trim(symget(cats(''fmt'',i)))); ';
+  put '        label=quote(trim(symget(cats(''label'',i)))); ';
+  put '        length=quote(trim(symget(cats(''length'',i)))); ';
+  put '        type=quote(trim(symget(cats(''typelong'',i)))); ';
+  put '        if i>1 then put "," @@; ';
+  put '        put name '':{"format":'' format '',"label":'' label ';
+  put '          '',"length":'' length '',"type":'' type ''}''; ';
+  put '      end; ';
+  put '      put ''}}''; ';
+  put '    run; ';
+  put '  %end; ';
+  put '%end; ';
+  put ' ';
+  put '%else %if &action=CLOSE %then %do; ';
+  put '  data _null_; file &jref encoding=''utf-8'' mod ; ';
+  put '    put "}"; ';
+  put '  run; ';
+  put '%end; ';
+  put '%mend mp_jsonout; ';
+  put ' ';
+  put '%macro mf_getuser(type=META ';
+  put ')/*/STORE SOURCE*/; ';
+  put '  %local user metavar; ';
+  put '  %if &type=OS %then %let metavar=_secureusername; ';
+  put '  %else %let metavar=_metaperson; ';
+  put ' ';
+  put '  %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %let user=&SYS_COMPUTE_SESSION_OWNER; ';
+  put '  %else %if %symexist(&metavar) %then %do; ';
+  put '    %if %length(&&&metavar)=0 %then %let user=&sysuserid; ';
+  put '    /* sometimes SAS will add @domain extension - remove for consistency */ ';
+  put '    /* but be sure to quote in case of usernames with commas */ ';
+  put '    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@)); ';
+  put '  %end; ';
+  put '  %else %let user=&sysuserid; ';
+  put ' ';
+  put '  %quote(&user) ';
+  put ' ';
+  put '%mend mf_getuser; ';
+  put ' ';
+  put '%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
+  put '  ,showmeta=NO ';
+  put '); ';
+  put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
+  put '  sasjs_tables; ';
+  put ' ';
+  put '%local i tempds; ';
+  put '%let action=%upcase(&action); ';
+  put ' ';
+  put '%if &action=FETCH %then %do; ';
+  put '  %if %str(&_debug) ge 131 %then %do; ';
+  put '    options mprint notes mprintnest; ';
+  put '  %end; ';
+  put '  %let _webin_file_count=%eval(&_webin_file_count+0); ';
+  put '  /* now read in the data */ ';
+  put '  %do i=1 %to &_webin_file_count; ';
+  put '    %if &_webin_file_count=1 %then %do; ';
+  put '      %let _webin_fileref1=&_webin_fileref; ';
+  put '      %let _webin_name1=&_webin_name; ';
+  put '    %end; ';
+  put '    data _null_; ';
+  put '      infile &&_webin_fileref&i termstr=crlf lrecl=32767; ';
+  put '      input; ';
+  put '      call symputx(''input_statement'',_infile_); ';
+  put '      putlog "&&_webin_name&i input statement: "  _infile_; ';
+  put '      stop; ';
+  put '    data &&_webin_name&i; ';
+  put '      infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding=''utf-8'' ';
+  put '        lrecl=32767; ';
+  put '      input &input_statement; ';
+  put '      %if %str(&_debug) ge 131 %then %do; ';
+  put '        if _n_<20 then putlog _infile_; ';
+  put '      %end; ';
+  put '    run; ';
+  put '    %let sasjs_tables=&sasjs_tables &&_webin_name&i; ';
+  put '  %end; ';
+  put '%end; ';
+  put ' ';
+  put '%else %if &action=OPEN %then %do; ';
+  put '  /* fix encoding and ensure enough lrecl */ ';
+  put '  OPTIONS NOBOMFILE lrecl=32767; ';
+  put ' ';
+  put '  /* set the header */ ';
+  put '  %mfs_httpheader(Content-type,application/json) ';
+  put ' ';
+  put '  /* setup json.  */ ';
+  put '  data _null_;file &fref encoding=''utf-8'' termstr=lf ; ';
+  put '    put ''{"SYSDATE" : "'' "&SYSDATE" ''"''; ';
+  put '    put '',"SYSTIME" : "'' "&SYSTIME" ''"''; ';
+  put '  run; ';
+  put ' ';
+  put '%end; ';
+  put ' ';
+  put '%else %if &action=ARR or &action=OBJ %then %do; ';
+  put '  %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do; ';
+  put '    /* functions in formats unsupported */ ';
+  put '    %put &sysmacroname: forcing missing back to NULL as feature not supported; ';
+  put '    %let missing=NULL; ';
+  put '  %end; ';
+  put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
+  put '    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta ';
+  put '  ) ';
+  put '%end; ';
+  put '%else %if &action=CLOSE %then %do; ';
+  put '  %if %str(&_debug) ge 131 %then %do; ';
+  put '    /* if debug mode, send back first 10 records of each work table also */ ';
+  put '    options obs=10; ';
+  put '    data;run;%let tempds=%scan(&syslast,2,.); ';
+  put '    ods output Members=&tempds; ';
+  put '    proc datasets library=WORK memtype=data; ';
+  put '    %local wtcnt;%let wtcnt=0; ';
+  put '    data _null_; ';
+  put '      set &tempds; ';
+  put '      if not (upcase(name) =:"DATA"); /* ignore temp datasets */ ';
+  put '      if not (upcase(name)=:"_DATA_"); ';
+  put '      i+1; ';
+  put '      call symputx(cats(''wt'',i),name,''l''); ';
+  put '      call symputx(''wtcnt'',i,''l''); ';
+  put '    data _null_; file &fref mod encoding=''utf-8'' termstr=lf; ';
+  put '      put ",""WORK"":{"; ';
+  put '    %do i=1 %to &wtcnt; ';
+  put '      %let wt=&&wt&i; ';
+  put '      data _null_; file &fref mod encoding=''utf-8'' termstr=lf; ';
+  put '        dsid=open("WORK.&wt",''is''); ';
+  put '        nlobs=attrn(dsid,''NLOBS''); ';
+  put '        nvars=attrn(dsid,''NVARS''); ';
+  put '        rc=close(dsid); ';
+  put '        if &i>1 then put '',''@; ';
+  put '        put " ""&wt"" : {"; ';
+  put '        put ''"nlobs":'' nlobs; ';
+  put '        put '',"nvars":'' nvars; ';
+  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) ';
+  put '      data _null_; file &fref mod encoding=''utf-8'' termstr=lf; ';
+  put '        put "}"; ';
+  put '    %end; ';
+  put '    data _null_; file &fref mod encoding=''utf-8'' termstr=lf; ';
+  put '      put "}"; ';
+  put '    run; ';
+  put '  %end; ';
+  put '  /* close off json */ ';
+  put '  data _null_;file &fref mod encoding=''utf-8'' termstr=lf lrecl=32767; ';
+  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
+  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
+  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
+  put '    put ",""_DEBUG"" : ""&_debug"" "; ';
+  put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
+  put '    put ",""SYSCC"" : ""&syscc"" "; ';
+  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
+  put '    SYSHOSTINFOLONG=quote(trim(symget(''SYSHOSTINFOLONG''))); ';
+  put '    put '',"SYSHOSTINFOLONG" : '' SYSHOSTINFOLONG; ';
+  put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
+  put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
+  put '    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; ';
+  put '    length SYSPROCESSNAME $512; ';
+  put '    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); ';
+  put '    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; ';
+  put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
+  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
+  put '    put ",""SYSSITE"" : ""&syssite"" "; ';
+  put '    put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" "; ';
+  put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
+  put '    put '',"SYSVLONG" : '' sysvlong; ';
+  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
+  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
+  put '    length autoexec $512; ';
+  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
+  put '    put '',"AUTOEXEC" : '' autoexec; ';
+  put '    length memsize $32; ';
+  put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
+  put '    memsize=quote(cats(memsize)); ';
+  put '    put '',"MEMSIZE" : '' memsize; ';
+  put '    put "}" @; ';
+  put '  run; ';
+  put '%end; ';
+  put ' ';
+  put '%mend ms_webout; ';
+  put ' ';
+  put '%macro mfs_httpheader(header_name ';
+  put '  ,header_value ';
+  put ')/*/STORE SOURCE*/; ';
+  put '%global sasjs_stpsrv_header_loc; ';
+  put '%local fref fid i; ';
+  put ' ';
+  put '%if %sysfunc(filename(fref,&sasjs_stpsrv_header_loc)) ne 0 %then %do; ';
+  put '  %put &=fref &=sasjs_stpsrv_header_loc; ';
+  put '  %put %str(ERR)OR: %sysfunc(sysmsg()); ';
+  put '  %return; ';
+  put '%end; ';
+  put ' ';
+  put '%let fid=%sysfunc(fopen(&fref,A)); ';
+  put ' ';
+  put '%if &fid=0 %then %do; ';
+  put '  %put %str(ERR)OR: %sysfunc(sysmsg()); ';
+  put '  %return; ';
+  put '%end; ';
+  put ' ';
+  put '%let rc=%sysfunc(fput(&fid,%str(&header_name): %str(&header_value))); ';
+  put '%let rc=%sysfunc(fwrite(&fid)); ';
+  put ' ';
+  put '%let rc=%sysfunc(fclose(&fid)); ';
+  put '%let rc=%sysfunc(filename(&fref)); ';
+  put ' ';
+  put '%mend mfs_httpheader; ';
+/* WEBOUT END */
+  put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO);';
+  put '  %ms_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt,missing=&missing';
+  put '    ,showmeta=&showmeta';
+  put '  )';
+  put '%mend;';
+run;
+
+/* add precode and code */
+%local x fref freflist;
+%let freflist=&precode &code ;
+%do x=1 %to %sysfunc(countw(&freflist));
+  %let fref=%scan(&freflist,&x);
+  %put &sysmacroname: adding &fref;
+  data _null_;
+    file &sasjsref lrecl=3000 termstr=crlf mod;
+    infile &fref lrecl=3000;
+    input;
+    put _infile_;
+  run;
+%end;
+
+/* create the web service */
+%ms_createfile(&path/&name..sas, inref=&sasjsref, mdebug=&mdebug)
+
+%put ;%put ;%put ;%put ;%put ;%put ;
+%put &sysmacroname: STP &name successfully created in &path;
+%put ;%put ;%put ;
+%put Check it out here:;
+%put ;%put ;%put ;
+%put &_sasjs_apiserverurl.&_sasjs_apipath?_PROGRAM=&path/&name;
+%put ;%put ;%put ;%put ;%put ;%put ;
+
+%mend ms_createwebservice;

server/ms_deletefile.sas

@@ -0,0 +1,48 @@
+/**
+  @file
+  @brief Deletes a file from SASjs Drive
+  @details Deletes a file from SASjs Drive, if it exists.
+
+  Example:
+
+      filename stpcode temp;
+      data _null_;
+        file stpcode;
+        put '%put hello world;';
+      run;
+      %ms_createfile(/some/stored/program.sas, inref=stpcode)
+
+      %ms_deletefile(/some/stored/program.sas)
+
+  @param [in] driveloc The full path to the file in SASjs Drive
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+
+**/
+
+%macro ms_deletefile(driveloc
+    ,mdebug=0
+  );
+
+%local headref;
+%let headref=%mf_getuniquefileref();
+
+data _null_;
+  file &headref lrecl=1000;
+  infile "&_sasjs_tokenfile" lrecl=1000;
+  input;
+  put _infile_;
+run;
+
+proc http method='DELETE' headerin=&headref
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file?_filePath=&driveloc";
+%if &mdebug=1 %then %do;
+  debug level=2;
+%end;
+run;
+
+filename &headref clear;
+
+%mend ms_deletefile;

server/ms_getfile.sas

@@ -0,0 +1,50 @@
+/**
+  @file
+  @brief Gets a file from SASjs Drive
+  @details Fetches a file on SASjs Drive and stores it in the output fileref.
+
+  Example:
+
+      %ms_getfile(/Public/app/dc/services/public/settings.sas, outref=myfile)
+
+  @param [in] driveloc The full path to the file in SASjs Drive
+  @param [out] outref= (msgetfil) The fileref to contain the file.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+
+**/
+
+%macro ms_getfile(driveloc
+    ,outref=msgetfil
+    ,mdebug=0
+  );
+
+/* use the recfm in a separate fileref to avoid issues with subsequent reads */
+%local binaryfref floc headref;
+%let binaryfref=%mf_getuniquefileref();
+%let headref=%mf_getuniquefileref();
+%let floc=%sysfunc(pathname(work))/%mf_getuniquename().txt;
+filename &outref "&floc" lrecl=32767;
+filename &binaryfref "&floc" recfm=n;
+
+data _null_;
+  file &headref lrecl=1000;
+  infile "&_sasjs_tokenfile" lrecl=1000;
+  input;
+  put _infile_;
+run;
+
+proc http method='GET' out=&binaryfref headerin=&headref
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file?_filePath=&driveloc";
+%if &mdebug=1 %then %do;
+  debug level=2;
+%end;
+run;
+
+filename &binaryfref clear;
+filename &headref clear;
+
+%mend ms_getfile;

server/ms_getgroups.sas

@@ -0,0 +1,105 @@
+/**
+  @file
+  @brief Fetches the list of groups from SASjs Server
+  @details Fetches the list of groups from SASjs Server and writes them to an
+  output dataset.
+
+  Example:
+
+      %ms_getgroups(outds=userlist)
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outds= (work.ms_getgroups) This output dataset will contain the
+    list of groups. Format:
+|NAME:$32.|DESCRIPTION:$64.|GROUPID:best.|
+|---|---|---|
+|`SomeGroup `|`A group `|`1`|
+|`Another Group`|`this is a different group`|`2`|
+|`admin`|`Administrators `|`3`|
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mp_abort.sas
+
+  <h4> Related Files </h4>
+  @li ms_creategroup.sas
+  @li ms_getusers.test.sas
+
+**/
+
+%macro ms_getgroups(
+    outds=work.ms_getgroups
+    ,mdebug=0
+  );
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_getusers.sas
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+
+%local fref0 fref1 libref optval rc msg;
+%let fref0=%mf_getuniquefileref();
+%let fref1=%mf_getuniquefileref();
+%let libref=%mf_getuniquelibref();
+
+/* avoid sending bom marker to API */
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
+data _null_;
+  file &fref0 lrecl=1000;
+  infile "&_sasjs_tokenfile" lrecl=1000;
+  input;
+  if _n_=1 then put "accept: application/json";
+  put _infile_;
+run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fref0;
+    input;
+    put _infile_;
+  run;
+%end;
+
+proc http method='GET' headerin=&fref0 out=&fref1
+  url="&_sasjs_apiserverurl/SASjsApi/group";
+%if &mdebug=1 %then %do;
+  debug level=1;
+%end;
+run;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_getgroups.sas
+  ,msg=%str(Issue submitting GET query to SASjsApi/group)
+)
+
+libname &libref JSON fileref=&fref1;
+
+data &outds;
+  length NAME $32 DESCRIPTION $64. GROUPID 8;
+  if _n_=1 then call missing(of _all_);
+  set &libref..root;
+  drop ordinal_root;
+run;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_getusers.sas
+  ,msg=%str(Issue reading response JSON)
+)
+
+/* reset options */
+options &optval;
+
+%if &mdebug=1 %then %do;
+  filename &fref0 clear;
+  filename &fref1 clear;
+  libname &libref clear;
+%end;
+
+%mend ms_getgroups;

server/ms_getusers.sas

@@ -0,0 +1,107 @@
+/**
+  @file
+  @brief Fetches the list of users from SASjs Server
+  @details Fetches the list of users from SASjs Server and writes them to an
+  output dataset.
+
+  Example:
+
+      %ms_getusers(outds=userlist)
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outds= (work.ms_getusers) This output dataset will contain the
+    list of user accounts. Format:
+|DISPLAYNAME:$18.|USERNAME:$10.|ID:best.|
+|---|---|---|
+|`Super Admin `|`secretuser `|`1`|
+|`Sabir Hassan`|`sabir`|`2`|
+|`Mihajlo Medjedovic `|`mihajlo `|`3`|
+|`Ivor Townsend `|`ivor `|`4`|
+|`New User `|`newuser `|`5`|
+
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mp_abort.sas
+
+  <h4> Related Files </h4>
+  @li ms_createuser.sas
+  @li ms_getusers.test.sas
+
+**/
+
+%macro ms_getusers(
+    outds=work.ms_getusers
+    ,mdebug=0
+  );
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_getusers.sas
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+
+%local fref0 fref1 libref optval rc msg;
+%let fref0=%mf_getuniquefileref();
+%let fref1=%mf_getuniquefileref();
+%let libref=%mf_getuniquelibref();
+
+/* avoid sending bom marker to API */
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
+data _null_;
+  file &fref0 lrecl=1000;
+  infile "&_sasjs_tokenfile" lrecl=1000;
+  input;
+  if _n_=1 then put "accept: application/json";
+  put _infile_;
+run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fref0;
+    input;
+    put _infile_;
+  run;
+%end;
+
+proc http method='GET' headerin=&fref0 out=&fref1
+  url="&_sasjs_apiserverurl/SASjsApi/user";
+%if &mdebug=1 %then %do;
+  debug level=1;
+%end;
+run;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_getusers.sas
+  ,msg=%str(Issue submitting GET query to SASjsApi/user)
+)
+
+libname &libref JSON fileref=&fref1;
+
+data &outds;
+  set &libref..root;
+  drop ordinal_root;
+run;
+
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_getusers.sas
+  ,msg=%str(Issue reading response JSON)
+)
+
+/* reset options */
+options &optval;
+
+%if &mdebug=1 %then %do;
+  filename &fref0 clear;
+  filename &fref1 clear;
+  libname &libref clear;
+%end;
+
+%mend ms_getusers;

server/ms_runstp.sas

@@ -0,0 +1,199 @@
+/**
+  @file
+  @brief Executes a SASjs Server Stored Program
+  @details Runs a Stored Program (using POST method) and extracts the webout and
+  log from the response JSON.
+
+  Example:
+
+      %ms_runstp(/some/stored/program
+        ,debug=131
+        ,outref=weboot
+      )
+
+  @param [in] pgm The full path to the Stored Program in SASjs Drive (_program
+    parameter)
+  @param [in] debug= (131) The value to supply to the _debug URL parameter
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] inputparams=(_null_) A dataset containing name/value pairs in the
+    following format:
+    |name:$32|value:$10000|
+    |---|---|
+    |stpmacname|some value|
+    |mustbevalidname|can be anything, oops, %abort!!|
+  @param [in] inputfiles= (_null_) A dataset containing fileref/name/filename in
+    the following format:
+    |fileref:$8|name:$32|filename:$256|
+    |---|---|--|
+    |someref|some_name|some_filename.xls|
+    |fref2|another_file|zyx_v2.csv|
+
+  @param [out] outref= (outweb) The output fileref to contain the response JSON
+    (will be created using temp engine)
+  @param [out] outlogds= (_null_) Set to the name of a dataset to contain the
+    log. Table format:
+    |line:$2000|
+    |---|
+    |log line 1|
+    |log line 2|
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mp_abort.sas
+
+**/
+
+%macro ms_runstp(pgm
+    ,debug=131
+    ,inputparams=_null_
+    ,inputfiles=_null_
+    ,outref=outweb
+    ,outlogds=_null_
+    ,mdebug=0
+  );
+%local dbg mainref authref boundary;
+%let mainref=%mf_getuniquefileref();
+%let authref=%mf_getuniquefileref();
+%let boundary=%mf_getuniquename();
+%if &inputparams=0 %then %let inputparams=_null_;
+
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+
+%mp_abort(iftrue=("&pgm"="")
+  ,mac=&sysmacroname
+  ,msg=%str(Program not provided)
+)
+
+/* avoid sending bom marker to API */
+%local optval;
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
+/* add params */
+data _null_;
+  file &mainref termstr=crlf lrecl=32767 mod;
+  length line $1000 name $32 value $32767;
+  if _n_=1 then call missing(of _all_);
+  set &inputparams;
+  put "--&boundary";
+  line=cats('Content-Disposition: form-data; name="',name,'"');
+  put line;
+  put ;
+  put value;
+run;
+
+/* parse input file list */
+%local webcount;
+%let webcount=0;
+data _null_;
+  set &inputfiles end=last;
+  length fileref $8 name $32 filename $256;
+  call symputx(cats('webref',_n_),fileref,'l');
+  call symputx(cats('webname',_n_),name,'l');
+  call symputx(cats('webfilename',_n_),filename,'l');
+  if last then do;
+    call symputx('webcount',_n_);
+    call missing(of _all_);
+  end;
+run;
+
+/* write out the input files */
+%local i;
+%do i=1 %to &webcount;
+  data _null_;
+    file &mainref termstr=crlf lrecl=32767 mod;
+    infile &&webref&i lrecl=32767;
+    if _n_ = 1 then do;
+      length line $32767;
+      line=cats(
+        'Content-Disposition: form-data; name="'
+        ,"&&webname&i"
+        ,'"; filename="'
+        ,"&&webfilename&i"
+        ,'"'
+      );
+      put "--&boundary";
+      put line;
+      put "Content-Type: text/plain";
+      put ;
+    end;
+    input;
+    put _infile_; /* add the actual file to be sent */
+  run;
+%end;
+
+data _null_;
+  file &mainref termstr=crlf mod;
+  put "--&boundary--";
+run;
+
+data _null_;
+  file &authref lrecl=1000;
+  infile "&_sasjs_tokenfile" lrecl=1000;
+  input;
+  if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary";
+  put _infile_;
+run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &authref;
+    input;
+    put _infile_;
+  data _null_;
+    infile &mainref;
+    input;
+    put _infile_;
+  run;
+%end;
+
+filename &outref temp lrecl=32767;
+/* prepare request*/
+proc http method='POST' headerin=&authref in=&mainref out=&outref
+  url="&_sasjs_apiserverurl.&_sasjs_apipath?_program=&pgm%str(&)_debug=131";
+%if &mdebug=1 %then %do;
+  debug level=2;
+%end;
+run;
+%if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+or &mdebug=1
+%then %do;
+  data _null_;infile &outref;input;putlog _infile_;run;
+%end;
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  ,mac=&sysmacroname
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+/* reset options */
+options &optval;
+
+%if &outlogds ne _null_ or &mdebug=1 %then %do;
+  %local dumplib;
+  %let dumplib=%mf_getuniquelibref();
+  libname &dumplib json fileref=&outref;
+  data &outlogds;
+    set &dumplib..log;
+  %if &mdebug=1 %then %do;
+    putlog line=;
+  %end;
+  run;
+%end;
+
+%if &mdebug=1 %then %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%else %do;
+  /* clear refs */
+  filename &authref;
+  filename &mainref;
+%end;
+%mend ms_runstp;

server/ms_testservice.sas

@@ -0,0 +1,154 @@
+/**
+  @file
+  @brief Will execute a SASjs web service on SASjs Server
+  @details Prepares the input files and retrieves the resulting datasets from
+  the response JSON.
+
+  @param [in] program The Stored Program endpoint to test
+  @param [in] inputfiles=(0) A list of space seperated fileref:filename pairs as
+    follows:
+        inputfiles=inref:filename inref2:filename2
+  @param [in] inputdatasets= (0) All datasets in this space seperated list are
+    converted into SASJS-formatted CSVs (see mp_ds2csv.sas) files and added to
+    the list of `inputfiles` for ingestion.  The dataset will be sent with the
+    same name (no need for a colon modifier).
+  @param [in] inputparams=(0) A dataset containing name/value pairs in the
+    following format:
+    |name:$32|value:$1000|
+    |---|---|
+    |stpmacname|some value|
+    |mustbevalidname|can be anything, oops, %abort!!|
+
+  @param [in] debug= (131) Provide the _debug value to pass to the STP
+  @param [in] mdebug= (0) Set to 1 to provide macro debugging (this macro)
+  @param [out] outlib= (0) Output libref to contain the final tables.  Set to
+    0 if the service output is not in JSON format.
+  @param [out] outref= (0) Output fileref to create, to contain the full _webout
+    response.
+  @param [out] outlogds= (_null_) Set to the name of a dataset to contain the
+    log. Table format:
+    |line:$2000|
+    |---|
+    |log line 1|
+    |log line 2|
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+  @li mp_chop.sas
+  @li mp_ds2csv.sas
+  @li ms_runstp.sas
+
+  <h4> Related Programs </h4>
+  @li mp_testservice.test.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro ms_testservice(program,
+  inputfiles=0,
+  inputdatasets=0,
+  inputparams=0,
+  debug=0,
+  mdebug=0,
+  outlib=0,
+  outref=0,
+  outlogds=_null_
+)/*/STORE SOURCE*/;
+%local dbg i var ds1 fref1 chopout1 chopout2;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+/* convert inputdatasets to filerefs */
+%if "&inputdatasets" ne "0" %then %do;
+  %if %quote(&inputfiles)=0 %then %let inputfiles=;
+  %do i=1 %to %sysfunc(countw(&inputdatasets,%str( )));
+    %let var=%scan(&inputdatasets,&i,%str( ));
+    %local dsref&i;
+    %let dsref&i=%mf_getuniquefileref();
+    %mp_ds2csv(&var,outref=&&dsref&i,headerformat=SASJS)
+    %let inputfiles=&inputfiles &&dsref&i:%scan(&var,-1,.);
+  %end;
+%end;
+
+/* parse the filerefs - convert to a dataset */
+%let ds1=%mf_getuniquename();
+data &ds1;
+  length fileref $8 name $32 filename $256 var $300;
+  if "&inputfiles" ne "0" then do;
+    webcount=countw("&inputfiles");
+    do i=1 to webcount;
+      var=scan("&inputfiles",i,' ');
+      fileref=scan(var,1,':');
+      name=scan(var,2,':');
+      filename=cats(name,'.csv');
+      output;
+    end;
+  end;
+run;
+
+
+/* execute the STP */
+%let fref1=%mf_getuniquefileref();
+
+%ms_runstp(&program
+  ,debug=&debug
+  ,inputparams=&inputparams
+  ,inputfiles=&ds1
+  ,outref=&fref1
+  ,mdebug=&mdebug
+  ,outlogds=&outlogds
+)
+
+
+/* SASjs services have the _webout embedded in wrapper JSON */
+/* Files can also be very large - so use a dedicated macro to chop it out */
+%local matchstr1 matchstr2 ;
+%let matchstr1={"status":"success","_webout":{;
+%let matchstr2=},"log":[{;
+%let chopout1=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop1);
+%let chopout2=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop2);
+
+%mp_chop("%sysfunc(pathname(&fref1,F))"
+  ,matchvar=matchstr1
+  ,keep=LAST
+  ,matchpoint=END
+  ,offset=-1
+  ,outfile="&chopout1"
+  ,mdebug=&mdebug
+)
+
+%mp_chop("&chopout1"
+  ,matchvar=matchstr2
+  ,keep=FIRST
+  ,matchpoint=START
+  ,offset=1
+  ,outfile="&chopout2"
+  ,mdebug=&mdebug
+)
+
+%if &outlib ne 0 %then %do;
+  libname &outlib json "&chopout2";
+%end;
+%if &outref ne 0 %then %do;
+  filename &outref "&chopout2";
+%end;
+
+%if &mdebug=0 %then %do;
+  filename &webref clear;
+  filename &fref1 clear;
+  filename &fref2 clear;
+%end;
+%else %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+
+%mend ms_testservice;

server/ms_webout.sas

@@ -1,6 +1,6 @@
 /**
   @file
-  @brief Send data to/from @sasjs/server
+  @brief Send data to/from sasjs/server
   @details This macro should be added to the start of each web service,
   **immediately** followed by a call to:
 
@@ -33,8 +33,9 @@
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
 
   <h4> SAS Macros </h4>
-  @li mp_jsonout.sas
   @li mf_getuser.sas
+  @li mp_jsonout.sas
+  @li mfs_httpheader.sas
 
   <h4> Related Macros </h4>
   @li mv_webout.sas
@@ -66,13 +67,14 @@
       %let _webin_name1=&_webin_name;
     %end;
     data _null_;
-      infile &&_webin_fileref&i termstr=crlf;
+      infile &&_webin_fileref&i termstr=crlf lrecl=32767;
       input;
       call symputx('input_statement',_infile_);
       putlog "&&_webin_name&i input statement: "  _infile_;
       stop;
     data &&_webin_name&i;
-      infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding='utf-8';
+      infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding='utf-8'
+        lrecl=32767;
       input &input_statement;
       %if %str(&_debug) ge 131 %then %do;
         if _n_<20 then putlog _infile_;
@@ -83,14 +85,14 @@
 %end;
 
 %else %if &action=OPEN %then %do;
-  /* fix encoding */
-  OPTIONS NOBOMFILE;
+  /* fix encoding and ensure enough lrecl */
+  OPTIONS NOBOMFILE lrecl=32767;
 
-  /* setup json */
-  data _null_;file &fref encoding='utf-8' termstr=lf;
-  %if %str(&_debug) ge 131 %then %do;
-    put '>>weboutBEGIN<<';
-  %end;
+  /* set the header */
+  %mfs_httpheader(Content-type,application/json)
+
+  /* setup json.  */
+  data _null_;file &fref encoding='utf-8' termstr=lf ;
     put '{"SYSDATE" : "' "&SYSDATE" '"';
     put ',"SYSTIME" : "' "&SYSTIME" '"';
   run;
@@ -98,6 +100,11 @@
 %end;
 
 %else %if &action=ARR or &action=OBJ %then %do;
+  %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do;
+    /* functions in formats unsupported */
+    %put &sysmacroname: forcing missing back to NULL as feature not supported;
+    %let missing=NULL;
+  %end;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
     ,engine=DATASTEP,missing=&missing,showmeta=&showmeta
   )
@@ -113,6 +120,7 @@
     data _null_;
       set &tempds;
       if not (upcase(name) =:"DATA"); /* ignore temp datasets */
+      if not (upcase(name)=:"_DATA_");
       i+1;
       call symputx(cats('wt',i),name,'l');
       call symputx('wtcnt',i,'l');
@@ -133,19 +141,20 @@
       data _null_; file &fref mod encoding='utf-8' termstr=lf;
         put "}";
     %end;
-    data _null_; file &fref mod encoding='utf-8' termstr=lf termstr=lf;
+    data _null_; file &fref mod encoding='utf-8' termstr=lf;
       put "}";
     run;
   %end;
   /* close off json */
-  data _null_;file &fref mod encoding='utf-8' termstr=lf;
+  data _null_;file &fref mod encoding='utf-8' termstr=lf lrecl=32767;
     _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ",""SYSUSERID"" : ""&sysuserid"" ";
     put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""_DEBUG"" : ""&_debug"" ";
     put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
     SYSHOSTINFOLONG=quote(trim(symget('SYSHOSTINFOLONG')));
     put ',"SYSHOSTINFOLONG" : ' SYSHOSTINFOLONG;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
@@ -160,7 +169,8 @@
     put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length autoexec $512;
     autoexec=quote(urlencode(trim(getoption('autoexec'))));
@@ -170,9 +180,6 @@
     memsize=quote(cats(memsize));
     put ',"MEMSIZE" : ' memsize;
     put "}" @;
-  %if %str(&_debug) ge 131 %then %do;
-    put '>>weboutEND<<';
-  %end;
   run;
 %end;
 

tests/base/mf_deletefile.test.sas

@@ -0,0 +1,29 @@
+/**
+  @file
+  @brief Testing mf_deletefile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_deletefile.sas
+  @li mf_writefile.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%let test1file=&sasjswork/myfile1.txt;
+
+%mf_writefile(&test1file,l1=some content)
+
+%mp_assert(
+  iftrue=(%sysfunc(fileexist(&test1file))=1),
+  desc=Check &test1file exists
+)
+
+%mp_assertscope(SNAPSHOT)
+%mf_deletefile(&test1file)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(%sysfunc(fileexist(&test1file))=0),
+  desc=Check &test1file no longer exists
+)

tests/base/mf_fmtdttm.test.sas

@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Testing mf_fmtdttm macro
+
+  <h4> SAS Macros </h4>
+  @li mf_fmtdttm.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%global test1;
+
+%mp_assertscope(SNAPSHOT)
+%let test1=%mf_fmtdttm();
+%mp_assertscope(COMPARE,ignorelist=test1)
+
+%mp_assert(
+  iftrue=("&test1"="DATETIME19.3" or "&test1"="E8601DT26.6"),
+  desc=Basic test,
+  outds=work.test_results
+)