Merge pull request #323 from sasjs/ms_getgroups

fix: increasing desc length to 256 in ms_getgroups

.all-contributorsrc

@@ -117,6 +117,24 @@
       "contributions": [
         "bug"
       ]
+    },
+    {
+      "login": "yabwon",
+      "name": "Bart Jablonski",
+      "avatar_url": "https://avatars.githubusercontent.com/u/9314894?v=4",
+      "profile": "https://github.com/yabwon",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "eltociear",
+      "name": "Ikko Ashimine",
+      "avatar_url": "https://avatars.githubusercontent.com/u/22633385?v=4",
+      "profile": "https://bandism.net/",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.devcontainer/Dockerfile

@@ -0,0 +1,8 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/javascript-node/.devcontainer/base.Dockerfile
+
+# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
+ARG VARIANT="18-bullseye"
+FROM mcr.microsoft.com/vscode/devcontainers/javascript-node:0-${VARIANT}
+
+RUN apt-get update \
+    && apt-get install -y doxygen

.devcontainer/devcontainer.json

@@ -0,0 +1,26 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/typescript-node
+{
+	"name": "Node.js & TypeScript",
+	"build": {
+		"dockerfile": "Dockerfile",
+		// Update 'VARIANT' to pick a Node version: 18, 16, 14.
+		// Append -bullseye or -buster to pin to an OS version.
+		// Use -bullseye variants on local on arm64/Apple Silicon.
+		"args": {
+			"VARIANT": "16-bullseye"
+		}
+	},
+	// Set *default* container specific settings.json values on container create.
+	"settings": {},
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"SASjs.sasjs-for-vscode"
+	],
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "npm i && npm i -g @sasjs/cli",
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "node"
+}

.git-hooks/pre-commit

@@ -1,5 +1,12 @@
 #!/bin/bash
-sasjs lint
+
+# Ensure lint is passing
+LINT=`sasjs lint`
+if [[ "$LINT" != "✔ All matched files use @sasjs/lint code style!" ]]; then
+  echo "$LINT"
+  echo "To commit in spite of these warnings, use the -n parameter."
+  exit 1
+fi
 
 # Avoid commits to the master branch
 BRANCH=`git rev-parse --abbrev-ref HEAD`

.github/CONTRIBUTING.md

@@ -12,7 +12,7 @@ This repository makes use of the [SASjs](https://sasjs.io) framework for code or
 * [VSCode](https://sasjs.io/windows/#vscode) - feature packed IDE for code editing (warning - highly effective!)
 * [GIT](https://sasjs.io/windows/#git) - a safety net you cannot (and should not) do without.
 
-For generating the documentation (`sasjs doc`) it is also necessary to install [doxygen](https://www.doxygen.nl/manual/install.html).
+For generating the documentation (`sasjs doc`) it is also necessary to install [doxygen](https://www.doxygen.nl/manual/install.html) and GraphViz (`sudo port install graphviz` on mac, or `sudo apt-get install graphviz` on Ubuntu).
 
 
 To get configured:

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [sasjs]

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,17 @@
+## Issue
+
+Link any related issue(s) in this section.
+
+## Intent
+
+What this PR intends to achieve.
+
+## Implementation
+
+What code changes have been made to achieve the intent.
+
+## Checks
+
+- [ ] Code is formatted correctly (`sasjs lint`).
+- [ ] Any new functionality has been unit tested.
+- [ ] All unit tests are passing (`sasjs test`).

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+  - package-ecosystem: npm
+    directory: '/'
+    schedule:
+      interval: monthly
+    open-pull-requests-limit: 3
+    allow:
+      - dependency-type: "production"

.github/vpn/config.ovpn

@@ -1,30 +0,0 @@
-cipher AES-256-CBC
-setenv FORWARD_COMPATIBLE 1
-client
-server-poll-timeout 4
-nobind
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 443 tcp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-dev tun
-dev-type tun
-ns-cert-type server
-setenv opt tls-version-min 1.0 or-highest
-reneg-sec 604800
-sndbuf 0
-rcvbuf 0
-# NOTE: LZO commands are pushed by the Access Server at connect time.
-# NOTE: The below line doesn't disable LZO.
-comp-lzo no
-verb 3
-setenv PUSH_PEER_INFO
-
-ca ca.crt
-cert user.crt
-key user.key
-tls-auth tls.key 1

.github/workflows/main.yml

@@ -19,3 +19,10 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+      - name: SAS Packages Release
+        run: |
+          npx @sasjs/cli compile job -s sasjs/utils/create_sas_package.sas -o sasjsbuild
+          # this part depends on https://github.com/sasjs/server/issues/307
+          # sasjs run sasjsbuild/jobs/utils/create_sas_package.sas -t sas9
+
+

.github/workflows/run-tests.yml

@@ -21,31 +21,6 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
 
-      - name: Write VPN Files
-        run: |
-          echo "$CA_CRT" > .github/vpn/ca.crt
-          echo "$USER_CRT" > .github/vpn/user.crt
-          echo "$USER_KEY" > .github/vpn/user.key
-          echo "$TLS_KEY" > .github/vpn/tls.key
-        shell: bash
-        env:
-          CA_CRT: ${{ secrets.CA_CRT}}
-          USER_CRT: ${{ secrets.USER_CRT }}
-          USER_KEY: ${{ secrets.USER_KEY }}
-          TLS_KEY: ${{ secrets.TLS_KEY }}
-
-      - name: Install Open VPN
-        run: |
-          sudo apt install apt-transport-https
-          sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
-          sudo apt-key add openvpn-repo-pkg-key.pub
-          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-focal.list
-          sudo apt update
-          sudo apt install openvpn3
-
-      - name: Start Open VPN 3
-        run: openvpn3 session-start --config .github/vpn/config.ovpn
-
       - name: Install Doxygen
         run: sudo apt-get install doxygen
 
@@ -78,7 +53,5 @@ jobs:
           SECRET: ${{secrets.SECRET}}
           SAS_USERNAME: ${{secrets.SAS_USERNAME}}
           SAS_PASSWORD: ${{secrets.SAS_PASSWORD}}
-          SERVER_URL: ${{secrets.SERVER_URL}}
-          SERVER_TYPE: ${{secrets.SERVER_TYPE}}
           ACCESS_TOKEN: ${{secrets.ACCESS_TOKEN}}
           REFRESH_TOKEN: ${{secrets.REFRESH_TOKEN}}

.gitignore

@@ -10,4 +10,6 @@ sasjsresults/
 mc_*
 
 # ignore .env files as they can contain sasjs access tokens
-*.env*
+*.env*
+
+~

.gitpod.dockerfile

@@ -1,6 +1,6 @@
 FROM gitpod/workspace-full
 
 RUN sudo apt-get update \
- && sudo apt-get install -y \
-    doxygen \
+ && sudo apt-get install -y doxygen \
+ && sudo apt-get install -y graphviz \
  && sudo rm -rf /var/lib/apt/lists/*

.gitpod.yml

@@ -1,5 +1,7 @@
 tasks:
-  - init: nvm install --lts && npm i -g @sasjs/cli
+  - init: npm install -g npm
+  - command: npm i
+  - command: npm i -g @sasjs/cli
 
 image:
   file: .gitpod.dockerfile
@@ -22,4 +24,4 @@ github:
     # add a "Review in Gitpod" button to pull requests (defaults to false)
     addBadge: false
     # add a label once the prebuild is ready to pull requests (defaults to false)
-    addLabel: prebuilt-in-gitpod
+    addLabel: prebuilt-in-gitpod

.npmignore

@@ -6,7 +6,6 @@ sasjs/
 .github/
 .git-hooks/
 .vscode/
-main.dox
 make_singlefile.sh
 *.md
 .all-contributorsrc

.sasjslint

@@ -1,13 +1,14 @@
 {
-    "noTrailingSpaces": true,
-    "noEncodedPasswords": true,
-    "hasDoxygenHeader": true,
-    "hasMacroNameInMend": true,
-    "hasMacroParentheses": true,
-    "noNestedMacros": false,
-    "noSpacesInFileNames": true,
-    "maxLineLength": 300,
-    "lowerCaseFileNames": true,
-    "noTabIndentation": true,
-    "indentationMultiple": 2
-}
+  "noTrailingSpaces": true,
+  "noEncodedPasswords": true,
+  "hasDoxygenHeader": true,
+  "hasMacroNameInMend": true,
+  "hasMacroParentheses": true,
+  "noGremlins": true,
+  "noNestedMacros": false,
+  "noSpacesInFileNames": true,
+  "maxLineLength": 300,
+  "lowerCaseFileNames": true,
+  "noTabs": true,
+  "indentationMultiple": 2
+}

.vscode/settings.json

@@ -6,5 +6,7 @@
   "editor.rulers": [
     80
   ],
-  "files.trimTrailingWhitespace": true
+  "files.trimTrailingWhitespace": true,
+  "sasjs-for-vscode.target": "docsonly",
+  "sasjs-for-vscode.isLocal": true
 }

README.md

@@ -18,7 +18,7 @@
 [dependency-url]:https://github.com/sasjs/core/blob/main/package.json
 
 
-Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
+Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/.github/CONTRIBUTING.md) are welcome.
 
 You can download and compile them all in just two lines of SAS code:
 
@@ -31,57 +31,34 @@ Documentation: https://core.sasjs.io
 
 ## Components
 
-### BASE library (SAS9/Viya)
+### BASE folder (All Platforms)
 
 - OS independent
-- Not metadata aware
+- Works on all SAS Platforms
 - No X command
-- Prefixes: _mf_, _mp_
+- Prefixes: `mf_`, `mp_`
+
+### DDL folder (All Platforms)
+
+- OS independent
+- Works on all SAS Platforms
+- No X command
+- Prefixes: `mddl_(lib)_` -> where lib can be "SAS" (in relation to a SAS component) or "DC" (in relation to a Data Controller component)
+
+This library will not be used for storing data entries (such as formats or datalines).  Where this becomes necessary in the future, a new repo will be created, in order to keep the NPM bundle size down (for the benefit of those looking to embed purely macros in their applications).
+
+### FCMP folder (All Platforms)
 
-#### FCMP library (SAS9/Viya)
 - Function and macro names are identical, except for special cases
-- Prefixes: _mcf_
+- Prefixes: `mcf_`
 
-The fcmp macros are used to generate fcmp functions, and can be used with or
-without the `proc fcmp` wrapper.
+The fcmp macros are used to generate fcmp functions, and can be used with or without the `proc fcmp` wrapper.
 
-### META library (SAS9 only)
-
-Macros used in SAS EBI, which connect to the metadata server.
-
-- OS independent
-- Metadata aware
-- No X command
-- Prefixes: _mm_
-
-### SERVER library (@sasjs/server only)
-These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
-
-- OS independent
-- @sasjs/server aware
-- No X command
-- Prefixes: _ms_
-
-### VIYA library (Viya only)
-
-Macros used for interfacing with SAS Viya.
-
-- OS independent
-- No X command
-- Prefixes: _mv_, _mvf_
-
-### METAX library (SAS9 only)
-
-- OS specific
-- Metadata aware
-- X command enabled
-- Prefixes: _mmw_,_mmu_,_mmx_
-
-### LUA library
+### LUA folder
 
 Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
 
-To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert your LUA into a data step with put statements, and create the macro wrapper with a `ml_` prefix.  You can then use your module in any program by running:
+To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert all files with a ".lua" extension into a macro wrapper with an `ml_` prefix (embedding the necessary data step put statements).  You can then use your module in any program by running:
 
 ```sas
 /* compile the lua module */
@@ -95,16 +72,63 @@ endsubmit;
 run;
 ```
 
+- Prefixes: `ml_`
+
+### META folder (SAS9 only)
+
+Macros used in SAS EBI, which connect to the metadata server.
+
+- OS independent
+- Metadata aware
+- No X command
+- Prefixes: `mm_`
+
+### METAX folder (SAS9 only)
+
+- OS specific
+- Metadata aware
 - X command enabled
-- Prefixes: _mmw_,_mmu_,_mmx_
+- Prefixes: `mmx_`
+
+### SERVER folder (@sasjs/server only)
+These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
+
+- OS independent
+- @sasjs/server aware
+- No X command
+- Prefixes: `ms_`
+
+### VIYA folder (Viya only)
+
+Macros used for interfacing with SAS Viya.
+
+- OS independent
+- No X command
+- Prefixes: `mv_`, `mvf_`
+
+### XPLATFORM folder (Viya, Meta, and Server)
+
+Sometimes it is helpful to use a macro that can be used interchangeably regardless of the server type on which is is running (SASVIYA, SAS9, SASJS).
+
+- OS independent
+- No X command
+- Prefixes: `mx_`
 
 ## Installation
 
 First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available, eg:
 
 ```sas
-options insert=(sasautos="/your/path/macrocore/base");
-options insert=(sasautos="/your/path/macrocore/meta");
+%let repoloc=/your/path/core;
+options insert=(sasautos="&repoloc/base");
+options insert=(sasautos="&repoloc/ddl");
+options insert=(sasautos="&repoloc/fcmp");
+options insert=(sasautos="&repoloc/lua");
+options insert=(sasautos="&repoloc/meta");
+options insert=(sasautos="&repoloc/metax");
+options insert=(sasautos="&repoloc/server");
+options insert=(sasautos="&repoloc/viya");
+options insert=(sasautos="&repoloc/xplatform");
 ```
 
 The above can be done directly in your sas program, via an autoexec, or an initialisation program.
@@ -125,14 +149,16 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 - macro names must be lowercase
 - one macro per file
 - prefixes:
+  - _mcf_ for macro compiled functions (proc fcmp)
+  - _mddl_ for macros containing DDL (Data Definition Language)
   - _mf_ for macro functions (can be used in open code).
   - _ml_ for macros that are used to compile LUA modules
   - _mm_ for metadata macros (interface with the metadata server).
-  - _mmx_ for macros that use metadata and are XCMD enabled
+  - _mmx_ for macros that use metadata and are XCMD enabled (working on both windows and unix)
   - _mp_ for macro procedures (which generate sas code)
   - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
   - _mv_ for macro procedures that will only work in Viya
-  - _mx_ for macros that are XCMD enabled
+  - _mx_ for macros that work on Viya, SAS 9 EBI and SASjs Server
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
@@ -166,7 +192,7 @@ SAS code can contain one of two types of dependency - SAS Macros, and SAS Includ
   @li someprogram.sas FREFTWO
 ```
 
-The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Includes) when creating SAS Jobs and Services.
+The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Includes) when creating SAS Jobs and Services (and Tests).
 
 When contributing to this library, it is therefore important to ensure that all dependencies are listed in the header in this format.
 
@@ -180,25 +206,49 @@ When contributing to this library, it is therefore important to ensure that all
 - The closing `%mend;` should **not** contain the macro name.
 - All macros should be defined with brackets, even if no variables are needed - ie `%macro x();` not `%macro x;`
 - Mandatory parameters should be positional, all optional parameters should be keyword (var=) style.
-- All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect.
+- All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect, or the [USER](https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/lrcon/n18m1vkqmeo4esn1moikt23zhp8s.htm) library is active.
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
+- Where global macro variables are absolutely necessary, they should make use of `&sasjs_prefix` - see mp_init.sas
 - The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
+- Use [sasjs lint](https://github.com/sasjs/lint)!
 
 ## General Notes
 
 - All macros should be compatible with SAS versions from support level B and above (so currently 9.2 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully (eg `%if %sysevalf(&sysver<9.3) %then %return`).
 
+## Breaking Changes
+
+We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major/breaking release (v5) becomes necessary:
+
+* mf_getuniquelibref.sas to have the deprecated maxtried parameter removed (no longer needed)
+* mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
+* `insert_cmplib` option of mcf_xxx macros will be deprecated (the option is now checked automatically with value inserted only if needed)
+* mcf_xxx macros to have `wrap=` option defaulted to YES for convenience.  Set this option explicitly to avoid issues.
+* mp_getddl.sas to be renamed to mp_ds2ddl.sas (consistent with other ds2xxx macros).  A wrapper macro is already in place, and you are able to use this immediately.  The default for SHOWLOG will also be YES instead of NO.
+* mp_coretable.sas will be replaced by the standalone macros in the `ddl` folder (which are already available)
+
 ## Star Gazing
 
 If you find this library useful, please leave a [star](https://github.com/sasjs/core/stargazers) and help us grow our star graph!
 
 ![](https://starchart.cc/sasjs/core.svg)
 
+## Other SAS Repositories
 
+The following repositories are also worth checking out:
+
+* [xieliaing/SAS](https://github.com/xieliaing/SAS)
+* [SASJedi/sas-macros](https://github.com/SASJedi/sas-macros)
+* [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
+* [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)
+* [KatjaGlassConsulting/SMILE-SmartSASMacros](https://github.com/KatjaGlassConsulting/SMILE-SmartSASMacros)
+* [rogerjdeangelis](https://github.com/rogerjdeangelis)
+* [scottbass/sas](https://github.com/scottbass/SAS)
+* [yabwon/sas_packages](https://github.com/yabwon/SAS_PACKAGES)
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-10-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-12-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -219,6 +269,8 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
     <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
     <td align="center"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
+    <td align="center"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
+    <td align="center"><a href="https://bandism.net/"><img src="https://avatars.githubusercontent.com/u/22633385?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Ikko Ashimine</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=eltociear" title="Code">💻</a></td>
   </tr>
 </table>
 

base/mf_abort.sas

@@ -11,7 +11,7 @@
   @cond
 **/
 
-%macro mf_abort(mac=mf_abort.sas, type=deprecated, msg=, iftrue=%str(1=1)
+%macro mf_abort(mac=mf_abort.sas, msg=, iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
   %if not(%eval(%unquote(&iftrue))) %then %return;

base/mf_dedup.sas

@@ -23,7 +23,6 @@
 
   <h4> Related Macros </h4>
   @li mf_trimstr.sas
-  @li mf_wordsinstr1butnotstr2.sas
 
   @version 9.2
   @author Allan Bowe

base/mf_deletefile.sas

@@ -0,0 +1,31 @@
+/**
+  @file
+  @brief Deletes a physical file, if it exists
+  @details Usage:
+
+      %mf_writefile(&sasjswork/myfile.txt,l1=some content)
+
+      %mf_deletefile(&sasjswork/myfile.txt)
+
+      %mf_deletefile(&sasjswork/myfile.txt)
+
+
+  @param filepath Full path to the target file
+
+  @returns The return code from the fdelete() invocation
+
+  <h4> Related Macros </h4>
+  @li mf_deletefile.test.sas
+  @li mf_writefile.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_deletefile(file
+)/*/STORE SOURCE*/;
+  %local rc fref;
+  %let rc= %sysfunc(filename(fref,&file));
+  %if %sysfunc(fdelete(&fref)) ne 0 %then %put %sysfunc(sysmsg());
+  %let rc= %sysfunc(filename(fref));
+%mend mf_deletefile;

base/mf_existfeature.sas

@@ -9,19 +9,17 @@
 
         %put %mf_existfeature(PROCLUA);
 
-  @param feature the feature to detect.  Leave blank to list all in log.
+  @param [in] feature The feature to detect.
 
   @return output returns 1 or 0 (or -1 if not found)
 
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
 
-
   @version 8
   @author Allan Bowe
 **/
 /** @cond */
-
 %macro mf_existfeature(feature
 )/*/STORE SOURCE*/;
   %let feature=%upcase(&feature);
@@ -29,7 +27,11 @@
   %let platform=%mf_getplatform();
 
   %if &feature= %then %do;
-    %put Supported features:  PROCLUA;
+    %put No feature was requested for detection;
+  %end;
+  %else %if &feature=COLCONSTRAINTS %then %do;
+    %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 0;
+    %else 1;
   %end;
   %else %if &feature=PROCLUA %then %do;
     /* https://blogs.sas.com/content/sasdummy/2015/08/03/using-lua-within-your-sas-programs */
@@ -38,10 +40,20 @@
     %else %if "&SYSVLONG" < "9.04.01M3" %then 0;
     %else 1;
   %end;
+  %else %if &feature=DBMS_MEMTYPE %then %do;
+    /* does dbms_memtype exist in dictionary.tables? */
+    %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 0;
+    %else 1;
+  %end;
+  %else %if &feature=EXPORTXLS %then %do;
+    /* is it possible to PROC EXPORT an excel file? */
+    %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 1;
+    %else %if %sysfunc(sysprod(SAS/ACCESS Interface to PC Files)) = 1 %then 1;
+    %else 0;
+  %end;
   %else %do;
     -1
     %put &sysmacroname: &feature not found;
   %end;
 %mend mf_existfeature;
-
-/** @endcond */
+/** @endcond */

base/mf_existfileref.sas

@@ -30,4 +30,4 @@
     0
   %end;
 
-%mend mf_existfileref;
+%mend mf_existfileref;

base/mf_existvar.sas

@@ -1,14 +1,18 @@
 /**
   @file
   @brief Checks if a variable exists in a data set.
-  @details Returns 0 if the variable does NOT exist, and return the position of
-    the var if it does.
-    Usage:
+  @details Returns 0 if the variable does NOT exist, and the position of the var
+  if it does.
+  Usage:
 
-        %put %mf_existvar(work.someds, somevar)
+      %put %mf_existvar(work.someds, somevar)
+
+  @param [in] libds 2 part dataset or view reference
+  @param [in] var variable name
+
+  <h4> Related Macros </h4>
+  @li mf_existvar.test.sas
 
-  @param libds (positional) - 2 part dataset or view reference
-  @param var (positional) - variable name
   @version 9.2
   @author Allan Bowe
 **/

base/mf_existvarlist.sas

@@ -6,9 +6,6 @@
 
       %put %mf_existVarList(sashelp.class, age sex name dummyvar);
 
-  <h4> SAS Macros </h4>
-  @li mf_abort.sas
-
   @param libds 2 part dataset or view reference
   @param varlist space separated variable names
 

base/mf_fmtdttm.sas

@@ -0,0 +1,42 @@
+/**
+  @file
+  @brief Returns E8601DT26.6 if compatible else DATETIME19.3
+  @details From our experience in [Data Controller for SAS]
+  (https://datacontroller.io) deployments, the E8601DT26.6 datetime format has
+  the widest support when it comes to pass-through SQL queries.
+
+  However, it is not supported in WPS or early versions of SAS 9 (M3 and below)
+  when used as a datetime literal, eg:
+
+      data _null_;
+        demo="%sysfunc(datetime(),E8601DT26.6)"dt;
+        demo=;
+      run;
+
+  This macro will therefore return DATEITME19.3 as an alternative format
+  based on the runtime environment so that it can be used in such cases, eg:
+
+      data _null_;
+        demo="%sysfunc(datetime(),%mf_fmtdttm())"dt;
+        demo=;
+      run;
+
+  <h4> Related Macros </h4>
+  @li mf_fmtdttm.test.sas
+
+  @author Allan Bowe
+**/
+
+%macro mf_fmtdttm(
+)/*/STORE SOURCE*/;
+
+%if "&sysver"="9.2" or "&sysver"="9.3"
+  or ("&sysver"="9.4" and "%substr(&SYSVLONG,9,1)" le "3")
+  or "%substr(&sysver,1,1)"="4"
+  or "%substr(&sysver,1,1)"="5"
+%then %do;DATETIME19.3%end;
+%else %do;E8601DT26.6%end;
+
+%mend mf_fmtdttm;
+
+

base/mf_getapploc.sas

@@ -15,6 +15,7 @@
   @li /data
   @li /jobs
   @li /services
+  @li /tests
   @li /tests/jobs
   @li /tests/services
   @li /tests/macros
@@ -46,9 +47,13 @@
 
 /**
   * First check we are not in the tests/macros folder (which has no subfolders)
+  * or specifically in the testsetup or testteardown services
   */
-%if %index(&pgm,/tests/macros/) %then %do;
-  %let root=%substr(&pgm,1,%index(&pgm,/tests/macros)-1);
+%if %index(&pgm,/tests/macros/)
+or %index(&pgm,/tests/testsetup)
+or %index(&pgm,/tests/testteardown)
+%then %do;
+  %let root=%substr(&pgm,1,%index(&pgm,/tests)-1);
   &root
   %return;
 %end;

base/mf_getengine.sas

@@ -12,9 +12,10 @@
   contributors of Chris Hemedingers blog [post](
   http://blogs.sas.com/content/sasdummy/2013/06/04/find-a-sas-library-engine/)
 
-  @param libref Library reference (also accepts a 2 level libds ref).
+  @param [in] libref Library reference (also accepts a 2 level libds ref).
 
-  @return output returns the library engine for the FIRST library encountered.
+  @return output returns the library engine (uppercase) for the FIRST library
+    encountered.
 
   @warning will only return the FIRST library engine - for concatenated
     libraries, with different engines, inconsistent results may be encountered.
@@ -46,7 +47,7 @@
     %let rc= %sysfunc(close(&dsid));
   %end;
 
-  &engine
+  %upcase(&engine)
 
 %mend mf_getengine;
 

base/mf_getfilesize.sas

@@ -5,18 +5,19 @@
 
       %put %mf_getfilesize(fpath=C:\temp\myfile.txt);
 
-      or
+  or, provide a libds value as follows:
 
       data x;do x=1 to 100000;y=x;output;end;run;
       %put %mf_getfilesize(libds=work.x,format=yes);
 
-      gives:
+  Which gives:
 
-      2mb
+  > 2mb
+
+  @param [in] fpath= Full path and filename.  Provide this OR the libds value.
+  @param [in] libds= (0) Library.dataset value (assumes library is BASE engine)
+  @param [in] format= (NO) Set to yes to apply sizekmg. format
 
-  @param fpath= full path and filename.  Provide this OR the libds value.
-  @param libds= library.dataset value (assumes library is BASE engine)
-  @param format=  set to yes to apply sizekmg. format
   @returns bytes
 
   @version 9.2
@@ -26,16 +27,32 @@
 %macro mf_getfilesize(fpath=,libds=0,format=NO
 )/*/STORE SOURCE*/;
 
-  %if &libds ne 0 %then %do;
-    %let fpath=%sysfunc(pathname(%scan(&libds,1,.)))/%scan(&libds,2,.).sas7bdat;
-  %end;
+  %local rc fid fref bytes dsid lib vnum;
 
-  %local rc fid fref bytes;
-  %let rc=%sysfunc(filename(fref,&fpath));
-  %let fid=%sysfunc(fopen(&fref));
-  %let bytes=%sysfunc(finfo(&fid,File Size (bytes)));
-  %let rc=%sysfunc(fclose(&fid));
-  %let rc=%sysfunc(filename(fref));
+  %if &libds ne 0 %then %do;
+    %let libds=%upcase(&libds);
+    %if %index(&libds,.)=0 %then %let lib=WORK;
+    %else %let lib=%scan(&libds,1,.);
+    %let dsid=%sysfunc(open(
+      sashelp.vtable(where=(libname="&lib" and memname="%scan(&libds,-1,.)")
+        keep=libname memname filesize
+      )
+    ));
+    %if (&dsid ^= 0) %then %do;
+      %let vnum=%sysfunc(varnum(&dsid,FILESIZE));
+      %let rc=%sysfunc(fetch(&dsid));
+      %let bytes=%sysfunc(getvarn(&dsid,&vnum));
+      %let rc= %sysfunc(close(&dsid));
+    %end;
+    %else %put &sysmacroname: &libds could not be opened! %sysfunc(sysmsg());
+  %end;
+  %else %do;
+    %let rc=%sysfunc(filename(fref,&fpath));
+    %let fid=%sysfunc(fopen(&fref));
+    %let bytes=%sysfunc(finfo(&fid,File Size (bytes)));
+    %let rc=%sysfunc(fclose(&fid));
+    %let rc=%sysfunc(filename(fref));
+  %end;
 
   %if &format=NO %then %do;
     &bytes

base/mf_getgitbranch.sas

@@ -0,0 +1,37 @@
+/**
+  @file
+  @brief Retrieves the current branch from a local GIT repo
+  @details In a local git repository, the current branch is always available in
+  the `.git/HEAD` file in a format like this:  `ref: refs/heads/master`
+
+  This macro simply reads the file and returns the last word (eg `master`).
+
+  Example usage:
+
+      %let gitdir=%sysfunc(pathname(work))/core;
+      %let repo=https://github.com/sasjs/core;
+      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&gitdir));
+
+      %put The current branch is %mf_getgitbranch(&gitdir);
+
+  @param [in] gitdir The directory containing the GIT repository
+
+  <h4> SAS Macros </h4>
+  @li mf_readfile.sas
+
+  <h4> Related Macros </h4>
+  @li mp_gitadd.sas
+  @li mp_gitlog.sas
+  @li mp_gitreleaseinfo.sas
+  @li mp_gitstatus.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_getgitbranch(gitdir
+)/*/STORE SOURCE*/;
+
+  %scan(%mf_readfile(&gitdir/.git/HEAD),-1)
+
+%mend mf_getgitbranch;

base/mf_getplatform.sas

@@ -5,8 +5,12 @@
 
       %put %mf_getplatform();
 
-    returns:
-      SASMETA  (or SASVIYA)
+  returns one of:
+
+  @li SASMETA
+  @li SASVIYA
+  @li SASJS
+  @li BASESAS
 
   @param switch the param for which to return a platform specific variable
 
@@ -22,6 +26,12 @@
 )/*/STORE SOURCE*/;
 %local a b c;
 %if &switch.NONE=NONE %then %do;
+  %if %symexist(sasjsprocessmode) %then %do;
+    %if &sasjsprocessmode=Stored Program %then %do;
+      SASJS
+      %return;
+    %end;
+  %end;
   %if %symexist(sysprocessmode) %then %do;
     %if "&sysprocessmode"="SAS Object Server"
     or "&sysprocessmode"= "SAS Compute Server" %then %do;
@@ -62,4 +72,4 @@
 %else %if &switch=VIYARESTAPI %then %do;
   %mf_trimstr(%sysfunc(getoption(servicesbaseurl)),/)
 %end;
-%mend mf_getplatform;
+%mend mf_getplatform;

base/mf_getuniquefileref.sas

@@ -28,15 +28,17 @@
     be 8 characters, so a 7 letter prefix would mean `maxtries` should be 10.
     if using zero (0) as the prefix, a native assignment is used.
   @param [in] maxtries= (1000) the last part of the libref. Must be an integer.
+  @param [in] lrecl= (32767) Provide a default lrecl with which to initialise
+    the generated fileref.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-%macro mf_getuniquefileref(prefix=_,maxtries=1000);
+%macro mf_getuniquefileref(prefix=_,maxtries=1000,lrecl=32767);
   %local rc fname;
   %if &prefix=0 %then %do;
-    %let rc=%sysfunc(filename(fname,,temp));
+    %let rc=%sysfunc(filename(fname,,temp,lrecl=&lrecl));
     %if &rc %then %put %sysfunc(sysmsg());
     &fname
   %end;
@@ -47,7 +49,7 @@
     %do x=0 %to &maxtries;
       %let fname=&prefix%substr(%sysfunc(ranuni(0)),3,&len);
       %if %sysfunc(fileref(&fname)) > 0 %then %do;
-        %let rc=%sysfunc(filename(fname,,temp));
+        %let rc=%sysfunc(filename(fname,,temp,lrecl=&lrecl));
         %if &rc %then %put %sysfunc(sysmsg());
         &fname
         %return;

base/mf_getuniquelibref.sas

@@ -3,38 +3,55 @@
   @brief Returns an unused libref
   @details Use as follows:
 
-    libname mclib0 (work);
-    libname mclib1 (work);
-    libname mclib2 (work);
+      libname mclib0 (work);
+      libname mclib1 (work);
+      libname mclib2 (work);
 
-    %let libref=%mf_getuniquelibref();
-    %put &=libref;
+      %let libref=%mf_getuniquelibref();
+      %put &=libref;
 
   which returns:
 
 > mclib3
 
-  @param prefix= first part of libref.  Remember that librefs can only be 8 characters,
-    so a 7 letter prefix would mean that maxtries should be 10.
-  @param maxtries= the last part of the libref.  Provide an integer value.
+  A blank value is returned if no usable libname is determined.
+
+  @param [in] prefix= (mclib) first part of the returned libref. As librefs can
+    be as long as 8 characters, a maximum length of 7 characters is premitted
+    for this prefix.
+  @param [in] maxtries= Deprecated parameter. Remains here to ensure a
+    non-breaking change.  Will be removed in v5.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-
 %macro mf_getuniquelibref(prefix=mclib,maxtries=1000);
-  %local x libref;
-  %let x=0;
-  %do x=0 %to &maxtries;
-  %if %sysfunc(libref(&prefix&x)) ne 0 %then %do;
-    %let libref=&prefix&x;
-    %let rc=%sysfunc(libname(&libref,%sysfunc(pathname(work))));
-    %if &rc %then %put %sysfunc(sysmsg());
-    &prefix&x
-    %*put &sysmacroname: Libref &libref assigned as WORK and returned;
+  %local x;
+
+  %if ( %length(&prefix) gt 7 ) %then %do;
+    %put %str(ERR)OR: The prefix parameter cannot exceed 7 characters.;
+    0
     %return;
   %end;
+  %else %if (%sysfunc(NVALID(&prefix,v7))=0) %then %do;
+    %put %str(ERR)OR: Invalid prefix (&prefix);
+    0
+    %return;
   %end;
-  %put unable to find available libref in range &prefix.0-&maxtries;
+
+  /* Set maxtries equal to '10 to the power of [# unused characters] - 1' */
+  %let maxtries=%eval(10**(8-%length(&prefix))-1);
+
+  %do x = 0 %to &maxtries;
+    %if %sysfunc(libref(&prefix&x)) ne 0 %then %do;
+      &prefix&x
+      %return;
+    %end;
+    %let x = %eval(&x + 1);
+  %end;
+
+  %put %str(ERR)OR: No usable libref in range &prefix.0-&maxtries;
+  %put %str(ERR)OR- Try reducing the prefix or deleting some libraries!;
+  0
 %mend mf_getuniquelibref;

base/mf_getuser.sas

@@ -23,17 +23,19 @@
   @author Allan Bowe
 **/
 
-%macro mf_getuser(type=META
+%macro mf_getuser(
 )/*/STORE SOURCE*/;
-  %local user metavar;
-  %if &type=OS %then %let metavar=_secureusername;
-  %else %let metavar=_metaperson;
+  %local user;
 
-  %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %let user=&SYS_COMPUTE_SESSION_OWNER;
-  %else %if %symexist(&metavar) %then %do;
-    %if %length(&&&metavar)=0 %then %let user=&sysuserid;
+  %if %symexist(_sasjs_username) %then %let user=&_sasjs_username;
+  %else %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %do;
+    %let user=&SYS_COMPUTE_SESSION_OWNER;
+  %end;
+  %else %if %symexist(_metaperson) %then %do;
+    %if %length(&_metaperson)=0 %then %let user=&sysuserid;
     /* sometimes SAS will add @domain extension - remove for consistency */
-    %else %let user=%scan(&&&metavar,1,@);
+    /* but be sure to quote in case of usernames with commas */
+    %else %let user=%unquote(%scan(%quote(&_metaperson),1,@));
   %end;
   %else %let user=&sysuserid;
 

base/mf_getvarcount.sas

@@ -2,31 +2,48 @@
   @file
   @brief Returns number of variables in a dataset
   @details Useful to identify those renagade datasets that have no columns!
+  Can also be used to count for numeric, or character columns
 
-        %put Number of Variables=%mf_getvarcount(sashelp.class);
+      %put Number of Variables=%mf_getvarcount(sashelp.class);
+      %put Character Variables=%mf_getvarcount(sashelp.class,typefilter=C);
+      %put Numeric Variables = %mf_getvarcount(sashelp.class,typefilter=N);
 
   returns:
   > Number of Variables=4
 
-  @param libds Two part dataset (or view) reference.
+
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] typefilter= (A) Filter for certain types of column.  Valid values:
+    @li A Count All columns
+    @li C Count Character columns only
+    @li N Count Numeric columns only
 
   @version 9.2
   @author Allan Bowe
 
 **/
 
-%macro mf_getvarcount(libds
+%macro mf_getvarcount(libds,typefilter=A
 )/*/STORE SOURCE*/;
-  %local dsid nvars rc ;
+  %local dsid nvars rc outcnt x;
   %let dsid=%sysfunc(open(&libds));
   %let nvars=.;
+  %let outcnt=0;
+  %let typefilter=%upcase(&typefilter);
   %if &dsid %then %do;
     %let nvars=%sysfunc(attrn(&dsid,NVARS));
+    %if &typefilter=A %then %let outcnt=&nvars;
+    %else %if &nvars>0 %then %do x=1 %to &nvars;
+      /* increment based on variable type */
+      %if %sysfunc(vartype(&dsid,&x))=&typefilter %then %do;
+        %let outcnt=%eval(&outcnt+1);
+      %end;
+    %end;
     %let rc=%sysfunc(close(&dsid));
   %end;
   %else %do;
     %put unable to open &libds (rc=&dsid);
     %let rc=%sysfunc(close(&dsid));
   %end;
-  &nvars
+  &outcnt
 %mend mf_getvarcount;

base/mf_increment.sas

@@ -0,0 +1,29 @@
+/**
+  @file
+  @brief Increments a macro variable
+  @details Useful outside of do-loops - will increment a macro variable every
+  time it is called.
+
+  Example:
+
+      %let cnt=1;
+      %put We have run %mf_increment(cnt) lines;
+      %put Now we have run %mf_increment(cnt) lines;
+      %put There are %mf_increment(cnt) lines in total;
+
+  @param [in] MACRO_NAME the name of the macro variable to increment
+  @param [in] ITER= The amount to add or subtract to the macro
+
+  <h4> Related Files </h4>
+  @li mf_increment.test.sas
+
+**/
+
+%macro mf_increment(macro_name,incr=1);
+
+  /* iterate the value */
+  %let &macro_name=%eval(&&&macro_name+&incr);
+  /* return the value */
+  &&&macro_name
+
+%mend mf_increment;

base/mf_isint.sas

@@ -20,8 +20,11 @@
 
 %macro mf_isint(arg
 )/*/STORE SOURCE*/;
-  /* remove minus sign if exists */
 
+  /* blank val is not an integer */
+  %if "&arg"="" %then %do;0%return;%end;
+
+  /* remove minus sign if exists */
   %local val;
   %if "%substr(%str(&arg),1,1)"="-" %then %let val=%substr(%str(&arg),2);
   %else %let val=&arg;
@@ -30,4 +33,4 @@
   %if %sysfunc(findc(%str(&val),,kd)) %then %do;0%end;
   %else %do;1%end;
 
-%mend mf_isint;
+%mend mf_isint;

base/mf_islibds.sas

@@ -0,0 +1,40 @@
+/**
+  @file
+  @brief Checks whether a string follows correct library.dataset format
+  @details Many macros in the core library accept a library.dataset parameter
+  referred to as 'libds'.  This macro validates the structure of that parameter,
+  eg:
+
+    @li 8 character libref?
+    @li 32 character dataset?
+    @li contains a period?
+
+  It does NOT check whether the dataset exists, or if the library is assigned.
+
+  Usage:
+
+      %put %mf_islibds(work.something)=1;
+      %put %mf_islibds(nolib)=0;
+      %put %mf_islibds(badlibref.ds)=0;
+      %put %mf_islibds(w.t.f)=0;
+
+  @param [in] libds The string to be checked
+
+  @return output Returns 1 if libds is valid, 0 if it is not
+
+  <h4> Related Macros </h4>
+  @li mf_islibds.test.sas
+  @li mp_validatecol.sas
+
+  @version 9.2
+**/
+
+%macro mf_islibds(libds
+)/*/STORE SOURCE*/;
+
+%local regex;
+%let regex=%sysfunc(prxparse(%str(/^[_a-z]\w{0,7}\.[_a-z]\w{0,31}$/i)));
+
+%sysfunc(prxmatch(&regex,&libds))
+
+%mend mf_islibds;

base/mf_readfile.sas

@@ -0,0 +1,63 @@
+/**
+  @file
+  @brief Reads the first line of a file using pure macro
+  @details Reads the first line of a file and returns it.  Future versions may
+  read each line into a macro variable array.
+
+  Generally, reading data into macro variables is not great as certain
+  nonprintable characters (such as CR, LF) may be dropped in the conversion.
+
+    Usage:
+
+        %mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=more content)
+
+        %put %mf_readfile(&sasjswork/myfile.txt);
+
+
+  @param [in] fpath Full path to file to be read
+
+  <h4> Related Macros </h4>
+  @li mf_deletefile.sas
+  @li mf_writefile.sas
+  @li mf_readfile.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mf_readfile(fpath
+)/*/STORE SOURCE*/;
+%local fref rc fid fcontent;
+
+/* check file exists */
+%if %sysfunc(filename(fref,&fpath)) ne 0 %then %do;
+  %put &=fref &=fpath;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let fid=%sysfunc(fopen(&fref,I));
+
+%if &fid=0 %then %do;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%if %sysfunc(fread(&fid)) = 0 %then %do;
+  %let rc=%sysfunc(fget(&fid,fcontent,65534));
+  &fcontent
+%end;
+
+/*
+%do %while(%sysfunc(fread(&fid)) = 0);
+  %let rc=%sysfunc(fget(&fid,fcontent,65534));
+  &fcontent
+%end;
+*/
+
+%let rc=%sysfunc(fclose(&fid));
+%let rc=%sysfunc(filename(&fref));
+
+%mend mf_readfile;
+/** @endcond */

base/mf_verifymacvars.sas

@@ -1,6 +1,6 @@
 /**
   @file
-  @brief Checks if a set of macro variables exist / contain values.
+  @brief Checks if a set of macro variables exist AND contain values.
   @details Writes ERROR to log if abortType is SOFT, else will call %mf_abort.
   Usage:
 
@@ -14,10 +14,11 @@
   <h4> SAS Macros </h4>
   @li mf_abort.sas
 
-  @param verifyvars space separated list of macro variable names
-  @param makeupcase= set to YES to convert all variable VALUES to
+  @param [in] verifyvars Space separated list of macro variable names
+  @param [in] makeupcase= (NO) Set to YES to convert all variable VALUES to
     uppercase.
-  @param mAbort= Abort Type.  Default is SOFT (writes err to log).
+  @param [in] mAbort= (SOFT) Abort Type.  When SOFT, simply writes an err
+    message to the log.
     Set to any other value to call mf_abort (which can be configured to abort in
     various fashions according to context).
 
@@ -58,8 +59,14 @@
 
   %goto exit_success;
   %exit_err:
-    %if &mAbort=SOFT %then %put %str(ERR)OR: &abortmsg;
-    %else %mf_abort(mac=mf_verifymacvars,type=&mabort,msg=&abortmsg);
+    %put &abortmsg;
+    %mf_abort(iftrue=(&mabort ne SOFT),
+      mac=mf_verifymacvars,
+      msg=%str(&abortmsg)
+    )
+    0
+    %return;
   %exit_success:
+  1
 
 %mend mf_verifymacvars;

base/mf_wordsinstr1andstr2.sas

@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Returns words that are in both string 1 and string 2
+  @details  Compares two space separated strings and returns the words that are
+  in both.
+  Usage:
+
+      %put %mf_wordsInStr1andStr2(
+        Str1=blah sss blaaah brah bram boo
+        ,Str2=   blah blaaah brah ssss
+      );
+
+  returns:
+  > blah blaaah brah
+
+  @param str1= string containing words to extract
+  @param str2= used to compare with the extract string
+
+  @warning CASE SENSITIVE!
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_wordsInStr1andStr2(
+  Str1= /* string containing words to extract */
+  ,Str2= /* used to compare with the extract string */
+)/*/STORE SOURCE*/;
+
+%local count_base count_extr i i2 extr_word base_word match outvar;
+%if %length(&str1)=0 or %length(&str2)=0 %then %do;
+  %put base string (str1)= &str1;
+  %put compare string (str2) = &str2;
+  %return;
+%end;
+%let count_base=%sysfunc(countw(&Str2));
+%let count_extr=%sysfunc(countw(&Str1));
+
+%do i=1 %to &count_extr;
+  %let extr_word=%scan(&Str1,&i,%str( ));
+  %let match=0;
+  %do i2=1 %to &count_base;
+    %let base_word=%scan(&Str2,&i2,%str( ));
+    %if &extr_word=&base_word %then %let match=1;
+  %end;
+  %if &match=1 %then %let outvar=&outvar &extr_word;
+%end;
+
+  &outvar
+
+%mend mf_wordsInStr1andStr2;
+

base/mf_wordsinstr1butnotstr2.sas

@@ -3,6 +3,9 @@
   @brief Returns words that are in string 1 but not in string 2
   @details  Compares two space separated strings and returns the words that are
   in the first but not in the second.
+
+  Note - case sensitive!
+
   Usage:
 
       %let x= %mf_wordsInStr1ButNotStr2(
@@ -13,10 +16,8 @@
   returns:
   > sss bram boo
 
-  @param str1= string containing words to extract
-  @param str2= used to compare with the extract string
-
-  @warning CASE SENSITIVE!
+  @param [in] str1= string containing words to extract
+  @param [in] str2= used to compare with the extract string
 
   @version 9.2
   @author Allan Bowe
@@ -30,7 +31,6 @@
 
 %local count_base count_extr i i2 extr_word base_word match outvar;
 %if %length(&str1)=0 or %length(&str2)=0 %then %do;
-  %put %str(WARN)ING: empty string provided!;
   %put base string (str1)= &str1;
   %put compare string (str2) = &str2;
   %return;

base/mf_writefile.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Creates a text file using pure macro
+  @details Creates a text file of up to 10 lines.  If further lines are
+    desired, feel free to [create an issue](
+    https://github.com/sasjs/core/issues/new), or make a pull request!
+
+    The use of PARMBUFF was considered for this macro, but it would have made
+    things problematic for writing lines containing commas.
+
+    Usage:
+
+        %mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=more content)
+        data _null_;
+          infile "&sasjswork/myfile.txt";
+          input;
+          list;
+        run;
+
+  @param [in] fpath Full path to file to be created or appended to
+  @param [in] mode= (O) Available options are A or O as follows:
+    @li A APPEND mode, writes new records after the current end of the file.
+    @li O OUTPUT mode, writes new records from the beginning of the file.
+  @param [in] l1= First line
+  @param [in] l2= Second line (etc through to l10)
+
+  <h4> Related Macros </h4>
+  @li mf_writefile.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mf_writefile(fpath,mode=O,l1=,l2=,l3=,l4=,l5=,l6=,l7=,l8=,l9=,l10=
+)/*/STORE SOURCE*/;
+%local fref rc fid i total_lines;
+
+/* find number of lines by reference to first non-blank param */
+%do i=10 %to 1 %by -1;
+  %if %str(&&l&i) ne %str() %then %goto continue;
+%end;
+%continue:
+%let total_lines=&i;
+
+%if %sysfunc(filename(fref,&fpath)) ne 0 %then %do;
+  %put &=fref &=fpath;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let fid=%sysfunc(fopen(&fref,&mode));
+
+%if &fid=0 %then %do;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%do i=1 %to &total_lines;
+  %let rc=%sysfunc(fput(&fid, &&l&i));
+  %let rc=%sysfunc(fwrite(&fid));
+%end;
+%let rc=%sysfunc(fclose(&fid));
+%let rc=%sysfunc(filename(&fref));
+
+%mend mf_writefile;
+/** @endcond */

base/mp_abort.sas

@@ -8,23 +8,29 @@
 
   The method used varies according to the context.  Important points:
 
-  @li should not use endsas or abort cancel in 9.4m3 environments as this can
-    cause hung multibridge sessions and result in a frozen STP server
+  @li should not use endsas or abort cancel in 9.4m3 WIN environments as this
+    can cause hung multibridge sessions and result in a frozen STP server
+  @li The use of endsas in 9.4m6+ windows environments for POST requests to the
+    STP server can result in an empty response body
   @li should not use endsas in viya 3.5 as this destroys the session and cannot
     fetch results (although both mv_getjoblog.sas and the @sasjs/adapter will
     recognise this and fetch the log of the parent session instead)
   @li STP environments must finish cleanly to avoid the log being sent to
     _webout.  To assist with this, we also run stpsrvset('program error', 0)
-    and set SYSCC=0.  We take a unique "soft abort" approach - we open a macro
+    and set SYSCC=0.
+    Where possible, we take a unique "soft abort" approach - we open a macro
     but don't close it!  This works everywhere EXCEPT inside a \%include inside
     a macro.  For that, we recommend you use mp_include.sas to perform the
     include, and then call \%mp_abort(mode=INCLUDE) from the source program (ie,
     OUTSIDE of the top-parent macro).
+    The soft abort has become ineffective in 9.4m6 WINDOWS environments.  We are
+    currently investigating approaches to deal with this.
 
 
-  @param mac= to contain the name of the calling macro
+  @param mac= (mp_abort.sas) To contain the name of the calling macro. Do not
+    use &sysmacroname as this will always resolve to MP_ABORT.
   @param msg= message to be returned
-  @param iftrue= supply a condition under which the macro should be executed.
+  @param iftrue= (1=1) Supply a condition for which the macro should be executed
   @param errds= (work.mp_abort_errds) There is no clean way to end a process
     within a %include called within a macro.  Furthermore, there is no way to
     test if a macro is called within a %include.  To handle this particular
@@ -45,11 +51,12 @@
     @li REGULAR (default)
     @li INCLUDE
 
+  @version 9.4
+  @author Allan Bowe
+
   <h4> Related Macros </h4>
   @li mp_include.sas
 
-  @version 9.4
-  @author Allan Bowe
   @cond
 **/
 
@@ -58,160 +65,233 @@
   , mode=REGULAR
 )/*/STORE SOURCE*/;
 
-  %global sysprocessmode sysprocessname;
+%global sysprocessmode sysprocessname sasjs_stpsrv_header_loc sasjsprocessmode;
+%local fref fid i;
 
-  %if not(%eval(%unquote(&iftrue))) %then %return;
+%if not(%eval(%unquote(&iftrue))) %then %return;
 
-  %put NOTE: ///  mp_abort macro executing //;
-  %if %length(&mac)>0 %then %put NOTE- called by &mac;
-  %put NOTE - &msg;
+%put NOTE: ///  mp_abort macro executing //;
+%if %length(&mac)>0 %then %put NOTE- called by &mac;
+%put NOTE - &msg;
 
-  %if %symexist(_SYSINCLUDEFILEDEVICE) %then %do;
-    %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do;
-      data &errds;
-        iftrue='1=1';
-        length mac $100 msg $5000;
-        mac=symget('mac');
-        msg=symget('msg');
-      run;
+%if %symexist(_SYSINCLUDEFILEDEVICE)
+/* abort cancel FILE does not restart outside the INCLUDE on Viya 3.5 */
+and %superq(SYSPROCESSNAME) ne %str(Compute Server)
+%then %do;
+  %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do;
+    data &errds;
+      iftrue='1=1';
+      length mac $100 msg $5000;
+      mac=symget('mac');
+      msg=symget('msg');
+    run;
+    data _null_;
+      abort cancel FILE;
+    run;
+    %return;
+  %end;
+%end;
+
+/* Web App Context */
+%if %symexist(_PROGRAM)
+  or %superq(SYSPROCESSNAME) = %str(Compute Server)
+  or &mode=INCLUDE
+%then %do;
+  options obs=max replace mprint;
+  %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5"
+  %then %do;
+    options nosyntaxcheck;
+  %end;
+
+  %if &mode=INCLUDE %then %do;
+    %if %sysfunc(exist(&errds))=1 %then %do;
       data _null_;
-        abort cancel FILE;
+        set &errds;
+        call symputx('iftrue',iftrue,'l');
+        call symputx('mac',mac,'l');
+        call symputx('msg',msg,'l');
+        putlog (_all_)(=);
       run;
+      %if (&iftrue)=0 %then %return;
+    %end;
+    %else %do;
+      %put &sysmacroname: No include errors found;
       %return;
     %end;
   %end;
 
-  /* Stored Process Server web app context */
-  %if %symexist(_metaperson)
-    or "&SYSPROCESSNAME "="Compute Server "
-    or &mode=INCLUDE
-  %then %do;
-    options obs=max replace nosyntaxcheck mprint;
-    %if &mode=INCLUDE %then %do;
-      %if %sysfunc(exist(&errds))=1 %then %do;
-        data _null_;
-          set &errds;
-          call symputx('iftrue',iftrue,'l');
-          call symputx('mac',mac,'l');
-          call symputx('msg',msg,'l');
-          putlog (_all_)(=);
-        run;
-        %if (&iftrue)=0 %then %return;
-      %end;
-      %else %do;
-        %put &sysmacroname: No include errors found;
-        %return;
-      %end;
-    %end;
-
-    /* extract log errs / warns, if exist */
-    %local logloc logline;
-    %global logmsg; /* capture global messages */
-    %if %symexist(SYSPRINTTOLOG) %then %let logloc=&SYSPRINTTOLOG;
-    %else %let logloc=%qsysfunc(getoption(LOG));
-    proc printto log=log;run;
-    %if %length(&logloc)>0 %then %do;
-      %let logline=0;
+  /* extract log errs / warns, if exist */
+  %local logloc logline;
+  %global logmsg; /* capture global messages */
+  %if %symexist(SYSPRINTTOLOG) %then %let logloc=&SYSPRINTTOLOG;
+  %else %let logloc=%qsysfunc(getoption(LOG));
+  proc printto log=log;run;
+  %let logline=0;
+  %if %length(&logloc)>0 %then %do;
+    data _null_;
+      infile &logloc lrecl=5000;
+      input; putlog _infile_;
+      i=1;
+      retain logonce 0;
+      if (
+          _infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR"
+        ) and logonce=0 then
+      do;
+        call symputx('logline',_n_);
+        logonce+1;
+      end;
+    run;
+    /* capture log including lines BEFORE the err */
+    %if &logline>0 %then %do;
       data _null_;
         infile &logloc lrecl=5000;
-        input; putlog _infile_;
+        input;
         i=1;
-        retain logonce 0;
-        if (
-            _infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR"
-          ) and logonce=0 then
-        do;
-          call symputx('logline',_n_);
-          logonce+1;
-        end;
-      run;
-      /* capture log including lines BEFORE the err */
-      %if &logline>0 %then %do;
-        data _null_;
-          infile &logloc lrecl=5000;
+        stoploop=0;
+        if _n_ ge &logline-15 and stoploop=0 then do until (i>22);
+          call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
           input;
-          i=1;
-          stoploop=0;
-          if _n_ ge &logline-15 and stoploop=0 then do until (i>22);
-            call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
-            input;
-            i+1;
-            stoploop=1;
-          end;
-          if stoploop=1 then stop;
-        run;
-      %end;
-    %end;
-
-    %if %symexist(SYS_JES_JOB_URI) %then %do;
-      /* setup webout */
-      OPTIONS NOBOMFILE;
-      %if "X&SYS_JES_JOB_URI.X"="XX" %then %do;
-          filename _webout temp lrecl=999999 mod;
-      %end;
-      %else %do;
-        filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
-          name="_webout.json" lrecl=999999 mod;
-      %end;
-    %end;
-
-    /* send response in SASjs JSON format */
-    data _null_;
-      file _webout mod lrecl=32000 encoding='utf-8';
-      length msg $32767 ;
-      sasdatetime=datetime();
-      msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
-      /* escape the quotes */
-      msg=tranwrd(msg,'"','\"');
-      /* ditch the CRLFs as chrome complains */
-      msg=compress(msg,,'kw');
-      /* quote without quoting the quotes (which are escaped instead) */
-      msg=cats('"',msg,'"');
-      if symexist('_debug') then debug=quote(trim(symget('_debug')));
-      else debug='""';
-      put '>>weboutBEGIN<<';
-      put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
-      put ',"sasjsAbort" : [{';
-      put ' "MSG":' msg ;
-      put ' ,"MAC": "' "&mac" '"}]';
-      put ",""SYSUSERID"" : ""&sysuserid"" ";
-      put ',"_DEBUG":' debug ;
-      if symexist('_metauser') then do;
-        _METAUSER=quote(trim(symget('_METAUSER')));
-        put ",""_METAUSER"": " _METAUSER;
-        _METAPERSON=quote(trim(symget('_METAPERSON')));
-        put ',"_METAPERSON": ' _METAPERSON;
-      end;
-      if symexist('SYS_JES_JOB_URI') then do;
-        SYS_JES_JOB_URI=quote(trim(symget('SYS_JES_JOB_URI')));
-        put ',"SYS_JES_JOB_URI": ' SYS_JES_JOB_URI;
-      end;
-      _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
-      put ',"_PROGRAM" : ' _PROGRAM ;
-      put ",""SYSCC"" : ""&syscc"" ";
-      syserrortext=quote(trim(symget('syserrortext')));
-      put ",""SYSERRORTEXT"" : " syserrortext;
-      put ",""SYSHOSTNAME"" : ""&syshostname"" ";
-      put ",""SYSJOBID"" : ""&sysjobid"" ";
-      put ",""SYSSCPL"" : ""&sysscpl"" ";
-      put ",""SYSSITE"" : ""&syssite"" ";
-      sysvlong=quote(trim(symget('sysvlong')));
-      put ',"SYSVLONG" : ' sysvlong;
-      syswarningtext=quote(trim(symget('syswarningtext')));
-      put ",""SYSWARNINGTEXT"" : " syswarningtext;
-      put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
-      put "}" @;
-      put '>>weboutEND<<';
-    run;
-
-    %put _all_;
-
-    %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
-      data _null_;
-        putlog 'stpsrvset program err and syscc';
-        rc=stpsrvset('program error', 0);
-        call symputx("syscc",0,"g");
+          i+1;
+          stoploop=1;
+        end;
+        if stoploop=1 then stop;
       run;
+    %end;
+  %end;
+
+  %if %symexist(SYS_JES_JOB_URI) %then %do;
+    /* setup webout for Viya */
+    options nobomfile;
+    %if "X&SYS_JES_JOB_URI.X"="XX" %then %do;
+        filename _webout temp lrecl=999999 mod;
+    %end;
+    %else %do;
+      filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
+        name="_webout.json" lrecl=999999 mod;
+    %end;
+  %end;
+  %else %if %sysfunc(filename(fref,&sasjs_stpsrv_header_loc))=0 %then %do;
+    options nobomfile;
+    /* set up http header for SASjs Server */
+    %let fid=%sysfunc(fopen(&fref,A));
+    %if &fid=0 %then %do;
+      %put %str(ERR)OR: %sysfunc(sysmsg());
+      %return;
+    %end;
+    %let rc=%sysfunc(fput(&fid,%str(Content-Type: application/json)));
+    %let rc=%sysfunc(fwrite(&fid));
+    %let rc=%sysfunc(fclose(&fid));
+    %let rc=%sysfunc(filename(&fref));
+  %end;
+
+  /* send response in SASjs JSON format */
+  data _null_;
+    file _webout mod lrecl=32000 encoding='utf-8';
+    length msg syswarningtext syserrortext $32767 mode $10 ;
+    sasdatetime=datetime();
+    msg=symget('msg');
+  %if &logline>0 %then %do;
+    msg=cats(msg,'\n\nLog Extract:\n',symget('logmsg'));
+  %end;
+    /* escape the escapes */
+    msg=tranwrd(msg,'\','\\');
+    /* escape the quotes */
+    msg=tranwrd(msg,'"','\"');
+    /* ditch the CRLFs as chrome complains */
+    msg=compress(msg,,'kw');
+    /* quote without quoting the quotes (which are escaped instead) */
+    msg=cats('"',msg,'"');
+    if symexist('_debug') then debug=quote(trim(symget('_debug')));
+    else debug='""';
+    if symget('sasjsprocessmode')='Stored Program' then mode='SASJS';
+    if mode ne 'SASJS' then put '>>weboutBEGIN<<';
+    put '{"SYSDATE" : "' "&SYSDATE" '"';
+    put ',"SYSTIME" : "' "&SYSTIME" '"';
+    put ',"sasjsAbort" : [{';
+    put ' "MSG":' msg ;
+    put ' ,"MAC": "' "&mac" '"}]';
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
+    put ',"_DEBUG":' debug ;
+    if symexist('_metauser') then do;
+      _METAUSER=quote(trim(symget('_METAUSER')));
+      put ",""_METAUSER"": " _METAUSER;
+      _METAPERSON=quote(trim(symget('_METAPERSON')));
+      put ',"_METAPERSON": ' _METAPERSON;
+    end;
+    if symexist('SYS_JES_JOB_URI') then do;
+      SYS_JES_JOB_URI=quote(trim(symget('SYS_JES_JOB_URI')));
+      put ',"SYS_JES_JOB_URI": ' SYS_JES_JOB_URI;
+    end;
+    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
+    put ',"_PROGRAM" : ' _PROGRAM ;
+    put ",""SYSCC"" : ""&syscc"" ";
+    syserrortext=cats(symget('syserrortext'));
+    if findc(syserrortext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syserrortext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syserrortext)
+      )))))))))))))!!'"';
+    end;
+    else syserrortext=cats('"',syserrortext,'"');
+    put ',"SYSERRORTEXT" : ' syserrortext;
+    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSJOBID"" : ""&sysjobid"" ";
+    put ",""SYSSCPL"" : ""&sysscpl"" ";
+    put ",""SYSSITE"" : ""&syssite"" ";
+    sysvlong=quote(trim(symget('sysvlong')));
+    put ',"SYSVLONG" : ' sysvlong;
+    syswarningtext=cats(symget('syswarningtext'));
+    if findc(syswarningtext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syswarningtext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syswarningtext)
+      )))))))))))))!!'"';
+    end;
+    else syswarningtext=cats('"',syswarningtext,'"');
+    put ",""SYSWARNINGTEXT"" : " syswarningtext;
+    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
+    put "}" ;
+    if mode ne 'SASJS' then put '>>weboutEND<<';
+  run;
+
+  %put _all_;
+
+  %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
+    data _null_;
+      putlog 'stpsrvset program err and syscc';
+      rc=stpsrvset('program error', 0);
+      call symputx("syscc",0,"g");
+    run;
+    %if &sysscp=WIN
+    and 1=0 /* deprecating this logic until we figure out a consistent abort */
+    and "%substr(%str(&sysvlong         ),1,8)"="9.04.01M"
+    and "%substr(%str(&sysvlong         ),9,1)">"5" %then %do;
+      /* skip approach (below) does not work in windows m6+ envs */
+      endsas;
+    %end;
+    %else %do;
       /**
         * endsas kills 9.4m3 deployments by orphaning multibridges.
         * Abort variants are ungraceful (non zero return code)
@@ -229,28 +309,29 @@
       run;
       %inc skip;
     %end;
-    %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
-      /* endsas kills the session making it harder to fetch results */
-      data _null_;
-        syswarningtext=symget('syswarningtext');
-        syserrortext=symget('syserrortext');
-        abort_msg=symget('msg');
-        syscc=symget('syscc');
-        sysuserid=symget('sysuserid');
-        iftrue=symget('iftrue');
-        put (_all_)(/=);
-        call symputx('syscc',0);
-        abort cancel nolist;
-      run;
-    %end;
-    %else %do;
-      %abort cancel;
-    %end;
+  %end;
+  %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
+    /* endsas kills the session making it harder to fetch results */
+    data _null_;
+      syswarningtext=symget('syswarningtext');
+      syserrortext=symget('syserrortext');
+      abort_msg=symget('msg');
+      syscc=symget('syscc');
+      sysuserid=symget('sysuserid');
+      iftrue=symget('iftrue');
+      put (_all_)(/=);
+      call symputx('syscc',0);
+      abort cancel nolist;
+    run;
   %end;
   %else %do;
-    %put _all_;
     %abort cancel;
   %end;
+%end;
+%else %do;
+  %put _all_;
+  %abort cancel;
+%end;
 %mend mp_abort;
 
-/** @endcond */
+/** @endcond */

base/mp_applyformats.sas

@@ -0,0 +1,181 @@
+/**
+  @file
+  @brief Apply a set of formats to a table
+  @details Applies a set of formats to the metadata of one or more SAS datasets.
+    Can be used to migrate formats from one table to another.  The input table
+    must contain the following columns:
+
+    @li lib - the libref of the table to be updated
+    @li ds - the dataset to be updated
+    @li var - the variable to be updated
+    @li fmt - the format to apply.  Missing or default ($CHAR, 8.) formats are
+      ignored.
+
+  The macro will abort in the following scenarios:
+
+    @li Libref not assigned
+    @li Dataset does not exist
+    @li Input table contains null or invalid values
+
+  Example usage:
+
+      data work.example;
+        set sashelp.prdsale;
+        format _all_ clear;
+      run;
+
+      %mp_getcols(sashelp.prdsale,outds=work.cols)
+
+      data work.cols2;
+        set work.cols;
+        lib='WORK';
+        ds='EXAMPLE';
+        var=name;
+        fmt=format;
+        keep lib ds var fmt;
+      run;
+
+      %mp_applyformats(work.cols2)
+
+  @param [in] inds The input dataset containing the formats to apply (and where
+    to apply them).  Example structure:
+  |LIB:$8.|DS:$32.|VAR:$32.|FMT:$49.|
+  |---|---|---|---|
+  |`WORK `|`EXAMPLE `|`ACTUAL `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`COUNTRY `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`DIVISION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`MONTH `|`MONNAME3. `|
+  |`WORK `|`EXAMPLE `|`PREDICT `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`PRODTYPE `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`PRODUCT `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`QUARTER `|`8. `|
+  |`WORK `|`EXAMPLE `|`REGION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`YEAR `|`8. `|
+
+  @param [out] errds= (0) Provide a libds reference here to export the
+    error messages to a table.  In this case, they will not be printed to the
+    log.
+
+  <h4> SAS Macros </h4>
+  @li mf_getengine.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_validatecol.sas
+
+
+  <h4> Related Macros </h4>
+  @li mp_getformats.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_applyformats(inds,errds=0
+)/*/STORE SOURCE*/;
+%local outds liblist i engine lib msg ;
+
+/**
+  * Validations
+  */
+proc sort data=&inds;
+  by lib ds var fmt;
+run;
+
+%if &errds=0 %then %let outds=%mf_getuniquename(prefix=mp_applyformats);
+%else %let outds=&errds;
+
+data &outds;
+  set &inds;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  length msg $128;
+  by lib ds var fmt;
+  if libref(lib) ne 0 then do;
+    msg=catx(' ','libref',lib,'is not assigned!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  if exist(cats(lib,'.',ds)) ne 1 then do;
+    msg=catx(' ','libds',lib,'.',ds,'does not exist!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  %mp_validatecol(fmt,FORMAT,is_fmt)
+  if is_fmt=0 then do;
+    msg=catx(' ','format',fmt,'on libds',lib,'.',ds,'.',var,'is not valid!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+
+  if first.ds then do;
+    retain dsid;
+    dsid=open(cats(lib,'.',ds));
+    if dsid=0 then do;
+      msg=catx(' ','libds',lib,'.',ds,' could not be opened!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+      return;
+    end;
+    if varnum(dsid,var)<1 then do;
+      msg=catx(' ','Variable',lib,'.',ds,'.',var,' was not found!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+    end;
+  end;
+  if last.ds then rc=close(dsid);
+run;
+
+proc sql noprint;
+select distinct lib into: liblist separated by ' ' from &inds;
+%put &=liblist;
+%if %length(&liblist)>0 %then %do i=1 %to %sysfunc(countw(&liblist));
+  %let lib=%scan(&liblist,1);
+  %let engine=%mf_getengine(&lib);
+  %if &engine ne V9 and &engine ne BASE %then %do;
+    %let msg=&lib has &engine engine - formats cannot be applied;
+    insert into &outds set lib="&lib",ds="_all_",var="_all", msg="&msg" ;
+    %if &errds=0 %then %put %str(ERR)OR: &msg;
+  %end;
+%end;
+quit;
+
+%if %mf_nobs(&outds)>0 %then %return;
+
+/**
+  * Validations complete - now apply the actual formats!
+  */
+%let fref=%mf_getuniquefileref();
+data _null_;
+  set &inds;
+  by lib ds var fmt;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  file &fref;
+  if first.lib then put 'proc datasets nolist lib=' lib ';';
+  if first.ds then put '  modify ' ds ';';
+  put '    format ' var fmt ';';
+  if last.ds then put '  run;';
+  if last.lib then put 'quit;';
+run;
+
+%inc &fref/source2;
+
+%if &errds=0 %then %do;
+  proc sql;
+  drop table &outds;
+%end;
+
+%mend mp_applyformats;

base/mp_assertdsobs.sas

@@ -12,10 +12,6 @@
 
       %mp_assertdsobs(sashelp.class,test=ATMOST 20) %* pass if <21 obs present;
 
-  <h4> SAS Macros </h4>
-  @li mf_nobs.sas
-  @li mp_abort.sas
-
 
   @param [in] inds input dataset to test for presence of observations
   @param [in] desc= (Testing observations) The user provided test description
@@ -33,6 +29,11 @@
   |---|---|---|
   |User Provided description|PASS|Dataset &inds has XX obs|
 
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
   <h4> Related Macros </h4>
   @li mp_assertcolvals.sas
   @li mp_assert.sas
@@ -49,9 +50,10 @@
   outds=work.test_results
 )/*/STORE SOURCE*/;
 
-  %local nobs;
+  %local nobs ds;
   %let nobs=%mf_nobs(&inds);
   %let test=%upcase(&test);
+  %let ds=%mf_getuniquename(prefix=mp_assertdsobs);
 
   %if %substr(&test.xxxxx,1,6)=EQUALS %then %do;
     %let val=%scan(&test,2,%str( ));
@@ -84,7 +86,7 @@
     )
   %end;
 
-  data;
+  data &ds;
     length test_description $256 test_result $4 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
@@ -110,9 +112,6 @@
   %end;
   run;
 
-  %local ds;
-  %let ds=&syslast;
-
   proc append base=&outds data=&ds;
   run;
 

base/mp_assertscope.sas

@@ -0,0 +1,147 @@
+/**
+  @file
+  @brief Used to capture scope leakage of macro variables
+  @details
+
+  A common 'difficult to detect' bug in macros is where a nested macro
+  over-writes variables in a higher level macro.
+
+  This assertion takes a snapshot of the macro variables before and after
+  a macro invocation.  Differences are captured in the `&outds` table. This
+  makes it easy to detect whether any macro variables were modified or
+  changed.
+
+  The following variables are NOT tested (as they are known, global variables
+  used in SASjs):
+
+  @li &sasjs_prefix._FUNCTIONS
+
+  Global variables are initialised in mp_init.sas - which will also trigger
+  "strict mode" in your SAS session.  Whilst this is a default in SASjs
+  produced apps, if you prefer not to use this mode, simply instantiate the
+  following variable to prevent the macro from running:  `SASJS_PREFIX`
+
+  Example usage:
+
+      %mp_assertscope(SNAPSHOT)
+
+      %let oops=I did it again;
+
+      %mp_assertscope(COMPARE,
+        desc=Checking macro variables against previous snapshot
+      )
+
+  This macro is designed to work alongside `sasjs test` - for more information
+  about this facility, visit [cli.sasjs.io/test](https://cli.sasjs.io/test).
+
+  @param [in] action (SNAPSHOT) The action to take.  Valid values:
+    @li SNAPSHOT - take a copy of the current macro variables
+    @li COMPARE - compare the current macro variables against previous values
+  @param [in] scope= (GLOBAL) The scope of the variables to be checked.  This
+    corresponds to the values in the SCOPE column in `sashelp.vmacro`.
+  @param [in] desc= (Testing scope leakage) The user provided test description
+  @param [in] ignorelist= Provide a list of macro variable names to ignore from
+    the comparison
+  @param [in,out] scopeds= (work.mp_assertscope) The dataset to contain the
+    scope snapshot
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|No out of scope variables created or modified|
+
+  <h4> SAS Macros </h4>
+  @li mf_getquotedstr.sas
+  @li mp_init.sas
+
+  <h4> Related Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertcols.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertscope(action,
+  desc=Testing Scope Leakage,
+  scope=GLOBAL,
+  scopeds=work.mp_assertscope,
+  ignorelist=,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+%local ds test_result test_comments del add mod ilist;
+%let ilist=%upcase(&sasjs_prefix._FUNCTIONS SYS_PROCHTTP_STATUS_CODE
+  SYS_PROCHTTP_STATUS_CODE SYS_PROCHTTP_STATUS_PHRASE &ignorelist);
+
+/**
+  * this sets up the global vars, it will also enter STRICT mode.  If this
+  * behaviour is not desired, simply initiate the following global macro
+  * variable to prevent the macro from running: SASJS_PREFIX
+  */
+%mp_init()
+
+/* get current variables */
+%if &action=SNAPSHOT %then %do;
+  proc sql;
+  create table &scopeds as
+    select name,offset,value
+    from dictionary.macros
+    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
+    order by name,offset;
+%end;
+%else %if &action=COMPARE %then %do;
+
+  proc sql;
+  create table _data_ as
+    select name,offset,value
+    from dictionary.macros
+    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
+    order by name,offset;
+
+  %let ds=&syslast;
+
+  proc compare
+    base=&scopeds(where=(upcase(name) not in (%mf_getquotedstr(&ilist))))
+    compare=&ds noprint;
+  run;
+
+  %if &sysinfo=0 %then %do;
+    %let test_result=PASS;
+    %let test_comments=&scope Variables Unmodified;
+  %end;
+  %else %do;
+    proc sql noprint undo_policy=none;
+    select distinct name into: del separated by ' '  from &scopeds
+      where name not in (select name from &ds);
+    select distinct name into: add separated by ' '  from &ds
+      where name not in (select name from &scopeds);
+    select distinct a.name into: mod separated by ' '
+      from &scopeds a
+      inner join &ds b
+      on a.name=b.name
+        and a.offset=b.offset
+      where a.value ne b.value;
+    %let test_result=FAIL;
+    %let test_comments=%str(Mod:(&mod) Add:(&add) Del:(&del));
+  %end;
+
+
+  data ;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_comments=symget('test_comments');
+    test_result=symget('test_result');
+  run;
+
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+%end;
+
+%mend mp_assertscope;

base/mp_binarycopy.sas

@@ -4,8 +4,8 @@
   @details Reads in a file byte by byte and writes it back out.  Is an
   os-independent method to copy files.  In case of naming collision, the
   default filerefs can be modified.
-  Based on:
-  https://stackoverflow.com/questions/13046116/using-sas-to-copy-a-text-file
+  Note that if you have a new enough version of SAS, and you don't need features
+  such as APPEND, you may be better of using the fcopy() function instead.
 
         %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
 
@@ -31,6 +31,7 @@
   @param [in] mode (CREATE) Valid values:
     @li CREATE - Create the file (even if it already exists)
     @li APPEND - Append to the file (don't overwrite)
+  @param iftrue= (1=1) Supply a condition for which the macro should be executed
 
   @returns nothing
 
@@ -44,15 +45,14 @@
     ,inref=____in   /* override default to use own filerefs */
     ,outref=____out /* override default to use own filerefs */
     ,mode=CREATE
+    ,iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
-  %local mod outmode;
-  %if &mode=APPEND %then %do;
-    %let mod=mod;
-    %let outmode='a';
-  %end;
-  %else %do;
-    %let outmode='o';
-  %end;
+  %local mod;
+
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
+  %if &mode=APPEND %then %let mod=mod;
+
   /* these IN and OUT filerefs can point to anything */
   %if &inref = ____in %then %do;
     filename &inref &inloc lrecl=1048576 ;
@@ -63,22 +63,17 @@
 
   /* copy the file byte-for-byte  */
   data _null_;
-    length filein 8 fileid 8;
-    filein = fopen("&inref",'I',1,'B');
-    fileid = fopen("&outref",&outmode,1,'B');
-    rec = '20'x;
-    do while(fread(filein)=0);
-      rc = fget(filein,rec,1);
-      rc = fput(fileid, rec);
-      rc =fwrite(fileid);
-    end;
-    rc = fclose(filein);
-    rc = fclose(fileid);
+    infile &inref lrecl=1 recfm=n;
+    file &outref &mod recfm=n;
+    input sourcechar $char1. @@;
+    format sourcechar hex2.;
+    put sourcechar char1. @@;
   run;
+
   %if &inref = ____in %then %do;
     filename &inref clear;
   %end;
   %if &outref=____out %then %do;
     filename &outref clear;
   %end;
-%mend mp_binarycopy;
+%mend mp_binarycopy;

base/mp_chop.sas

@@ -0,0 +1,194 @@
+/**
+  @file
+  @brief Splits a file of ANY SIZE by reference to a search string.
+  @details Provide a fileref and a search string to chop off part of a file.
+
+  Works by reading in the file byte by byte, then marking the beginning and end
+  of each matched string, before finally doing the chop.
+
+  Choose whether to keep the FIRST or the LAST section of the file.  Optionally,
+  use an OFFSET to fix the precise chop point.
+
+  Usage:
+
+      %let src="%sysfunc(pathname(work))/file.txt";
+      %let str=Chop here!;
+      %let out1="%sysfunc(pathname(work))/file1.txt";
+      %let out2="%sysfunc(pathname(work))/file2.txt";
+      %let out3="%sysfunc(pathname(work))/file3.txt";
+      %let out4="%sysfunc(pathname(work))/file4.txt";
+
+      data _null_;
+        file &src;
+        put "startsection&str.endsection";
+      run;
+
+      %mp_chop(&src, matchvar=str, keep=FIRST, outfile=&out1)
+      %mp_chop(&src, matchvar=str, keep=LAST, outfile=&out2)
+      %mp_chop(&src, matchvar=str, keep=FIRST, matchpoint=END, outfile=&out3)
+      %mp_chop(&src, matchvar=str, keep=LAST, matchpoint=END, outfile=&out4)
+
+      filename results (&out1 &out2 &out3 &out4);
+      data _null_;
+        infile results;
+        input;
+        list;
+      run;
+
+  Results:
+    @li `startsection`
+    @li `Chop here!endsection`
+    @li `startsectionChop here!`
+    @li `endsection`
+
+  For more examples, see mp_chop.test.sas
+
+  @param [in] infile The QUOTED path to the file on which to perform the chop
+  @param [in] matchvar= Macro variable NAME containing the string to split by
+  @param [in] matchpoint= (START) Valid values:
+    @li START - chop at the beginning of the string in `matchvar`.
+    @li END - chop at the end of the string in `matchvar`.
+  @param [in] offset= (0) An adjustment to the precise chop location, by
+    by reference to the `matchpoint`. Should be a positive or negative integer.
+  @param [in] keep= (FIRST) Valid values:
+    @li FIRST - keep the section of the file before the chop
+    @li LAST - keep the section of the file after the chop
+  @param [in] mdebug= (0) Set to 1 to provide macro debugging
+  @param outfile= (0) Optional QUOTED path to the adjusted output file (avoids
+    overwriting the first file).
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mp_abort.sas
+  @li mp_gsubfile.sas
+  @li mp_replace.sas
+  @li mp_chop.test.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mp_chop(infile,
+  matchvar=,
+  matchpoint=START,
+  keep=FIRST,
+  offset=0,
+  mdebug=0,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+%local fref0 dttm ds1 outref;
+%let fref0=%mf_getuniquefileref();
+%let ds1=%mf_getuniquename(prefix=allchars);
+%let ds2=%mf_getuniquename(prefix=startmark);
+
+%if &outfile=0 %then %let outfile=&infile;
+
+%mp_abort(iftrue= (%length(%superq(&matchvar))=0)
+  ,mac=mp_chop.sas
+  ,msg=%str(&matchvar is an empty variable)
+)
+
+/* START */
+%let dttm=%sysfunc(datetime());
+
+filename &fref0 &infile lrecl=1 recfm=n;
+
+/* create dataset with one char per row */
+data &ds1;
+  infile &fref0;
+  input sourcechar $char1. @@;
+  format sourcechar hex2.;
+run;
+
+/* get start & stop position of first matchvar string (one row, two vars) */
+data &ds2;
+  /* set find string to length in bytes to cover trailing spaces */
+  length string $ %length(%superq(&matchvar));
+  string =symget("&matchvar");
+  drop string;
+
+  firstchar=char(string,1);
+  findlen=lengthm(string); /* <- for trailing bytes */
+
+  do _N_=1 to nobs;
+    set &ds1 nobs=nobs point=_N_;
+    if sourcechar=firstchar then do;
+      pos=1;
+      s=0;
+      do point=_N_ to min(_N_ + findlen -1,nobs);
+        set &ds1 point=point;
+        if sourcechar=char(string, pos) then s + 1;
+        else goto _leave_;
+        pos+1;
+      end;
+      _leave_:
+      if s=findlen then do;
+        START =_N_;
+        _N_ =_N_+ s - 1;
+        STOP =_N_;
+        output;
+        /* matched! */
+        stop;
+      end;
+    end;
+  end;
+  stop;
+  keep START STOP;
+run;
+
+%local split;
+%let split=0;
+data _null_;
+  set &ds2;
+  if "&matchpoint"='START' then do;
+    if "&keep"='FIRST' then mp=start;
+    else if "&keep"='LAST' then mp=start-1;
+  end;
+  else if "&matchpoint"='END' then do;
+    if "&keep"='FIRST' then mp=stop+1;
+    else if "&keep"='LAST' then mp=stop;
+  end;
+  split=mp+&offset;
+  call symputx('split',split,'l');
+%if &mdebug=1 %then %do;
+  put (_all_)(=);
+  %put &=offset;
+%end;
+run;
+%if &split=0 %then %do;
+  %put &sysmacroname: No match found in &infile for string %superq(&matchvar);
+  %return;
+%end;
+
+data _null_;
+  file &outfile recfm=n;
+  set &ds1;
+%if &keep=FIRST %then %do;
+  if _n_ ge &split then stop;
+%end;
+%else %do;
+  if _n_ gt &split;
+%end;
+  put sourcechar char1.;
+run;
+
+%if &mdebug=0 %then %do;
+  filename &fref0 clear;
+%end;
+%else %do;
+  data _null_;
+    infile &outfile lrecl=32767;
+    input;
+    list;
+    if _n_>50 then stop;
+  run;
+%end;
+/* END */
+%put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) seconds to run;
+
+%mend mp_chop;

base/mp_cleancsv.sas

@@ -1,5 +1,5 @@
 /**
-  @file mp_cleancsv.sas
+  @file
   @brief Fixes embedded cr / lf / crlf in CSV
   @details CSVs will sometimes contain lf or crlf within quotes (eg when
   saved by excel).  When the termstr is ALSO lf or crlf that can be tricky
@@ -7,14 +7,16 @@
   This macro converts any csv to follow the convention of a windows excel file,
   applying CRLF line endings and converting embedded cr and crlf to lf.
 
-  usage:
+  Usage:
 
       fileref mycsv "/path/your/csv";
       %mp_cleancsv(in=mycsv,out=/path/new.csv)
 
-  @param in= provide path or fileref to input csv
-  @param out= output path or fileref to output csv
-  @param qchar= quote char - hex code 22 is the double quote.
+  @param in= (NOTPROVIDED) Provide path or fileref to input csv. If a period is
+    found, it is assumed to be a file.
+  @param out= (NOTPROVIDED) Output path or fileref to output csv.  If a period
+    is found, it is assumed to be a file.
+  @param qchar= ('22'x) Quote char - hex code 22 is the double quote.
 
   @version 9.2
   @author Allan Bowe
@@ -56,9 +58,14 @@
     else do;
       /* outside a quote, change cr and lf to crlf */
       if inchar='0D'x then do;
+        crblank:
         put '0D0A'x;
         input inchar $char1.;
-        if inchar ne '0A'x then do;
+        if inchar='0D'x then do;
+          /* multiple CR indicates CR formatted file with blank lines */
+          goto crblank;
+        end;
+        else if inchar ne '0A'x then do;
           put inchar $char1.;
           if inchar=qchar then isq = mod(isq+1,2);
         end;

base/mp_cntlout.sas

@@ -0,0 +1,85 @@
+/**
+  @file mp_cntlout.sas
+  @brief Creates a cntlout dataset in a consistent format
+  @details The dataset produced by proc format in the cntlout option will vary
+  according to its contents.
+
+  When dealing with formats from an ETL perspective (eg in [Data Controller for
+  SAS](https://datacontroller.io)), it is important that the output dataset
+  has a consistent model (and compariable values).
+
+  This macro makes use of mddl_sas_cntlout.sas to provide the consistent model,
+  and will left-align the start and end values when dealing with numeric ranges
+  to enable consistency when checking for differences.
+
+  usage:
+
+      %mp_cntlout(libcat=yourlib.cat,cntlout=work.formatexport)
+
+  @param [in] libcat The library.catalog reference
+  @param [in] fmtlist= (0) provide a space separated list of specific formats to
+    extract
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [out] cntlout= (work.fmtextract) Libds reference for the output dataset
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mf_getvarformat.sas
+  @li mp_getformats.sas
+  @li mp_loadformat.sas
+  @li mp_ds2fmtds.sas
+
+  @version 9.2
+  @author Allan Bowe
+  @cond
+**/
+
+%macro mp_cntlout(
+  iftrue=(1=1)
+  ,libcat=
+  ,cntlout=work.fmtextract
+  ,fmtlist=0
+)/*/STORE SOURCE*/;
+%local ddlds cntlds i;
+
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+%let ddlds=%mf_getuniquename();
+%let cntlds=%mf_getuniquename();
+
+%mddl_sas_cntlout(libds=&ddlds)
+
+%if %index(&libcat,-)>0 and %scan(&libcat,2,-)=FC %then %do;
+  %let libcat=%scan(&libcat,1,-);
+%end;
+
+proc format lib=&libcat cntlout=&cntlds;
+%if "&fmtlist" ne "0" %then %do;
+  select
+  %do i=1 %to %sysfunc(countw(&fmtlist));
+    %scan(&fmtlist,&i,%str( ))
+  %end;
+  ;
+%end;
+run;
+
+data &cntlout;
+  if 0 then set &ddlds;
+  set &cntlds;
+  if type="N" then do;
+    start=cats(start);
+    end=cats(end);
+  end;
+run;
+proc sort;
+  by fmtname start;
+run;
+
+proc sql;
+drop table &ddlds,&cntlds;
+
+%mend mp_cntlout;
+/** @endcond */

base/mp_copyfolder.sas

@@ -16,8 +16,11 @@
 
       %mp_copyfolder(&rootdir,&copydir)
 
-  @param source Unquoted path to the folder to copy from.
-  @param target Unquoted path to the folder to copy to.
+  @param [in] source Unquoted path to the folder to copy from.
+  @param [out] target Unquoted path to the folder to copy to.
+  @param [in] copymax=(MAX) Set to a positive integer to indicate the level of
+    subdirectory copy recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas
@@ -31,7 +34,7 @@
 
 **/
 
-%macro mp_copyfolder(source,target);
+%macro mp_copyfolder(source,target,copymax=MAX);
 
   %mp_abort(iftrue=(%mf_isdir(&source)=0)
     ,mac=&sysmacroname
@@ -50,7 +53,7 @@
   %let tempds=%mf_getuniquename();
 
   /* recursive directory listing */
-  %mp_dirlist(path=&source,outds=work.&tempds, maxdepth=MAX)
+  %mp_dirlist(path=&source,outds=work.&tempds,maxdepth=&copymax)
 
   /* create folders and copy content */
   data _null_;
@@ -78,4 +81,4 @@
   proc sql;
   drop table work.&tempds;
 
-%mend mp_copyfolder;
+%mend mp_copyfolder;

base/mp_coretable.sas

@@ -0,0 +1,73 @@
+/**
+  @file
+  @brief Create the permanent Core tables
+  @details Several macros in the [core](https://github.com/sasjs/core) library
+    make use of permanent tables.  To avoid duplication in definitions, this
+    macro provides a central location for managing the corresponding DDL.
+
+  Note - this macro is likely to be deprecated in future in favour of a
+  dedicated "datamodel" folder (prefix mddl)
+
+  Any corresponding data would go in a seperate repo, to avoid this one
+  ballooning in size!
+
+  Example usage:
+
+      %mp_coretable(LOCKTABLE,libds=work.locktable)
+
+  @param [in] table_ref The type of table to create.  Example values:
+    @li DIFFTABLE
+    @li FILTER_DETAIL
+    @li FILTER_SUMMARY
+    @li LOCKANYTABLE
+    @li MAXKEYTABLE
+  @param [in] libds= (0) The library.dataset reference used to create the table.
+    If not provided, then the DDL is simply printed to the log.
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mddl_dc_filterdetail.sas
+  @li mddl_dc_filtersummary.sas
+  @li mddl_dc_locktable.sas
+  @li mddl_dc_maxkeytable.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filterstore.sas
+  @li mp_lockanytable.sas
+  @li mp_retainedkey.sas
+  @li mp_storediffs.sas
+  @li mp_stackdiffs.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_coretable(table_ref,libds=0
+)/*/STORE SOURCE*/;
+%local outds ;
+%let outds=%sysfunc(ifc(&libds=0,%mf_getuniquename(),&libds));
+proc sql;
+%if &table_ref=DIFFTABLE %then %do;
+  %mddl_dc_difftable(libds=&outds)
+%end;
+%else %if &table_ref=LOCKTABLE %then %do;
+  %mddl_dc_locktable(libds=&outds)
+%end;
+%else %if &table_ref=FILTER_SUMMARY %then %do;
+  %mddl_dc_filtersummary(libds=&outds)
+%end;
+%else %if &table_ref=FILTER_DETAIL %then %do;
+  %mddl_dc_filterdetail(libds=&outds)
+%end;
+%else %if &table_ref=MAXKEYTABLE %then %do;
+  %mddl_dc_maxkeytable(libds=&outds)
+%end;
+
+%if &libds=0 %then %do;
+  proc sql;
+  describe table &syslast;
+  drop table &syslast;
+%end;
+%mend mp_coretable;

base/mp_createconstraints.sas

@@ -18,11 +18,14 @@
       %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
       %mp_createconstraints(inds=work.constraints,outds=created,execute=YES)
 
-  @param inds= The input table containing the constraint info
-  @param outds= a table containing the create statements (create_statement column)
-  @param execute= `YES|NO` - default is NO. To actually create, use YES.
+  @param inds= (work.mp_getconstraints) The input table containing the
+    constraint info
+  @param outds= (work.mp_createconstraints) A table containing the create
+    statements (create_statement column)
+  @param execute= (NO) To actually create, use YES.
 
-  <h4> SAS Macros </h4>
+  <h4> Related Files </h4>
+  @li mp_getconstraints.sas
 
   @version 9.2
   @author Allan Bowe
@@ -30,7 +33,7 @@
 **/
 
 %macro mp_createconstraints(inds=mp_getconstraints
-  ,outds=mp_createconstraints
+  ,outds=work.mp_createconstraints
   ,execute=NO
 )/*/STORE SOURCE*/;
 
@@ -64,4 +67,4 @@ data &outds;
   output;
 run;
 
-%mend mp_createconstraints;
+%mend mp_createconstraints;

base/mp_createwebservice.sas

@@ -1,47 +1,13 @@
 /**
   @file mp_createwebservice.sas
-  @brief Create a web service in SAS 9 or Viya
-  @details Creates a SASJS ready Stored Process in SAS 9 or Job Execution
-  Service in SAS Viya
+  @brief Create a web service in SAS 9, Viya or SASjs Server
+  @details This is actually a wrapper for mx_createwebservice.sas, remaining
+  for legacy purposes.  For new apps, use mx_createwebservice.sas.
 
-Usage:
-
-    %* compile macros ;
-    filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
-    %inc mc;
-
-    %* write some code;
-    filename ft15f001 temp;
-    parmcards4;
-        %* fetch any data from frontend ;
-        %webout(FETCH)
-        data example1 example2;
-          set sashelp.class;
-        run;
-        %* send data back;
-        %webout(OPEN)
-        %webout(ARR,example1) * Array format, fast, suitable for large tables ;
-        %webout(OBJ,example2) * Object format, easier to work with ;
-        %webout(CLOSE)
-    ;;;;
-    %mp_createwebservice(path=/Public/app/common,name=appInit,code=ft15f001,replace=YES)
 
   <h4> SAS Macros </h4>
-  @li mf_getplatform.sas
-  @li mm_createwebservice.sas
-  @li mv_createwebservice.sas
+  @li mx_createwebservice.sas
 
-  @param path= The full folder path where the service will be created
-  @param name= Service name.  Avoid spaces.
-  @param desc= The description of the service (optional)
-  @param precode= Space separated list of filerefs, pointing to the code that
-    needs to be attached to the beginning of the service (optional)
-  @param code= Space seperated fileref(s) of the actual code to be added
-  @param replace= select YES to replace any existing service in that location
-
-
-  @version 9.2
-  @author Allan Bowe
 
 **/
 
@@ -51,33 +17,16 @@ Usage:
     ,code=ft15f001
     ,desc=This service was created by the mp_createwebservice macro
     ,replace=YES
+    ,mdebug=0
 )/*/STORE SOURCE*/;
 
-%if &syscc ge 4 %then %do;
-  %put syscc=&syscc - &sysmacroname will not execute in this state;
-  %return;
-%end;
-
-%local platform; %let platform=%mf_getplatform();
-%if &platform=SASVIYA %then %do;
-  %if "&path"="HOME" %then %let path=/Users/&sysuserid/My Folder;
-  %mv_createwebservice(path=&path
+  %mx_createwebservice(path=&path
     ,name=&name
-    ,code=&code
     ,precode=&precode
+    ,code=&code
     ,desc=&desc
     ,replace=&replace
+    ,mdebug=&mdebug
   )
-%end;
-%else %do;
-  %if "&path"="HOME" %then %let path=/User Folders/&sysuserid/My Folder;
-  %mm_createwebservice(path=&path
-    ,name=&name
-    ,code=&code
-    ,precode=&precode
-    ,desc=&desc
-    ,replace=&replace
-  )
-%end;
 
 %mend mp_createwebservice;

base/mp_csv2ds.sas

@@ -49,10 +49,6 @@
   ,mac=&sysmacroname
   ,msg=%str(the BASEDS variable must be provided)
 )
-%mp_abort(iftrue=( &baseds=0 )
-  ,mac=&sysmacroname
-  ,msg=%str(the BASEDS variable must be provided)
-)
 %mp_abort(iftrue=( %mf_existds(&baseds)=0 )
   ,mac=&sysmacroname
   ,msg=%str(the BASEDS dataset (&baseds) needs to be assigned, and to exist)

base/mp_deletefolder.sas

@@ -51,6 +51,7 @@
     data _null_;
       set work.&tempds end=last;
       length fref $8;
+      fref='';
       rc=filename(fref,filepath);
       rc=fdelete(fref);
       if rc then do;

base/mp_dictionary.sas

@@ -0,0 +1,52 @@
+/**
+  @file mp_dictionary.sas
+  @brief Creates a portal (libref) into the SQL Dictionary Views
+  @details Provide a libref and the macro will create a series of views against
+  each view in the special PROC SQL dictionary libref.
+
+  This is useful if you would like to visualise (navigate) the views in a SAS
+  client such as Base SAS, Enterprise Guide, or Studio (or [Data Controller](
+  https://datacontroller.io)).
+
+  It works by extracting the dictionary.dictionaries view into
+  YOURLIB.dictionaries, then uses that to create a YOURLIB.{viewName} for every
+  other dictionary.view, eg:
+
+      proc sql;
+      create view YOURLIB.columns as select * from dictionary.columns;
+
+  Usage:
+
+      libname demo "/lib/directory";
+      %mp_dictionary(lib=demo)
+
+  Or, to just create them in WORK:
+
+      %mp_dictionary()
+
+  If you'd just like to browse the dictionary data model, you can also check
+  out [this article](https://rawsas.com/dictionary-of-dictionaries/).
+
+  ![](https://user-images.githubusercontent.com/4420615/188278365-2987db97-0594-4a39-ac81-dbacdef5cdc8.png)
+
+  @param lib= (WORK) The libref in which to create the views
+
+  <h4> Related Files </h4>
+  @li mp_dictionary.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_dictionary(lib=WORK)/*/STORE SOURCE*/;
+  %local list i mem;
+  proc sql noprint;
+  create view &lib..dictionaries as select * from dictionary.dictionaries;
+  select distinct memname into: list separated by ' '  from &lib..dictionaries;
+  %do i=1 %to %sysfunc(countw(&list,%str( )));
+    %let mem=%scan(&list,&i,%str( ));
+    create view &lib..&mem as select * from dictionary.&mem;
+  %end;
+  quit;
+%mend mp_dictionary;

base/mp_dirlist.sas

@@ -6,8 +6,7 @@
   Credit for the rename approach:
   https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003
 
-
-  usage:
+  Usage:
 
       %mp_dirlist(path=/some/location, outds=myTable, maxdepth=MAX)
 
@@ -23,12 +22,15 @@
   X CMD) do please raise an issue!
 
 
-  @param [in] path= for which to return contents
-  @param [in] fref= Provide a DISK engine fileref as an alternative to PATH
+  @param [in] path= (%sysfunc(pathname(work))) Path for which to return contents
+  @param [in] fref= (0) Provide a DISK engine fileref as an alternative to PATH
   @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
     subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
     recursion, set to MAX.
-  @param [out] outds= the output dataset to create
+  @param [in] showparent= (NO) By default, the initial parent directory is not
+    part of the results.  Set to YES to include it.  For this record only,
+    directory=filepath.
+  @param [out] outds= (work.mp_dirlist) The output dataset to create
   @param [out] getattrs= (NO)  If getattrs=YES then the doptname / foptname
     functions are used to scan all properties - any characters that are not
     valid in a SAS name (v7) are simply stripped, and the table is transposed
@@ -49,19 +51,22 @@
     - OS SPECIFIC variables, if <code>getattrs=</code> is used.
 
   <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_getvarlist.sas
+  @li mf_wordsinstr1butnotstr2.sas
   @li mp_dropmembers.sas
 
   <h4> Related Macros </h4>
   @li mp_dirlist.test.sas
 
   @version 9.2
-  @author Allan Bowe
 **/
 
 %macro mp_dirlist(path=%sysfunc(pathname(work))
     , fref=0
     , outds=work.mp_dirlist
     , getattrs=NO
+    , showparent=NO
     , maxdepth=0
     , level=0 /* The level of recursion to perform.  For internal use only. */
 )/*/STORE SOURCE*/;
@@ -81,7 +86,8 @@ data &out_ds(compress=no
     keep=file_or_folder filepath filename ext msg directory level
   );
   length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
-    ext $20 msg $200;
+    ext $20 msg $200 foption $16;
+  if _n_=1 then call missing(of _all_);
   retain level &level;
   %if &fref=0 %then %do;
     rc = filename(fref, "&path");
@@ -92,13 +98,19 @@ data &out_ds(compress=no
   %end;
   if rc = 0 then do;
     did = dopen(fref);
-    directory=dinfo(did,'Directory');
     if did=0 then do;
-      putlog "NOTE: This directory is empty - " directory;
+      putlog "NOTE: This directory is empty, or does not exist - &path";
       msg=sysmsg();
-      put _all_;
+      put (_all_)(=);
       stop;
     end;
+    /* attribute is OS-dependent - could be "Directory" or "Directory Name" */
+    numopts=doptnum(did);
+    do i=1 to numopts;
+      foption=doptname(did,i);
+      if foption=:'Directory' then i=numopts;
+    end;
+    directory=dinfo(did,foption);
     rc = filename(fref);
   end;
   else do;
@@ -136,6 +148,15 @@ data &out_ds(compress=no
     output;
   end;
   rc = dclose(did);
+  %if &showparent=YES and &level=0 %then %do;
+    filepath=directory;
+    file_or_folder='folder';
+    ext='';
+    filename=scan(directory,-1,'/\');
+    msg='';
+    level=&level;
+    output;
+  %end;
   stop;
 run;
 
@@ -143,6 +164,7 @@ run;
   data &out_ds;
     set &out_ds;
     length infoname infoval $60 fref $8;
+    if _n_=1 then call missing(fref);
     rc=filename(fref,filepath);
     drop rc infoname fid i close fref;
     if file_or_folder='file' then do;
@@ -193,15 +215,38 @@ data &out_ds;
   set &out_ds(where=(filepath ne ''));
 run;
 
-/* update main table */
-proc append base=&outds data=&out_ds;
-run;
+/**
+  * The above transpose can mean that some updates create additional columns.
+  * This necessitates the occasional use of datastep over proc append.
+  */
+%if %mf_existds(&outds) %then %do;
+  %local basevars appvars newvars;
+  %let basevars=%mf_getvarlist(&outds);
+  %let appvars=%mf_getvarlist(&out_ds);
+  %let newvars=%length(%mf_wordsinstr1butnotstr2(Str1=&appvars,Str2=&basevars));
+  %if &newvars>0 %then %do;
+    data &outds;
+      set &outds &out_ds;
+    run;
+  %end;
+  %else %do;
+    proc append base=&outds data=&out_ds force nowarn;
+    run;
+  %end;
+%end;
+%else %do;
+  proc append base=&outds data=&out_ds;
+  run;
+%end;
 
 /* recursive call */
 %if &maxdepth>&level or &maxdepth=MAX %then %do;
   data _null_;
     set &out_ds;
     where file_or_folder='folder';
+  %if &showparent=YES and &level=0 %then %do;
+    if filepath ne directory;
+  %end;
     length code $10000;
     code=cats('%nrstr(%mp_dirlist(path=',filepath,",outds=&outds"
       ,",getattrs=&getattrs,level=%eval(&level+1),maxdepth=&maxdepth))");
@@ -214,4 +259,4 @@ run;
 proc sql;
 drop table &out_ds;
 
-%mend mp_dirlist;
+%mend mp_dirlist;

base/mp_ds2cards.sas

@@ -11,7 +11,7 @@
 
   Usage:
 
-      %mp_ds2cards(base_ds=sashelp.class
+      %mp_ds2cards(sashelp.class
         , tgt_ds=work.class
         , cards_file= "C:\temp\class.sas"
         , showlog=NO
@@ -23,7 +23,7 @@
     - explicity setting a unix LF
     - constraints / indexes etc
 
-  @param [in] base_ds= Should be two level - eg work.blah.  This is the table
+  @param [in] base_ds Should be two level - eg work.blah.  This is the table
                   that is converted to a cards file.
   @param [in] tgt_ds= Table that the generated cards file would create.
     Optional - if omitted, will be same as BASE_DS.
@@ -48,9 +48,10 @@
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
-%macro mp_ds2cards(base_ds=, tgt_ds=
+%macro mp_ds2cards(base_ds, tgt_ds=
     ,cards_file="%sysfunc(pathname(work))/cardgen.sas"
     ,maxobs=max
     ,random_sample=NO
@@ -139,8 +140,9 @@ create table datalines1 as
 /**
   Due to long decimals cannot use best. format
   So - use bestd. format and then use character functions to strip trailing
-    zeros, if NOT an integer!!
-  resolved code = ifc(int(VARIABLE)=VARIABLE
+    zeros, if NOT an integer or missing!!  Cannot use int() as it upsets
+    note2err when there are missings.
+  resolved code = ifc( mod(coalesce(VARIABLE,0),1)=0
     ,put(VARIABLE,best32.)
     ,substrn(put(VARIABLE,bestd32.),1
     ,findc(put(VARIABLE,bestd32.),'0','TBK')));
@@ -151,7 +153,7 @@ data datalines_2;
   set datalines1 (where=(upcase(name) not in
     ('PROCESSED_DTTM','VALID_FROM_DTTM','VALID_TO_DTTM')));
   if type='num' then dataline=
-    cats('ifc(int(',name,')=',name,'
+    cats('ifc(mod(coalesce(',name,',0),1)=0
       ,put(',name,',best32.-l)
       ,substrn(put(',name,',bestd32.-l),1
       ,findc(put(',name,',bestd32.-l),"0","TBK")))');
@@ -218,7 +220,8 @@ data _null_;
     put '  @file';
     put "  @brief Datalines for %upcase(%scan(&base_ds,2)) dataset";
     put "  @details Generated by %nrstr(%%)mp_ds2cards()";
-    put "  Available on github.com/sasjs/core";
+    put "    Source: https://github.com/sasjs/core";
+    put '  @cond ';
     put '**/';
     put "data &tgt_ds &indexes;";
     put "attrib ";
@@ -251,6 +254,7 @@ data _null_;
         ;
       %end;
       put ";";
+      put 'missing a b c d e f g h i j k l m n o p q r s t u v w x y z _;';
       put "datalines4;";
     end;
   end;
@@ -263,6 +267,7 @@ data _null_;
   if __lastobs then do;
     put ';;;;';
     put 'run;';
+    put '/** @endcond **/';
     stop;
   end;
 run;
@@ -282,4 +287,5 @@ quit;
 %put NOTE-;%put NOTE-;
 %put NOTE- %sysfunc(dequote(&cards_file.));
 %put NOTE-;%put NOTE-;
-%mend mp_ds2cards;
+%mend mp_ds2cards;
+/** @endcond **/

base/mp_ds2csv.sas

@@ -1,23 +1,82 @@
 /**
   @file
-  @brief Export a dataset to a CSV file
-  @details Export to a file or a fileref
+  @brief Export a dataset to a CSV file WITH leading blanks
+  @details Export a dataset to a file or fileref, retaining leading blanks.
+
+  When using SASJS headerformat, the input statement is provided in the first
+  row of the CSV.
+
   Usage:
 
       %mp_ds2csv(sashelp.class,outref="%sysfunc(pathname(work))/file.csv")
 
-  @param ds The dataset to be exported
-  @param outfile= The output filename - should be quoted.
-  @param outref= The output fileref (takes precedence if provided)
-  @param outencoding= The output encoding to use (unquoted)
+      filename example temp;
+      %mp_ds2csv(sashelp.air,outref=example,headerformat=SASJS)
+      data; infile example; input;put _infile_; if _n_>5 then stop;run;
+
+      data _null_;
+        infile example;
+        input;
+        call symputx('stmnt',_infile_);
+        stop;
+      run;
+      data work.want;
+        infile example dsd firstobs=2;
+        input &stmnt;
+      run;
+
+  Why use mp_ds2csv over, say, proc export?
+
+  1. Ability to retain leading blanks (this is a major one)
+  2. Control the header format
+  3. Simple one-liner
+
+  @param [in] ds The dataset to be exported
+  @param [in] dlm= (COMMA) The delimeter to apply.  For SASJS, will always be
+    COMMA. Supported values:
+    @li COMMA
+    @li SEMICOLON
+  @param [in] headerformat= (LABEL) The format to use for the header section.
+    Valid values:
+    @li LABEL - Use the variable label (or name, if blank)
+    @li NAME - Use the variable name
+    @li SASJS - Used to create sasjs-formatted input CSVs, eg for use in
+      mp_testservice.sas.  This format will supply an input statement in the
+      first row, making ingestion by datastep a breeze.  Special misisng values
+      will be prefixed with a period (eg `.A`) to enable ingestion on both SAS 9
+      and Viya.  Dates / Datetimes etc are identified by the format type (lookup
+      with mcf_getfmttype.sas) and converted to human readable formats (not
+      numbers).
+  @param [out] outfile= The output filename - should be quoted.
+  @param [out] outref= (0) The output fileref (takes precedence if provided)
+  @param [in] outencoding= (0) The (quoted) output encoding to use, eg `"UTF-8"`
+  @param [in] termstr= (CRLF) The line seperator to use.  For SASJS, will
+    always be CRLF.  Valid values:
+    @li CRLF
+    @li LF
+
+  <h4> SAS Macros </h4>
+  @li mcf_getfmttype.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarformat.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
 
   @version 9.2
   @author Allan Bowe (credit mjsq)
 **/
 
-%macro mp_ds2csv(ds, outref=0, outfile=, outencoding=0
+%macro mp_ds2csv(ds
+  ,dlm=COMMA
+  ,outref=0
+  ,outfile=
+  ,outencoding=0
+  ,headerformat=LABEL
+  ,termstr=CRLF
 )/*/STORE SOURCE*/;
 
+%local outloc delim i varlist var vcnt vat dsv vcom vmiss fmttype vfmt;
+
 %if not %sysfunc(exist(&ds)) %then %do;
   %put %str(WARN)ING:  &ds does not exist;
   %return;
@@ -26,33 +85,128 @@
 %if %index(&ds,.)=0 %then %let ds=WORK.&ds;
 
 %if &outencoding=0 %then %let outencoding=;
-%else %let outencoding=encoding="&outencoding";
+%else %let outencoding=encoding=&outencoding;
 
-%local outloc;
 %if &outref=0 %then %let outloc=&outfile;
 %else %let outloc=&outref;
 
+%if &headerformat=SASJS %then %do;
+  %let delim=",";
+  %let termstr=CRLF;
+  %mcf_getfmttype(wrap=YES)
+%end;
+%else %if &dlm=COMMA %then %let delim=",";
+%else %let delim=";";
+
 /* credit to mjsq - https://stackoverflow.com/a/55642267 */
 
 /* first get headers */
 data _null_;
-  file &outloc dlm=',' dsd &outencoding lrecl=32767;
-  length header $ 2000;
+  file &outloc &outencoding lrecl=32767 termstr=&termstr;
+  length header $ 2000 varnm vfmt $32 dlm $1 fmttype $8;
+  call missing(of _all_);
   dsid=open("&ds.","i");
   num=attrn(dsid,"nvars");
+  dlm=&delim;
   do i=1 to num;
-    header = trim(left(coalescec(varlabel(dsid,i),varname(dsid,i))));
+    varnm=upcase(varname(dsid,i));
+    if i=num then dlm='';
+  %if &headerformat=NAME %then %do;
+    header=cats(varnm,dlm);
+  %end;
+  %else %if &headerformat=LABEL %then %do;
+    header = cats(coalescec(varlabel(dsid,i),varnm),dlm);
+  %end;
+  %else %if &headerformat=SASJS %then %do;
+    if vartype(dsid,i)='C' then header=cats(varnm,':$char',varlen(dsid,i),'.');
+    else do;
+      vfmt=coalescec(varfmt(dsid,i),'0');
+      fmttype=mcf_getfmttype(vfmt);
+      if fmttype='DATE' then header=cats(varnm,':date9.');
+      else if fmttype='DATETIME' then header=cats(varnm,':E8601DT26.6');
+      else if fmttype='TIME' then header=cats(varnm,':TIME12.');
+      else header=cats(varnm,':best.');
+    end;
+  %end;
+  %else %do;
+    %put &sysmacroname: Invalid headerformat value (&headerformat);
+    %return;
+  %end;
     put header @;
   end;
   rc=close(dsid);
 run;
 
+%let varlist=%mf_getvarlist(&ds);
+%let vcnt=%sysfunc(countw(&varlist));
+
+/**
+  * The $quote modifier (without a width) will take the length from the variable
+  * and increase by two.  However this will lead to truncation where the value
+  * contains double quotes (which are doubled up).  To get around this, scan the
+  * data to see the max number of double quotes, so that the appropriate width
+  * can be applied in the subsequent step.
+  */
+data _null_;
+  set &ds end=last;
+%do i=1 %to &vcnt;
+  %let var=%scan(&varlist,&i);
+  %if %mf_getvartype(&ds,&var)=C %then %do;
+    %let dsv1=%mf_getuniquename(prefix=csvcol1_);
+    %let dsv2=%mf_getuniquename(prefix=csvcol2_);
+    retain &dsv1 0;
+    &dsv2=length(&var)+countc(&var,'"');
+    if &dsv2>&dsv1 then &dsv1=&dsv2;
+    if last then call symputx(
+      "vlen&i"
+      /* should be no shorter than varlen, and no longer than 32767 */
+      ,cats('$quote',min(&dsv1+2,32767),'.')
+      ,'l'
+    );
+  %end;
+%end;
+
+%let vat=@;
+%let vcom=&delim;
+%let vmiss=%mf_getuniquename(prefix=csvcol3_);
 /* next, export data */
 data _null_;
   set &ds.;
-  file &outloc mod dlm=',' dsd &outencoding lrecl=32767;
-  put (_all_) (+0);
+  file &outloc mod dlm=&delim dsd &outencoding lrecl=32767 termstr=&termstr;
+  if _n_=1 then &vmiss='  ';
+  %do i=1 %to &vcnt;
+    %let var=%scan(&varlist,&i);
+    %if &i=&vcnt %then %do;
+      %let vat=;
+      %let vcom=;
+    %end;
+    %if %mf_getvartype(&ds,&var)=N %then %do;
+      %if &headerformat = SASJS %then %do;
+        %let vcom=&delim;
+        %let fmttype=%sysfunc(mcf_getfmttype(%mf_getvarformat(&ds,&var)0));
+        %if &fmttype=DATE %then %let vfmt=DATE9.;
+        %else %if &fmttype=DATETIME %then %let vfmt=E8601DT26.6;
+        %else %if &fmttype=TIME %then %let vfmt=TIME12.;
+        %else %do;
+          %let vfmt=;
+          %let vcom=;
+        %end;
+      %end;
+      %else %let vcom=;
+
+      /* must use period - in order to work in both 9.4 and Viya 3.5 */
+      if missing(&var) and &var ne %sysfunc(getoption(MISSING)) then do;
+        &vmiss=cats('.',&var);
+        put &vmiss &vat;
+      end;
+      else put &var &vfmt &vcom &vat;
+
+    %end;
+    %else %do;
+      %if &i ne &vcnt %then %let vcom=&delim;
+      put &var &&vlen&i &vcom &vat;
+    %end;
+  %end;
 run;
 
-
 %mend mp_ds2csv;

base/mp_ds2ddl.sas

@@ -0,0 +1,30 @@
+/**
+  @file
+  @brief A wrapper for mp_getddl.sas
+  @details In the next release, this will be the main version.
+
+
+  <h4> SAS Macros </h4>
+  @li mp_getddl.sas
+
+**/
+
+%macro mp_ds2ddl(libds,fref=getddl,flavour=SAS,showlog=YES,schema=
+  ,applydttm=NO
+)/*/STORE SOURCE*/;
+
+%local libref;
+%let libds=%upcase(&libds);
+%let libref=%scan(&libds,1,.);
+%if &libref=&libds %then %let libds=WORK.&libds;
+
+%mp_getddl(%scan(&libds,1,.)
+  ,%scan(&libds,2,.)
+  ,fref=&fref
+  ,flavour=SAS
+  ,showlog=&showlog
+  ,schema=&schema
+  ,applydttm=&applydttm
+)
+
+%mend mp_ds2ddl;

base/mp_ds2fmtds.sas

@@ -17,7 +17,7 @@
   <h4> SAS Macros </h4>
   @li mf_existds.sas
 
-  <h4> Related Macros <h4>
+  <h4> Related Macros </h4>
   @li mp_jsonout.sas
 
   @version 9.2

base/mp_ds2md.sas

@@ -57,6 +57,11 @@
 %local vars;
 %let vars=%upcase(%mf_getvarlist(&libds));
 
+%if %trim(X&vars)=X %then %do;
+  %put &sysmacroname: Table &libds has no columns!!;
+  %return;
+%end;
+
 /* create the header row */
 data _null_;
   file &outref;
@@ -87,7 +92,7 @@ data _null_;
 run;
 
 %if %upcase(&showlog)=YES %then %do;
-  options ps=max;
+  options ps=max lrecl=max;
   data _null_;
     infile &outref;
     input;
@@ -95,4 +100,4 @@ run;
   run;
 %end;
 
-%mend mp_ds2md;
+%mend mp_ds2md;

base/mp_ds2squeeze.sas

@@ -0,0 +1,120 @@
+/**
+  @file
+  @brief Create a smaller version of a dataset, without data loss
+  @details This macro will scan the input dataset and create a new one, that
+  has the minimum variable lengths needed to store the data without data loss.
+
+  Inspiration was taken from [How to Reduce the Disk Space Required by a
+  SAS® Data Set](https://www.lexjansen.com/nesug/nesug06/io/io18.pdf) by
+  Selvaratnam Sridharma.  The end of the referenced paper presents a macro named
+  "squeeze", hence the nomenclature.
+
+  Usage:
+
+      data big;
+        length my big $32000;
+        do i=1 to 1e4;
+          my=repeat('oh my',100);
+          big='dawg';
+          special=._;
+          output;
+        end;
+      run;
+
+      %mp_ds2squeeze(work.big,outds=work.smaller)
+
+  The following will also be printed to the log (exact values may differ
+  depending on your OS and COMPRESS settings):
+
+  > MP_DS2SQUEEZE: work.big was  625MB
+
+  > MP_DS2SQUEEZE: work.smaller is    5MB
+
+  @param [in] libds The library.dataset to be squeezed
+  @param [out] outds= (work.mp_ds2squeeze) The squeezed dataset to create
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> SAS Macros </h4>
+  @li mf_getfilesize.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_getmaxvarlengths.sas
+
+  <h4> Related Programs </h4>
+  @li mp_ds2squeeze.test.sas
+
+  @version 9.3
+  @author Allan Bowe
+**/
+
+%macro mp_ds2squeeze(
+  libds,
+  outds=work.mp_ds2squeeze,
+  mdebug=0
+)/*/STORE SOURCE*/;
+%local dbg source;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %do;
+  %let dbg=*;
+  %let source=/source2;
+%end;
+
+%local optval ds fref startsize;
+%let ds=%mf_getuniquename();
+%let fref=%mf_getuniquefileref();
+%let startsize=%mf_getfilesize(libds=&libds,format=yes);
+
+%mp_getmaxvarlengths(&libds,outds=&ds)
+
+data _null_;
+  set &ds end=last;
+  file &fref;
+  /* grab the types */
+  retain dsid;
+  if _n_=1 then dsid=open("&libds",'is');
+  if dsid le 0 then do;
+    msg=sysmsg();
+    put msg=;
+    stop;
+  end;
+  type=vartype(dsid,varnum(dsid, name));
+  if last then rc=close(dsid);
+  /* write out the length statement */
+  if _n_=1 then put 'length ';
+  length len $6;
+  if type='C' then do;
+    if maxlen=0 then len='$1';
+    else len=cats('$',maxlen);
+  end;
+  else do;
+    if maxlen=0 then len='3';
+    else len=cats(maxlen);
+  end;
+  put '  ' name ' ' len;
+  if last then put ';';
+run;
+
+/* configure varlenchk - as we are explicitly shortening the variables */
+%let optval=%sysfunc(getoption(varlenchk));
+options varlenchk=NOWARN;
+
+data &outds;
+  %inc &fref &source;
+  set &libds;
+run;
+
+options varlenchk=&optval;
+
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &ds;
+  filename &fref clear;
+%end;
+
+%put &sysmacroname: &libds was &startsize;
+%put &sysmacroname: &outds is %mf_getfilesize(libds=&outds,format=yes);
+
+%mend mp_ds2squeeze;

base/mp_dsmeta.sas

@@ -0,0 +1,110 @@
+/**
+  @file
+  @brief Export dataset metadata to a single output table
+  @details Exports the dataset attributes and enginehost information, then
+  converts the datasets into a single output table in the following format:
+
+|ODS_TABLE:$10.|NAME:$100.|VALUE:$1000.|
+|---|---|---|
+|`ATTRIBUTES `|`Data Set Name `|`SASHELP.CLASS `|
+|`ATTRIBUTES `|`Observations `|`19 `|
+|`ATTRIBUTES `|`Member Type `|`DATA `|
+|`ATTRIBUTES `|`Variables `|`5 `|
+|`ATTRIBUTES `|`Engine `|`V9 `|
+|`ATTRIBUTES `|`Indexes `|`0 `|
+|`ATTRIBUTES `|`Created `|`06/08/2020 00:59:14 `|
+|`ATTRIBUTES `|`Observation Length `|`40 `|
+|`ATTRIBUTES `|`Last Modified `|`06/08/2020 00:59:14 `|
+|`ATTRIBUTES `|`Deleted Observations `|`0 `|
+|`ATTRIBUTES `|`Protection `|`. `|
+|`ATTRIBUTES `|`Compressed `|`NO `|
+|`ATTRIBUTES `|`Data Set Type `|`. `|
+|`ATTRIBUTES `|`Sorted `|`NO `|
+|`ATTRIBUTES `|`Label `|`Student Data `|
+|`ATTRIBUTES `|`Data Representation `|`SOLARIS_X86_64, LINUX_X86_64, ALPHA_TRU64, LINUX_IA64 `|
+|`ATTRIBUTES `|`Encoding `|`us-ascii  ASCII (ANSI) `|
+|`ENGINEHOST `|`Data Set Page Size `|`65536 `|
+|`ENGINEHOST `|`Number of Data Set Pages `|`1 `|
+|`ENGINEHOST `|`First Data Page `|`1 `|
+|`ENGINEHOST `|`Max Obs per Page `|`1632 `|
+|`ENGINEHOST `|`Obs in First Data Page `|`19 `|
+|`ENGINEHOST `|`Number of Data Set Repairs `|`0 `|
+|`ENGINEHOST `|`Filename `|`/opt/sas/sas9/SASHome/SASFoundation/9.4/sashelp/class.sas7bdat `|
+|`ENGINEHOST `|`Release Created `|`9.0401M7 `|
+|`ENGINEHOST `|`Host Created `|`Linux `|
+|`ENGINEHOST `|`Inode Number `|`28314616 `|
+|`ENGINEHOST `|`Access Permission `|`rw-r--r-- `|
+|`ENGINEHOST `|`Owner Name `|`sas `|
+|`ENGINEHOST `|`File Size `|`128KB `|
+|`ENGINEHOST `|`File Size (bytes) `|`131072 `|
+
+  Example usage:
+
+      %mp_dsmeta(sashelp.class,outds=work.mymeta)
+      proc print data=work.mymeta;
+      run;
+
+  For more details on creating datasets from PROC CONTENTS check out this
+  excellent [paper](
+  https://support.sas.com/resources/papers/proceedings14/1549-2014.pdf) by
+  [Louise Hadden](https://www.linkedin.com/in/louisehadden/).
+
+  @param libds The library.dataset to export the metadata for
+  @param outds= (work.dsmeta) The output table to contain the metadata
+
+  <h4> Related Files </h4>
+  @li mp_dsmeta.test.sas
+  @li mp_getcols.sas
+  @li mp_getdbml.sas
+  @li mp_getddl.sas
+  @li mp_getformats.sas
+  @li mp_getpk.sas
+  @li mp_guesspk.sas
+
+**/
+
+%macro mp_dsmeta(libds,outds=work.dsmeta);
+
+%local ds1 ds2;
+data;run; %let ds1=&syslast;
+data;run; %let ds2=&syslast;
+
+/* setup the ODS capture */
+ods output attributes=&ds1 enginehost=&ds2;
+
+/* export the metadata */
+proc contents data=&libds;
+run;
+
+/* load it into a single table */
+data &outds (keep=ods_table name value);
+  length ods_table $10 name label2 label1 label $100
+    value cvalue cvalue1 cvalue2 $1000
+    nvalue nvalue1 nvalue2 8;
+  if _n_=1 then call missing (of _all_);
+  * putlog (_all_)(=);
+  set &ds1 (in=atrs) &ds2 (in=eng);
+  if atrs then do;
+    ods_table='ATTRIBUTES';
+    name=coalescec(label1,label);
+    value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.));
+    output;
+    if label2 ne '' then do;
+    name=label2;
+    value=coalescec(cvalue2,put(nvalue2,best.));
+    output;
+    end;
+  end;
+  else if eng then do;
+    ods_table='ENGINEHOST';
+    name=coalescec(label1,label);
+    value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.));
+    output;
+  end;
+run;
+
+proc sql;
+drop table &ds1, &ds2;
+
+%mend mp_dsmeta;
+

base/mp_filtercheck.sas

@@ -6,7 +6,8 @@
   SYSCC to 1008 if bad records are found, and call mp_abort.sas for a
   graceful service exit (configurable).
 
-  Used for dynamic filtering in [Data Controller for SAS®](https://datacontroller.io).
+  Used for dynamic filtering in [Data Controller for SAS®](
+  https://datacontroller.io).
 
   Usage:
 
@@ -91,7 +92,37 @@ data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
-  length reason_cd $4032;
+  length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000;
+  drop tmp;
+
+  /* quick check to ensure column exists */
+  if upcase(VARIABLE_NM) not in
+  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
+  then do;
+    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
+    putlog REASON_CD= VARIABLE_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
+    output;
+    return;
+  end;
+
+  /* need to open the dataset to get the column type */
+  dsid=open("&targetds","i");
+  if dsid>0 then do;
+    vnum=varnum(dsid,VARIABLE_NM);
+    if vnum<1 then do;
+      /* should not happen as was also tested for above */
+      REASON_CD=cats("Variable (",VARIABLE_NM,") not found in &targetds");
+      putlog REASON_CD= dsid=;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
+      output;
+      return;
+    end;
+    /* now we can get the type */
+    else vtype=vartype(dsid,vnum);
+  end;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
@@ -109,44 +140,61 @@ data &outds;
     output;
   end;
   if mod(SUBGROUP_ID,1) ne 0 then do;
-    REASON_CD='SUBGROUP_ID should be integer, not '!!left(subgroup_id);
+    REASON_CD='SUBGROUP_ID should be integer, not '!!cats(subgroup_id);
     putlog REASON_CD= SUBGROUP_ID=;
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');
     output;
   end;
-  if upcase(VARIABLE_NM) not in
-  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
-  then do;
-    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
-    putlog REASON_CD= VARIABLE_NM=;
-    call symputx('reason_cd',reason_cd,'l');
-    call symputx('nobs',_n_,'l');
-    output;
-  end;
   if OPERATOR_NM not in
-  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
+  ('=','>','<','<=','>=','NE','GE','LE','BETWEEN','IN','NOT IN','CONTAINS')
   then do;
-    REASON_CD='Invalid OPERATOR_NM: '!!left(OPERATOR_NM);
+    REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');
     output;
   end;
 
+  /* special missing logic */
+  if vtype='N'
+    and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE')
+    and cats(upcase(raw_value)) in (
+      '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N'
+      '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._'
+    )
+  then do;
+    /* valid numeric - exit data step loop */
+    return;
+  end;
+
   /* special logic */
-  if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ','');
-  else if OPERATOR_NM in ('IN','NOT IN') then do;
-    if substr(raw_value,1,1) ne '('
-    or substr(cats(reverse(raw_value)),1,1) ne ')'
-    then do;
-      REASON_CD='Missing start/end bracket in RAW_VALUE';
-      putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
-      call symputx('reason_cd',reason_cd,'l');
-      call symputx('nobs',_n_,'l');
-      output;
+  if OPERATOR_NM in ('IN','NOT IN','BETWEEN') then do;
+    if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ',',');
+    else do;
+      if substr(raw_value,1,1) ne '('
+      or substr(cats(reverse(raw_value)),1,1) ne ')'
+      then do;
+        REASON_CD='Missing start/end bracket in RAW_VALUE';
+        putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+        call symputx('reason_cd',reason_cd,'l');
+        call symputx('nobs',_n_,'l');
+        output;
+      end;
+      else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
+    end;
+    /* we now have a comma seperated list of values */
+    if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
+      tmp=scan(raw_value1,i,',');
+      if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
+        REASON_CD='Non Numeric value provided';
+        putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+        call symputx('reason_cd',reason_cd,'l');
+        call symputx('nobs',_n_,'l');
+        output;
+      end;
+      return;
     end;
-    else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
   end;
   else raw_value1=raw_value;
 

base/mp_filtergenerate.sas

@@ -84,6 +84,9 @@ filename &outref temp;
   run;
 %end;
 %else %do;
+  proc sort data=&inds;
+    by SUBGROUP_ID;
+  run;
   data _null_;
     file &outref lrecl=32800;
     set &inds end=last;

base/mp_filterstore.sas

@@ -0,0 +1,254 @@
+/**
+  @file
+  @brief Checks & Stores an input filter table and returns the Filter Key
+  @details Used to generate a FILTER_RK from an input query dataset.  This
+  process requires several permanent tables (names are configurable).  The
+  benefit of storing query values at backend is to enable stored 'views' of
+  filtered tables at frontend (ie, when building [SAS-Powered Apps](
+  https://sasapps.io)).  This macro is also used in [Data Controller for SAS](
+  https://datacontroller.io).
+
+  A more recent feature of this macro is the ability to support filter queries
+  on Format Catalogs.  This is achieved by adding a `-FC` suffix to the `libds`
+  parameter - where the "ds" in this case is the catalog name.
+
+
+  @param [in] libds= The target dataset to be filtered (lib should be assigned).
+    If filtering a format catalog, add the following suffix:  `-FC`.
+  @param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to
+    be validated.  Has the following format:
+|GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
+|---|---|---|---|---|---|
+|AND|AND|1|SOME_BESTNUM|>|1|
+|AND|AND|1|SOME_TIME|=|77333|
+  @param [in] filter_summary= (PERM.FILTER_SUMMARY) Permanent table containing
+    summary filter values.  The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(FILTER_SUMMARY)`. Example
+    values:
+|FILTER_RK:best.|FILTER_HASH:$32.|FILTER_TABLE:$41.|PROCESSED_DTTM:datetime19.|
+|---|---|---|---|
+|`1 `|`540E96F566D194AB58DD4C413C99C9DB `|`VIYA6014.MPE_TABLES `|`1956084246 `|
+|`2 `|`87737DB9EEE2650F5C89956CEAD0A14F `|`VIYA6014.MPE_X_TEST `|`1956084452.1`|
+|`3 `|`8048BD908DBBD83D013560734E90D394 `|`VIYA6014.MPE_TABLES `|`1956093620.6`|
+  @param [in] filter_detail= (PERM.FILTER_DETAIL) Permanent table containing
+    detailed (raw) filter values. The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(FILTER_DETAIL)`. Example
+    values:
+|FILTER_HASH:$32.|FILTER_LINE:best.|GROUP_LOGIC:$3.|SUBGROUP_LOGIC:$3.|SUBGROUP_ID:best.|VARIABLE_NM:$32.|OPERATOR_NM:$12.|RAW_VALUE:$4000.|PROCESSED_DTTM:datetime19.|
+|---|---|---|---|---|---|---|---|---|
+|`540E96F566D194AB58DD4C413C99C9DB `|`1 `|`AND `|`AND `|`1 `|`LIBREF `|`CONTAINS `|`DC`|`1956084245.8 `|
+|`540E96F566D194AB58DD4C413C99C9DB `|`2 `|`AND `|`OR `|`2 `|`DSN `|`= `|` MPE_LOCK_ANYTABLE `|`1956084245.8 `|
+|`87737DB9EEE2650F5C89956CEAD0A14F `|`1 `|`AND `|`AND `|`1 `|`PRIMARY_KEY_FIELD `|`IN `|`(1,2,3) `|`1956084451.9 `|
+  @param [in] lock_table= (PERM.LOCK_TABLE) Permanent locking table.  Used to
+    manage concurrent access.  The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(LOCKTABLE)`.
+  @param [in] maxkeytable= (0) Optional permanent reference table used for
+    retained key tracking.  Described in mp_retainedkey.sas.
+  @param [in] mdebug= set to 1 to enable DEBUG messages
+  @param [out] outresult= The result table with the FILTER_RK
+  @param [out] outquery= The original query, taken as extract after table load
+
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+  @li mf_getvalue.sas
+  @li mf_islibds.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_filtercheck.sas
+  @li mp_hashdataset.sas
+  @li mp_retainedkey.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filtercheck.sas
+  @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
+  @li mp_filterstore.test.sas
+
+  @version 9.2
+  @author [Allan Bowe](https://www.linkedin.com/in/allanbowe)
+
+**/
+
+%macro mp_filterstore(libds=,
+  queryds=work.filterquery,
+  filter_summary=PERM.FILTER_SUMMARY,
+  filter_detail=PERM.FILTER_DETAIL,
+  lock_table=PERM.LOCK_TABLE,
+  maxkeytable=PERM.MAXKEYTABLE,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+);
+%put &sysmacroname entry vars:;
+%put _local_;
+
+%local ds0 ds1 ds2 ds3 ds4 filter_hash orig_libds;
+%let libds=%upcase(&libds);
+%let orig_libds=&libds;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+%mp_abort(iftrue= (%mf_islibds(&filter_summary)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid filter_summary value: &filter_summary)
+)
+%mp_abort(iftrue= (%mf_islibds(&filter_detail)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid filter_detail value: &filter_detail)
+)
+%mp_abort(iftrue= (%mf_islibds(&lock_table)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid lock_table value: &lock_table)
+)
+
+/**
+  * validate query
+  * use format catalog export, if a format
+  */
+%if "%substr(&libds,%length(&libds)-2,3)"="-FC" %then %do;
+  %let libds=%scan(&libds,1,-); /* chop off -FC extension */
+  %let ds0=%mf_getuniquename(prefix=fmtds_);
+  %let libds=&ds0;
+  /*
+    There is no need to export the entire format catalog here - the validations
+    are done against the data model, not the data values.  So we can simply
+    hardcode the structure based on the cntlout dataset.
+  */
+  %mddl_sas_cntlout(libds=&ds0)
+
+%end;
+%mp_filtercheck(&queryds,targetds=&libds,abort=YES)
+
+/* hash the result */
+%let ds1=%mf_getuniquename(prefix=hashds);
+%mp_hashdataset(&queryds,outds=&ds1,salt=&orig_libds)
+%let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
+%if &mdebug=1 %then %do;
+  data _null_;
+    putlog "filter_hash=&filter_hash";
+    set &ds1;
+    putlog (_all_)(=);
+  run;
+%end;
+
+/* check if data already exists for this hash */
+data &outresult;
+  set &filter_summary;
+  where filter_hash="&filter_hash";
+run;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc after hash check)
+)
+%mp_abort(iftrue= ("&filter_hash "=" ")
+  ,mac=mp_filterstore
+  ,msg=%str(problem with filter_hash generation)
+)
+
+%if %mf_nobs(&outresult)=0 %then %do;
+
+  /* first update summary table */
+  %let ds3=%mf_getuniquename(prefix=filtersum);
+  data work.&ds3;
+    if 0 then set &filter_summary;
+    filter_table="&orig_libds";
+    filter_hash="&filter_hash";
+    PROCESSED_DTTM=%sysfunc(datetime());
+    output;
+    stop;
+  run;
+
+  %mp_lockanytable(LOCK,
+    lib=%scan(&filter_summary,1,.)
+    ,ds=%scan(&filter_summary,2,.)
+    ,ref=MP_FILTERSTORE summary update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  %let ds4=%mf_getuniquename(prefix=filtersumappend);
+  %mp_retainedkey(
+    base_lib=%scan(&filter_summary,1,.)
+    ,base_dsn=%scan(&filter_summary,2,.)
+    ,append_lib=work
+    ,append_dsn=&ds3
+    ,retained_key=filter_rk
+    ,business_key=filter_hash
+    ,maxkeytable=&maxkeytable
+    ,locktable=&lock_table
+    ,outds=work.&ds4
+  )
+  proc append base=&filter_summary data=&ds4;
+  run;
+
+  %mp_lockanytable(UNLOCK,
+    lib=%scan(&filter_summary,1,.)
+    ,ds=%scan(&filter_summary,2,.)
+    ,ref=MP_FILTERSTORE summary update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  %if &syscc ne 0 %then %do;
+    data _null_;
+      set &ds4;
+      putlog (_all_)(=);
+    run;
+    %goto err;
+  %end;
+
+  data &outresult;
+    set &filter_summary;
+    where filter_hash="&filter_hash";
+  run;
+
+  /* Next, update detail table */
+  %let ds2=%mf_getuniquename(prefix=filterdetail);
+  data &ds2;
+    if 0 then set &filter_detail;
+    set &queryds;
+    format filter_hash $hex32. filter_line 8.;
+    filter_hash="&filter_hash";
+    filter_line=_n_;
+    PROCESSED_DTTM=%sysfunc(datetime());
+  run;
+  %mp_lockanytable(LOCK,
+    lib=%scan(&filter_detail,1,.)
+    ,ds=%scan(&filter_detail,2,.)
+    ,ref=MP_FILTERSTORE update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+  proc append base=&filter_detail data=&ds2;
+  run;
+
+  %mp_lockanytable(UNLOCK,
+    lib=%scan(&filter_detail,1,.)
+    ,ds=%scan(&filter_detail,2,.)
+    ,ref=MP_FILTERSTORE detail update &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  %if &syscc ne 0 %then %do;
+    data _null_;
+      set &ds2;
+      putlog (_all_)(=);
+    run;
+    %goto err;
+  %end;
+
+%end;
+
+proc sort data=&filter_detail(where=(filter_hash="&filter_hash")) out=&outquery;
+  by filter_line;
+run;
+
+%err:
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc on macro exit)
+)
+
+%mend mp_filterstore;

base/mp_filtervalidate.sas

@@ -96,8 +96,7 @@ filename &fref1 clear;
     run;
     %mp_abort(
       mac=&sysmacroname,
-      msg=%str(Filter validation issues. ERR=%superq(SYSERRORTEXT)
-        , WARN=%superq(SYSWARNINGTEXT) )
+      msg=%str(Filter validation issues.)
     )
   %end;
   %let syscc=1008;

base/mp_getcols.sas

@@ -14,11 +14,11 @@
 
   @param ds The dataset from which to obtain column metadata
   @param outds= (work.cols) The output dataset to create. Sample data:
-  |NAME $|LENGTH 8|VARNUM 8|LABEL $|FORMAT $49|TYPE $1 |DDTYPE $|
-  |---|---|---|---|---|---|---|
-  |AIR|8|2|international airline travel (thousands)|8.|N|NUMERIC|
-  |DATE|8|1|DATE|MONYY.|N|DATE|
-  |REGION|3|3|REGION|$3.|C|CHARACTER|
+|NAME:$32.|LENGTH:best.|VARNUM:best.|LABEL:$256.|FMTNAME:$32.|FORMAT:$49.|TYPE:$1.|DDTYPE:$9.|
+|---|---|---|---|---|---|---|---|
+|`AIR `|`8 `|`2 `|`international airline travel (thousands) `|` `|`8. `|`N `|`NUMERIC `|
+|`DATE `|`8 `|`1 `|`DATE `|`MONYY `|`MONYY. `|`N `|`DATE `|
+|`REGION `|`3 `|`3 `|`REGION `|` `|`$3. `|`C `|`CHARACTER `|
 
   <h4> Related Macros </h4>
   @li mf_getvarlist.sas
@@ -30,26 +30,27 @@
 **/
 
 %macro mp_getcols(ds, outds=work.cols);
-
+%local dropds;
 proc contents noprint data=&ds
   out=_data_ (keep=name type length label varnum format:);
 run;
-data &outds(keep=name type length varnum format label ddtype);
-  set &syslast(rename=(format=format2 type=type2));
+%let dropds=&syslast;
+data &outds(keep=name type length varnum format label ddtype fmtname);
+  set &dropds(rename=(format=fmtname type=type2));
   name=upcase(name);
   if type2=2 then do;
     length format $49.;
-    if format2='' then format=cats('$',length,'.');
-    else if formatl=0 then format=cats(format2,'.');
-    else format=cats(format2,formatl,'.');
+    if fmtname='' then format=cats('$',length,'.');
+    else if formatl=0 then format=cats(fmtname,'.');
+    else format=cats(fmtname,formatl,'.');
     type='C';
     ddtype='CHARACTER';
   end;
   else do;
-    if format2='' then format=cats(length,'.');
-    else if formatl=0 then format=cats(format2,'.');
-    else if formatd=0 then format=cats(format2,formatl,'.');
-    else format=cats(format2,formatl,'.',formatd);
+    if fmtname='' then format=cats(length,'.');
+    else if formatl=0 then format=cats(fmtname,'.');
+    else if formatd=0 then format=cats(fmtname,formatl,'.');
+    else format=cats(fmtname,formatl,'.',formatd);
     type='N';
     if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
     else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
@@ -61,5 +62,6 @@ data &outds(keep=name type length varnum format label ddtype);
   end;
   if label='' then label=name;
 run;
-
+proc sql;
+drop table &dropds;
 %mend mp_getcols;

base/mp_getconstraints.sas

@@ -40,6 +40,22 @@
 %let lib=%upcase(&lib);
 %let ds=%upcase(&ds);
 
+/**
+  * Cater for environments where sashelp.vcncolu is not available
+  */
+%if %sysfunc(exist(sashelp.vcncolu,view))=0 %then %do;
+  proc sql;
+  create table &outds(
+      libref char(8)
+    ,TABLE_NAME char(32)
+    ,constraint_type char(8)     label='Constraint Type'
+    ,constraint_name char(32)     label='Constraint Name'
+    ,column_name char(32)     label='Column'
+    ,constraint_order num
+  );
+  %return;
+%end;
+
 /**
   * Neither dictionary tables nor sashelp provides a constraint order column,
   * however they DO arrive in the correct order.  So, create the col.
@@ -78,8 +94,11 @@ create table &outds as
 /**
   * We cannot apply this clause to the underlying dictionary table.  See:
   * https://communities.sas.com/t5/SAS-Programming/Unexpected-Where-Clause-behaviour-in-dictionary-TABLE/m-p/771554#M244867
+  * cannot use`where calculated libref="&lib"` either as it will STILL execute
+  * all the underlying constraint queries, causing exception errors in some
+  * cases: https://github.com/sasjs/core/issues/283
   */
-  where calculated libref="&lib"
+  where a.TABLE_CATALOG="&lib"
   %if "&ds" ne "" %then %do;
     and upcase(a.TABLE_NAME)="&ds"
     and upcase(b.TABLE_NAME)="&ds"

base/mp_getddl.sas

@@ -130,13 +130,13 @@ run;
 
 %local x curds;
 %if &flavour=SAS %then %do;
-  data _null_;
-    file &fref mod;
-    put "/* SAS Flavour DDL for %upcase(&libref).&curds */";
-    put "proc sql;";
-  run;
   %do x=1 %to %sysfunc(countw(&dsnlist));
     %let curds=%scan(&dsnlist,&x);
+    data _null_;
+      file &fref mod;
+      put "/* SAS Flavour DDL for %upcase(&libref).&curds */";
+      put "proc sql;";
+    run;
     data _null_;
       file &fref mod;
       length lab $1024 typ $20;
@@ -158,7 +158,7 @@ run;
         lab=" label="!!cats("'",tranwrd(label,"'","''"),"'");
       if notnull='yes' then notnul=' not null';
       if type='char' then typ=cats('char(',length,')');
-      else if length ne 8 then typ='num length='!!left(length);
+      else if length ne 8 then typ='num length='!!cats(length);
       else typ='num';
       put name typ fmt notnul lab;
     run;

base/mp_getformats.sas

@@ -40,6 +40,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
   |`WHICHPATH `|`**OTHER** `|`**OTHER** `|`big fat problem if not path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|`O `|` `|` `|` `|` `|
 
   <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
   @li mf_dedup.sas
   @li mf_getfmtlist.sas
   @li mf_getfmtname.sas
@@ -48,6 +49,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 
 
   <h4> Related Macros </h4>
+  @li mp_applyformats.sas
   @li mp_getformats.test.sas
 
   @version 9.2
@@ -93,30 +95,7 @@ create table &outsummary as
 
 %if "&outdetail" ne "0" %then %do;
   /* ensure base table always exists */
-  proc sql;
-  create table &outdetail(
-      FMTNAME char(32)     label='Format name'
-      ,START char(16)     label='Starting value for format'
-      ,END char(16)     label='Ending value for format'
-      ,LABEL char(256)     label='Format value label'
-      ,MIN num length=3     label='Minimum length'
-      ,MAX num length=3     label='Maximum length'
-      ,DEFAULT num length=3     label='Default length'
-      ,LENGTH num length=3     label='Format length'
-      ,FUZZ num     label='Fuzz value'
-      ,PREFIX char(2)     label='Prefix characters'
-      ,MULT num     label='Multiplier'
-      ,FILL char(1)     label='Fill character'
-      ,NOEDIT num length=3     label='Is picture string noedit?'
-      ,TYPE char(1)     label='Type of format'
-      ,SEXCL char(1)     label='Start exclusion'
-      ,EEXCL char(1)     label='End exclusion'
-      ,HLO char(13)     label='Additional information'
-      ,DECSEP char(1)     label='Decimal separator'
-      ,DIG3SEP char(1)     label='Three-digit separator'
-      ,DATATYPE char(8)     label='Date/time/datetime?'
-      ,LANGUAGE char(8)     label='Language for date strings'
-  );
+  %mddl_sas_cntlout(libds=&outdetail)
   /* grab the location of each format */
   %let fmtcnt=0;
   data _null_;
@@ -133,7 +112,11 @@ create table &outsummary as
     proc format library=&&fmtloc&i CNTLOUT=&tempds;
       select &&fmtname&i;
     run;
-    proc append base=&outdetail data=&tempds;
+    data &tempds;
+      if 0 then set &outdetail;
+      set &tempds;
+    run;
+    proc append base=&outdetail data=&tempds ;
     run;
   %end;
 %end;

base/mp_getmaxvarlengths.sas

@@ -1,28 +1,47 @@
 /**
-  @file mp_getmaxvarlengths.sas
+  @file
   @brief Scans a dataset to find the max length of the variable values
   @details
   This macro will scan a base dataset and produce an output dataset with two
   columns:
 
   - NAME    Name of the base dataset column
-  - MAXLEN Maximum length of the data contained therein.
+  - MAXLEN  Maximum length of the data contained therein.
 
-  Character fields may be allocated very large widths (eg 32000) of which the
-  maximum  value is likely to be much narrower.  This macro was designed to
-  enable a HTML table to be appropriately sized however this could be used as
-  part of a data audit to ensure we aren't over-sizing our tables in relation to
-  the data therein.
+  Character fields are often allocated very large widths (eg 32000) of which the
+  maximum  value is likely to be much narrower.  Identifying such cases can be
+  helpful in the following scenarios:
+
+  @li Enabling a HTML table to be appropriately sized (`num2char=YES`)
+  @li Reducing the size of a dataset to save on storage (mp_ds2squeeze.sas)
+  @li Identifying columns containing nothing but missing values (`MAXLEN=0` in
+    the output table)
+
+  If the entire column is made up of (non-special) missing values then a value
+  of 0 is returned.
 
-  Numeric fields are converted using the relevant format to determine the width.
   Usage:
 
       %mp_getmaxvarlengths(sashelp.class,outds=work.myds)
 
-  @param libds Two part dataset (or view) reference.
-  @param outds= The output dataset to create
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] num2char= (NO) When set to NO, numeric fields are sized according
+    to the number of bytes used (or set to zero in the case of non-special
+    missings). When YES, the numeric field is converted to character (using the
+    format, if available), and that is sized instead, using `lengthn()`.
+  @param [out] outds= The output dataset to create, eg:
+  |NAME:$8.|MAXLEN:best.|
+  |---|---|
+  |`Name `|`7 `|
+  |`Sex `|`1 `|
+  |`Age `|`3 `|
+  |`Height `|`8 `|
+  |`Weight `|`3 `|
 
   <h4> SAS Macros </h4>
+  @li mcf_length.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarcount.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
   @li mf_getvarformat.sas
@@ -30,20 +49,50 @@
   @version 9.2
   @author Allan Bowe
 
+  <h4> Related Macros </h4>
+  @li mp_ds2squeeze.sas
+  @li mp_getmaxvarlengths.test.sas
+
 **/
 
 %macro mp_getmaxvarlengths(
-    libds      /* libref.dataset to analyse */
-    ,outds=work.mp_getmaxvarlengths /* name of output dataset to create */
+  libds
+  ,num2char=NO
+  ,outds=work.mp_getmaxvarlengths
 )/*/STORE SOURCE*/;
 
-%local vars x var fmt;
+%local vars prefix x var fmt srcds;
 %let vars=%mf_getvarlist(libds=&libds);
+%let prefix=%substr(%mf_getuniquename(),1,25);
+%let num2char=%upcase(&num2char);
+
+%if &num2char=NO %then %do;
+  /* compile length function for numeric fields */
+  %mcf_length(wrap=YES, insert_cmplib=YES)
+%end;
+
+%if &num2char=NO
+  and ("%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5")
+  and %mf_getvarcount(&libds,typefilter=N) gt 0
+%then %do;
+  /* custom functions not supported in summary operations */
+  %let srcds=%mf_getuniquename();
+  data &srcds/view=&srcds;
+    set &libds;
+  %do x=1 %to %sysfunc(countw(&vars,%str( )));
+    %let var=%scan(&vars,&x);
+    %if %mf_getvartype(&libds,&var)=N %then %do;
+      &prefix.&x=mcf_length(&var);
+    %end;
+  %end;
+  run;
+%end;
+%else %let srcds=&libds;
 
 proc sql;
 create table &outds (rename=(
     %do x=1 %to %sysfunc(countw(&vars,%str( )));
-      ________&x=%scan(&vars,&x)
+      &prefix.&x=%scan(&vars,&x)
     %end;
     ))
   as select
@@ -51,20 +100,28 @@ create table &outds (rename=(
       %let var=%scan(&vars,&x);
       %if &x>1 %then ,;
       %if %mf_getvartype(&libds,&var)=C %then %do;
-        max(length(&var)) as ________&x
+        max(lengthn(&var)) as &prefix.&x
       %end;
-      %else %do;
+      %else %if &num2char=YES %then %do;
         %let fmt=%mf_getvarformat(&libds,&var);
         %put fmt=&fmt;
         %if %str(&fmt)=%str() %then %do;
-          max(length(cats(&var))) as ________&x
+          max(lengthn(cats(&var))) as &prefix.&x
         %end;
         %else %do;
-          max(length(put(&var,&fmt))) as ________&x
+          max(lengthn(put(&var,&fmt))) as &prefix.&x
+        %end;
+      %end;
+      %else %do;
+        %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do;
+          max(&prefix.&x) as &prefix.&x
+        %end;
+        %else %do;
+          max(mcf_length(&var)) as &prefix.&x
         %end;
       %end;
     %end;
-  from &libds;
+  from &srcds;
 
   proc transpose data=&outds
     out=&outds(rename=(_name_=NAME COL1=MAXLEN));

base/mp_getpk.sas

@@ -34,6 +34,7 @@
   @param [out] outds= (work.mp_getpk) The name of the output table to create.
 
   <h4> SAS Macros </h4>
+  @li mf_existfeature.sas
   @li mf_getengine.sas
   @li mf_getschema.sas
   @li mp_dropmembers.sas
@@ -55,7 +56,8 @@
 )/*/STORE SOURCE*/;
 
 
-%local engine schema ds1 ds2 ds3 dsn tabs1 tabs2 sum pk4sure pkdefault finalpks;
+%local engine schema ds1 ds2 ds3 dsn tabs1 tabs2 sum pk4sure pkdefault finalpks
+  pkfromindex;
 
 %let lib=%upcase(&lib);
 %let ds=%upcase(&ds);
@@ -70,6 +72,7 @@
 %let sum=%mf_getuniquename(prefix=getpk_sum);
 %let pk4sure=%mf_getuniquename(prefix=getpk_pk4sure);
 %let pkdefault=%mf_getuniquename(prefix=getpk_pkdefault);
+%let pkfromindex=%mf_getuniquename(prefix=getpk_pkfromindex);
 %let finalpks=%mf_getuniquename(prefix=getpk_finalpks);
 
 %local dbg;
@@ -180,9 +183,23 @@ create table &ds1 as
       and a.constraint_name=b.constraint_name
     order by 1,2,3,4;
 
+  /* extract cols from the relevant unique INDEXES */
+  create table &pkfromindex as
+    select libname as libref
+      ,memname as table_name
+      ,indxname as constraint_name
+      ,indxpos as constraint_order
+      ,name
+    from dictionary.indexes
+    where nomiss='yes' and unique='yes' and upcase(libname)="&lib"
+  %if &ds ne 0 %then %do;
+      and upcase(memname)="&ds"
+  %end;
+    order by 1,2,3,4;
+
   /* create one table */
   data &finalpks;
-    set &pkdefault &pk4sure ;
+    set &pkdefault &pk4sure &pkfromindex;
     pk_ind=1;
     /* if there are multiple unique constraints, take the first */
     by libref table_name constraint_name;
@@ -213,7 +230,12 @@ create table work.&tabs1 as select
   libname as libref
   ,upcase(memname) as dsn
   ,memtype
+%if %mf_existfeature(DBMS_MEMTYPE)=1 %then %do;
   ,dbms_memtype
+%end;
+%else %do;
+  ,'n/a' as dbms_memtype format=$32.
+%end;
   ,typemem
   ,memlabel
   ,nvar
@@ -256,4 +278,4 @@ create table &outds as
   iftrue=(&mdebug=0)
 )
 
-%mend mp_getpk;
+%mend mp_getpk;

base/mp_gitadd.sas

@@ -0,0 +1,45 @@
+/**
+  @file
+  @brief Stages files in a GIT repo
+  @details Uses the output dataset from mp_gitstatus.sas to determine the files
+  that should be staged.
+
+  If `STAGED ne "TRUE"` then the file is staged.
+
+  Usage:
+
+      %let dir=%sysfunc(pathname(work))/core;
+      %let repo=https://github.com/sasjs/core;
+      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir));
+      %mf_writefile(&dir/somefile.txt,l1=some content)
+      %mf_deletefile(&dir/package.json)
+      %mp_gitstatus(&dir,outds=work.gitstatus)
+
+      %mp_gitadd(&dir,inds=work.gitstatus)
+
+  @param [in] gitdir The directory containing the GIT repository
+  @param [in] inds= (work.mp_gitadd) The input dataset with the list of files
+  to stage.  Will accept the output from mp_gitstatus(), else just use a table
+  with the following columns:
+    @li path $1024 - relative path to the file in the repo
+    @li staged $32 - whether the file is staged (TRUE or FALSE)
+    @li status $64 - either new, deleted, or modified
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> Related Files </h4>
+  @li mp_gitadd.test.sas
+  @li mp_gitstatus.sas
+
+**/
+
+%macro mp_gitadd(gitdir,inds=work.mp_gitadd,mdebug=0);
+
+data _null_;
+  set &inds;
+  if STAGED ne "TRUE";
+  rc=git_index_add("&gitdir",cats(path),status);
+  if rc ne 0 or &mdebug=1 then put rc=;
+run;
+
+%mend mp_gitadd;

base/mp_gitlog.sas

@@ -0,0 +1,104 @@
+/**
+  @file
+  @brief Creates a dataset with the commit history of a local repository
+  @details Returns the commit history from a local repository. The name of the
+  branch is also returned.
+
+  More details here:
+https://documentation.sas.com/doc/ko/pgmsascdc/v_033/lefunctionsref/n1qo5miyvry1nen111js203hlwrh.htm
+
+  Usage:
+
+      %let gitdir=%sysfunc(pathname(work))/core;
+      %let repo=https://github.com/sasjs/core;
+      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir));
+
+      %mp_gitlog(&gitdir,outds=work.mp_gitlog)
+
+  @param [in] gitdir The directory containing the GIT repository
+  @param [in] filter= (BRANCHONLY) To return only the commits for the current
+  branch, use BRANCHONLY (the default).  Anything else will return the entire
+  commit history.
+  @param [out] outds= (work.mp_gitlog) The output dataset to create.
+  All vars are $128 except `message` which is $4000.
+    @li author returns the author who submitted the commit.
+    @li children_ids returns a list of the children commit IDs
+    @li committer returns the name of the committer.
+    @li committer_email returns the email of the committer.
+    @li email returns the email of the commit author.
+    @li id returns the commit ID of the commit object.
+    @li in_current_branch returns "TRUE" or "FALSE" to indicate if the commit is
+      in the current branch.
+    @li message returns the commit message.
+    @li parent_ids returns a list of the parent commit IDs.
+    @li stash returns "TRUE" or "FALSE" to indicate if the commit is a stash
+      commit.
+    @li time returns the time of the commit as numeric string
+    @li commit_time_num time of the commit as numeric SAS datetime
+    @li commit_time_str the commit_time_num variable cast as string
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] nobs= (0) Set to an integer greater than 0 to restrict the number
+  of rows returned
+
+  <h4> SAS Macros </h4>
+  @li mf_getgitbranch.sas
+
+  <h4> Related Files </h4>
+  @li mp_gitadd.sas
+  @li mp_gitreleaseinfo.sas
+  @li mp_gitstatus.sas
+
+**/
+
+%macro mp_gitlog(gitdir,outds=work.mp_gitlog,mdebug=0,filter=BRANCHONLY,nobs=0);
+
+%local varlist i var;
+%let varlist=author children_ids committer committer_email email id
+  in_current_branch parent_ids stash time ;
+
+data &outds;
+  LENGTH gitdir branch $ 1024 message $4000 &varlist $128 commit_time_num 8.
+    commit_time_str $32;
+  call missing (of _all_);
+  branch="%mf_getgitbranch(&gitdir)";
+  gitdir=symget('gitdir');
+  rc=git_status_free(trim(gitdir));
+  if rc=-1 then do;
+    put "The libgit2 library is unavailable and no Git operations can be used.";
+    put "See: https://stackoverflow.com/questions/74082874";
+    stop;
+  end;
+  else if rc=-2 then do;
+    put "The libgit2 library is available, but the status function failed.";
+    put "See the log for details.";
+    stop;
+  end;
+  entries=git_commit_log(trim(gitdir));
+  do n=1 to entries;
+
+  %do i=1 %to %sysfunc(countw(&varlist message));
+    %let var=%scan(&varlist message,&i,%str( ));
+    rc=git_commit_get(n,trim(gitdir),"&var",&var);
+  %end;
+    /* convert unix time to SAS time - https://4gl.uk/corelink0 */
+    /* Number of seconds between 01JAN1960 and 01JAN1970: 315619200 */
+    format commit_time_num datetime19.;
+    commit_time_num=sum(input(cats(time),best.),315619200);
+    commit_time_str=put(commit_time_num,datetime19.);
+  %if &mdebug=1 %then %do;
+    putlog (_all_)(=);
+  %end;
+    if "&filter"="BRANCHONLY" then do;
+      if cats(in_current_branch)='TRUE' then output;
+    end;
+    else output;
+  %if &nobs>0 %then %do;
+    if n ge &nobs then stop;
+  %end;
+  end;
+  rc=git_commit_free(trim(gitdir));
+  keep gitdir branch &varlist message time commit_time_num commit_time_str;
+run;
+
+%mend mp_gitlog;

base/mp_gitreleaseinfo.sas

@@ -0,0 +1,74 @@
+/**
+  @file
+  @brief Pulls latest release info from a GIT repository
+  @details Useful for grabbing the latest version number or other attributes
+  from a GIT server.  Supported providers are GitLab and GitHub. Pull requests
+  are welcome if you'd like to see additional providers!
+
+  Note that each provider provides slightly different JSON output.  Therefore
+  the macro simply extracts the JSON and assigns the libname (using the JSON
+  engine).
+
+  Example usage (eg, to grab latest release version from github):
+
+      %mp_gitreleaseinfo(GITHUB,sasjs/core,outlib=mylibref)
+
+      data _null_;
+        set mylibref.root;
+        putlog TAG_NAME=;
+      run;
+
+  @param [in] provider The GIT provider for the release info.  Accepted values:
+    @li GITLAB
+    @li GITHUB - Tables include root, assets, author, alldata
+  @param [in] project The link to the repository.  This has different formats
+    depending on the vendor:
+    @li GITHUB - org/repo, eg sasjs/core
+    @li GITLAB - project, eg 1343223
+  @param [in] server= (0) If your repo is self-hosted, then provide the domain
+    here.  Otherwise it will default to the provider domain (eg gitlab.com).
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outlib= (GITREL) The JSON-engine libref to be created, which will
+    point at the returned JSON
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+
+  <h4> Related Files </h4>
+  @li mp_gitreleaseinfo.test.sas
+
+**/
+
+%macro mp_gitreleaseinfo(provider,project,server=0,outlib=GITREL,mdebug=0);
+%local url fref;
+
+%let provider=%upcase(&provider);
+
+%if &provider=GITHUB %then %do;
+  %if "&server"="0" %then %let server=https://api.github.com;
+  %let url=&server/repos/&project/releases/latest;
+%end;
+%else %if &provider=GITLAB %then %do;
+  %if "&server"="0" %then %let server=https://gitlab.com;
+  %let url=&server/api/v4/projects/&project/releases;
+%end;
+
+%let fref=%mf_getuniquefileref();
+
+proc http method='GET' out=&fref url="&url";
+%if &mdebug=1 %then %do;
+  debug level = 3;
+%end;
+run;
+
+libname &outlib JSON fileref=&fref;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fref;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_gitreleaseinfo;

base/mp_gitstatus.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Creates a dataset with the output from `GIT_STATUS()`
+  @details Uses `git_status()` to fetch the number of changed files, then
+  iterates with `git_status_get()`, inserting all attributes into an output
+  dataset.
+
+  Usage:
+
+      %let dir=%sysfunc(pathname(work))/core;
+      %let repo=https://github.com/sasjs/core;
+      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir));
+      %mf_writefile(&dir/somefile.txt,l1=some content)
+      %mf_deletefile(&dir/package.json)
+
+      %mp_gitstatus(&dir,outds=work.gitstatus)
+
+  More info on these functions is in this [helpful paper]
+(https://www.sas.com/content/dam/SAS/support/en/sas-global-forum-proceedings/2019/3057-2019.pdf)
+  by Danny Zimmerman.
+
+  @param [in] gitdir The directory containing the GIT repository
+  @param [out] outds= (work.git_status) The output dataset to create.  Vars:
+    @li gitdir $1024 - directory of repo
+    @li path $1024 - relative path to the file in the repo
+    @li staged $32 - whether the file is staged (TRUE or FALSE)
+    @li status $64 - either new, deleted, or modified
+    @li cnt - number of files
+    @li n - the "nth" file in the list from git_status()
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> Related Files </h4>
+  @li mp_gitstatus.test.sas
+  @li mp_gitadd.sas
+
+**/
+
+%macro mp_gitstatus(gitdir,outds=work.mp_gitstatus,mdebug=0);
+
+data &outds;
+  LENGTH gitdir path $ 1024 STATUS $ 64 STAGED $ 32;
+  call missing (of _all_);
+  gitdir=symget('gitdir');
+  cnt=git_status(trim(gitdir));
+  if cnt=-1 then do;
+    put "The libgit2 library is unavailable and no Git operations can be used.";
+    put "See: https://stackoverflow.com/questions/74082874";
+  end;
+  else if cnt=-2 then do;
+    put "The libgit2 library is available, but the status function failed.";
+    put "See the log for details.";
+  end;
+  else do n=1 to cnt;
+    rc=GIT_STATUS_GET(n,gitdir,'PATH',path);
+    rc=GIT_STATUS_GET(n,gitdir,'STAGED',staged);
+    rc=GIT_STATUS_GET(n,gitdir,'STATUS',status);
+    output;
+  %if &mdebug=1 %then %do;
+    putlog (_all_)(=);
+  %end;
+  end;
+  rc=git_status_free(trim(gitdir));
+  drop rc cnt;
+run;
+
+%mend mp_gitstatus;

base/mp_gsubfile.sas

@@ -48,6 +48,11 @@
   outfile=0
 )/*/STORE SOURCE*/;
 
+  %if "%substr(&sysver.XX,1,4)"="V.04" %then %do;
+    %put %str(ERR)OR: Viya 4 does not support the IO library in lua;
+    %return;
+  %end;
+
   %ml_gsubfile()
 
 %mend mp_gsubfile;

base/mp_guesspk.sas

@@ -17,16 +17,21 @@
       %inc mc;
       %mp_guesspk(sashelp.class,outds=classpks)
 
-  @param baseds The dataset to analyse
-  @param outds= The output dataset to contain the possible PKs
-  @param max_guesses= (3) The total number of possible primary keys to generate.
-    A table may have multiple unlikely PKs, so no need to list them all.
-  @param min_rows= (5) The minimum number of rows a table should have in order
-    to try and guess the PK.
+  @param [in] baseds The dataset to analyse
+  @param [out] outds= The output dataset to contain the possible PKs
+  @param [in] max_guesses= (3) The total number of possible primary keys to
+    generate. A table may have multiple (unlikely) PKs, so no need to list them
+    all.
+  @param [in] min_rows= (5) The minimum number of rows a table should have in
+    order to try and guess the PK.
+  @param [in] ignore_cols (0) Space seperated list of columns which you are
+    sure are not part of the primary key (helps to avoid false positives)
+  @param [in] mdebug= Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
   @li mf_getvarlist.sas
   @li mf_getuniquename.sas
+  @li mf_wordsInstr1butnotstr2.sas
   @li mf_nobs.sas
 
   <h4> Related Macros </h4>
@@ -38,179 +43,226 @@
 **/
 
 %macro mp_guesspk(baseds
-      ,outds=mp_guesspk
-      ,max_guesses=3
-      ,min_rows=5
+  ,outds=mp_guesspk
+  ,max_guesses=3
+  ,min_rows=5
+  ,ignore_cols=0
+  ,mdebug=0
 )/*/STORE SOURCE*/;
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
 
-  /* declare local vars */
-  %local var vars vcnt i j k l tmpvar tmpds rows posspks ppkcnt;
-  %let vars=%mf_getvarlist(&baseds);
-  %let vcnt=%sysfunc(countw(&vars));
+/* declare local vars */
+%local var vars vcnt i j k l tmpvar tmpds rows posspks ppkcnt;
+%let vars=%upcase(%mf_getvarlist(&baseds));
+%let vars=%mf_wordsInStr1ButNotStr2(str1=&vars,str2=%upcase(&ignore_cols));
+%let vcnt=%sysfunc(countw(&vars));
 
-  %if &vcnt=0 %then %do;
-    %put &sysmacroname: &baseds has no variables!  Exiting.;
-    %return;
+%if &vcnt=0 %then %do;
+  %put &sysmacroname: &baseds has no variables!  Exiting.;
+  %return;
+%end;
+
+/* get null count and row count */
+%let tmpvar=%mf_getuniquename();
+proc sql noprint;
+create table _data_ as select
+  count(*) as &tmpvar
+%do i=1 %to &vcnt;
+  %let var=%scan(&vars,&i);
+  ,sum(case when &var is missing then 1 else 0 end) as &var
+%end;
+  from &baseds;
+
+/* transpose table and scan for not null cols */
+proc transpose;
+data _null_;
+  set &syslast end=last;
+  length vars $32767;
+  retain vars ;
+  if _name_="&tmpvar" then call symputx('rows',col1,'l');
+  else if col1=0 then vars=catx(' ',vars,_name_);
+  if last then call symputx('posspks',vars,'l');
+run;
+
+%let ppkcnt=%sysfunc(countw(&posspks));
+%if &ppkcnt=0 %then %do;
+  %put &sysmacroname: &baseds has no non-missing variables!  Exiting.;
+  %return;
+%end;
+
+proc sort data=&baseds(keep=&posspks) out=_data_ noduprec;
+  by _all_;
+run;
+%local pkds; %let pkds=&syslast;
+
+%if &rows > %mf_nobs(&pkds) %then %do;
+  %put &sysmacroname: &baseds has no combination of unique records! Exiting.;
+  %return;
+%end;
+
+/* now check cardinality */
+proc sql noprint;
+create table _data_ as select
+%do i=1 %to &ppkcnt;
+  %let var=%scan(&posspks,&i);
+  count(distinct &var) as &var
+  %if &i<&ppkcnt %then ,;
+%end;
+  from &pkds;
+
+/* transpose and sort by cardinality */
+proc transpose;
+proc sort; by descending col1;
+run;
+
+/* create initial PK list and re-order posspks list */
+data &outds(keep=pkguesses);
+  length pkguesses $5000 vars $5000;
+  set &syslast end=last;
+  retain vars ;
+  vars=catx(' ',vars,_name_);
+  if col1=&rows then do;
+    pkguesses=_name_;
+    output;
+  end;
+  if last then call symputx('posspks',vars,'l');
+run;
+
+%if %mf_nobs(&outds) ge &max_guesses %then %do;
+  %put &sysmacroname: %mf_nobs(&outds) possible primary key values found;
+  %return;
+%end;
+
+%if &ppkcnt=1 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %return;
+%end;
+
+/* begin scanning for uniques on pairs of PKs */
+%let tmpds=%mf_getuniquename();
+%local lev1 lev2;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do;
+      /* check for two level uniqueness */
+      proc sort data=&pkds(keep=&lev1 &lev2) out=&tmpds noduprec;
+        by _all_;
+      run;
+      %if %mf_nobs(&tmpds)=&rows %then %do;
+        proc sql;
+        insert into &outds values("&lev1 &lev2");
+        %if %mf_nobs(&outds) ge &max_guesses %then %do;
+          %put &sysmacroname: Max PKs reached at Level 2 for &baseds;
+          %goto exit;
+        %end;
+      %end;
+    %end;
   %end;
+%end;
 
-  /* get null count and row count */
-  %let tmpvar=%mf_getuniquename();
-  proc sql noprint;
-  create table _data_ as select
-    count(*) as &tmpvar
-  %do i=1 %to &vcnt;
-    %let var=%scan(&vars,&i);
-    ,sum(case when &var is missing then 1 else 0 end) as &var
-  %end;
-    from &baseds;
+%if &ppkcnt=2 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* transpose table and scan for not null cols */
-  proc transpose;
-  data _null_;
-    set &syslast end=last;
-    length vars $32767;
-    retain vars ;
-    if _name_="&tmpvar" then call symputx('rows',col1,'l');
-    else if col1=0 then vars=catx(' ',vars,_name_);
-    if last then call symputx('posspks',vars,'l');
-  run;
-
-  %let ppkcnt=%sysfunc(countw(&posspks));
-  %if &ppkcnt=0 %then %do;
-    %put &sysmacroname: &baseds has no non-missing variables!  Exiting.;
-    %return;
-  %end;
-
-  proc sort data=&baseds(keep=&posspks) out=_data_ noduprec;
-    by _all_;
-  run;
-  %local pkds; %let pkds=&syslast;
-
-  %if &rows > %mf_nobs(&pkds) %then %do;
-    %put &sysmacroname: &baseds has no combination of unique records! Exiting.;
-    %return;
-  %end;
-
-  /* now check cardinality */
-  proc sql noprint;
-  create table _data_ as select
-  %do i=1 %to &ppkcnt;
-    %let var=%scan(&posspks,&i);
-    count(distinct &var) as &var
-    %if &i<&ppkcnt %then ,;
-  %end;
-    from &pkds;
-
-  /* transpose and sort by cardinality */
-  proc transpose;
-  proc sort; by descending col1;
-  run;
-
-  /* create initial PK list and re-order posspks list */
-  data &outds(keep=pkguesses);
-    length pkguesses $5000 vars $5000;
-    set &syslast end=last;
-    retain vars ;
-    vars=catx(' ',vars,_name_);
-    if col1=&rows then do;
-      pkguesses=_name_;
-      output;
-    end;
-    if last then call symputx('posspks',vars,'l');
-  run;
-
-  %if %mf_nobs(&outds) ge &max_guesses %then %do;
-    %put &sysmacroname: %mf_nobs(&outds) possible primary key values found;
-    %return;
-  %end;
-
-  %if &ppkcnt=1 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
-
-  /* begin scanning for uniques on pairs of PKs */
-  %let tmpds=%mf_getuniquename();
-  %local lev1 lev2;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do;
-        /* check for two level uniqueness */
-        proc sort data=&pkds(keep=&lev1 &lev2) out=&tmpds noduprec;
+/* begin scanning for uniques on PK triplets */
+%local lev3;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
+      %let lev3=%scan(&posspks,&k);
+      %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do;
+        /* check for three level uniqueness */
+        proc sort data=&pkds(keep=&lev1 &lev2 &lev3) out=&tmpds noduprec;
           by _all_;
         run;
         %if %mf_nobs(&tmpds)=&rows %then %do;
           proc sql;
-          insert into &outds values("&lev1 &lev2");
+          insert into &outds values("&lev1 &lev2 &lev3");
           %if %mf_nobs(&outds) ge &max_guesses %then %do;
-            %put &sysmacroname: Max PKs reached at Level 2 for &baseds;
-            %return;
+            %put &sysmacroname: Max PKs reached at Level 3 for &baseds;
+            %goto exit;
           %end;
         %end;
       %end;
     %end;
   %end;
+%end;
 
-  %if &ppkcnt=2 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%if &ppkcnt=3 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* begin scanning for uniques on PK triplets */
-  %local lev3;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
-        %let lev3=%scan(&posspks,&k);
-        %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do;
-          /* check for three level uniqueness */
-          proc sort data=&pkds(keep=&lev1 &lev2 &lev3) out=&tmpds noduprec;
+/* scan for uniques on up to 4 PK fields */
+%local lev4;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
+      %let lev3=%scan(&posspks,&k);
+      %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
+        %let lev4=%scan(&posspks,&l);
+        %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then %do;
+          /* check for four level uniqueness */
+          proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4)
+              out=&tmpds noduprec;
             by _all_;
           run;
           %if %mf_nobs(&tmpds)=&rows %then %do;
             proc sql;
-            insert into &outds values("&lev1 &lev2 &lev3");
+            insert into &outds values("&lev1 &lev2 &lev3 &lev4");
             %if %mf_nobs(&outds) ge &max_guesses %then %do;
-              %put &sysmacroname: Max PKs reached at Level 3 for &baseds;
-              %return;
+              %put &sysmacroname: Max PKs reached at Level 4 for &baseds;
+              %goto exit;
             %end;
           %end;
         %end;
       %end;
     %end;
   %end;
+%end;
 
-  %if &ppkcnt=3 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%if &ppkcnt=4 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* scan for uniques on up to 4 PK fields */
-  %local lev4;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
-        %let lev3=%scan(&posspks,&k);
-        %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
-          %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then %do;
+/* scan for uniques on up to 4 PK fields */
+%local lev5 m;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
+      %let lev3=%scan(&posspks,&k);
+      %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
+        %let lev4=%scan(&posspks,&l);
+        %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
+        %do m=5 %to &ppkcnt;
+          %let lev5=%scan(&posspks,&m);
+          %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then %do;
             /* check for four level uniqueness */
-            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4)
+            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5)
                 out=&tmpds noduprec;
               by _all_;
             run;
             %if %mf_nobs(&tmpds)=&rows %then %do;
               proc sql;
-              insert into &outds values("&lev1 &lev2 &lev3 &lev4");
+              insert into &outds values("&lev1 &lev2 &lev3 &lev4 &lev5");
               %if %mf_nobs(&outds) ge &max_guesses %then %do;
-                %put &sysmacroname: Max PKs reached at Level 4 for &baseds;
-                %return;
+                %put &sysmacroname: Max PKs reached at Level 5 for &baseds;
+                %goto exit;
               %end;
             %end;
           %end;
@@ -218,37 +270,44 @@
       %end;
     %end;
   %end;
+%end;
 
-  %if &ppkcnt=4 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%if &ppkcnt=5 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* scan for uniques on up to 4 PK fields */
-  %local lev5 m;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
-        %let lev3=%scan(&posspks,&k);
-        %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
-          %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
-          %do m=5 %to &ppkcnt;
-            %let lev5=%scan(&posspks,&m);
-            %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then %do;
+/* scan for uniques on up to 4 PK fields */
+%local lev6 n;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
+      %let lev3=%scan(&posspks,&k);
+      %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
+        %let lev4=%scan(&posspks,&l);
+        %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
+        %do m=5 %to &ppkcnt;
+          %let lev5=%scan(&posspks,&m);
+          %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5
+          %then %do n=6 %to &ppkcnt;
+            %let lev6=%scan(&posspks,&n);
+            %if &lev1 ne &lev6 & &lev2 ne &lev6 & &lev3 ne &lev6
+            & &lev4 ne &lev6 & &lev5 ne &lev6 %then
+            %do;
               /* check for four level uniqueness */
-              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5)
-                  out=&tmpds noduprec;
+              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5 &lev6)
+                out=&tmpds noduprec;
                 by _all_;
               run;
               %if %mf_nobs(&tmpds)=&rows %then %do;
                 proc sql;
-                insert into &outds values("&lev1 &lev2 &lev3 &lev4 &lev5");
+                insert into &outds
+                  values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
                 %if %mf_nobs(&outds) ge &max_guesses %then %do;
-                  %put &sysmacroname: Max PKs reached at Level 5 for &baseds;
-                  %return;
+                  %put &sysmacroname: Max PKs reached at Level 6 for &baseds;
+                  %goto exit;
                 %end;
               %end;
             %end;
@@ -257,56 +316,17 @@
       %end;
     %end;
   %end;
+%end;
 
-  %if &ppkcnt=5 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%if &ppkcnt=6 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* scan for uniques on up to 4 PK fields */
-  %local lev6 n;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
-        %let lev3=%scan(&posspks,&k);
-        %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
-          %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
-          %do m=5 %to &ppkcnt;
-            %let lev5=%scan(&posspks,&m);
-            %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then
-            %do n=6 %to &ppkcnt;
-              %let lev6=%scan(&posspks,&n);
-              %if &lev1 ne &lev6 & &lev2 ne &lev6 & &lev3 ne &lev6
-              & &lev4 ne &lev6 & &lev5 ne &lev6 %then
-              %do;
-                /* check for four level uniqueness */
-                proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5 &lev6)
-                  out=&tmpds noduprec;
-                  by _all_;
-                run;
-                %if %mf_nobs(&tmpds)=&rows %then %do;
-                  proc sql;
-                  insert into &outds
-                    values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
-                  %if %mf_nobs(&outds) ge &max_guesses %then %do;
-                    %put &sysmacroname: Max PKs reached at Level 6 for &baseds;
-                    %return;
-                  %end;
-                %end;
-              %end;
-            %end;
-          %end;
-        %end;
-      %end;
-    %end;
-  %end;
-
-  %if &ppkcnt=6 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%exit:
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &tmpds;
+%end;
 
 %mend mp_guesspk;

base/mp_hashdataset.sas

@@ -1,8 +1,8 @@
 /**
   @file
   @brief Returns a unique hash for a dataset
-  @details Ignores metadata attributes, used only to hash values. Compared
-  datasets must be in the same order.
+  @details Ignores metadata attributes, used only to hash values. If used to
+  compare datasets, they must have their columns and rows in the same order.
 
       %mp_hashdataset(sashelp.class,outds=myhash)
 
@@ -11,19 +11,23 @@
         put hashkey=;
       run;
 
-  ![sas md5 hash dataset log results](https://i.imgur.com/MqF98vk.png)
+  ![sas md5 hash dataset log results](https://i.4gl.io/1/KorUKoyE05.png/raw)
 
   <h4> SAS Macros </h4>
   @li mf_getattrn.sas
   @li mf_getuniquename.sas
   @li mf_getvarlist.sas
-  @li mf_getvartype.sas
+  @li mp_md5.sas
+
+  <h4> Related Files </h4>
+  @li mp_hashdataset.test.sas
+  @li mp_hashdirectory.sas
 
   @param [in] libds dataset to hash
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
-  @param [in] iftrue= A condition under which the macro should be executed.
-  @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
-  will contain one column (hashkey) with one observation (a hex32.
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [out] outds= (work._data_) The output dataset to create. This
+  will contain one column (hashkey) with one observation (a $hex32.
   representation of the input hash)
   |hashkey:$32.|
   |---|
@@ -35,48 +39,53 @@
 
 %macro mp_hashdataset(
   libds,
-  outds=,
+  outds=work._data_,
   salt=,
   iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
-  %if not(%eval(%unquote(&iftrue))) %then %return;
+%local keyvar /* roll up the md5 */
+  prevkeyvar /* retain prev record md5 */
+  lastvar /* last var in input ds */
+  cvars nvars;
 
-  %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
-    %put %str(WARN)ING: Dataset &libds is empty, or is not a dataset;
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+/* avoid naming conflict for hash key vars */
+%let keyvar=%mf_getuniquename();
+%let prevkeyvar=%mf_getuniquename();
+%let lastvar=%mf_getuniquename();
+
+%if %mf_getattrn(&libds,NLOBS)=0 %then %do;
+  data &outds;
+    length hashkey $32;
+    hashkey=put(md5("&salt"),$hex32.);
+    output;
+    stop;
+  run;
+  %put &sysmacroname: Dataset &libds is empty, or is not a dataset;
+  %put &sysmacroname: hashkey of &outds is based on salt (&salt) only;
+%end;
+%else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
+  %put %str(ERR)OR: Dataset &libds is not a dataset;
+%end;
+%else %do;
+  data &outds(rename=(&keyvar=hashkey) keep=&keyvar)
+  %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+    /nonote2err
   %end;
-  %else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
-    %put %str(ERR)OR: Dataset &libds is not a dataset;
-  %end;
-  %else %do;
-    %local keyvar /* roll up the md5 */
-      prevkeyvar /* retain prev record md5 */
-      lastvar /* last var in input ds */
-      varlist var i;
-    /* avoid naming conflict for hash key vars */
-    %let keyvar=%mf_getuniquename();
-    %let prevkeyvar=%mf_getuniquename();
-    %let lastvar=%mf_getuniquename();
-    %let varlist=%mf_getvarlist(&libds);
-    data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
-      length &prevkeyvar &keyvar $32;
-      retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
-      set &libds end=&lastvar;
-      /* hash should include previous row */
-      &keyvar=put(md5(&prevkeyvar
-      /* loop every column, hashing every individual value */
-    %do i=1 %to %sysfunc(countw(&varlist));
-      %let var=%scan(&varlist,&i,%str( ));
-      %if %mf_getvartype(&libds,&var)=C %then %do;
-          !!put(md5(trim(&var)),$hex32.)
-      %end;
-      %else %do;
-          !!put(md5(trim(put(&var*1,binary64.))),$hex32.)
-      %end;
-    %end;
-      ),$hex32.);
-      &prevkeyvar=&keyvar;
-      if &lastvar then output;
-    run;
-  %end;
-%mend mp_hashdataset;
+  ;
+    length &prevkeyvar &keyvar $32;
+    retain &prevkeyvar;
+    if _n_=1 then &prevkeyvar=put(md5("&salt"),$hex32.);
+    set &libds end=&lastvar;
+    /* hash should include previous row */
+    &keyvar=%mp_md5(
+      cvars=%mf_getvarlist(&libds,typefilter=C) &prevkeyvar,
+      nvars=%mf_getvarlist(&libds,typefilter=N)
+    );
+    &prevkeyvar=&keyvar;
+    if &lastvar then output;
+  run;
+%end;
+%mend mp_hashdataset;

base/mp_hashdirectory.sas

@@ -0,0 +1,164 @@
+/**
+  @file
+  @brief Returns a unique hash for each file in a directory
+  @details Hashes each file in each directory, and then hashes the hashes to
+  create a hash for each directory also.
+
+  This makes use of the new `hashing_file()` and `hashing` functions, available
+  since 9.4m6. Interestingly, those functions can be used in pure macro, eg:
+
+      %put %sysfunc(hashing_file(md5,/path/to/file.blob,0));
+
+  Actual usage:
+
+      %let fpath=/some/directory;
+
+      %mp_hashdirectory(&fpath,outds=myhash,maxdepth=2)
+
+      data _null_;
+        set work.myhash;
+        put (_all_)(=);
+      run;
+
+  Whilst files are hashed in their entirety, the logic for creating a folder
+  hash is as follows:
+
+  @li Sort the files by filename (case sensitive, uppercase then lower)
+  @li Take the first 100 hashes, concatenate and hash
+  @li Concatenate this hash with another 100 hashes and hash again
+  @li Continue until the end of the folder.  This is the folder hash
+  @li If a folder contains other folders, start from the bottom of the tree -
+    the folder hashes cascade upwards so you know immediately if there is a
+    change in a sub/sub directory
+  @li If a subfolder has no content (empty) then it is ignored. No hash created.
+  @li If the file is empty, it is also ignored / no hash created.
+  @li If the target directory (&inloc) is empty, &outds will also be empty
+
+  <h4> SAS Macros </h4>
+  @li mp_dirlist.sas
+
+  <h4> Related Files </h4>
+  @li mp_hashdataset.sas
+  @li mp_hashdirectory.test.sas
+  @li mp_md5.sas
+
+  @param [in] inloc Full filepath of the file to be hashed (unquoted)
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
+    subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
+  @param [in] method= (MD5) the hashing method to use.  Available options:
+    @li MD5
+    @li SH1
+    @li SHA256
+    @li SHA384
+    @li SHA512
+    @li CRC32
+  @param [out] outds= (work.mp_hashdirectory) The output dataset.  Contains:
+    @li directory - the parent folder
+    @li file_hash - the hash output
+    @li hash_duration - how long the hash took (first hash always takes longer)
+    @li file_path - /full/path/to/each/file.ext
+    @li file_or_folder - contains either "file" or "folder"
+    @li level - the depth of the directory (top level is 0)
+
+  @version 9.4m6
+  @author Allan Bowe
+**/
+
+%macro mp_hashdirectory(inloc,
+  outds=work.mp_hashdirectory,
+  method=MD5,
+  maxdepth=0,
+  iftrue=%str(1=1)
+)/*/STORE SOURCE*/;
+
+%local curlevel tempds maxlevel;
+
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+/* get the directory listing */
+%mp_dirlist(path=&inloc, outds=&outds, maxdepth=&maxdepth, showparent=YES)
+
+/* create the hashes */
+data &outds;
+  set &outds (rename=(filepath=file_path));
+  length FILE_HASH $32 HASH_DURATION 8;
+  keep directory file_hash hash_duration file_path file_or_folder level;
+
+  ts=datetime();
+  if file_or_folder='file' then do;
+    /* if file is empty, hashing_file will break - so ignore / delete */
+    length fname val $8;
+    drop fname val fid is_empty;
+    rc=filename(fname,file_path);
+    fid=fopen(fname);
+    if fid > 0 then do;
+      rc=fread(fid);
+      is_empty=fget(fid,val);
+    end;
+    rc=fclose(fid);
+    rc=filename(fname);
+    if is_empty ne 0 then delete;
+    else file_hash=hashing_file("&method",cats(file_path),0);
+  end;
+  hash_duration=datetime()-ts;
+run;
+
+proc sort data=&outds ;
+  by descending level directory file_path;
+run;
+
+%let maxlevel=0;
+data _null_;
+  set &outds;
+  call symputx('maxlevel',level,'l');
+  stop;
+run;
+
+/* now hash the hashes to populate folder hashes, starting from the bottom */
+%do curlevel=&maxlevel %to 0 %by -1;
+  data work._data_ (keep=directory file_hash);
+    set &outds;
+    where level=&curlevel;
+    by descending level directory file_path;
+    length str $32767 tmp_hash $32;
+    retain str tmp_hash ;
+    /* reset vars when starting a new directory */
+    if first.directory then do;
+      str='';
+      tmp_hash='';
+      i=0;
+    end;
+    /* hash each chunk of 100 file paths */
+    i+1;
+    str=cats(str,file_hash);
+    if mod(i,100)=0 or last.directory then do;
+      tmp_hash=hashing("&method",cats(tmp_hash,str));
+      str='';
+    end;
+    /* output the hash at directory level */
+    if last.directory then do;
+      file_hash=tmp_hash;
+      output;
+    end;
+    if last.level then stop;
+  run;
+  %let tempds=&syslast;
+  /* join the hash back into the main table */
+  proc sql undo_policy=none;
+  create table &outds as
+    select a.directory
+      ,coalesce(b.file_hash,a.file_hash) as file_hash
+      ,a.hash_duration
+      ,a.file_path
+      ,a.file_or_folder
+      ,a.level
+    from &outds a
+    left join &tempds b
+    on a.file_path=b.directory
+    order by level desc, directory, file_path;
+  drop table &tempds;
+%end;
+
+%mend mp_hashdirectory;

base/mp_include.sas

@@ -51,9 +51,10 @@ https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/mcrolref/n1j5tcc0n2xczyn1
     this dataset.
     It will then run an abort cancel FILE to stop the include running, and pass
     the dataset back.
-    NOTE - it is NOT possible to read this dataset as part of _this_ macro -
-    when running abort cancel FILE, ALL macros are closed, so instead it is
-    necessary to invoke "%mp_abort(mode=INCLUDE)" OUTSIDE of any macro wrappers.
+
+    IMPORTANT NOTE - it is NOT possible to read this dataset as part of _this_
+    macro! When running abort cancel FILE, ALL macros are closed, so instead it
+    is necessary to invoke "%mp_abort(mode=INCLUDE)" OUTSIDE of macro wrappers.
 
 
   @version 9.4

base/mp_init.sas

@@ -33,36 +33,44 @@
 %macro mp_init(prefix=SASJS
 )/*/STORE SOURCE*/;
 
-  %global
-    &prefix._INIT_NUM   /* initialisation time as numeric                   */
-    &prefix._INIT_DTTM  /* initialisation time in E8601DT26.6 format        */
-    &prefix.WORK        /* avoid typing %sysfunc(pathname(work)) every time */
-  ;
-  %if %eval(&&&prefix._INIT_NUM>0) %then %return;  /* only run once */
+%if %symexist(SASJS_PREFIX) %then %return;  /* only run once */
 
-  data _null_;
-    dttm=datetime();
-    call symputx("&prefix._init_num",dttm,'g');
-    call symputx("&prefix._init_dttm",put(dttm,E8601DT26.6),'g');
-    call symputx("&prefix.work",pathname('WORK'),'g');
-  run;
+%global
+  SASJS_PREFIX       /* the ONLY hard-coded global macro variable in SASjs    */
+  &prefix._FUNCTIONS /* used in mcf_init() to track core function compilation */
+  &prefix._INIT_NUM  /* initialisation time as numeric                        */
+  &prefix._INIT_DTTM /* initialisation time in E8601DT26.6 format             */
+  &prefix.WORK       /* avoid typing %sysfunc(pathname(work)) every time      */
+;
 
-  options
-    noautocorrect           /* disallow misspelled procedure names            */
-    compress=CHAR           /* default is none so ensure we have something!   */
-    datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
-    %str(err)orcheck=STRICT /* catch errs in libname/filename statements      */
-    fmterr                  /* ensure err when a format cannot be found       */
-    mergenoby=%str(ERR)OR   /* throw err when a merge has no BY variables     */
-    missing=.               /* changing this can cause hard to detect errs    */
-    noquotelenmax           /* avoid warnings for long strings                */
-    noreplace               /* avoid overwriting permanent datasets           */
-    ps=max                  /* reduce log size slightly                       */
-    ls=max                  /* reduce log even more and avoid word truncation */
-    validmemname=COMPATIBLE /* avoid special characters etc in table names    */
-    validvarname=V7         /* avoid special characters etc in variable names */
-    varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
-    varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
-  ;
+%let sasjs_prefix=&prefix;
 
-%mend mp_init;
+data _null_;
+  dttm=datetime();
+  call symputx("&sasjs_prefix._init_num",dttm,'g');
+  call symputx("&sasjs_prefix._init_dttm",put(dttm,E8601DT26.6),'g');
+  call symputx("&sasjs_prefix.work",pathname('WORK'),'g');
+run;
+
+options
+  compress=CHAR           /* default is none so ensure we have something!   */
+  datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
+  errorcheck=STRICT       /* catch errs in libname/filename statements      */
+  fmterr                  /* ensure err when a format cannot be found       */
+  mergenoby=%str(ERR)OR   /* throw err when a merge has no BY variables     */
+  missing=.               /* changing this can cause hard to detect errs    */
+  noquotelenmax           /* avoid warnings for long strings                */
+  noreplace               /* avoid overwriting permanent datasets           */
+  ps=max                  /* reduce log size slightly                       */
+  ls=max                  /* reduce log even more and avoid word truncation */
+  validmemname=COMPATIBLE /* avoid special characters etc in table names    */
+  validvarname=V7         /* avoid special characters etc in variable names */
+  varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
+  varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  noautocorrect           /* disallow misspelled procedure names            */
+  dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
+%end;
+;
+
+%mend mp_init;

base/mp_jsonout.sas

@@ -1,7 +1,12 @@
 /**
   @file mp_jsonout.sas
   @brief Writes JSON in SASjs format to a fileref
-  @details PROC JSON is faster but will produce errs like the ones below if
+  @details This macro can be used to OPEN a JSON stream and send one or more
+  tables as arrays of rows, where each row can be an object or a nested array.
+
+  There are two engines available - DATASTEP or PROCJSON.
+
+  PROC JSON is fast but will produce errs like the ones below if
   special chars are encountered.
 
   > (ERR)OR: Some code points did not transcode.
@@ -12,6 +17,10 @@
 
   If this happens, try running with ENGINE=DATASTEP.
 
+  The DATASTEP engine is used to handle special SAS missing numerics, and
+  can also convert entire datasets to formatted values.  Output JSON is always
+  in UTF-8.
+
   Usage:
 
         filename tmp temp;
@@ -19,11 +28,12 @@
 
         %mp_jsonout(OPEN,jref=tmp)
         %mp_jsonout(OBJ,class,jref=tmp)
+        %mp_jsonout(OBJ,class,dslabel=class2,jref=tmp,showmeta=Y)
         %mp_jsonout(CLOSE,jref=tmp)
 
         data _null_;
         infile tmp;
-        input;list;
+        input;putlog _infile_;
         run;
 
   If you are building web apps with SAS then you are strongly encouraged to use
@@ -31,24 +41,28 @@
   [sasjs adapter](https://github.com/sasjs/adapter).
   For more information see https://sasjs.io
 
-  @param action Valid values:
+  @param [in] action Valid values:
     @li OPEN - opens the JSON
     @li OBJ - sends a table with each row as an object
     @li ARR - sends a table with each row in an array
     @li CLOSE - closes the JSON
-
-  @param ds the dataset to send.  Must be a work table.
-  @param jref= the fileref to which to send the JSON
-  @param dslabel= the name to give the table in the exported JSON
-  @param fmt= Whether to keep or strip formats from the table
-  @param engine= Which engine to use to send the JSON, valid options are:
+  @param [in] ds The dataset to send.  Must be a work table.
+  @param [out] jref= (_webout) The fileref to which to send the JSON
+  @param [out] dslabel= The name to give the table in the exported JSON
+  @param [in] fmt= (Y) Whether to keep (Y) or strip (N) formats from the table
+  @param [in] engine= (DATASTEP) Which engine to use to send the JSON. Options:
     @li PROCJSON (default)
     @li DATASTEP (more reliable when data has non standard characters)
+  @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
+    (eg `null`) or as STRING values (eg `".a"` or `".b"`)
+  @param [in] showmeta= (N) Set to Y to output metadata alongside each table,
+    such as the column formats and types.  The metadata is contained inside an
+    object with the same name as the table but prefixed with a dollar sign - ie,
+    `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to JSON
 
-  @param dbg= DEPRECATED - was used to conditionally add PRETTY to
-    proc json but this can cause line truncation in large files.
-
-  <h4> Related Macros <h4>
+  <h4> Related Files </h4>
   @li mp_ds2fmtds.sas
 
   @version 9.2
@@ -56,177 +70,336 @@
   @source https://github.com/sasjs/core
 
 **/
-
-%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0
+%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y
+  ,engine=DATASTEP
+  ,missing=NULL
+  ,showmeta=N
+  ,maxobs=MAX
 )/*/STORE SOURCE*/;
-%put output location=&jref;
+%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval
+  tmpds1 tmpds2 tmpds3 tmpds4;
+%let numcols=0;
+%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;);
+
 %if &action=OPEN %then %do;
   options nobomfile;
-  data _null_;file &jref encoding='utf-8';
+  data _null_;file &jref encoding='utf-8' lrecl=200;
     put '{"PROCESSED_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '"';
   run;
 %end;
 %else %if (&action=ARR or &action=OBJ) %then %do;
+  /* force variable names to always be uppercase in the JSON */
   options validvarname=upcase;
-  data _null_;file &jref mod encoding='utf-8';
+  /* To avoid issues with _webout on EBI - such as encoding diffs and truncation
+    (https://support.sas.com/kb/49/325.html) we use temporary files */
+  filename _sjs1 temp lrecl=200 ;
+  data _null_; file _sjs1 encoding='utf-8';
     put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":";
+  run;
+  /* now write to _webout 1 char at a time */
+  data _null_;
+    infile _sjs1 lrecl=1 recfm=n;
+    file &jref mod lrecl=1 recfm=n;
+    input sourcechar $char1. @@;
+    format sourcechar hex2.;
+    put sourcechar char1. @@;
+  run;
+  filename _sjs1 clear;
+
+  /* grab col defs */
+  proc contents noprint data=&ds
+    out=_data_(keep=name type length format formatl formatd varnum label);
+  run;
+  %let colinfo=%scan(&syslast,2,.);
+  proc sort data=&colinfo;
+    by varnum;
+  run;
+  /* move meta to mac vars */
+  data &colinfo;
+    if _n_=1 then call symputx('numcols',nobs,'l');
+    set &colinfo end=last nobs=nobs;
+    name=upcase(name);
+    /* fix formats */
+    if type=2 or type=6 then do;
+      typelong='char';
+      length fmt $49.;
+      if format='' then fmt=cats('$',length,'.');
+      else if formatl=0 then fmt=cats(format,'.');
+      else fmt=cats(format,formatl,'.');
+    end;
+    else do;
+      typelong='num';
+      if format='' then fmt='best.';
+      else if formatl=0 then fmt=cats(format,'.');
+      else if formatd=0 then fmt=cats(format,formatl,'.');
+      else fmt=cats(format,formatl,'.',formatd);
+    end;
+    /* 32 char unique name */
+    newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
+
+    call symputx(cats('name',_n_),name,'l');
+    call symputx(cats('newname',_n_),newname,'l');
+    call symputx(cats('length',_n_),length,'l');
+    call symputx(cats('fmt',_n_),fmt,'l');
+    call symputx(cats('type',_n_),type,'l');
+    call symputx(cats('typelong',_n_),typelong,'l');
+    call symputx(cats('label',_n_),coalescec(label,name),'l');
+    /* overwritten when fmt=Y and a custom format exists in catalog */
+    if typelong='num' then call symputx(cats('fmtlen',_n_),200,'l');
+    else call symputx(cats('fmtlen',_n_),min(32767,ceil((length+10)*1.5)),'l');
+  run;
+
+  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+  proc sql;
+  select count(*) into: lastobs from &ds;
+  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs));
 
   %if &engine=PROCJSON %then %do;
-    data;run;%let tempds=&syslast;
-    proc sql;drop table &tempds;
-    data &tempds /view=&tempds;set &ds;
+    %if &missing=STRING %then %do;
+      %put &sysmacroname: Special Missings not supported in proc json.;
+      %put &sysmacroname: Switching to DATASTEP engine;
+      %goto datastep;
+    %end;
+    data &tempds;
+      set &ds;
+      &stmt_obs;
     %if &fmt=N %then format _numeric_ best32.;;
-    proc json out=&jref pretty
+    /* PRETTY is necessary to avoid line truncation in large files */
+    filename _sjs2 temp lrecl=131068 encoding='utf-8';
+    proc json out=_sjs2 pretty
         %if &action=ARR %then nokeys ;
         ;export &tempds / nosastags fmtnumeric;
     run;
-    proc sql;drop view &tempds;
+    /* send back to webout */
+    data _null_;
+      infile _sjs2 lrecl=1 recfm=n;
+      file &jref mod lrecl=1 recfm=n;
+      input sourcechar $char1. @@;
+      format sourcechar hex2.;
+      put sourcechar char1. @@;
+    run;
+    filename _sjs2 clear;
   %end;
   %else %if &engine=DATASTEP %then %do;
-    %local cols i tempds;
-    %let cols=0;
-    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 %then %do;
+    %datastep:
+    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1
+    %then %do;
       %put &sysmacroname:  &ds NOT FOUND!!!;
       %return;
     %end;
-    %if &fmt=Y %then %do;
-      %put converting every variable to a formatted variable;
-      /* see mp_ds2fmtds.sas for source */
-      proc contents noprint data=&ds
-        out=_data_(keep=name type length format formatl formatd varnum);
-      run;
-      proc sort;
-        by varnum;
-      run;
-      %local fmtds;
-      %let fmtds=%scan(&syslast,2,.);
-      /* prepare formats and varnames */
-      data _null_;
-        if _n_=1 then call symputx('nobs',nobs,'l');
-        set &fmtds end=last nobs=nobs;
-        name=upcase(name);
-        /* fix formats */
-        if type=2 or type=6 then do;
-          length fmt $49.;
-          if format='' then fmt=cats('$',length,'.');
-          else if formatl=0 then fmt=cats(format,'.');
-          else fmt=cats(format,formatl,'.');
-          newlen=max(formatl,length);
-        end;
-        else do;
-          if format='' then fmt='best.';
-          else if formatl=0 then fmt=cats(format,'.');
-          else if formatd=0 then fmt=cats(format,formatl,'.');
-          else fmt=cats(format,formatl,'.',formatd);
-          /* needs to be wide, for datetimes etc */
-          newlen=max(length,formatl,24);
-        end;
-        /* 32 char unique name */
-        newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
 
-        call symputx(cats('name',_n_),name,'l');
-        call symputx(cats('newname',_n_),newname,'l');
-        call symputx(cats('len',_n_),newlen,'l');
-        call symputx(cats('fmt',_n_),fmt,'l');
-        call symputx(cats('type',_n_),type,'l');
+    %if &fmt=Y %then %do;
+      /**
+        * Extract format definitions
+        * First, by getting library locations from dictionary.formats
+        * Then, by exporting the width using proc format
+        * Cannot use maxw from sashelp.vformat as not always populated
+        * Cannot use fmtinfo() as not supported in all flavours
+        */
+      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      proc sql noprint;
+      create table &tmpds1 as
+          select cats(libname,'.',memname) as FMTCAT,
+          FMTNAME
+        from dictionary.formats
+        where fmttype='F' and libname is not null
+          and fmtname in (select format from &colinfo where format is not null)
+        order by 1;
+      create table &tmpds2(
+          FMTNAME char(32),
+          LENGTH num
+      );
+      %local catlist cat fmtlist i;
+      select distinct fmtcat into: catlist separated by ' ' from &tmpds1;
+      %do i=1 %to %sysfunc(countw(&catlist,%str( )));
+        %let cat=%scan(&catlist,&i,%str( ));
+        proc sql;
+        select distinct fmtname into: fmtlist separated by ' '
+          from &tmpds1 where fmtcat="&cat";
+        proc format lib=&cat cntlout=&tmpds3(keep=fmtname length);
+          select &fmtlist;
+        run;
+        proc sql;
+        insert into &tmpds2 select distinct fmtname,length from &tmpds3;
+      %end;
+
+      proc sql;
+      create table &tmpds4 as
+        select a.*, b.length as MAXW
+        from &colinfo a
+        left join &tmpds2 b
+        on cats(a.format)=cats(upcase(b.fmtname))
+        order by a.varnum;
+      data _null_;
+        set &tmpds4;
+        if not missing(maxw);
+        call symputx(
+          cats('fmtlen',_n_),
+          /* vars need extra padding due to JSON escaping of special chars */
+          min(32767,ceil((max(length,maxw)+10)*1.5))
+          ,'l'
+        );
       run;
-      data &fmtds;
+
+      /* configure varlenchk - as we are explicitly shortening the variables */
+      %let optval=%sysfunc(getoption(varlenchk));
+      options varlenchk=NOWARN;
+      data _data_(compress=char);
+        /* shorten the new vars */
+        length
+      %do i=1 %to &numcols;
+          &&name&i $&&fmtlen&i
+      %end;
+          ;
         /* rename on entry */
         set &ds(rename=(
-      %local i;
-      %do i=1 %to &nobs;
-        &&name&i=&&newname&i
+      %do i=1 %to &numcols;
+          &&name&i=&&newname&i
       %end;
         ));
-      %do i=1 %to &nobs;
-        length &&name&i $&&len&i;
-        &&name&i=left(put(&&newname&i,&&fmt&i));
-        drop &&newname&i;
+      &stmt_obs;
+
+      drop
+      %do i=1 %to &numcols;
+        &&newname&i
       %end;
-        if _error_ then call symputx('syscc',1012);
-      run;
-      %let ds=&fmtds;
-    %end; /* &fmt=Y */
-    data _null_;file &jref mod encoding='utf-8';
-      put "["; call symputx('cols',0,'l');
-    proc sort
-      data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
-      out=_data_;
-      by varnum;
-
-    data _null_;
-      set _last_ end=last;
-      call symputx(cats('name',_n_),name,'l');
-      call symputx(cats('type',_n_),type,'l');
-      call symputx(cats('len',_n_),length,'l');
-      if last then call symputx('cols',_n_,'l');
-    run;
-
-    proc format; /* credit yabwon for special null removal */
-      value bart ._ - .z = null
-      other = [best.];
-
-    data;run; %let tempds=&syslast; /* temp table for spesh char management */
-    proc sql; drop table &tempds;
-    data &tempds/view=&tempds;
-      attrib _all_ label='';
-      %do i=1 %to &cols;
-        %if &&type&i=char %then %do;
-          length &&name&i $32767;
-          format &&name&i $32767.;
+        ;
+      %do i=1 %to &numcols;
+        %if &&typelong&i=num %then %do;
+          &&name&i=cats(put(&&newname&i,&&fmt&i));
+        %end;
+        %else %do;
+          &&name&i=put(&&newname&i,&&fmt&i);
         %end;
       %end;
-      set &ds;
+        if _error_ then do;
+          call symputx('syscc',1012);
+          stop;
+        end;
+      run;
+      %let fmtds=&syslast;
+      options varlenchk=&optval;
+    %end;
+
+    proc format; /* credit yabwon for special null removal */
+    value bart (default=40)
+    %if &missing=NULL %then %do;
+      ._ - .z = null
+    %end;
+    %else %do;
+      ._ = [quote()]
+      . = null
+      .a - .z = [quote()]
+    %end;
+      other = [best.];
+
+    data &tempds;
+      attrib _all_ label='';
+      %do i=1 %to &numcols;
+        %if &&typelong&i=char or &fmt=Y %then %do;
+          length &&name&i $&&fmtlen&i...;
+          format &&name&i $&&fmtlen&i...;
+        %end;
+      %end;
+      %if &fmt=Y %then %do;
+        set &fmtds;
+      %end;
+      %else %do;
+        set &ds;
+      %end;
+      &stmt_obs;
       format _numeric_ bart.;
-    %do i=1 %to &cols;
-      %if &&type&i=char %then %do;
-        &&name&i='"'!!trim(prxchange('s/"/\"/',-1,
-                    prxchange('s/'!!'0A'x!!'/\n/',-1,
-                    prxchange('s/'!!'0D'x!!'/\r/',-1,
-                    prxchange('s/'!!'09'x!!'/\t/',-1,
-                    prxchange('s/\\/\\\\/',-1,&&name&i)
-        )))))!!'"';
+    %do i=1 %to &numcols;
+      %if &&typelong&i=char or &fmt=Y %then %do;
+        if findc(&&name&i,'"\'!!'0A0D09000E0F010210111A'x) then do;
+          &&name&i='"'!!trim(
+            prxchange('s/"/\\"/',-1,        /* double quote */
+            prxchange('s/\x0A/\n/',-1,      /* new line */
+            prxchange('s/\x0D/\r/',-1,      /* carriage return */
+            prxchange('s/\x09/\\t/',-1,     /* tab */
+            prxchange('s/\x00/\\u0000/',-1, /* NUL */
+            prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+            prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+            prxchange('s/\x01/\\u0001/',-1, /* SOH */
+            prxchange('s/\x02/\\u0002/',-1, /* STX */
+            prxchange('s/\x10/\\u0010/',-1, /* DLE */
+            prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+            prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+            prxchange('s/\\/\\\\/',-1,&&name&i)
+          )))))))))))))!!'"';
+        end;
+        else &&name&i=quote(cats(&&name&i));
       %end;
     %end;
     run;
-    /* write to temp loc to avoid _webout truncation
-      - https://support.sas.com/kb/49/325.html */
-    filename _sjs temp lrecl=131068 encoding='utf-8';
-    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod;
+
+    filename _sjs3 temp lrecl=131068 ;
+    data _null_;
+      file _sjs3 encoding='utf-8';
+      if _n_=1 then put "[";
       set &tempds;
       if _n_>1 then put "," @; put
       %if &action=ARR %then "[" ; %else "{" ;
-      %do i=1 %to &cols;
+      %do i=1 %to &numcols;
         %if &i>1 %then  "," ;
         %if &action=OBJ %then """&&name&i"":" ;
-        &&name&i
+        "&&name&i"n /* name literal for reserved variable names */
       %end;
       %if &action=ARR %then "]" ; %else "}" ; ;
-    proc sql;
-    drop view &tempds;
-    /* now write the long strings to _webout 1 byte at a time */
+
+    /* close out the table */
     data _null_;
-      length filein 8 fileid 8;
-      filein = fopen("_sjs",'I',1,'B');
-      fileid = fopen("&jref",'A',1,'B');
-      rec = '20'x;
-      do while(fread(filein)=0);
-        rc = fget(filein,rec,1);
-        rc = fput(fileid, rec);
-        rc =fwrite(fileid);
+      file _sjs3 mod encoding='utf-8';
+      put ']';
+    run;
+    data _null_;
+      infile _sjs3 lrecl=1 recfm=n;
+      file &jref mod lrecl=1 recfm=n;
+      input sourcechar $char1. @@;
+      format sourcechar hex2.;
+      put sourcechar char1. @@;
+    run;
+    filename _sjs3 clear;
+  %end;
+
+  proc sql;
+  drop table &colinfo, &tempds;
+
+  %if %substr(&showmeta,1,1)=Y %then %do;
+    filename _sjs4 temp lrecl=131068 encoding='utf-8';
+    data _null_;
+      file _sjs4;
+      length label $350;
+      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{";
+      do i=1 to &numcols;
+        name=quote(trim(symget(cats('name',i))));
+        format=quote(trim(symget(cats('fmt',i))));
+        label=quote(prxchange('s/\\/\\\\/',-1,trim(symget(cats('label',i)))));
+        length=quote(trim(symget(cats('length',i))));
+        type=quote(trim(symget(cats('typelong',i))));
+        if i>1 then put "," @@;
+        put name ':{"format":' format ',"label":' label
+          ',"length":' length ',"type":' type '}';
       end;
-      rc = fclose(filein);
-      rc = fclose(fileid);
+      put '}}';
     run;
-    filename _sjs clear;
-    data _null_; file &jref mod encoding='utf-8';
-      put "]";
+    /* send back to webout */
+    data _null_;
+      infile _sjs4 lrecl=1 recfm=n;
+      file &jref mod lrecl=1 recfm=n;
+      input sourcechar $char1. @@;
+      format sourcechar hex2.;
+      put sourcechar char1. @@;
     run;
+    filename _sjs4 clear;
   %end;
 %end;
 
 %else %if &action=CLOSE %then %do;
-  data _null_;file &jref encoding='utf-8' mod;
+  data _null_; file &jref encoding='utf-8' mod ;
     put "}";
   run;
 %end;

base/mp_loadformat.sas

@@ -0,0 +1,303 @@
+/**
+  @file
+  @brief Loads a format catalog from a staging dataset
+  @details When loading staged data, it is common to receive only the records
+  that have actually changed.  However, when loading a format catalog, if
+  records are missing they are presumed to be no longer required.
+
+  This macro will augment a staging dataset with other records from the same
+  format, to prevent loss of data - UNLESS the input dataset contains a marker
+  column, specifying that a particular row needs to be deleted (`delete_col=`).
+
+  This macro can also be used to identify which records would be (or were)
+  considered new, modified or deleted (`loadtarget=`) by creating the following
+  tables:
+
+  @li work.outds_add
+  @li work.outds_del
+  @li work.outds_mod
+
+  For example usage, see mp_loadformat.test.sas
+
+  @param [in] libcat The format catalog to be loaded
+  @param [in] libds The staging table to load
+  @param [in] loadtarget= (NO) Set to YES to actually load the target catalog
+  @param [in] delete_col= (_____DELETE__THIS__RECORD_____) The column used to
+    mark a record for deletion.  Values should be "Yes" or "No".
+  @param [out] auditlibds= (0) For change tracking, set to the libds of an audit
+    table as defined in mddl_dc_difftable.sas
+  @param [in] locklibds= (0) For multi-user (parallel) situations, set to the
+    libds of the DC lock table as defined in the mddl_dc_locktable.sas macro.
+  @param [out] outds_add= (0) Set a libds here to see the new records added
+  @param [out] outds_del= (0) Set a libds here to see the records deleted
+  @param [out] outds_mod= (0) Set a libds here to see the modified records
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_cntlout.sas
+  @li mp_lockanytable.sas
+  @li mp_storediffs.sas
+
+  <h4> Related Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mddl_dc_locktable.sas
+  @li mp_loadformat.test.sas
+  @li mp_lockanytable.sas
+  @li mp_stackdiffs.sas
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_loadformat(libcat,libds
+  ,loadtarget=NO
+  ,auditlibds=0
+  ,locklibds=0
+  ,delete_col=_____DELETE__THIS__RECORD_____
+  ,outds_add=0
+  ,outds_del=0
+  ,outds_mod=0
+  ,mdebug=0
+);
+/* set up local macro variables and temporary tables (with a prefix) */
+%local err msg prefix dslist i var fmtlist ibufsize;
+%let dslist=base_fmts template inlibds ds1 stagedata storediffs;
+%if &outds_add=0 %then %let dslist=&dslist outds_add;
+%if &outds_del=0 %then %let dslist=&dslist outds_del;
+%if &outds_mod=0 %then %let dslist=&dslist outds_mod;
+%let prefix=%substr(%mf_getuniquename(),1,21);
+%do i=1 %to %sysfunc(countw(&dslist));
+  %let var=%scan(&dslist,&i);
+  %local &var;
+  %let &var=%upcase(&prefix._&var);
+%end;
+
+/*
+format values can be up to 32767 wide.  SQL joins on such a wide column can
+cause buffer issues.  Update ibufsize and reset at the end.
+*/
+%let ibufsize=%sysfunc(getoption(ibufsize));
+options ibufsize=32767 ;
+
+/* in DC, format catalogs maybe specified in the libds with a -FC extension */
+%let libcat=%scan(&libcat,1,-);
+
+/* perform input validations */
+%let err=0;
+%let msg=0;
+data _null_;
+  if _n_=1 then putlog "&sysmacroname entry vars:";
+  set sashelp.vmacro;
+  where scope="&sysmacroname";
+  value=upcase(value);
+  if &mdebug=0 then put name '=' value;
+  if name=:'LOAD' and value not in ('YES','NO') then do;
+    call symputx('msg',"invalid value for "!!name!!":"!!value);
+    call symputx('err',1);
+    stop;
+  end;
+  else if name='LIBCAT' then do;
+    if exist(value,'CATALOG') le 0 then do;
+      call symputx('msg',"Unable to open catalog: "!!value);
+      call symputx('err',1);
+      stop;
+    end;
+  end;
+  else if name='LIBDS' then do;
+    if exist(value) le 0 then do;
+      call symputx('msg',"Unable to open staging table: "!!value);
+      call symputx('err',1);
+      stop;
+    end;
+  end;
+  else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
+  and missing(value) then do;
+    call symputx('msg',"missing value in var: "!!name);
+    call symputx('err',1);
+    stop;
+  end;
+run;
+
+%mp_abort(
+  iftrue=(&err ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(&msg)
+)
+
+/**
+  * First, extract only relevant formats from the catalog
+  */
+proc sql noprint;
+select distinct upcase(fmtname) into: fmtlist separated by ' ' from &libds;
+
+%mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
+
+
+/**
+  * Ensure input table and base_formats have consistent lengths and types
+  */
+%mddl_sas_cntlout(libds=&template)
+data &inlibds;
+  length &delete_col $3;
+  if 0 then set &template;
+  set &libds;
+  if &delete_col='' then &delete_col='No';
+  fmtname=upcase(fmtname);
+  if missing(type) then do;
+    if substr(fmtname,1,1)='$' then type='C';
+    else type='N';
+  end;
+  if type='N' then do;
+    start=cats(start);
+    end=cats(end);
+  end;
+run;
+
+/**
+  * Identify new records
+  */
+proc sql;
+create table &outds_add(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  left join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where b.fmtname is null
+    and upcase(a.&delete_col) ne "YES"
+  order by fmtname, start;;
+
+/**
+  * Identify deleted records
+  */
+create table &outds_del(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where upcase(a.&delete_col)="YES"
+  order by fmtname, start;
+
+/**
+  * Identify modified records
+  */
+create table &outds_mod (drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where upcase(a.&delete_col) ne "YES"
+  order by fmtname, start;
+
+options ibufsize=&ibufsize;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc prior to load prep)
+)
+
+%if &loadtarget=YES %then %do;
+  data &ds1;
+    merge &base_fmts(in=base)
+      &outds_mod(in=mod)
+      &outds_add(in=add)
+      &outds_del(in=del);
+    if not del and not mod;
+    by fmtname start;
+  run;
+  data &stagedata;
+    set &ds1 &outds_mod;
+  run;
+  proc sort;
+    by fmtname start;
+  run;
+%end;
+/* mp abort needs to run outside of conditional blocks */
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc prior to actual load)
+)
+%if &loadtarget=YES %then %do;
+  %if %mf_nobs(&stagedata)=0 %then %do;
+    %put There are no changes to load in &libcat!;
+    %return;
+  %end;
+  %if &locklibds ne 0 %then %do;
+    /* prevent parallel updates */
+    %mp_lockanytable(LOCK
+      ,lib=%scan(&libcat,1,.)
+      ,ds=%scan(&libcat,2,.)-FC
+      ,ref=MP_LOADFORMAT commencing format load
+      ,ctl_ds=&locklibds
+    )
+  %end;
+  /* do the actual load */
+  proc format lib=&libcat cntlin=&stagedata;
+  run;
+  %if &locklibds ne 0 %then %do;
+    /* unlock the table */
+    %mp_lockanytable(UNLOCK
+      ,lib=%scan(&libcat,1,.)
+      ,ds=%scan(&libcat,2,.)-FC
+      ,ref=MP_LOADFORMAT completed format load
+      ,ctl_ds=&locklibds
+    )
+  %end;
+  /* track the changes */
+  %if &auditlibds ne 0 %then %do;
+    %if &locklibds ne 0 %then %do;
+      %mp_lockanytable(LOCK
+        ,lib=%scan(&auditlibds,1,.)
+        ,ds=%scan(&auditlibds,2,.)
+        ,ref=MP_LOADFORMAT commencing audit table load
+        ,ctl_ds=&locklibds
+      )
+    %end;
+
+    %mp_storediffs(&libcat-FC
+      ,&base_fmts
+      ,FMTNAME START
+      ,delds=&outds_del
+      ,modds=&outds_mod
+      ,appds=&outds_add
+      ,outds=&storediffs
+      ,mdebug=&mdebug
+    )
+
+    proc append base=&auditlibds data=&storediffs;
+    run;
+
+    %if &locklibds ne 0 %then %do;
+      %mp_lockanytable(UNLOCK
+        ,lib=%scan(&auditlibds,1,.)
+        ,ds=%scan(&auditlibds,2,.)
+        ,ref=MP_LOADFORMAT commencing audit table load
+        ,ctl_ds=&locklibds
+      )
+    %end;
+  %end;
+%end;
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc after load)
+)
+
+%if &mdebug=0 %then %do;
+  proc datasets lib=work;
+    delete &prefix:;
+  run;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%mend mp_loadformat;

base/mp_lockanytable.sas

@@ -1,27 +1,28 @@
 /**
   @file
   @brief Mechanism for locking tables to prevent parallel modifications
-  @details Uses a control table to enable ANY table to be locked for updates.
+  @details Uses a control table to enable ANY table to be locked for updates
+  (not just SAS datasets).
   Only useful if every update uses the macro!   Used heavily within
   [Data Controller for SAS](https://datacontroller.io).
 
-  The underlying table is structured as per the MAKETABLE action.
-
   @param [in] action The action to be performed.  Valid values:
     @li LOCK - Sets the lock flag, also confirms if a SAS lock is available
     @li UNLOCK - Unlocks the table
-    @li MAKETABLE - creates the control table (ctl_ds)
   @param [in] lib= (WORK) The libref of the table to lock.  Should already be
     assigned.
   @param [in] ds= The dataset to lock
   @param [in] ref= A meaningful reference to enable the lock to be traced. Max
     length is 200 characters.
   @param [out] ctl_ds= (0) The control table which controls the actual locking.
-    Should already be assigned and available.
+    Should already be assigned and available.  The definition is available by
+    running the mddl_dc_locktable.sas macro.
+
   @param [in] loops= (25) Number of times to check for a lock.
   @param [in] loop_secs= (1) Seconds to wait between each lock attempt
 
   <h4> SAS Macros </h4>
+  @li mf_fmtdttm.sas
   @li mp_abort.sas
   @li mp_lockfilecheck.sas
   @li mf_getuser.sas
@@ -49,7 +50,7 @@ data _null_;
   put name '=' value;
 run;
 
-%mp_abort(iftrue= (&ds=0 and &action ne MAKETABLE)
+%mp_abort(iftrue= ("&ds"="0" and &action ne MAKETABLE)
   ,mac=&sysmacroname
   ,msg=%str(dataset was not provided)
 )
@@ -86,7 +87,7 @@ run;
 /* do not proceed if no observations can be processed */
 %mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)
   ,mac=&sysmacroname
-  ,msg=%str(options obs = 0. syserrortext=&syserrortext)
+  ,msg=%str(cannot continue when options obs = 0)
 )
 
 %if &ACTION=LOCK %then %do;
@@ -111,7 +112,7 @@ run;
       LOCK_LIB ="&lib";
       LOCK_DS="&ds";
       LOCK_STATUS_CD='LOCKED';
-      LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt;
+      LOCK_START_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt;
       LOCK_USER_NM="&user";
       LOCK_PID="&sysjobid";
       LOCK_REF="&ref";
@@ -131,7 +132,7 @@ run;
     proc sql;
     update &ctl_ds
       set LOCK_STATUS_CD='LOCKED'
-        , LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_START_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
         , LOCK_USER_NM="&user"
         , LOCK_PID="&sysjobid"
         , LOCK_REF="&ref"
@@ -206,7 +207,7 @@ run;
     proc sql;
     update &ctl_ds
       set LOCK_STATUS_CD='UNLOCKED'
-        , LOCK_END_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_END_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
         , LOCK_USER_NM="&user"
         , LOCK_PID="&sysjobid"
         , LOCK_REF="&ref"
@@ -221,19 +222,6 @@ run;
     %let abortme=1;
   %end;
 %end;
-%else %if &action=MAKETABLE %then %do;
-  proc sql;
-  create table &ctl_ds(
-      lock_lib char(8),
-      lock_ds char(32),
-      lock_status_cd char(10) not null,
-      lock_user_nm char(100) not null ,
-      lock_ref char(200),
-      lock_pid char(10),
-      lock_start_dttm num format=E8601DT26.6,
-      lock_end_dttm num format=E8601DT26.6,
-    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
-%end;
 %else %do;
   %let msg=lock_anytable given unsupported action (&action);
   %let abortme=1;

base/mp_lockfilecheck.sas

@@ -37,7 +37,7 @@ run;
   ,mac=checklock.sas
   ,msg=Aborting with syscc=&syscc on entry.
 )
-%mp_abort(iftrue= (&libds=0)
+%mp_abort(iftrue= ("&libds"="0")
   ,mac=&sysmacroname
   ,msg=%str(libds not provided)
 )
@@ -46,6 +46,12 @@ run;
 %let lib=%upcase(%scan(&libds,1,.));
 %let ds=%upcase(%scan(&libds,2,.));
 
+/* in DC, format catalogs are passed with a -FC suffix. No saslock here! */
+%if %scan(&libds,2,-)=FC %then %do;
+  %put &sysmacroname: Format Catalog detected, no lockfile applied to &libds;
+  %return;
+%end;
+
 /* do not proceed if no observations can be processed */
 %let msg=options obs = 0. syserrortext=%superq(syserrortext);
 %mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)

base/mp_makedata.sas

@@ -10,8 +10,6 @@
     according to the variable types and formats.
 
     TODO:
-      @li Respect PKs
-      @li Respect NOT NULLs
       @li Consider dates, datetimes, times, integers etc
 
   Usage:
@@ -27,16 +25,23 @@
       );
       %mp_makedata(work.example)
 
-  @param [in] libds The empty table in which to create data
-  @param [out] obs= (500) The number of records to create.
+  @param [in] libds The empty table (libref.dataset) in which to create data
+  @param [out] obs= (500) The maximum number of records to create.  The table
+    is sorted with nodup on the primary key, so the actual number of records may
+    be lower than this.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas
   @li mf_getvarlen.sas
+  @li mf_getvarlist.sas
+  @li mf_islibds.sas
   @li mf_nobs.sas
   @li mp_getcols.sas
   @li mp_getpk.sas
 
+  <h4> Related Macros </h4>
+  @li mp_makedata.test.sas
+
   @version 9.2
   @author Allan Bowe
 
@@ -44,44 +49,59 @@
 
 %macro mp_makedata(libds
   ,obs=500
+  ,seed=1
 )/*/STORE SOURCE*/;
 
-%local ds1 c1 n1 i col charvars numvars;
+%local ds1 ds2 lib ds pk_fields i col charvars numvars ispk;
 
-%if %mf_nobs(&libds)>0 %then %do;
+%if %mf_islibds(&libds)=0 %then %do;
+  %put &sysmacroname: Invalid libds (&libds) - should be library.dataset format;
+  %return;
+%end;
+%else %if %mf_nobs(&libds)>0 %then %do;
   %put &sysmacroname: &libds has data, it will not be recreated;
   %return;
 %end;
 
-%local ds1 c1 n1;
-%let ds1=%mf_getuniquename(prefix=mp_makedata);
-%let c1=%mf_getuniquename(prefix=mp_makedatacol);
-%let n1=%mf_getuniquename(prefix=mp_makedatacol);
-data &ds1;
+/* set up temporary vars */
+%let ds1=%mf_getuniquename(prefix=mp_makedatads1);
+%let ds2=%mf_getuniquename(prefix=mp_makedatads2);
+%let lib=%scan(&libds,1,.);
+%let ds=%scan(&libds,2,.);
+
+/* grab the primary key vars */
+%mp_getpk(&lib,ds=&ds,outds=&ds1)
+
+proc sql noprint;
+select coalescec(pk_fields,'_all_') into: pk_fields from &ds1;
+
+data &ds2;
   if 0 then set &libds;
   do _n_=1 to &obs;
-    &c1=repeat(uuidgen(),10);
-    &n1=ranuni(1)*5000000;
-    drop &c1 &n1;
     %let charvars=%mf_getvarlist(&libds,typefilter=C);
     %if &charvars ^= %then %do i=1 %to %sysfunc(countw(&charvars));
       %let col=%scan(&charvars,&i);
-      &col=subpad(&c1,1,%mf_getvarlen(&libds,&col));
+      /* create random value based on observation number and colum length */
+      &col=repeat(put(md5(cats(_n_)),$hex32.),%mf_getvarlen(&libds,&col)/32);
     %end;
 
     %let numvars=%mf_getvarlist(&libds,typefilter=N);
     %if &numvars ^= %then %do i=1 %to %sysfunc(countw(&numvars));
       %let col=%scan(&numvars,&i);
-      &col=&n1;
+      &col=_n_;
     %end;
     output;
   end;
+  stop;
+run;
+proc sort data=&ds2 nodupkey;
+  by &pk_fields;
 run;
 
-proc append base=&libds data=&ds1;
+proc append base=&libds data=&ds2;
 run;
 
 proc sql;
-drop table &ds1;
+drop table &ds1, &ds2;
 
 %mend mp_makedata;

base/mp_md5.sas

@@ -0,0 +1,58 @@
+/**
+  @file
+  @brief Generates an md5 expression for hashing a set of variables
+  @details This is the same algorithm used to hash records in
+  [Data Controller for SAS](https://datacontroller.io) (free for up
+  to 5 users).
+
+  It is not designed to be efficient - it is designed to be effective,
+  given the range of edge cases (large floating points, special missing
+  numerics, thousands of columns, very wide columns).
+
+  It can be used only in data step, eg as follows:
+
+      data _null_;
+        set sashelp.class;
+        hashvar=%mp_md5(cvars=name sex, nvars=age height weight);
+        put hashvar=;
+      run;
+
+  Unfortunately it will not run in SQL - it fails with the following message:
+
+  > The width value for HEX is out of bounds. It should be between 1 and 16
+
+  The macro will also cause errors if the data contains (non-special) missings
+  and the (undocumented) `options dsoptions=nonote2err;` is in effect.
+
+  This can be avoided in two ways:
+
+  @li Global option:  `options dsoptions=nonote2err;`
+  @li Data step option: `data YOURLIB.YOURDATASET /nonote2err;`
+
+  @param cvars= Space seperated list of character variables
+  @param nvars= Space seperated list of numeric variables
+
+  <h4> Related Programs </h4>
+  @li mp_init.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_md5(cvars=,nvars=);
+%local i var sep;
+put(md5(
+  %do i=1 %to %sysfunc(countw(&cvars));
+    %let var=%scan(&cvars,&i,%str( ));
+    &sep put(md5(trim(&var)),$hex32.)
+    %let sep=!!;
+  %end;
+  %do i=1 %to %sysfunc(countw(&nvars));
+    %let var=%scan(&nvars,&i,%str( ));
+    /* multiply by 1 to strip precision errors (eg 0 != 0) */
+    /* but ONLY if not missing, else will lose any special missing values */
+    &sep put(md5(trim(put(ifn(missing(&var),&var,&var*1),binary64.))),$hex32.)
+    %let sep=!!;
+  %end;
+),$hex32.)
+%mend mp_md5;

base/mp_perflog.sas

@@ -1,13 +1,14 @@
 /**
   @file
-  @brief Logs the time the macro was executed in a control dataset.
-  @details If the dataset does not exist, it is created.  Usage:
+  @brief Logs a message in a dataset every time it is invoked
+  @details If the dataset does not exist, it is created.
+  Usage:
 
-    %mp_perflog(started)
-    %mp_perflog()
-    %mp_perflog(startanew,libds=work.newdataset)
-    %mp_perflog(finished,libds=work.newdataset)
-    %mp_perflog(finished)
+      %mp_perflog(started)
+      %mp_perflog()
+      %mp_perflog(startanew,libds=work.newdataset)
+      %mp_perflog(finished,libds=work.newdataset)
+      %mp_perflog(finished)
 
 
   @param label Provide label to go into the control dataset

base/mp_replace.sas

@@ -0,0 +1,151 @@
+/**
+  @file
+  @brief Performs a text substitution on a file
+  @details Performs a find and replace on a file, either in place or to a new
+  file. Can be used on files where lines are longer than 32767.
+
+  Works by reading in the file byte by byte, then marking the beginning and end
+  of each matched string, before finally doing the replace.
+
+  Full credit for this highly efficient and syntactically satisfying SAS logic
+  goes to [Bartosz Jabłoński](https://www.linkedin.com/in/yabwon), founder of
+  the [SAS Packages](https://github.com/yabwon/SAS_PACKAGES) framework.
+
+  Usage:
+
+      %let file="%sysfunc(pathname(work))/file.txt";
+      %let str=replace/me;
+      %let rep=with/this;
+      data _null_;
+        file &file;
+        put 'blahblah';
+        put "blahblah&str.blah";
+        put 'blahblahblah';
+      run;
+      %mp_replace(&file, findvar=str, replacevar=rep)
+      data _null_;
+        infile &file;
+        input;
+        list;
+      run;
+
+  Note - if you are running a version of SAS that will allow the io package in
+  LUA, you can also use this macro: mp_gsubfile.sas
+
+  @param infile The QUOTED path to the file on which to perform the substitution
+  @param findvar= Macro variable NAME containing the string to search for
+  @param replacevar= Macro variable NAME containing the replacement string
+  @param outfile= (0) Optional QUOTED path to the adjusted output file (to
+    avoid overwriting the first file).
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mp_chop.sas
+  @li mp_gsubfile.sas
+  @li mp_replace.test.sas
+
+  @version 9.4
+  @author Bartosz Jabłoński
+  @author Allan Bowe
+**/
+
+%macro mp_replace(infile,
+  findvar=,
+  replacevar=,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+%local inref dttm ds1;
+%let inref=%mf_getuniquefileref();
+%let outref=%mf_getuniquefileref();
+%if &outfile=0 %then %let outfile=&infile;
+%let ds1=%mf_getuniquename(prefix=allchars);
+%let ds2=%mf_getuniquename(prefix=startmark);
+
+/* START */
+%let dttm=%sysfunc(datetime());
+
+filename &inref &infile lrecl=1 recfm=n;
+
+data &ds1;
+  infile &inref;
+  input sourcechar $char1. @@;
+  format sourcechar hex2.;
+run;
+
+data &ds2;
+  /* set find string to length in bytes to cover trailing spaces */
+  length string $ %length(%superq(&findvar));
+  string =symget("&findvar");
+  drop string;
+
+  firstchar=char(string,1);
+  findlen=lengthm(string); /* <- for trailing bytes */
+
+  do _N_=1 to nobs;
+    set &ds1 nobs=nobs point=_N_;
+    if sourcechar=firstchar then do;
+      pos=1;
+      s=0;
+      do point=_N_ to min(_N_ + findlen -1,nobs);
+        set &ds1 point=point;
+        if sourcechar=char(string, pos) then s + 1;
+        else goto _leave_;
+        pos+1;
+      end;
+      _leave_:
+      if s=findlen then do;
+        START =_N_;
+        _N_ =_N_+ s - 1;
+        STOP =_N_;
+        output;
+      end;
+    end;
+  end;
+  stop;
+  keep START STOP;
+run;
+
+data &ds1;
+  declare hash HS(dataset:"&ds2(keep=start)");
+  HS.defineKey("start");
+  HS.defineDone();
+  declare hash HE(dataset:"&ds2(keep=stop)");
+  HE.defineKey("stop");
+  HE.defineDone();
+  do until(eof);
+    set &ds1 end=eof curobs =n;
+    start = ^HS.check(key:n);
+    stop  = ^HE.check(key:n);
+    length strt $ 1;
+    strt =put(start,best. -L);
+    retain out 1;
+    if out   then output;
+    if start then out=0;
+    if stop  then out=1;
+  end;
+  stop;
+  keep sourcechar strt;
+run;
+
+filename &outref &outfile recfm=n;
+
+data _null_;
+  length replace $ %length(%superq(&replacevar));
+  replace=symget("&replacevar");
+  file &outref;
+  do until(eof);
+    set &ds1 end=eof;
+    if strt ="1" then put replace char.;
+    else put sourcechar char1.;
+  end;
+  stop;
+run;
+
+/* END */
+%put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) seconds to run;
+
+%mend mp_replace;

base/mp_resetoption.sas

@@ -3,13 +3,15 @@
   @brief Reset an option to original value
   @details Inspired by the SAS Jedi -
 https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options
-    Called as follows:
 
-    options obs=30;
-    %mp_resetoption(OBS)
+  Called as follows:
+
+      options obs=30 ps=max;
+      %mp_resetoption(OBS)
+      %mp_resetoption(PS)
 
 
-  @param option the option to reset
+  @param [in] option the option to reset
 
   @version 9.2
   @author Allan Bowe
@@ -19,15 +21,19 @@ https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-s
 %macro mp_resetoption(option /* the option to reset */
 )/*/STORE SOURCE*/;
 
-data _null_;
-  length code  $1500;
-  startup=getoption("&option",'startupvalue');
-  current=getoption("&option");
-  if startup ne current then do;
-    code =cat('OPTIONS ',getoption("&option",'keyword','startupvalue'),';');
-    putlog "NOTE: Resetting system option: " code ;
-    call execute(code );
-  end;
-run;
-
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  data _null_;
+    length code  $1500;
+    startup=getoption("&option",'startupvalue');
+    current=getoption("&option");
+    if startup ne current then do;
+      code =cat('OPTIONS ',getoption("&option",'keyword','startupvalue'),';');
+      putlog "NOTE: Resetting system option: " code ;
+      call execute(code );
+    end;
+  run;
+%end;
+%else %do;
+  %put &sysmacroname: reset option feature unavailable on &sysvlong;
+%end;
 %mend mp_resetoption;

base/mp_retainedkey.sas

@@ -0,0 +1,247 @@
+/**
+  @file
+  @brief Generate and apply retained key values to a staging table
+  @details This macro will populate a staging table with a Retained Key based on
+  a business key and a base (target) table.
+
+  Definition of retained key ([source](
+    http://bukhantsov.org/2012/04/what-is-data-vault/)):
+
+  > The retained key is a key which is mapped to business key one-to-one. In
+  > comparison,  the surrogate key includes time and there can be many surrogate
+  > keys corresponding to one business key. This explains the name of the key,
+  > it is retained with insertion of a new version of a row while surrogate key
+  > is increasing.
+
+  This macro is designed to be used as part of a wider load / ETL process (such
+  as the one in [Data Controller for SAS](https://datacontroller.io)).
+
+  Specifically, the macro assumes that the base table has already been 'locked'
+  (eg with the mp_lockanytable.sas macro) prior to invocation.  Also, several
+  tables are assumed to exist (names are configurable):
+
+  @li work.staging_table - the staged data, minus the retained key element
+  @li permlib.base_table - the target table to be loaded (**not** loaded by this
+    macro)
+  @li permlib.maxkeytable - optional, used to store load metaadata.
+    The definition is available by running mp_coretable.sas as follows:
+    `mp_coretable(MAXKEYTABLE)`.
+  @li permlib.locktable - Necessary if maxkeytable is being populated. The
+    definition is available by running mp_coretable.sas as follows:
+    `mp_coretable(LOCKTABLE)`.
+
+
+  @param [in] base_lib= (WORK) Libref of the base (target) table.
+  @param [in] base_dsn= (BASETABLE) Name of the base (target) table.
+  @param [in] append_lib= (WORK) Libref of the staging table
+  @param [in] append_dsn= (APPENDTABLE) Name of the staging table
+  @param [in] retained_key= (DEFAULT_RK) Name of RK to generate (should exist on
+    base table)
+  @param [in] business_key= (PK1 PK2) Business key against which to generate
+    RK values.  Should be unique and not null on the staging table.
+  @param [in] check_uniqueness=(NO) Set to yes to perform a uniqueness check.
+    Recommended if there is a chance that the staging data is not unique on the
+    business key.
+  @param [in] maxkeytable= (0) Provide a maxkeytable libds reference here, to
+    store load metadata (maxkey val, load time).  Set to zero if metadata is not
+    required, eg, when preparing a 'dummy' load. Structure is described above.
+    See below for sample data.
+    |KEYTABLE:$32.|KEYCOLUMN:$32.|MAX_KEY:best.|PROCESSED_DTTM:E8601DT26.6|
+    |---|---|---|---|
+    |`DC487173.MPE_SELECTBOX `|`SELECTBOX_RK `|`55 `|`1950427787.8 `|
+    |`DC487173.MPE_FILTERANYTABLE `|`filter_rk `|`14 `|`1951053886.8 `|
+  @param [in] locktable= (0) If updating the maxkeytable, provide the libds
+    reference to the lock table (per mp_lockanytable.sas macro)
+  @param [in] filter_str= Apply a filter - useful for SCD2 or BITEMPORAL loads.
+    Example: `filter_str=%str( (where=( &now < &tech_to)) )`
+  @param [out] outds= (WORK.APPEND) Output table (staging table + retained key)
+
+  <h4> SAS Macros </h4>
+  @li mf_existvar.sas
+  @li mf_fmtdttm.sas
+  @li mf_getquotedstr.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_lockanytable.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filterstore.sas
+  @li mp_retainedkey.test.sas
+
+  @version 9.2
+
+**/
+
+%macro mp_retainedkey(
+  base_lib=WORK
+  ,base_dsn=BASETABLE
+  ,append_lib=WORK
+  ,append_dsn=APPENDTABLE
+  ,retained_key=DEFAULT_RK
+  ,business_key= PK1 PK2
+  ,check_uniqueness=NO
+  ,maxkeytable=0
+  ,locktable=0
+  ,outds=WORK.APPEND
+  ,filter_str=
+);
+%put &sysmacroname entry vars:;
+%put _local_;
+
+%local base_libds app_libds key_field check maxkey idx_pk newkey_cnt iserr
+  msg x tempds1 tempds2 comma_pk appnobs checknobs dropvar tempvar idx_val;
+%let base_libds=%upcase(&base_lib..&base_dsn);
+%let app_libds=%upcase(&append_lib..&append_dsn);
+%let tempds1=%mf_getuniquename();
+%let tempds2=%mf_getuniquename();
+%let comma_pk=%mf_getquotedstr(in_str=%str(&business_key),dlm=%str(,),quote=);
+%let outds=%sysfunc(ifc(%index(&outds,.)=0,work.&outds,&outds));
+/* validation checks */
+%let iserr=0;
+%if &syscc>0 %then %do;
+  %let iserr=1;
+  %let msg=%str(SYSCC=&syscc on macro entry);
+%end;
+%else %if %sysfunc(exist(&base_libds))=0 %then %do;
+  %let iserr=1;
+  %let msg=%str(Base LIBDS (&base_libds) expected but NOT FOUND);
+%end;
+%else %if %sysfunc(exist(&app_libds))=0 %then %do;
+  %let iserr=1;
+  %let msg=%str(Append LIBDS (&app_libds) expected but NOT FOUND);
+%end;
+%else %if &maxkeytable ne 0 and %sysfunc(exist(&maxkeytable))=0  %then %do;
+  %let iserr=1;
+  %let msg=%str(Maxkeytable (&maxkeytable) expected but NOT FOUND);
+%end;
+%else %if &maxkeytable ne 0 and %sysfunc(exist(&locktable))=0  %then %do;
+  %let iserr=1;
+  %let msg=%str(Locktable (&locktable) expected but NOT FOUND);
+%end;
+%else %if %length(&business_key)=0 %then %do;
+  %let iserr=1;
+  %let msg=%str(Business key (&business_key) expected but NOT FOUND);
+%end;
+
+%do x=1 %to %sysfunc(countw(&business_key));
+  /* check business key values exist */
+  %let key_field=%scan(&business_key,&x,%str( ));
+  %if not %mf_existvar(&app_libds,&key_field) %then %do;
+    %let iserr=1;
+    %let msg=Business key (&key_field) not found on &app_libds!;
+    %goto err;
+  %end;
+  %else %if not %mf_existvar(&base_libds,&key_field) %then %do;
+    %let iserr=1;
+    %let msg=Business key (&key_field) not found on &base_libds!;
+    %goto err;
+  %end;
+%end;
+%err:
+%if &iserr=1 %then %do;
+  /* err case so first perform an unlock of the base table before exiting */
+  %mp_lockanytable(
+    UNLOCK,lib=&base_lib,ds=&base_dsn,ref=%superq(msg),ctl_ds=&locktable
+  )
+%end;
+%mp_abort(iftrue=(&iserr=1),mac=mp_retainedkey,msg=%superq(msg))
+
+proc sql noprint;
+select sum(max(&retained_key),0) into: maxkey from &base_libds;
+
+/**
+  * get base table RK and bus field values for lookup
+  */
+proc sql noprint;
+create table &tempds1 as
+  select distinct &comma_pk,&retained_key
+  from &base_libds &filter_str
+  order by &comma_pk,&retained_key;
+
+%if &check_uniqueness=YES %then %do;
+  select count(*) into:checknobs
+    from (select distinct &comma_pk from &app_libds);
+  select count(*) into: appnobs from &app_libds; /* might be view */
+  %if &checknobs ne &appnobs %then %do;
+    %let msg=Source table &app_libds is not unique on (&business_key);
+    %let iserr=1;
+  %end;
+%end;
+%if &iserr=1 %then %do;
+  /* err case so first perform an unlock of the base table before exiting */
+  %mp_lockanytable(
+    UNLOCK,lib=&base_lib,ds=&base_dsn,ref=%superq(msg),ctl_ds=&locktable
+  )
+%end;
+%mp_abort(iftrue= (&iserr=1),mac=mp_retainedkey,msg=%superq(msg))
+
+%if %mf_existvar(&app_libds,&retained_key)
+%then %let dropvar=(drop=&retained_key);
+
+/* prepare interim table with retained key populated for matching keys */
+proc sql noprint;
+create table &tempds2 as
+  select b.&retained_key, a.*
+  from &app_libds &dropvar a
+  left join &tempds1 b
+  on 1
+  %do idx_pk=1 %to %sysfunc(countw(&business_key));
+    %let idx_val=%scan(&business_key,&idx_pk);
+    and a.&idx_val=b.&idx_val
+  %end;
+  order by &retained_key;
+
+/* identify the number of entries without retained keys (new records) */
+select count(*) into: newkey_cnt
+  from &tempds2
+  where missing(&retained_key);
+quit;
+
+/**
+  * Update maxkey table if link provided
+  */
+%if &maxkeytable ne 0 %then %do;
+  proc sql noprint;
+  select count(*) into: check from &maxkeytable
+    where upcase(keytable)="&base_libds";
+
+  %mp_lockanytable(LOCK
+    ,lib=%scan(&maxkeytable,1,.)
+    ,ds=%scan(&maxkeytable,2,.)
+    ,ref=Updating maxkeyvalues with mp_retainedkey
+    ,ctl_ds=&locktable
+  )
+  proc sql;
+  %if &check=0 %then %do;
+  insert into &maxkeytable
+    set keytable="&base_libds"
+      ,keycolumn="&retained_key"
+      ,max_key=%eval(&maxkey+&newkey_cnt)
+      ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt;
+  %end;
+  %else %do;
+  update &maxkeytable
+    set max_key=%eval(&maxkey+&newkey_cnt)
+      ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt
+    where keytable="&base_libds";
+  %end;
+  %mp_lockanytable(UNLOCK
+    ,lib=%scan(&maxkeytable,1,.)
+    ,ds=%scan(&maxkeytable,2,.)
+    ,ref=Updating maxkeyvalues with maxkey=%eval(&maxkey+&newkey_cnt)
+    ,ctl_ds=&locktable
+  )
+%end;
+
+/* fill in the missing retained key values */
+%let tempvar=%mf_getuniquename();
+data &outds(drop=&tempvar);
+  retain &tempvar %eval(&maxkey+1);
+  set &tempds2;
+  if &retained_key =. then &retained_key=&tempvar;
+  &tempvar=&tempvar+1;
+run;
+
+%mend mp_retainedkey;
+

base/mp_sortinplace.sas

@@ -11,9 +11,7 @@
 
   This macro will only work for BASE (V9) engine libraries.  It works by
   creating a copy of the dataset (without data, WITH constraints) in the same
-  library, appending a sorted view into it, and finally - renaming it.  By
-  default, COMPRESS=CHAR and REUSE=YES will be applied, this behaviour can
-  be adjusted using the `dsoptions=` parameter.
+  library, appending a sorted view into it, and finally - renaming it.
 
   Example usage:
 
@@ -41,7 +39,6 @@
 
   @version 9.2
   @author Allan Bowe
-  @source https://github.com/sasjs/core
 
 **/
 

base/mp_stackdiffs.sas

@@ -0,0 +1,592 @@
+/**
+  @file
+  @brief Prepares an audit table for stacking (re-applying) the changes.
+  @details When the underlying data from a Base Table is refreshed, it can be
+    helpful to have any previously-applied changes, re-applied.
+
+    Such situation might arise if you are applying those changes using a tool
+    like [Data Controller for SAS®](https://datacontroller.io) - which records
+    all such changes in an audit table.
+    It may also apply if you are preparing a series of specific cell-level
+    transactions, that you would like to apply to multiple sets of (similarly
+    structured) Base Tables.
+
+    In both cases, it is necessary that the transactions are stored using
+    the mp_storediffs.sas macro, or at least that the underlying table is
+    structured as per the definition in mp_coretable.sas (DIFFTABLE entry)
+
+    <b>This</b> macro is used to convert the stored changes (tall format) into
+    staged changes (wide format), with base table values incorporated (in the
+    case of modified rows), ready for the subsequent load process.
+
+    Essentially then, what this macro does, is turn a table like this:
+
+|KEY_HASH:$32.|MOVE_TYPE:$1.|TGTVAR_NM:$32.|IS_PK:best.|IS_DIFF:best.|TGTVAR_TYPE:$1.|OLDVAL_NUM:best32.|NEWVAL_NUM:best32.|OLDVAL_CHAR:$32765.|NEWVAL_CHAR:$32765.|
+|---|---|---|---|---|---|---|---|---|---|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`NAME `|`1 `|`-1 `|`C `|`. `|`. `|` `|`Newbie `|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`AGE `|`0 `|`-1 `|`N `|`. `|`13 `|` `|` `|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`HEIGHT `|`0 `|`-1 `|`N `|`. `|`65.3 `|` `|` `|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`SEX `|`0 `|`-1 `|`C `|`. `|`. `|` `|`F `|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`WEIGHT `|`0 `|`-1 `|`N `|`. `|`98 `|` `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`NAME `|`1 `|`-1 `|`C `|`. `|`. `|`Alfred `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`AGE `|`0 `|`-1 `|`N `|`14 `|`. `|` `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`HEIGHT `|`0 `|`-1 `|`N `|`69 `|`. `|` `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`SEX `|`0 `|`-1 `|`C `|`. `|`. `|`M `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`WEIGHT `|`0 `|`-1 `|`N `|`112.5 `|`. `|` `|` `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`NAME `|`1 `|`0 `|`C `|`. `|`. `|`Alice `|`Alice `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`AGE `|`0 `|`1 `|`N `|`13 `|`99 `|` `|` `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`HEIGHT `|`0 `|`0 `|`N `|`56.5 `|`56.5 `|` `|` `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`SEX `|`0 `|`0 `|`C `|`. `|`. `|`F `|`F `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`WEIGHT `|`0 `|`0 `|`N `|`84 `|`84 `|` `|` `|
+
+    Into three tables like this:
+
+  <b> `work.outmod`: </b>
+  |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.|
+  |---|---|---|---|---|
+  |`Alice `|`F `|`99 `|`56.5 `|`84 `|
+
+  <b> `work.outadd`: </b>
+  |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.|
+  |---|---|---|---|---|
+  |`Newbie `|`F `|`13 `|`65.3 `|`98 `|
+
+  <b> `work.outdel`: </b>
+  |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.|
+  |---|---|---|---|---|
+  |`Alfred `|`M `|`14 `|`69 `|`112.5 `|
+
+    As you might expect, there are a bunch of extra features and checks.
+
+    The macro supports both SCD2 (TXTEMPORAL) and UPDATE loadtypes. If the
+    base table contains a PROCESSED_DTTM column (or similar), this can be
+    ignored by declaring it in the `processed_dttm_var` parameter.
+
+    The macro is also flexible where columns have been added or removed from
+    the base table UNLESS there is a change to the primary key.
+
+    Changes to the primary key fields are NOT supported, and are likely to cause
+    unexpected results.
+
+    The following pre-flight checks are made:
+
+    @li All primary key columns exist on the base table
+    @li There is no change in variable TYPE for any of the columns
+    @li There is no reduction in variable LENGTH below the max-length of the
+      supplied values
+
+    Rules for stacking changes are as follows:
+
+    <table>
+    <tr>
+      <th>Transaction Type</th><th>Key Behaviour</th><th>Column Behaviour</th>
+    </tr>
+    <tr>
+      <td>Deletes</td>
+      <td>
+        The row is added to `&outDEL.` UNLESS it no longer exists
+        in the base table, in which case it is added to `&errDS.` instead.
+      </td>
+      <td>
+        Deletes are unaffected by the addition or removal of non Primary-Key
+        columns.
+      </td>
+    </tr>
+    <tr>
+      <td>Inserts</td>
+      <td>
+        Previously newly added rows are added to the `outADD` table UNLESS they
+        are present in the Base table.<br>In this case they are added to the
+        `&errDS.` table instead.
+      </td>
+      <td>
+        Inserts are unaffected by the addition of columns in the Base Table
+        (they are padded with blanks).  Deleted columns are only a problem if
+        they appear on the previous insert - in which case the record is added
+        to `&errDS.`.
+      </td>
+    </tr>
+    <tr>
+      <td>Updates</td>
+      <td>
+        Previously modified rows are merged with base table values such that
+        only the individual cells that were _previously_ changed are re-applied.
+        Where the row contains cells that were not marked as having changed in
+        the prior transaction, the 'blanks' are filled with base table values in
+        the `outMOD` table.<br>
+        If the row no longer exists on the base table, then the row is added to
+        the `errDS` table instead.
+      </td>
+      <td>
+        Updates are unaffected by the addition of columns in the Base Table -
+        the new cells are simply populated with Base Table values.  Deleted
+        columns are only a problem if they relate to a modified cell
+        (`is_diff=1`) - in which case the record is added to `&errDS.`.
+      </td>
+    </tr>
+    </table>
+
+    To illustrate the above with a diagram:
+
+  @dot
+    digraph {
+      rankdir="TB"
+      start[label="Transaction Type?" shape=Mdiamond]
+      del[label="Does Base Row exist?" shape=rectangle]
+      add [label="Does Base Row exist?" shape=rectangle]
+      mod [label="Does Base Row exist?" shape=rectangle]
+      chkmod [label="Do all modified\n(is_diff=1) cells exist?" shape=rectangle]
+      chkadd [label="Do all inserted cells exist?" shape=rectangle]
+      outmod [label="outMOD\nTable" shape=Msquare style=filled]
+      outadd [label="outADD\nTable" shape=Msquare style=filled]
+      outdel [label="outDEL\nTable" shape=Msquare style=filled]
+      outerr [label="ErrDS Table" shape=Msquare fillcolor=Orange style=filled]
+      start -> del [label="Delete"]
+      start -> add [label="Insert"]
+      start -> mod [label="Update"]
+
+      del -> outdel [label="Yes"]
+      del -> outerr [label="No" color="Red" fontcolor="Red"]
+      add -> chkadd [label="No"]
+      add -> outerr [label="Yes" color="Red" fontcolor="Red"]
+      mod -> outerr [label="No" color="Red" fontcolor="Red"]
+      mod -> chkmod [label="Yes"]
+      chkmod -> outerr [label="No" color="Red" fontcolor="Red"]
+      chkmod -> outmod [label="Yes"]
+      chkadd -> outerr [label="No" color="Red" fontcolor="Red"]
+      chkadd -> outadd [label="Yes"]
+
+    }
+  @enddot
+
+    For examples of usage, check out the mp_stackdiffs.test.sas program.
+
+
+  @param [in] baselibds Base Table against which the changes will be applied,
+    in libref.dataset format.
+  @param [in] auditlibds Dataset with previously applied transactions, to be
+    re-applied. Use libref.dataset format.
+    DDL as follows:  %mp_coretable(DIFFTABLE)
+  @param [in] key Space seperated list of key variables
+  @param [in] mdebug= Set to 1 to enable DEBUG messages and preserve outputs
+  @param [in] processed_dttm_var= (0) If a variable is being used to mark
+    the processed datetime, put the name of the variable here.  It will NOT
+    be included in the staged dataset (the load process is expected to
+    provide this)
+  @param [out] errds= (work.errds) Output table containing problematic records.
+    The columns of this table are:
+    @li PK_VARS - Space separated list of primary key variable names
+    @li PK_VALS - Slash separted list of PK variable values
+    @li ERR_MSG - Explanation of why this record is problematic
+  @param [out] outmod= (work.outmod) Output table containing modified records
+  @param [out] outadd= (work.outadd) Output table containing additional records
+  @param [out] outdel= (work.outdel) Output table containing deleted records
+
+  <h4> SAS Macros </h4>
+  @li mf_existvarlist.sas
+  @li mf_getquotedstr.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_islibds.sas
+  @li mf_nobs.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_abort.sas
+  @li mp_ds2squeeze.sas
+
+  <h4> Related Macros </h4>
+  @li mp_coretable.sas
+  @li mp_stackdiffs.test.sas
+  @li mp_storediffs.sas
+
+  @todo The current approach assumes that a variable called KEY_HASH is not on
+    the base table.  This part will need to be refactored (eg using
+    mf_getuniquename.sas) when such a use case arises.
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mp_stackdiffs(baselibds
+  ,auditlibds
+  ,key
+  ,mdebug=0
+  ,processed_dttm_var=0
+  ,errds=work.errds
+  ,outmod=work.outmod
+  ,outadd=work.outadd
+  ,outdel=work.outdel
+)/*/STORE SOURCE*/;
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+/* input parameter validations */
+%mp_abort(iftrue= (%mf_islibds(&baselibds) ne 1)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid baselibds: &baselibds)
+)
+%mp_abort(iftrue= (%mf_islibds(&auditlibds) ne 1)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid auditlibds: &auditlibds)
+)
+%mp_abort(iftrue= (%length(&key)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Missing key variables!)
+)
+%mp_abort(iftrue= (
+    %mf_existVarList(&auditlibds,LIBREF DSN MOVE_TYPE KEY_HASH TGTVAR_NM IS_PK
+      IS_DIFF TGTVAR_TYPE OLDVAL_NUM NEWVAL_NUM OLDVAL_CHAR NEWVAL_CHAR)=0
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Input &auditlibds is missing required columns!)
+)
+
+
+/* set up macro vars */
+%local prefix dslist x var keyjoin commakey keepvars missvars fref;
+%let prefix=%substr(%mf_getuniquename(),1,25);
+%let dslist=ds1d ds2d ds3d ds1a ds2a ds3a ds1m ds2m ds3m pks dups base
+  delrec delerr addrec adderr modrec moderr;
+%do x=1 %to %sysfunc(countw(&dslist));
+  %let var=%scan(&dslist,&x);
+  %local &var;
+  %let &var=%upcase(&prefix._&var);
+%end;
+
+%let key=%upcase(&key);
+%let commakey=%mf_getquotedstr(&key,quote=N);
+
+%let keyjoin=1=1;
+%do x=1 %to %sysfunc(countw(&key));
+  %let var=%scan(&key,&x);
+  %let keyjoin=&keyjoin and a.&var=b.&var;
+%end;
+
+data &errds;
+  length pk_vars $256 pk_vals $4098 err_msg $512;
+  call missing (of _all_);
+  stop;
+run;
+
+/**
+  * Prepare raw DELETE table
+  * Records are in the OLDVAL_xxx columns
+  */
+%let keepvars=MOVE_TYPE KEY_HASH TGTVAR_NM TGTVAR_TYPE IS_PK
+              OLDVAL_NUM OLDVAL_CHAR
+              NEWVAL_NUM NEWVAL_CHAR;
+proc sort data=&auditlibds(where=(move_type='D') keep=&keepvars)
+  out=&ds1d(drop=move_type);
+by KEY_HASH TGTVAR_NM;
+run;
+proc transpose data=&ds1d(where=(tgtvar_type='N'))
+    out=&ds2d(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var OLDVAL_NUM;
+run;
+proc transpose data=&ds1d(where=(tgtvar_type='C'))
+    out=&ds3d(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var OLDVAL_CHAR;
+run;
+%mp_ds2squeeze(&ds2d,outds=&ds2d)
+%mp_ds2squeeze(&ds3d,outds=&ds3d)
+data &outdel;
+  if 0 then set &baselibds;
+  set &ds2d;
+  set &ds3d;
+  drop key_hash;
+  if not missing(%scan(&key,1));
+run;
+proc sort;
+  by &key;
+run;
+
+/**
+  * Prepare raw APPEND table
+  * Records are in the NEWVAL_xxx columns
+  */
+proc sort data=&auditlibds(where=(move_type='A') keep=&keepvars)
+    out=&ds1a(drop=move_type);
+  by KEY_HASH TGTVAR_NM;
+run;
+proc transpose data=&ds1a(where=(tgtvar_type='N'))
+    out=&ds2a(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var NEWVAL_NUM;
+run;
+proc transpose data=&ds1a(where=(tgtvar_type='C'))
+    out=&ds3a(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var NEWVAL_CHAR;
+run;
+%mp_ds2squeeze(&ds2a,outds=&ds2a)
+%mp_ds2squeeze(&ds3a,outds=&ds3a)
+data &outadd;
+  if 0 then set &baselibds;
+  set &ds2a;
+  set &ds3a;
+  drop key_hash;
+  if not missing(%scan(&key,1));
+run;
+proc sort;
+  by &key;
+run;
+
+/**
+  * Prepare raw MODIFY table
+  * Keep only primary key - will add modified values later
+  */
+proc sort data=&auditlibds(
+      where=(move_type='M' and is_pk=1) keep=&keepvars
+    ) out=&ds1m(drop=move_type);
+  by KEY_HASH TGTVAR_NM;
+run;
+proc transpose data=&ds1m(where=(tgtvar_type='N'))
+    out=&ds2m(drop=_name_);
+  by KEY_HASH ;
+  id TGTVAR_NM;
+  var NEWVAL_NUM;
+run;
+proc transpose data=&ds1m(where=(tgtvar_type='C'))
+    out=&ds3m(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var NEWVAL_CHAR;
+run;
+%mp_ds2squeeze(&ds2m,outds=&ds2m)
+%mp_ds2squeeze(&ds3m,outds=&ds3m)
+data &outmod;
+  if 0 then set &baselibds;
+  set &ds2m;
+  set &ds3m;
+  if not missing(%scan(&key,1));
+run;
+proc sort;
+  by &key;
+run;
+
+/**
+  * Extract matching records from the base table
+  * Do this in one join for efficiency.
+  * At a later date, this should be optimised for large database tables by using
+  * passthrough and a temporary table.
+  */
+data &pks;
+  if 0 then set &baselibds;
+  set &outadd &outmod &outdel;
+  keep &key;
+run;
+
+proc sort noduprec dupout=&dups;
+by &key;
+run;
+data _null_;
+  set &dups;
+  putlog (_all_)(=);
+run;
+%mp_abort(iftrue= (%mf_nobs(&dups) ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(duplicates (%mf_nobs(&dups)) found on &auditlibds!)
+)
+
+proc sql;
+create table &base as
+  select a.*
+  from &baselibds a, &pks b
+  where &keyjoin;
+
+/**
+  * delete check
+  * This is straightforward as it relates to records only
+  */
+proc sql;
+create table &delrec as
+  select a.*
+  from &outdel a
+  left join &base b
+  on &keyjoin
+  where b.%scan(&key,1) is null
+  order by &commakey;
+
+data &delerr;
+  if 0 then set &errds;
+  set &delrec;
+  PK_VARS="&key";
+  PK_VALS=catx('/',&commakey);
+  ERR_MSG="Rows cannot be deleted as they do not exist on the Base dataset";
+  keep PK_VARS PK_VALS ERR_MSG;
+run;
+proc append base=&errds data=&delerr;
+run;
+
+data &outdel;
+  merge &outdel (in=a) &delrec (in=b);
+  by &key;
+  if not b;
+run;
+
+/**
+  * add check
+  * Problems - where record already exists, or base table has columns missing
+  */
+%let missvars=%mf_wordsinstr1butnotstr2(
+  Str1=%upcase(%mf_getvarlist(&outadd)),
+  Str2=%upcase(%mf_getvarlist(&baselibds))
+);
+%if %length(&missvars)>0 %then %do;
+    /* add them to the err table */
+  data &adderr;
+    if 0 then set &errds;
+    set &outadd;
+    PK_VARS="&key";
+    PK_VALS=catx('/',&commakey);
+    ERR_MSG="Rows cannot be added due to missing base vars: &missvars";
+    keep PK_VARS PK_VALS ERR_MSG;
+  run;
+  proc append base=&errds data=&adderr;
+  run;
+  proc sql;
+  delete * from &outadd;
+%end;
+%else %do;
+  proc sql;
+  /* find records that already exist on base table */
+  create table &addrec as
+    select a.*
+    from &outadd a
+    inner join &base b
+    on &keyjoin
+    order by &commakey;
+
+  /* add them to the err table */
+  data &adderr;
+    if 0 then set &errds;
+    set &addrec;
+    PK_VARS="&key";
+    PK_VALS=catx('/',&commakey);
+    ERR_MSG="Rows cannot be added as they already exist on the Base dataset";
+    keep PK_VARS PK_VALS ERR_MSG;
+  run;
+  proc append base=&errds data=&adderr;
+  run;
+
+  /* remove invalid rows from the outadd table */
+  data &outadd;
+    merge &outadd (in=a) &addrec (in=b);
+    by &key;
+    if not b;
+  run;
+%end;
+
+/**
+  * mod check
+  * Problems - where record does not exist or baseds has modified cols missing
+  */
+proc sql noprint;
+select distinct tgtvar_nm into: missvars separated by ' '
+  from &auditlibds
+  where move_type='M' and is_diff=1;
+%let missvars=%mf_wordsinstr1butnotstr2(
+  Str1=&missvars,
+  Str2=%upcase(%mf_getvarlist(&baselibds))
+);
+%if %length(&missvars)>0 %then %do;
+    /* add them to the err table */
+  data &moderr;
+    if 0 then set &errds;
+    set &outmod;
+    PK_VARS="&key";
+    PK_VALS=catx('/',&commakey);
+    ERR_MSG="Rows cannot be modified due to missing base vars: &missvars";
+    keep PK_VARS PK_VALS ERR_MSG;
+  run;
+  proc append base=&errds data=&moderr;
+  run;
+  proc sql;
+  delete * from &outmod;
+%end;
+%else %do;
+  /* now check for records that do not exist (therefore cannot be modified) */
+  proc sql;
+  create table &modrec as
+    select a.*
+    from &outmod a
+    left join &base b
+    on &keyjoin
+    where b.%scan(&key,1) is null
+    order by &commakey;
+  data &moderr;
+    if 0 then set &errds;
+    set &modrec;
+    PK_VARS="&key";
+    PK_VALS=catx('/',&commakey);
+    ERR_MSG="Rows cannot be modified as they do not exist on the Base dataset";
+    keep PK_VARS PK_VALS ERR_MSG;
+  run;
+  proc append base=&errds data=&moderr;
+  run;
+  /* delete the above records from the outmod table */
+  data &outmod;
+    merge &outmod (in=a) &modrec (in=b);
+    by &key;
+    if not b;
+  run;
+  /* now - we can prepare the final MOD table (which is currently PK only) */
+  proc sql undo_policy=none;
+  create table &outmod as
+    select a.key_hash
+      ,b.*
+    from &outmod a
+    inner join &base b
+    on &keyjoin
+    order by &commakey;
+  /* now - to update outmod with modified (is_diff=1) values */
+  %let fref=%mf_getuniquefileref();
+  data _null_;
+    file &fref;
+    set &auditlibds(where=(move_type='M')) end=lastobs;
+    by key_hash;
+    retain comma 'N';
+    if _n_=1 then put 'proc sql;';
+    if first.key_hash then do;
+      comma='N';
+      put "update &outmod set " @@;
+    end;
+    if is_diff=1 then do;
+      if comma='N' then do;
+        put '  '@@;
+        comma='Y';
+      end;
+      else put '  ,'@@;
+      if tgtvar_type='C' then do;
+        length qstr $32767;
+        qstr=quote(trim(NEWVAL_CHAR));
+        put tgtvar_nm '=' qstr;
+      end;
+      else put tgtvar_nm '=' newval_num;
+      if comma='  ' then comma='  ,';
+    end;
+    if last.key_hash then put '  where key_hash=trim("' key_hash '");';
+    if lastobs then put "alter table &outmod drop key_hash;";
+  run;
+  %inc &fref/source2;
+%end;
+
+%if &mdebug=0 %then %do;
+  proc datasets lib=work;
+    delete &prefix:;
+  run;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%mend mp_stackdiffs;
+/** @endcond */

base/mp_storediffs.sas

@@ -49,41 +49,23 @@
   @param [in] appds= (0) Dataset with appended records
   @param [in] modds= (0) Dataset with modified records
   @param [out] outds= (work.mp_storediffs) Output table containing stored data.
-    Has the following format:
+    DDL as follows:  %mp_coretable(DIFFTABLE)
 
-        proc sql;
-        create table &outds(
-          load_ref char(36) label='unique load reference',
-          processed_dttm num format=E8601DT26.6 label='Processed at timestamp',
-          libref char(8) label='Library Reference (8 chars)',
-          dsn char(32) label='Dataset Name (32 chars)',
-          key_hash char(32) label=
-            'MD5 Hash of primary key values (pipe seperated)',
-          move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified',
-          is_pk num label='Is Primary Key Field? (1/0)',
-          is_diff num label=
-            'Did value change? (1/0/-1).  Always -1 for appends and deletes.',
-          tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
-          tgtvar_nm char(32) label='Target variable name (32 chars)',
-          oldval_num num format=best32. label='Old (numeric) value',
-          newval_num num format=best32. label='New (numeric) value',
-          oldval_char char(32765) label='Old (character) value',
-          newval_char char(32765) label='New (character) value',
-          constraint pk_mpe_audit
-            primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
-        );
-
-    @param [in] processed_dttm= (0) Provide a datetime constant in relation to
-      the actual load time.  If not provided, current timestamp is used.
-    @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
-    @param [out] loadref= (0) Provide a unique key to reference the load,
-      otherwise a UUID will be generated.
+  @param [in] processed_dttm= (0) Provide a datetime constant in relation to
+    the actual load time.  If not provided, current timestamp is used.
+  @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
+  @param [out] loadref= (0) Provide a unique key to reference the load,
+    otherwise a UUID will be generated.
 
   <h4> SAS Macros </h4>
   @li mf_getquotedstr.sas
   @li mf_getuniquename.sas
   @li mf_getvarlist.sas
 
+  <h4> Related Macros </h4>
+  @li mp_stackdiffs.sas
+  @li mp_storediffs.test.sas
+
   @version 9.2
   @author Allan Bowe
 **/
@@ -108,7 +90,7 @@
 %else %let dbg=*;
 
 /* set up unique and temporary vars */
-%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist;
+%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist vlist;
 %let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_ds1));
 %let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_ds2));
 %let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_ds3));
@@ -143,7 +125,7 @@
 data &ds1;
   set &dslist indsname=&inds_auto;
   &hashkey=put(md5(catx('|',%mf_getquotedstr(&key,quote=N))),$hex32.);
-  &inds_keep=&inds_auto;
+  &inds_keep=upcase(&inds_auto);
 proc sort;
   by &inds_keep &hashkey;
 run;
@@ -162,15 +144,26 @@ proc transpose data=&ds1
   by &inds_keep &hashkey;
   var _character_;
 run;
+
+%if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do;
+  /* this is a format catalog - cannot query cols directly */
+  %let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH"
+    ,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO"
+    ,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
+%end;
+%else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
+
 data &ds4;
-  length &inds_keep $41 tgtvar_nm $32;
+  length &inds_keep $41 tgtvar_nm $32 _label_ $256;
+  if _n_=1 then call missing(_label_);
+  drop _label_;
   set &ds2 &ds3 indsname=&inds_auto;
 
   tgtvar_nm=upcase(tgtvar_nm);
-  if tgtvar_nm in (%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE)));
+  if tgtvar_nm in (%upcase(&vlist));
 
-  if &inds_auto="&ds2" then tgtvar_type='N';
-  else if &inds_auto="&ds3" then tgtvar_type='C';
+  if upcase(&inds_auto)="&ds2" then tgtvar_type='N';
+  else if upcase(&inds_auto)="&ds3" then tgtvar_type='C';
   else do;
     putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
     call symputx('syscc',98);