fix: streaming output in mp_streamfile by default (eg when param not found)

fix: avoiding error and including test in mp_streamfile.sas

.all-contributorsrc

@@ -0,0 +1,124 @@
+{
+  "projectName": "core",
+  "projectOwner": "sasjs",
+  "repoType": "github",
+  "repoHost": "https://github.com",
+  "files": [
+    "README.md"
+  ],
+  "imageSize": 100,
+  "commit": false,
+  "commitConvention": "angular",
+  "contributors": [
+    {
+      "login": "allanbowe",
+      "name": "Allan Bowe",
+      "avatar_url": "https://avatars.githubusercontent.com/u/4420615?v=4",
+      "profile": "https://github.com/allanbowe",
+      "contributions": [
+        "business",
+        "code",
+        "content",
+        "doc",
+        "infra",
+        "maintenance",
+        "mentoring",
+        "question",
+        "review",
+        "test"
+      ]
+    },
+    {
+      "login": "rafgag",
+      "name": "rafgag",
+      "avatar_url": "https://avatars.githubusercontent.com/u/69139928?v=4",
+      "profile": "https://github.com/rafgag",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "tmoody",
+      "name": "Trevor Moody",
+      "avatar_url": "https://avatars.githubusercontent.com/u/79837106?v=4",
+      "profile": "https://github.com/tmoody",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "krishna-acondy",
+      "name": "Krishna Acondy",
+      "avatar_url": "https://avatars.githubusercontent.com/u/2980428?v=4",
+      "profile": "https://krishna-acondy.io/",
+      "contributions": [
+        "code",
+        "infra",
+        "blog",
+        "content",
+        "ideas",
+        "video"
+      ]
+    },
+    {
+      "login": "saadjutt01",
+      "name": "Muhammad Saad ",
+      "avatar_url": "https://avatars.githubusercontent.com/u/8914650?v=4",
+      "profile": "https://github.com/saadjutt01",
+      "contributions": [
+        "code",
+        "ideas"
+      ]
+    },
+    {
+      "login": "YuryShkoda",
+      "name": "Yury Shkoda",
+      "avatar_url": "https://avatars.githubusercontent.com/u/25773492?v=4",
+      "profile": "https://www.erudicat.com/",
+      "contributions": [
+        "code",
+        "infra",
+        "video"
+      ]
+    },
+    {
+      "login": "medjedovicm",
+      "name": "Mihajlo Medjedovic",
+      "avatar_url": "https://avatars.githubusercontent.com/u/18329105?v=4",
+      "profile": "https://github.com/medjedovicm",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "kkchandok",
+      "name": "kkchandok",
+      "avatar_url": "https://avatars.githubusercontent.com/u/46090627?v=4",
+      "profile": "https://github.com/kkchandok",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "VladislavParhomchik",
+      "name": "Vladislav Parhomchik",
+      "avatar_url": "https://avatars.githubusercontent.com/u/83717836?v=4",
+      "profile": "https://github.com/VladislavParhomchik",
+      "contributions": [
+        "test",
+        "review"
+      ]
+    },
+    {
+      "login": "vznesh",
+      "name": "Vignesh T.",
+      "avatar_url": "https://avatars.githubusercontent.com/u/28916792?v=4",
+      "profile": "https://github.com/vznesh",
+      "contributions": [
+        "bug"
+      ]
+    }
+  ],
+  "contributorsPerLine": 7,
+  "skipCi": true
+}

.editorconfig

@@ -1,12 +0,0 @@
-# http://editorconfig.org
-root = true
-
-[*]
-charset = utf-8
-indent_style = space
-indent_size = 2
-end_of_line = lf
-insert_final_newline = false
-trim_trailing_whitespace = true
-
-

.git-hooks/pre-commit

@@ -1,2 +1,11 @@
-#!/bin/sh
-sasjs lint
+#!/bin/bash
+sasjs lint
+
+# Avoid commits to the master branch
+BRANCH=`git rev-parse --abbrev-ref HEAD`
+
+if [[ "$BRANCH" =~ ^(master|main|develop)$ ]]; then
+  echo "You are on branch $BRANCH. Are you sure you want to commit to this branch?"
+  echo "If so, commit with -n to bypass the pre-commit hook."
+  exit 1
+fi

.github/CONTRIBUTING.md

@@ -12,7 +12,7 @@ This repository makes use of the [SASjs](https://sasjs.io) framework for code or
 * [VSCode](https://sasjs.io/windows/#vscode) - feature packed IDE for code editing (warning - highly effective!)
 * [GIT](https://sasjs.io/windows/#git) - a safety net you cannot (and should not) do without.
 
-For generating the documentation (`sasjs doc`) it is also necessary to install [doxygen](https://www.doxygen.nl/manual/install.html).
+For generating the documentation (`sasjs doc`) it is also necessary to install [doxygen](https://www.doxygen.nl/manual/install.html) and GraphViz (`sudo port install graphviz` on mac, or `sudo apt-get install graphviz` on Ubuntu).
 
 
 To get configured:

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+  - package-ecosystem: npm
+    directory: '/'
+    schedule:
+      interval: monthly
+    open-pull-requests-limit: 3
+    allow:
+      - dependency-type: "production"

.github/workflows/run-tests.yml

@@ -12,12 +12,12 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x]
+        node-version: [lts/fermium]
 
     steps:
       - uses: actions/checkout@v2
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node-version }}
 
@@ -39,7 +39,7 @@ jobs:
           sudo apt install apt-transport-https
           sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
           sudo apt-key add openvpn-repo-pkg-key.pub
-          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-bionic.list
+          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-focal.list
           sudo apt update
           sudo apt install openvpn3
 

.gitpod.dockerfile

@@ -1,6 +1,6 @@
 FROM gitpod/workspace-full
 
 RUN sudo apt-get update \
- && sudo apt-get install -y \
-    doxygen \
+ && sudo apt-get install -y doxygen \
+ && sudo apt-get install -y graphviz \
  && sudo rm -rf /var/lib/apt/lists/*

.gitpod.yml

@@ -1,8 +1,27 @@
 tasks:
-  - init: nvm install --latest-npm && npm i -g @sasjs/cli
+  - init: |
+      nvm install --lts
+      npm i -g @sasjs/cli
 
 image:
   file: .gitpod.dockerfile
 vscode:
   extensions:
     - sasjs.sasjs-for-vscode
+
+github:
+  prebuilds:
+    # enable for the master/default branch (defaults to true)
+    master: true
+    # enable for all branches in this repo (defaults to false)
+    branches: false
+    # enable for pull requests coming from this repo (defaults to true)
+    pullRequests: true
+    # enable for pull requests coming from forks (defaults to false)
+    pullRequestsFromForks: true
+    # add a "Review in Gitpod" button as a comment to pull requests (defaults to true)
+    addComment: true
+    # add a "Review in Gitpod" button to pull requests (defaults to false)
+    addBadge: false
+    # add a label once the prebuild is ready to pull requests (defaults to false)
+    addLabel: prebuilt-in-gitpod

.npmignore

@@ -9,3 +9,4 @@ sasjs/
 main.dox
 make_singlefile.sh
 *.md
+.all-contributorsrc

.sasjslint

@@ -6,7 +6,7 @@
     "hasMacroParentheses": true,
     "noNestedMacros": false,
     "noSpacesInFileNames": true,
-    "maxLineLength": 230,
+    "maxLineLength": 300,
     "lowerCaseFileNames": true,
     "noTabIndentation": true,
     "indentationMultiple": 2

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2020 (Allan Bowe)
+Copyright 2021 (Allan Bowe)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

README.md

@@ -1,8 +1,7 @@
 # Macro Core
 [![npm package][npm-image]][npm-url]
 [![Github Workflow][githubworkflow-image]][githubworkflow-url]
-[![Dependency Status][dependency-image]][dependency-url]
-[![npm](https://img.shields.io/npm/dt/@sasjs/core)]()
+![npm](https://img.shields.io/npm/dt/@sasjs/core)
 ![Snyk Vulnerabilities for npm package](https://img.shields.io/snyk/vulnerabilities/npm/@sasjs/core)
 [![License](https://img.shields.io/apm/l/atomic-design-ui.svg)](/LICENSE)
 ![GitHub top language](https://img.shields.io/github/languages/top/sasjs/core)
@@ -16,11 +15,9 @@
 [npm-url]:http://npmjs.org/package/@sasjs/core
 [githubworkflow-image]:https://github.com/sasjs/core/actions/workflows/main.yml/badge.svg
 [githubworkflow-url]:https://github.com/sasjs/core/blob/main/.github/workflows/main.yml
-[dependency-image]:https://david-dm.org/sasjs/core.svg
 [dependency-url]:https://github.com/sasjs/core/blob/main/package.json
 
 
-
 Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
 
 You can download and compile them all in just two lines of SAS code:
@@ -32,42 +29,61 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 
 Documentation: https://core.sasjs.io
 
-# Components
+## Components
 
-**base** library (SAS9/Viya)
+### BASE library (SAS9/Viya)
 
 - OS independent
 - Not metadata aware
 - No X command
 - Prefixes: _mf_, _mp_
 
-**meta** library (SAS9 only)
+#### FCMP library (SAS9/Viya)
+- Function and macro names are identical, except for special cases
+- Prefixes: _mcf_
+
+The fcmp macros are used to generate fcmp functions, and can be used with or
+without the `proc fcmp` wrapper.
+
+### META library (SAS9 only)
+
+Macros used in SAS EBI, which connect to the metadata server.
 
 - OS independent
 - Metadata aware
 - No X command
 - Prefixes: _mm_
 
-**viya** library (Viya only)
+### SERVER library (@sasjs/server only)
+These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
+
+- OS independent
+- @sasjs/server aware
+- No X command
+- Prefixes: _ms_
+
+### VIYA library (Viya only)
+
+Macros used for interfacing with SAS Viya.
 
 - OS independent
 - No X command
-- Prefixes: _mv_
+- Prefixes: _mv_, _mvf_
 
-**metax** library (SAS9 only)
+### METAX library (SAS9 only)
 
 - OS specific
 - Metadata aware
 - X command enabled
 - Prefixes: _mmw_,_mmu_,_mmx_
 
-**lua** library
+### LUA library
 
 Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
 
 To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert your LUA into a data step with put statements, and create the macro wrapper with a `ml_` prefix.  You can then use your module in any program by running:
 
-```
+```sas
 /* compile the lua module */
 %ml_yourmodule()
 
@@ -82,9 +98,9 @@ run;
 - X command enabled
 - Prefixes: _mmw_,_mmu_,_mmx_
 
-# Installation
+## Installation
 
-First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available,eg:
+First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available, eg:
 
 ```sas
 options insert=(sasautos="/your/path/macrocore/base");
@@ -100,29 +116,31 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 %inc mc;
 ```
 
-# Standards
+## Standards
 
-## File Properties
+### File Properties
 
 - filenames much match macro names
 - filenames must be lowercase, without spaces
 - macro names must be lowercase
 - one macro per file
 - prefixes:
+  - _mcf_ for macro compiled functions (proc fcmp)
   - _mf_ for macro functions (can be used in open code).
-  - _mp_ for macro procedures (which generate sas code)
+  - _ml_ for macros that are used to compile LUA modules
   - _mm_ for metadata macros (interface with the metadata server).
   - _mmx_ for macros that use metadata and are XCMD enabled
+  - _mp_ for macro procedures (which generate sas code)
+  - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
+  - _mv_ for macro procedures that will only work in Viya
   - _mx_ for macros that are XCMD enabled
-  - _ml_ for macros that are used to compile LUA modules
-  - _mv_ for macros that will only work in Viya
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
 - UTF-8
 
 
-## Header Properties
+### Header Properties
 
 The **Macro Core** documentation is created using [doxygen](http://www.doxygen.nl). A full list of attributes can be found [here](http://www.doxygen.nl/manual/commands.html) but the following are most relevant:
 
@@ -136,7 +154,7 @@ The **Macro Core** documentation is created using [doxygen](http://www.doxygen.n
 
 All macros must be commented in the doxygen format, to enable the [online documentation](https://core.sasjs.io).
 
-### Dependencies
+#### Dependencies
 SAS code can contain one of two types of dependency - SAS Macros, and SAS Includes.  When compiling projects using the [SASjs CLI](https://cli.sasjs.io) the doxygen header is scanned for `  @li` items under the following headers:
 
 ```sas
@@ -149,12 +167,12 @@ SAS code can contain one of two types of dependency - SAS Macros, and SAS Includ
   @li someprogram.sas FREFTWO
 ```
 
-The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Includes) when creating SAS Jobs and Services.
+The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Includes) when creating SAS Jobs and Services (and Tests).
 
 When contributing to this library, it is therefore important to ensure that all dependencies are listed in the header in this format.
 
 
-## Coding Standards
+### Coding Standards
 
 - Indentation = 2 spaces. No tabs!
 - no trailing white space
@@ -163,14 +181,24 @@ When contributing to this library, it is therefore important to ensure that all
 - The closing `%mend;` should **not** contain the macro name.
 - All macros should be defined with brackets, even if no variables are needed - ie `%macro x();` not `%macro x;`
 - Mandatory parameters should be positional, all optional parameters should be keyword (var=) style.
-- All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect.
+- All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect, or the [USER](https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/lrcon/n18m1vkqmeo4esn1moikt23zhp8s.htm) library is active.
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
+- Where global macro variables are absolutely necessary, they should make use of `&sasjs_prefix` - see mp_init.sas
 - The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
+- Use [sasjs lint](https://github.com/sasjs/lint)!
 
-# General Notes
+## General Notes
 
 - All macros should be compatible with SAS versions from support level B and above (so currently 9.2 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully (eg `%if %sysevalf(&sysver<9.3) %then %return`).
 
+## Breaking Changes
+
+We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major (breaking) release becomes necessary:
+
+* mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
+* `insert_cmplib` option of mcf_xxx macros will be deprecated (the option is now checked automatically with value inserted only if needed)
+* mcf_xxx macros to have `wrap=` option defaulted to YES for convenience
+
 ## Star Gazing
 
 If you find this library useful, please leave a [star](https://github.com/sasjs/core/stargazers) and help us grow our star graph!
@@ -179,3 +207,35 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 
 
+## Contributors ✨
+<!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
+[![All Contributors](https://img.shields.io/badge/all_contributors-10-orange.svg?style=flat-square)](#contributors-)
+<!-- ALL-CONTRIBUTORS-BADGE:END -->
+Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
+
+<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-disable -->
+<table>
+  <tr>
+    <td align="center"><a href="https://github.com/allanbowe"><img src="https://avatars.githubusercontent.com/u/4420615?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Allan Bowe</b></sub></a><br /><a href="#business-allanbowe" title="Business development">💼</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Code">💻</a> <a href="#content-allanbowe" title="Content">🖋</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Documentation">📖</a> <a href="#infra-allanbowe" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#maintenance-allanbowe" title="Maintenance">🚧</a> <a href="#mentoring-allanbowe" title="Mentoring">🧑‍🏫</a> <a href="#question-allanbowe" title="Answering Questions">💬</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3Aallanbowe" title="Reviewed Pull Requests">👀</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Tests">⚠️</a></td>
+    <td align="center"><a href="https://github.com/rafgag"><img src="https://avatars.githubusercontent.com/u/69139928?v=4?s=100" width="100px;" alt=""/><br /><sub><b>rafgag</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rafgag" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/tmoody"><img src="https://avatars.githubusercontent.com/u/79837106?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Trevor Moody</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=tmoody" title="Code">💻</a></td>
+    <td align="center"><a href="https://krishna-acondy.io/"><img src="https://avatars.githubusercontent.com/u/2980428?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Krishna Acondy</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=krishna-acondy" title="Code">💻</a> <a href="#infra-krishna-acondy" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#blog-krishna-acondy" title="Blogposts">📝</a> <a href="#content-krishna-acondy" title="Content">🖋</a> <a href="#ideas-krishna-acondy" title="Ideas, Planning, & Feedback">🤔</a> <a href="#video-krishna-acondy" title="Videos">📹</a></td>
+    <td align="center"><a href="https://github.com/saadjutt01"><img src="https://avatars.githubusercontent.com/u/8914650?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Muhammad Saad </b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=saadjutt01" title="Code">💻</a> <a href="#ideas-saadjutt01" title="Ideas, Planning, & Feedback">🤔</a></td>
+    <td align="center"><a href="https://www.erudicat.com/"><img src="https://avatars.githubusercontent.com/u/25773492?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Yury Shkoda</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=YuryShkoda" title="Code">💻</a> <a href="#infra-YuryShkoda" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#video-YuryShkoda" title="Videos">📹</a></td>
+    <td align="center"><a href="https://github.com/medjedovicm"><img src="https://avatars.githubusercontent.com/u/18329105?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Mihajlo Medjedovic</b></sub></a><br /><a href="#infra-medjedovicm" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+  </tr>
+  <tr>
+    <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
+    <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
+    <td align="center"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
+  </tr>
+</table>
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+
+<!-- ALL-CONTRIBUTORS-LIST:END -->
+
+This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!

base/mf_abort.sas

@@ -11,7 +11,7 @@
   @cond
 **/
 
-%macro mf_abort(mac=mf_abort.sas, type=deprecated, msg=, iftrue=%str(1=1)
+%macro mf_abort(mac=mf_abort.sas, msg=, iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
   %if not(%eval(%unquote(&iftrue))) %then %return;

base/mf_dedup.sas

@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief de-duplicates a macro string
+  @details Removes all duplicates from a string of words.  A delimeter can be
+  chosen.  Is inspired heavily by this excellent [macro](
+  https://github.com/scottbass/SAS/blob/master/Macro/dedup_mstring.sas) from
+  [Scott Base](https://www.linkedin.com/in/scottbass).  Case sensitive.
+
+  Usage:
+
+      %let str=One two one two and through and through;
+      %put %mf_dedup(&str);
+      %put %mf_dedup(&str,outdlm=%str(,));
+
+  Which returns:
+
+      > One two one and through
+      > One,two,one,and,through
+
+  @param [in] str String to be deduplicated
+  @param [in] indlm= ( ) Delimeter of the input string
+  @param [out] outdlm= ( ) Delimiter of the output string
+
+  <h4> Related Macros </h4>
+  @li mf_trimstr.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_dedup(str
+  ,indlm=%str( )
+  ,outdlm=%str( )
+)/*/STORE SOURCE*/;
+
+%local num word i pos out;
+
+%* loop over each token, searching the target for that token ;
+%let num=%sysfunc(countc(%superq(str),%str(&indlm)));
+%do i=1 %to %eval(&num+1);
+  %let word=%scan(%superq(str),&i,%str(&indlm));
+  %let pos=%sysfunc(indexw(&out,&word,%str(&outdlm)));
+  %if (&pos eq 0) %then %do;
+    %if (&i gt 1) %then %let out=&out%str(&outdlm);
+    %let out=&out&word;
+  %end;
+%end;
+
+%unquote(&out)
+
+%mend mf_dedup;
+
+

base/mf_existds.sas

@@ -12,6 +12,10 @@
 
   @param libds library.dataset
   @return output returns 1 or 0
+
+  <h4> Related Macros </h4>
+  @li mf_existds.test.sas
+
   @warning Untested on tables registered in metadata but not physically present
   @version 9.2
   @author Allan Bowe

base/mf_existfeature.sas

@@ -5,7 +5,7 @@
     Run without arguments to see a list of detectable features.
     Note - this list is based on known versions of SAS rather than
     actual feature detection, as that is tricky / impossible to do
-    without generating errors in most cases.
+    without generating errs in most cases.
 
         %put %mf_existfeature(PROCLUA);
 

base/mf_existfileref.sas

@@ -17,7 +17,13 @@
 %macro mf_existfileref(fref
 )/*/STORE SOURCE*/;
 
-  %if %sysfunc(fileref(&fref))=0 %then %do;
+  %local rc;
+  %let rc=%sysfunc(fileref(&fref));
+  %if &rc=0 %then %do;
+    1
+  %end;
+  %else %if &rc<0 %then %do;
+    %put &sysmacroname: Fileref &fref exists but the underlying file does not;
     1
   %end;
   %else %do;

base/mf_existfunction.sas

@@ -0,0 +1,37 @@
+/**
+  @file
+  @brief Checks if a function exists
+  @details Returns 1 if the function exists, else 0.  Note that this function
+  can be slow as it needs to open the sashelp.vfuncs table.
+
+  Usage:
+
+      %put %mf_existfunction(CAT);
+      %put %mf_existfunction(DOG);
+
+  Full credit to [Bart](https://sasensei.com/user/305) for the vfunc pointer
+  and the tidy approach for pure macro data set filtering.
+  Check out his [SAS Packages](https://github.com/yabwon/SAS_PACKAGES)
+  framework!  Where you can find the same [function](
+https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionexists-macro
+  ).
+
+  @param [in] name (positional) - function name
+
+  @author Allan Bowe
+**/
+/** @cond */
+%macro mf_existfunction(name
+)/*/STORE SOURCE*/;
+
+  %local dsid rc exist;
+  %let dsid=%sysfunc(open(sashelp.vfunc(where=(fncname="%upcase(&name)"))));
+  %let exist=1;
+  %let exist=%sysfunc(fetch(&dsid, NOSET));
+  %let rc=%sysfunc(close(&dsid));
+
+  %sysevalf(0 = &exist)
+
+%mend mf_existfunction;
+
+/** @endcond */

base/mf_existvar.sas

@@ -1,14 +1,18 @@
 /**
   @file
   @brief Checks if a variable exists in a data set.
-  @details Returns 0 if the variable does NOT exist, and return the position of
-    the var if it does.
-    Usage:
+  @details Returns 0 if the variable does NOT exist, and the position of the var
+  if it does.
+  Usage:
 
-        %put %mf_existvar(work.someds, somevar)
+      %put %mf_existvar(work.someds, somevar)
+
+  @param [in] libds 2 part dataset or view reference
+  @param [in] var variable name
+
+  <h4> Related Macros </h4>
+  @li mf_existvar.test.sas
 
-  @param libds (positional) - 2 part dataset or view reference
-  @param var (positional) - variable name
   @version 9.2
   @author Allan Bowe
 **/

base/mf_getapploc.sas

@@ -0,0 +1,73 @@
+/**
+  @file
+  @brief Returns the appLoc from the _program variable
+  @details When working with SASjs apps, web services / tests / jobs are always
+  deployed to a root (app) location in the SAS logical folder tree.
+
+  When building apps for use in other environments, you do not necessarily know
+  where the backend services will be deployed.  Therefore a function like this
+  is handy in order to dynamically figure out the appLoc, and enable other
+  services to be connected by a relative reference.
+
+  SASjs apps always have the same immediate substructure (one or more of the
+  following):
+
+  @li /data
+  @li /jobs
+  @li /services
+  @li /tests/jobs
+  @li /tests/services
+  @li /tests/macros
+
+  This function works by testing for the existence of any of the above in the
+  automatic _program variable, and returning the part to the left of it.
+
+  Usage:
+
+      %put %mf_getapploc(&_program)
+
+      %put %mf_getapploc(/some/location/services/admin/myservice);
+      %put %mf_getapploc(/some/location/jobs/extract/somejob/);
+      %put %mf_getapploc(/some/location/tests/jobs/somejob/);
+
+
+  @author Allan Bowe
+**/
+
+%macro mf_getapploc(pgm);
+%if "&pgm"="" %then %do;
+  %if %symexist(_program) %then %let pgm=&_program;
+  %else %do;
+    %put &sysmacroname: No value provided and no _program variable available;
+    %return;
+  %end;
+%end;
+%local root;
+
+/**
+  * First check we are not in the tests/macros folder (which has no subfolders)
+  */
+%if %index(&pgm,/tests/macros/) %then %do;
+  %let root=%substr(&pgm,1,%index(&pgm,/tests/macros)-1);
+  &root
+  %return;
+%end;
+
+/**
+  * Next, move up two levels to avoid matches on subfolder or service name
+  */
+%let root=%substr(&pgm,1,%length(&pgm)-%length(%scan(&pgm,-1,/))-1);
+%let root=%substr(&root,1,%length(&root)-%length(%scan(&root,-1,/))-1);
+
+%if %index(&root,/tests/) %then %do;
+  %let root=%substr(&root,1,%index(&root,/tests/)-1);
+%end;
+%else %if %index(&root,/services) %then %do;
+  %let root=%substr(&root,1,%index(&root,/services)-1);
+%end;
+%else %if %index(&root,/jobs) %then %do;
+  %let root=%substr(&root,1,%index(&root,/jobs)-1);
+%end;
+%else %put &sysmacroname: Could not find an app location from &pgm;
+  &root
+%mend mf_getapploc ;

base/mf_getattrc.sas

@@ -10,7 +10,7 @@
   @param attr full list in [documentation](
     https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000147794.htm)
   @return output returns result of the attrc value supplied, or -1 and log
-    message if error.
+    message if err.
 
   @version 9.2
   @author Allan Bowe

base/mf_getattrn.sas

@@ -10,7 +10,7 @@
   @param attr Common values are NLOBS and NVARS, full list in [documentation](
   http://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000212040.htm)
   @return output returns result of the attrn value supplied, or -1 and log
-    message if error.
+    message if err.
 
   @version 9.2
   @author Allan Bowe

base/mf_getengine.sas

@@ -12,9 +12,10 @@
   contributors of Chris Hemedingers blog [post](
   http://blogs.sas.com/content/sasdummy/2013/06/04/find-a-sas-library-engine/)
 
-  @param libref Library reference (also accepts a 2 level libds ref).
+  @param [in] libref Library reference (also accepts a 2 level libds ref).
 
-  @return output returns the library engine for the FIRST library encountered.
+  @return output returns the library engine (uppercase) for the FIRST library
+    encountered.
 
   @warning will only return the FIRST library engine - for concatenated
     libraries, with different engines, inconsistent results may be encountered.
@@ -46,7 +47,7 @@
     %let rc= %sysfunc(close(&dsid));
   %end;
 
-  &engine
+  %upcase(&engine)
 
 %mend mf_getengine;
 

base/mf_getfilesize.sas

@@ -5,18 +5,19 @@
 
       %put %mf_getfilesize(fpath=C:\temp\myfile.txt);
 
-      or
+  or, provide a libds value as follows:
 
       data x;do x=1 to 100000;y=x;output;end;run;
       %put %mf_getfilesize(libds=work.x,format=yes);
 
-      gives:
+  Which gives:
 
-      2mb
+  > 2mb
+
+  @param [in] fpath= Full path and filename.  Provide this OR the libds value.
+  @param [in] libds= (0) Library.dataset value (assumes library is BASE engine)
+  @param [in] format= (NO) Set to yes to apply sizekmg. format
 
-  @param fpath= full path and filename.  Provide this OR the libds value.
-  @param libds= library.dataset value (assumes library is BASE engine)
-  @param format=  set to yes to apply sizekmg. format
   @returns bytes
 
   @version 9.2
@@ -26,16 +27,32 @@
 %macro mf_getfilesize(fpath=,libds=0,format=NO
 )/*/STORE SOURCE*/;
 
-  %if &libds ne 0 %then %do;
-    %let fpath=%sysfunc(pathname(%scan(&libds,1,.)))/%scan(&libds,2,.).sas7bdat;
-  %end;
+  %local rc fid fref bytes dsid lib vnum;
 
-  %local rc fid fref bytes;
-  %let rc=%sysfunc(filename(fref,&fpath));
-  %let fid=%sysfunc(fopen(&fref));
-  %let bytes=%sysfunc(finfo(&fid,File Size (bytes)));
-  %let rc=%sysfunc(fclose(&fid));
-  %let rc=%sysfunc(filename(fref));
+  %if &libds ne 0 %then %do;
+    %let libds=%upcase(&libds);
+    %if %index(&libds,.)=0 %then %let lib=WORK;
+    %else %let lib=%scan(&libds,1,.);
+    %let dsid=%sysfunc(open(
+      sashelp.vtable(where=(libname="&lib" and memname="%scan(&libds,-1,.)")
+        keep=libname memname filesize
+      )
+    ));
+    %if (&dsid ^= 0) %then %do;
+      %let vnum=%sysfunc(varnum(&dsid,FILESIZE));
+      %let rc=%sysfunc(fetch(&dsid));
+      %let bytes=%sysfunc(getvarn(&dsid,&vnum));
+      %let rc= %sysfunc(close(&dsid));
+    %end;
+    %else %put &sysmacroname: &libds could not be opened! %sysfunc(sysmsg());
+  %end;
+  %else %do;
+    %let rc=%sysfunc(filename(fref,&fpath));
+    %let fid=%sysfunc(fopen(&fref));
+    %let bytes=%sysfunc(finfo(&fid,File Size (bytes)));
+    %let rc=%sysfunc(fclose(&fid));
+    %let rc=%sysfunc(filename(fref));
+  %end;
 
   %if &format=NO %then %do;
     &bytes

base/mf_getfmtlist.sas

@@ -0,0 +1,61 @@
+/**
+  @file
+  @brief Returns a distinct list of formats from a table
+  @details Reads the dataset header and returns a distinct list of formats
+  applied.
+
+        %put NOTE- %mf_getfmtlist(sashelp.prdsale);
+        %put NOTE- %mf_getfmtlist(sashelp.shoes);
+        %put NOTE- %mf_getfmtlist(sashelp.demographics);
+
+  returns:
+
+  > DOLLAR $CHAR W MONNAME
+  > $CHAR BEST DOLLAR
+  > BEST Z $CHAR COMMA PERCENTN
+
+
+  @param [in] libds Two part library.dataset reference.
+
+  <h4> SAS Macros </h4>
+  @li mf_getfmtname.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_getfmtlist(libds
+)/*/STORE SOURCE*/;
+/* declare local vars */
+%local out dsid nvars x rc fmt;
+
+/* open dataset in macro */
+%let dsid=%sysfunc(open(&libds));
+
+/* continue if dataset exists */
+%if &dsid %then %do;
+  /* loop each variable in the dataset */
+  %let nvars=%sysfunc(attrn(&dsid,NVARS));
+  %do x=1 %to &nvars;
+    /* grab format and check it exists */
+    %let fmt=%sysfunc(varfmt(&dsid,&x));
+    %if %quote(&fmt) ne %quote() %then %let fmt=%mf_getfmtname(&fmt);
+    %else %do;
+      /* assign default format depending on variable type */
+      %if %sysfunc(vartype(&dsid, &x))=C %then %let fmt=$CHAR;
+      %else %let fmt=BEST;
+    %end;
+    /* concatenate unique list of formats */
+    %if %sysfunc(indexw(&out,&fmt,%str( )))=0 %then %let out=&out &fmt;
+  %end;
+  %let rc=%sysfunc(close(&dsid));
+%end;
+%else %do;
+  %put &sysmacroname: Unable to open &libds (rc=&dsid);
+  %put &sysmacroname: SYSMSG= %sysfunc(sysmsg());
+  %let rc=%sysfunc(close(&dsid));
+%end;
+/* send them out without spaces or quote markers */
+%do;%unquote(&out)%end;
+%mend mf_getfmtlist;

base/mf_getfmtname.sas

@@ -0,0 +1,44 @@
+/**
+  @file
+  @brief Extracts a format name from a fully defined format
+  @details Converts formats in like $thi3. and th13.2 $THI and TH.
+  Usage:
+
+      %put %mf_getfmtname(8.);
+      %put %mf_getfmtname($4.);
+      %put %mf_getfmtname(comma14.10);
+
+  Returns:
+
+  > W
+  > $CHAR
+  > COMMA
+
+  Note that system defaults are inferred from the values provided.
+
+  @param [in] fmt The fully defined format. If left blank, nothing is returned.
+
+  @returns The name (without width or decimal) of the format.
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_getfmtname(fmt
+)/*/STORE SOURCE*/ /minoperator mindelimiter=' ';
+
+%local out dsid nvars x rc fmt;
+
+/* extract actual format name from the format definition */
+%let fmt=%scan(&fmt,1,.);
+%do %while(%substr(&fmt,%length(&fmt),1) in 1 2 3 4 5 6 7 8 9 0);
+  %if %length(&fmt)=1 %then %let fmt=W;
+  %else %let fmt=%substr(&fmt,1,%length(&fmt)-1);
+%end;
+
+%if &fmt=$ %then %let fmt=$CHAR;
+
+/* send them out without spaces or quote markers */
+%do;%unquote(%upcase(&fmt))%end;
+%mend mf_getfmtname;

base/mf_getquotedstr.sas

@@ -15,12 +15,12 @@
   for:
 > "these","words","are","double","quoted"
 
-  @param in_str the unquoted, spaced delimited string to transform
-  @param dlm= the delimeter to be applied to the output (default comma)
-  @param indlm= the delimeter used for the input (default is space)
-  @param quote= the quote mark to apply (S=Single, D=Double). If any other value
-    than uppercase S or D is supplied, then that value will be used as the
-    quoting character.
+  @param [in] in_str The unquoted, spaced delimited string to transform
+  @param [in] dlm= (,) The delimeter to be applied to the output (default comma)
+  @param [in] indlm= ( ) The delimeter used for the input (default is space)
+  @param [in] quote= (S) The quote mark to apply (S=Single, D=Double, N=None).
+    If any other value than uppercase S or D is supplied, then that value will
+    be used as the quoting character.
   @return output returns a string with the newly quoted / delimited output.
 
   @version 9.2
@@ -28,11 +28,15 @@
 **/
 
 
-%macro mf_getquotedstr(IN_STR,DLM=%str(,),QUOTE=S,indlm=%str( )
+%macro mf_getquotedstr(IN_STR
+  ,DLM=%str(,)
+  ,QUOTE=S
+  ,indlm=%str( )
 )/*/STORE SOURCE*/;
-  %if &quote=S %then %let quote=%str(%');
-  %else %if &quote=D %then %let quote=%str(%");
-  %else %let quote=%str();
+  /* credit Rowland Hale  - byte34 is double quote, 39 is single quote */
+  %if &quote=S %then %let quote=%qsysfunc(byte(39));
+  %else %if &quote=D %then %let quote=%qsysfunc(byte(34));
+  %else %if &quote=N %then %let quote=;
   %local i item buffer;
   %let i=1;
   %do %while (%qscan(&IN_STR,&i,%str(&indlm)) ne %str() ) ;

base/mf_getuniquefileref.sas

@@ -1,37 +1,58 @@
 /**
   @file
   @brief Assigns and returns an unused fileref
-  @details
+  @details  Using the native approach for assigning filerefs fails as some
+  procedures (such as proc http) do not recognise the temporary names (starting
+  with a hash), returning a message such as:
+
+  > ERROR 22-322: Expecting a name.
+
+  This macro works by attempting a random fileref (with a prefix), seeing if it
+  is already assigned, and if not - returning the fileref.
+
+  If your process can accept filerefs with the hash (#) prefix, then set
+  `prefix=0` to revert to the native approach - which is significantly faster
+  when there are a lot of filerefs in a session.
+
   Use as follows:
 
       %let fileref1=%mf_getuniquefileref();
-      %let fileref2=%mf_getuniquefileref();
+      %let fileref2=%mf_getuniquefileref(prefix=0);
       %put &fileref1 &fileref2;
 
-  which returns:
+  which returns filerefs similar to:
 
-> mcref0 mcref1
+> _7432233 #LN00070
 
-  @param prefix= first part of fileref. Remember that filerefs can only be 8
-    characters, so a 7 letter prefix would mean that `maxtries` should be 10.
-  @param maxtries= the last part of the libref.  Provide an integer value.
+  @param [in] prefix= (_) first part of fileref. Remember that filerefs can only
+    be 8 characters, so a 7 letter prefix would mean `maxtries` should be 10.
+    if using zero (0) as the prefix, a native assignment is used.
+  @param [in] maxtries= (1000) the last part of the libref. Must be an integer.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-%macro mf_getuniquefileref(prefix=mcref,maxtries=1000);
-  %local x fname;
-  %let x=0;
-  %do x=0 %to &maxtries;
-  %if %sysfunc(fileref(&prefix&x)) > 0 %then %do;
-    %let fname=&prefix&x;
+%macro mf_getuniquefileref(prefix=_,maxtries=1000);
+  %local rc fname;
+  %if &prefix=0 %then %do;
     %let rc=%sysfunc(filename(fname,,temp));
     %if &rc %then %put %sysfunc(sysmsg());
-    &prefix&x
-    %*put &sysmacroname: Fileref &prefix&x was assigned and returned;
-    %return;
+    &fname
   %end;
+  %else %do;
+    %local x len;
+    %let len=%eval(8-%length(&prefix));
+    %let x=0;
+    %do x=0 %to &maxtries;
+      %let fname=&prefix%substr(%sysfunc(ranuni(0)),3,&len);
+      %if %sysfunc(fileref(&fname)) > 0 %then %do;
+        %let rc=%sysfunc(filename(fname,,temp));
+        %if &rc %then %put %sysfunc(sysmsg());
+        &fname
+        %return;
+      %end;
+    %end;
+    %put unable to find available fileref after &maxtries attempts;
   %end;
-  %put unable to find available fileref in range &prefix.0-&maxtries;
 %mend mf_getuniquefileref;

base/mf_getvalue.sas

@@ -10,6 +10,9 @@
   <h4> SAS Macros </h4>
   @li mf_getattrn.sas
 
+  <h4> Related Macros </h4>
+  @li mp_setkeyvalue.sas
+
   @param libds dataset to query
   @param variable the variable which contains the value to return.
   @param filter contents of where clause

base/mf_getvarformat.sas

@@ -51,7 +51,8 @@
     %end;
   %end;
   %else %do;
-    %put dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
     %return;
   %end;
 

base/mf_getvarlen.sas

@@ -43,7 +43,11 @@
       %let vlen = %str( );
     %end;
   %end;
-  %else %put dataset &libds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));

base/mf_getvarlist.sas

@@ -70,5 +70,5 @@
     %put &sysmacroname: SYSMSG= %sysfunc(sysmsg());
     %let rc=%sysfunc(close(&dsid));
   %end;
-  &outvar
+  %do;%unquote(&outvar)%end;
 %mend mf_getvarlist;

base/mf_getvarnum.sas

@@ -43,7 +43,11 @@ returns:
       %let vnum = %str( );
     %end;
   %end;
-  %else %put dataset &ds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));

base/mf_getvartype.sas

@@ -39,7 +39,11 @@ Usage:
       %let vtype = %str( );
     %end;
   %end;
-  %else %put dataset &libds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));

base/mf_isint.sas

@@ -0,0 +1,33 @@
+/**
+  @file
+  @brief Returns 1 if the variable contains only digits 0-9, else 0
+  @details Note that numerics containing any punctuation (including decimals
+    or exponents) will be flagged zero.
+
+  If you'd like support for this, then do raise an issue (or even better, a
+  pull request!)
+
+  Usage:
+
+      %put %mf_isint(1) returns 1;
+      %put %mf_isint(1.1) returns 0;
+      %put %mf_isint(%str(1,1)) returns 0;
+
+  @param [in] arg input value to check
+
+  @version 9.2
+**/
+
+%macro mf_isint(arg
+)/*/STORE SOURCE*/;
+  /* remove minus sign if exists */
+
+  %local val;
+  %if "%substr(%str(&arg),1,1)"="-" %then %let val=%substr(%str(&arg),2);
+  %else %let val=&arg;
+
+  /* check remaining chars */
+  %if %sysfunc(findc(%str(&val),,kd)) %then %do;0%end;
+  %else %do;1%end;
+
+%mend mf_isint;

base/mf_islibds.sas

@@ -0,0 +1,40 @@
+/**
+  @file
+  @brief Checks whether a string follows correct library.dataset format
+  @details Many macros in the core library accept a library.dataset parameter
+  referred to as 'libds'.  This macro validates the structure of that parameter,
+  eg:
+
+    @li 8 character libref?
+    @li 32 character dataset?
+    @li contains a period?
+
+  It does NOT check whether the dataset exists, or if the library is assigned.
+
+  Usage:
+
+      %put %mf_islibds(work.something)=1;
+      %put %mf_islibds(nolib)=0;
+      %put %mf_islibds(badlibref.ds)=0;
+      %put %mf_islibds(w.t.f)=0;
+
+  @param [in] libds The string to be checked
+
+  @return output Returns 1 if libds is valid, 0 if it is not
+
+  <h4> Related Macros </h4>
+  @li mf_islibds.test.sas
+  @li mp_validatecol.sas
+
+  @version 9.2
+**/
+
+%macro mf_islibds(libds
+)/*/STORE SOURCE*/;
+
+%local regex;
+%let regex=%sysfunc(prxparse(%str(/^[_a-z]\w{0,7}\.[_a-z]\w{0,31}$/i)));
+
+%sysfunc(prxmatch(&regex,&libds))
+
+%mend mf_islibds;

base/mf_mkdir.sas

@@ -51,7 +51,7 @@ Usage:
     %end;
 
     /*
-      Now create the directory.  Complain loudly of any errors.
+      Now create the directory.  Complain loudly of any errs.
     */
 
     %let dname = %sysfunc(dcreate(&child, &parent));

base/mf_nobs.sas

@@ -12,7 +12,7 @@
   @param libds library.dataset
 
   @return output returns result of the attrn value supplied, or log message
-    if error.
+    if err.
 
 
   @version 9.2

base/mf_wordsinstr1andstr2.sas

@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Returns words that are in both string 1 and string 2
+  @details  Compares two space separated strings and returns the words that are
+  in both.
+  Usage:
+
+      %put %mf_wordsInStr1andStr2(
+        Str1=blah sss blaaah brah bram boo
+        ,Str2=   blah blaaah brah ssss
+      );
+
+  returns:
+  > blah blaaah brah
+
+  @param str1= string containing words to extract
+  @param str2= used to compare with the extract string
+
+  @warning CASE SENSITIVE!
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_wordsInStr1andStr2(
+  Str1= /* string containing words to extract */
+  ,Str2= /* used to compare with the extract string */
+)/*/STORE SOURCE*/;
+
+%local count_base count_extr i i2 extr_word base_word match outvar;
+%if %length(&str1)=0 or %length(&str2)=0 %then %do;
+  %put base string (str1)= &str1;
+  %put compare string (str2) = &str2;
+  %return;
+%end;
+%let count_base=%sysfunc(countw(&Str2));
+%let count_extr=%sysfunc(countw(&Str1));
+
+%do i=1 %to &count_extr;
+  %let extr_word=%scan(&Str1,&i,%str( ));
+  %let match=0;
+  %do i2=1 %to &count_base;
+    %let base_word=%scan(&Str2,&i2,%str( ));
+    %if &extr_word=&base_word %then %let match=1;
+  %end;
+  %if &match=1 %then %let outvar=&outvar &extr_word;
+%end;
+
+  &outvar
+
+%mend mf_wordsInStr1andStr2;
+

base/mf_wordsinstr1butnotstr2.sas

@@ -3,6 +3,9 @@
   @brief Returns words that are in string 1 but not in string 2
   @details  Compares two space separated strings and returns the words that are
   in the first but not in the second.
+
+  Note - case sensitive!
+
   Usage:
 
       %let x= %mf_wordsInStr1ButNotStr2(
@@ -13,10 +16,8 @@
   returns:
   > sss bram boo
 
-  @param str1= string containing words to extract
-  @param str2= used to compare with the extract string
-
-  @warning CASE SENSITIVE!
+  @param [in] str1= string containing words to extract
+  @param [in] str2= used to compare with the extract string
 
   @version 9.2
   @author Allan Bowe
@@ -30,7 +31,6 @@
 
 %local count_base count_extr i i2 extr_word base_word match outvar;
 %if %length(&str1)=0 or %length(&str2)=0 %then %do;
-  %put %str(WARN)ING: empty string provided!;
   %put base string (str1)= &str1;
   %put compare string (str2) = &str2;
   %return;

base/mf_writefile.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Creates a text file using pure macro
+  @details Creates a text file of up to 10 lines.  If further lines are
+    desired, feel free to [create an issue](
+    https://github.com/sasjs/core/issues/new), or make a pull request!
+
+    The use of PARMBUFF was considered for this macro, but it would have made
+    things problematic for writing lines containing commas.
+
+    Usage:
+
+        %mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=more content)
+        data _null_;
+          infile "&sasjswork/myfile.txt";
+          input;
+          list;
+        run;
+
+  @param [in] fpath Full path to file to be created or appended to
+  @param [in] mode= (O) Available options are A or O as follows:
+    @li A APPEND mode, writes new records after the current end of the file.
+    @li O OUTPUT mode, writes new records from the beginning of the file.
+  @param [in] l1= First line
+  @param [in] l2= Second line (etc through to l10)
+
+  <h4> Related Macros </h4>
+  @li mf_writefile.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mf_writefile(fpath,mode=O,l1=,l2=,l3=,l4=,l5=,l6=,l7=,l8=,l9=,l10=
+)/*/STORE SOURCE*/;
+%local fref rc fid i total_lines;
+
+/* find number of lines by reference to first non-blank param */
+%do i=10 %to 1 %by -1;
+  %if %str(&&l&i) ne %str() %then %goto continue;
+%end;
+%continue:
+%let total_lines=&i;
+
+%if %sysfunc(filename(fref,&fpath)) ne 0 %then %do;
+  %put &=fref &=fpath;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let fid=%sysfunc(fopen(&fref,&mode));
+
+%if &fid=0 %then %do;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%do i=1 %to &total_lines;
+  %let rc=%sysfunc(fput(&fid, &&l&i));
+  %let rc=%sysfunc(fwrite(&fid));
+%end;
+%let rc=%sysfunc(fclose(&fid));
+%let rc=%sysfunc(filename(&fref));
+
+%mend mf_writefile;
+/** @endcond */

base/mp_abort.sas

@@ -15,24 +15,47 @@
     recognise this and fetch the log of the parent session instead)
   @li STP environments must finish cleanly to avoid the log being sent to
     _webout.  To assist with this, we also run stpsrvset('program error', 0)
-    and set SYSCC=0.  For 9.4m3 we take a unique approach - we open a macro
-    but don't close it!  This provides a graceful abort, EXCEPT when called
-    called within a %include within a macro (and that macro contains additional
-    logic).  See mp_abort.test.nofix.sas for the example case.
-    If you know of another way to gracefully abort a 9.4m3 STP session, we'd
-    love to hear about it!
+    and set SYSCC=0.  We take a unique "soft abort" approach - we open a macro
+    but don't close it!  This works everywhere EXCEPT inside a \%include inside
+    a macro.  For that, we recommend you use mp_include.sas to perform the
+    include, and then call \%mp_abort(mode=INCLUDE) from the source program (ie,
+    OUTSIDE of the top-parent macro).
 
 
   @param mac= to contain the name of the calling macro
   @param msg= message to be returned
   @param iftrue= supply a condition under which the macro should be executed.
+  @param errds= (work.mp_abort_errds) There is no clean way to end a process
+    within a %include called within a macro.  Furthermore, there is no way to
+    test if a macro is called within a %include.  To handle this particular
+    scenario, the %include should be switched for the mp_include.sas macro.
+    This provides an indicator that we are running a macro within a \%include
+    (`_SYSINCLUDEFILEDEVICE`) and allows us to provide a dataset with the abort
+    values (msg, mac).
+    We can then run an abort cancel FILE to stop the include running, and pass
+    the dataset back to the calling program to run a regular \%mp_abort().
+    The dataset will contain the following fields:
+    @li iftrue (1=1)
+    @li msg (the message)
+    @li mac (the mac param)
 
-  @version 9.4M3
+  @param mode= (REGULAR) If mode=INCLUDE then the &errds dataset is checked for
+    an abort status.
+    Valid values:
+    @li REGULAR (default)
+    @li INCLUDE
+
+  <h4> Related Macros </h4>
+  @li mp_include.sas
+
+  @version 9.4
   @author Allan Bowe
   @cond
 **/
 
 %macro mp_abort(mac=mp_abort.sas, type=, msg=, iftrue=%str(1=1)
+  , errds=work.mp_abort_errds
+  , mode=REGULAR
 )/*/STORE SOURCE*/;
 
   %global sysprocessmode sysprocessname;
@@ -43,17 +66,55 @@
   %if %length(&mac)>0 %then %put NOTE- called by &mac;
   %put NOTE - &msg;
 
+  %if %symexist(_SYSINCLUDEFILEDEVICE)
+  /* abort cancel FILE does not restart outside the INCLUDE on Viya 3.5 */
+  and "&SYSPROCESSNAME " ne "Compute Server "
+  %then %do;
+    %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do;
+      data &errds;
+        iftrue='1=1';
+        length mac $100 msg $5000;
+        mac=symget('mac');
+        msg=symget('msg');
+      run;
+      data _null_;
+        abort cancel FILE;
+      run;
+      %return;
+    %end;
+  %end;
+
   /* Stored Process Server web app context */
-  %if %symexist(_metaperson) or "&SYSPROCESSNAME "="Compute Server " %then %do;
+  %if %symexist(_metaperson)
+    or "&SYSPROCESSNAME "="Compute Server "
+    or &mode=INCLUDE
+  %then %do;
     options obs=max replace nosyntaxcheck mprint;
+    %if &mode=INCLUDE %then %do;
+      %if %sysfunc(exist(&errds))=1 %then %do;
+        data _null_;
+          set &errds;
+          call symputx('iftrue',iftrue,'l');
+          call symputx('mac',mac,'l');
+          call symputx('msg',msg,'l');
+          putlog (_all_)(=);
+        run;
+        %if (&iftrue)=0 %then %return;
+      %end;
+      %else %do;
+        %put &sysmacroname: No include errors found;
+        %return;
+      %end;
+    %end;
+
     /* extract log errs / warns, if exist */
     %local logloc logline;
     %global logmsg; /* capture global messages */
     %if %symexist(SYSPRINTTOLOG) %then %let logloc=&SYSPRINTTOLOG;
     %else %let logloc=%qsysfunc(getoption(LOG));
     proc printto log=log;run;
+    %let logline=0;
     %if %length(&logloc)>0 %then %do;
-      %let logline=0;
       data _null_;
         infile &logloc lrecl=5000;
         input; putlog _infile_;
@@ -100,9 +161,12 @@
     /* send response in SASjs JSON format */
     data _null_;
       file _webout mod lrecl=32000 encoding='utf-8';
-      length msg $32767 debug $8;
+      length msg $32767 ;
       sasdatetime=datetime();
-      msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
+      msg=symget('msg');
+    %if &logline>0 %then %do;
+      msg=cats(msg,'\n\nLog Extract:\n',symget('logmsg'));
+    %end;
       /* escape the quotes */
       msg=tranwrd(msg,'"','\"');
       /* ditch the CRLFs as chrome complains */
@@ -111,8 +175,9 @@
       msg=cats('"',msg,'"');
       if symexist('_debug') then debug=quote(trim(symget('_debug')));
       else debug='""';
-      if debug ge '"131"' then put '>>weboutBEGIN<<';
-      put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
+      put '>>weboutBEGIN<<';
+      put '{"SYSDATE" : "' "&SYSDATE" '"';
+      put ',"SYSTIME" : "' "&SYSTIME" '"';
       put ',"sasjsAbort" : [{';
       put ' "MSG":' msg ;
       put ' ,"MAC": "' "&mac" '"}]';
@@ -131,29 +196,45 @@
       _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
       put ',"_PROGRAM" : ' _PROGRAM ;
       put ",""SYSCC"" : ""&syscc"" ";
-      put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+      syserrortext=quote(trim(symget('syserrortext')));
+      put ",""SYSERRORTEXT"" : " syserrortext;
+      put ",""SYSHOSTNAME"" : ""&syshostname"" ";
       put ",""SYSJOBID"" : ""&sysjobid"" ";
+      put ",""SYSSCPL"" : ""&sysscpl"" ";
+      put ",""SYSSITE"" : ""&syssite"" ";
       sysvlong=quote(trim(symget('sysvlong')));
       put ',"SYSVLONG" : ' sysvlong;
-      put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
-      put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+      syswarningtext=quote(trim(symget('syswarningtext')));
+      put ",""SYSWARNINGTEXT"" : " syswarningtext;
+      put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
       put "}" @;
-      if debug ge '"131"' then put '>>weboutEND<<';
+      put '>>weboutEND<<';
     run;
 
     %put _all_;
 
     %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
       data _null_;
-        putlog 'stpsrvset program error and syscc';
+        putlog 'stpsrvset program err and syscc';
         rc=stpsrvset('program error', 0);
         call symputx("syscc",0,"g");
       run;
-      %if "%substr(&sysvlong.xxxxxxxxx,1,9)" ne "9.04.01M3" %then %do;
-        %put NOTE: Ending SAS session due to:;
-        %put NOTE- &msg;
-        endsas;
-      %end;
+      /**
+        * endsas kills 9.4m3 deployments by orphaning multibridges.
+        * Abort variants are ungraceful (non zero return code)
+        * This approach lets SAS run silently until the end :-)
+        * Caution - fails when called within a %include within a macro
+        * Use mp_include() to handle this.
+        */
+      filename skip temp;
+      data _null_;
+        file skip;
+        put '%macro skip();';
+        comment '%mend skip; -> fix lint ';
+        put '%macro skippy();';
+        comment '%mend skippy; -> fix lint ';
+      run;
+      %inc skip;
     %end;
     %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
       /* endsas kills the session making it harder to fetch results */
@@ -169,24 +250,6 @@
         abort cancel nolist;
       run;
     %end;
-    %else %if "%substr(&sysvlong.xxxxxxxxx,1,9)" = "9.04.01M3" %then %do;
-      /**
-        * endsas kills 9.4m3 deployments by orphaning multibridges.
-        * Abort variants are ungraceful (non zero return code)
-        * This approach lets SAS run silently until the end :-)
-        * Caution - fails when called within a %include within a macro
-        * See tests/mp_abort.test.1 for an example case.
-        */
-      filename skip temp;
-      data _null_;
-        file skip;
-        put '%macro skip();';
-        comment '%mend skip; -> fix lint ';
-        put '%macro skippy();';
-        comment '%mend skippy; -> fix lint ';
-      run;
-      %inc skip;
-    %end;
     %else %do;
       %abort cancel;
     %end;

base/mp_appendfile.sas

@@ -0,0 +1,57 @@
+/**
+  @file
+  @brief Append (concatenate) two or more files.
+  @details Will append one more more `appendrefs` (filerefs) to a `baseref`.
+  Uses a binary mechanism, so will work with any file type.  For that reason -
+  use with care!   And supply your own trailing carriage returns in each file..
+
+  Usage:
+
+        filename tmp1 temp;
+        filename tmp2 temp;
+        filename tmp3 temp;
+        data _null_; file tmp1; put 'base file';
+        data _null_; file tmp2; put 'append1';
+        data _null_; file tmp3; put 'append2';
+        run;
+        %mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3)
+
+
+  @param [in] baseref= Fileref of the base file (should exist)
+  @param [in] appendrefs= One or more filerefs to be appended to the base
+    fileref.  Space separated.
+
+  @version 9.2
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+
+
+**/
+
+%macro mp_appendfile(
+  baseref=0,
+  appendrefs=0
+)/*/STORE SOURCE*/;
+
+%mp_abort(iftrue= (&baseref=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Baseref NOT specified!)
+)
+%mp_abort(iftrue= (&appendrefs=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Appendrefs NOT specified!)
+)
+
+%local i;
+%do i=1 %to %sysfunc(countw(&appendrefs));
+  %mp_abort(iftrue= (&syscc>0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc)
+  )
+  %mp_binarycopy(inref=%scan(&appendrefs,&i), outref=&baseref, mode=APPEND)
+%end;
+
+%mend mp_appendfile;

base/mp_applyformats.sas

@@ -0,0 +1,181 @@
+/**
+  @file
+  @brief Apply a set of formats to a table
+  @details Applies a set of formats to the metadata of one or more SAS datasets.
+    Can be used to migrate formats from one table to another.  The input table
+    must contain the following columns:
+
+    @li lib - the libref of the table to be updated
+    @li ds - the dataset to be updated
+    @li var - the variable to be updated
+    @li fmt - the format to apply.  Missing or default ($CHAR, 8.) formats are
+      ignored.
+
+  The macro will abort in the following scenarios:
+
+    @li Libref not assigned
+    @li Dataset does not exist
+    @li Input table contains null or invalid values
+
+  Example usage:
+
+      data work.example;
+        set sashelp.prdsale;
+        format _all_ clear;
+      run;
+
+      %mp_getcols(sashelp.prdsale,outds=work.cols)
+
+      data work.cols2;
+        set work.cols;
+        lib='WORK';
+        ds='EXAMPLE';
+        var=name;
+        fmt=format;
+        keep lib ds var fmt;
+      run;
+
+      %mp_applyformats(work.cols2)
+
+  @param [in] inds The input dataset containing the formats to apply (and where
+    to apply them).  Example structure:
+  |LIB:$8.|DS:$32.|VAR:$32.|FMT:$49.|
+  |---|---|---|---|
+  |`WORK `|`EXAMPLE `|`ACTUAL `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`COUNTRY `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`DIVISION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`MONTH `|`MONNAME3. `|
+  |`WORK `|`EXAMPLE `|`PREDICT `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`PRODTYPE `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`PRODUCT `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`QUARTER `|`8. `|
+  |`WORK `|`EXAMPLE `|`REGION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`YEAR `|`8. `|
+
+  @param [out] errds= (0) Provide a libds reference here to export the
+    error messages to a table.  In this case, they will not be printed to the
+    log.
+
+  <h4> SAS Macros </h4>
+  @li mf_getengine.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_validatecol.sas
+
+
+  <h4> Related Macros </h4>
+  @li mp_getformats.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_applyformats(inds,errds=0
+)/*/STORE SOURCE*/;
+%local outds liblist i engine lib msg ;
+
+/**
+  * Validations
+  */
+proc sort data=&inds;
+  by lib ds var fmt;
+run;
+
+%if &errds=0 %then %let outds=%mf_getuniquename(prefix=mp_applyformats);
+%else %let outds=&errds;
+
+data &outds;
+  set &inds;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  length msg $128;
+  by lib ds var fmt;
+  if libref(lib) ne 0 then do;
+    msg=catx(' ','libref',lib,'is not assigned!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  if exist(cats(lib,'.',ds)) ne 1 then do;
+    msg=catx(' ','libds',lib,'.',ds,'does not exist!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  %mp_validatecol(fmt,FORMAT,is_fmt)
+  if is_fmt=0 then do;
+    msg=catx(' ','format',fmt,'on libds',lib,'.',ds,'.',var,'is not valid!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+
+  if first.ds then do;
+    retain dsid;
+    dsid=open(cats(lib,'.',ds));
+    if dsid=0 then do;
+      msg=catx(' ','libds',lib,'.',ds,' could not be opened!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+      return;
+    end;
+    if varnum(dsid,var)<1 then do;
+      msg=catx(' ','Variable',lib,'.',ds,'.',var,' was not found!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+    end;
+  end;
+  if last.ds then rc=close(dsid);
+run;
+
+proc sql noprint;
+select distinct lib into: liblist separated by ' ' from &inds;
+%put &=liblist;
+%if %length(&liblist)>0 %then %do i=1 %to %sysfunc(countw(&liblist));
+  %let lib=%scan(&liblist,1);
+  %let engine=%mf_getengine(&lib);
+  %if &engine ne V9 and &engine ne BASE %then %do;
+    %let msg=&lib has &engine engine - formats cannot be applied;
+    insert into &outds set lib="&lib",ds="_all_",var="_all", msg="&msg" ;
+    %if &errds=0 %then %put %str(ERR)OR: &msg;
+  %end;
+%end;
+quit;
+
+%if %mf_nobs(&outds)>0 %then %return;
+
+/**
+  * Validations complete - now apply the actual formats!
+  */
+%let fref=%mf_getuniquefileref();
+data _null_;
+  set &inds;
+  by lib ds var fmt;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  file &fref;
+  if first.lib then put 'proc datasets nolist lib=' lib ';';
+  if first.ds then put '  modify ' ds ';';
+  put '    format ' var fmt ';';
+  if last.ds then put '  run;';
+  if last.lib then put 'quit;';
+run;
+
+%inc &fref/source2;
+
+%if &errds=0 %then %do;
+  proc sql;
+  drop table &outds;
+%end;
+
+%mend mp_applyformats;

base/mp_assertcolvals.sas

@@ -30,6 +30,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
+  @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
 
@@ -115,6 +116,26 @@
   select count(*) into: orig from &lib..&ds;
   quit;
 
+  %local notfound tmp1 tmp2;
+  %let tmp1=%mf_getuniquename();
+  %let tmp2=%mf_getuniquename();
+
+  /* this is a bit convoluted - but using sql outobs=10 throws warnings */
+  proc sql noprint;
+  create view &tmp1 as
+    select distinct &col
+    from &lib..&ds
+    where &col not in (
+      select &ccol from &clib..&cds
+    );
+  data &tmp2;
+    set &tmp1;
+    if _n_>10 then stop;
+  run;
+  proc sql;
+  select distinct &col  into: notfound separated by ' ' from &tmp2;
+
+
   %mp_abort(iftrue= (&syscc ne 0)
     ,mac=&sysmacroname
     ,msg=%str(syscc=&syscc after macro query)
@@ -125,7 +146,7 @@
     test_description=symget('desc');
     test_result='FAIL';
     test_comments="&sysmacroname: &lib..&ds..&col has &result values "
-      !!"not in &clib..&cds..&ccol ";
+      !!"not in &clib..&cds..&ccol.. First 10 vals:"!!symget('notfound');
   %if &test=ANYVAL %then %do;
     if &result < &orig then test_result='PASS';
   %end;

base/mp_assertdsobs.sas

@@ -8,9 +8,9 @@
 
       %mp_assertdsobs(sashelp.class) %* tests if any observations are present;
 
-  <h4> SAS Macros </h4>
-  @li mf_nobs.sas
-  @li mp_abort.sas
+      %mp_assertdsobs(sashelp.class,test=ATLEAST 10) %* pass if >9 obs present;
+
+      %mp_assertdsobs(sashelp.class,test=ATMOST 20) %* pass if <21 obs present;
 
 
   @param [in] inds input dataset to test for presence of observations
@@ -19,9 +19,9 @@
     @li HASOBS - Test is a PASS if the input dataset has any observations
     @li EMPTY - Test is a PASS if input dataset is empty
     @li EQUALS [integer] - Test passes if row count matches the provided integer
-    @LI ATLEAST [integer] - Test passes if row count is more than or equal to
+    @li ATLEAST [integer] - Test passes if row count is more than or equal to
       the provided integer
-    @LI ATMOST [integer] - Test passes if row count is less than or equal to
+    @li ATMOST [integer] - Test passes if row count is less than or equal to
       the provided integer
   @param [out] outds= (work.test_results) The output dataset to contain the
   results.  If it does not exist, it will be created, with the following format:
@@ -29,6 +29,11 @@
   |---|---|---|
   |User Provided description|PASS|Dataset &inds has XX obs|
 
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
   <h4> Related Macros </h4>
   @li mp_assertcolvals.sas
   @li mp_assert.sas
@@ -45,9 +50,10 @@
   outds=work.test_results
 )/*/STORE SOURCE*/;
 
-  %local nobs;
+  %local nobs ds;
   %let nobs=%mf_nobs(&inds);
   %let test=%upcase(&test);
+  %let ds=%mf_getuniquename(prefix=mp_assertdsobs);
 
   %if %substr(&test.xxxxx,1,6)=EQUALS %then %do;
     %let val=%scan(&test,2,%str( ));
@@ -80,7 +86,7 @@
     )
   %end;
 
-  data;
+  data &ds;
     length test_description $256 test_result $4 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
@@ -106,9 +112,6 @@
   %end;
   run;
 
-  %local ds;
-  %let ds=&syslast;
-
   proc append base=&outds data=&ds;
   run;
 

base/mp_assertscope.sas

@@ -0,0 +1,144 @@
+/**
+  @file
+  @brief Used to capture scope leakage of macro variables
+  @details
+
+  A common 'difficult to detect' bug in macros is where a nested macro
+  over-writes variables in a higher level macro.
+
+  This assertion takes a snapshot of the macro variables before and after
+  a macro invocation.  Differences are captured in the `&outds` table. This
+  makes it easy to detect whether any macro variables were modified or
+  changed.
+
+  The following variables are NOT tested (as they are known, global variables
+  used in SASjs):
+
+  @li &sasjs_prefix._FUNCTIONS
+
+  Global variables are initialised in mp_init.sas - which will also trigger
+  "strict mode" in your SAS session.  Whilst this is a default in SASjs
+  produced apps, if you prefer not to use this mode, simply instantiate the
+  following variable to prevent the macro from running:  `SASJS_PREFIX`
+
+  Example usage:
+
+      %mp_assertscope(SNAPSHOT)
+
+      %let oops=I did it again;
+
+      %mp_assertscope(COMPARE,
+        desc=Checking macro variables against previous snapshot
+      )
+
+  This macro is designed to work alongside `sasjs test` - for more information
+  about this facility, visit [cli.sasjs.io/test](https://cli.sasjs.io/test).
+
+  @param [in] action (SNAPSHOT) The action to take.  Valid values:
+    @li SNAPSHOT - take a copy of the current macro variables
+    @li COMPARE - compare the current macro variables against previous values
+  @param [in] scope= (GLOBAL) The scope of the variables to be checked.  This
+    corresponds to the values in the SCOPE column in `sashelp.vmacro`.
+  @param [in] desc= (Testing scope leakage) The user provided test description
+  @param [in] ignorelist= Provide a list of macro variable names to ignore from
+    the comparison
+  @param [in,out] scopeds= (work.mp_assertscope) The dataset to contain the
+    scope snapshot
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|No out of scope variables created or modified|
+
+  <h4> SAS Macros </h4>
+  @li mf_getquotedstr.sas
+  @li mp_init.sas
+
+  <h4> Related Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertcols.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertscope(action,
+  desc=Testing Scope Leakage,
+  scope=GLOBAL,
+  scopeds=work.mp_assertscope,
+  ignorelist=,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+%local ds test_result test_comments del add mod ilist;
+%let ilist=%upcase(&sasjs_prefix._FUNCTIONS &ignorelist);
+
+/**
+  * this sets up the global vars, it will also enter STRICT mode.  If this
+  * behaviour is not desired, simply initiate the following global macro
+  * variable to prevent the macro from running: SASJS_PREFIX
+  */
+%mp_init()
+
+/* get current variables */
+%if &action=SNAPSHOT %then %do;
+  proc sql;
+  create table &scopeds as
+    select name,offset,value
+    from dictionary.macros
+    where scope="&scope" and name not in (%mf_getquotedstr(&ilist))
+    order by name,offset;
+%end;
+%else %if &action=COMPARE %then %do;
+
+  proc sql;
+  create table _data_ as
+    select name,offset,value
+    from dictionary.macros
+    where scope="&scope" and name not in (%mf_getquotedstr(&ilist))
+    order by name,offset;
+
+  %let ds=&syslast;
+
+  proc compare base=&scopeds compare=&ds;
+  run;
+
+  %if &sysinfo=0 %then %do;
+    %let test_result=PASS;
+    %let test_comments=&scope Variables Unmodified;
+  %end;
+  %else %do;
+    proc sql noprint undo_policy=none;
+    select distinct name into: del separated by ' '  from &scopeds
+      where name not in (select name from &ds);
+    select distinct name into: add separated by ' '  from &ds
+      where name not in (select name from &scopeds);
+    select distinct a.name into: mod separated by ' '
+      from &scopeds a
+      inner join &ds b
+      on a.name=b.name
+        and a.offset=b.offset
+      where a.value ne b.value;
+    %let test_result=FAIL;
+    %let test_comments=%str(Mod:(&mod) Add:(&add) Del:(&del));
+  %end;
+
+
+  data ;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_comments=symget('test_comments');
+    test_result=symget('test_result');
+  run;
+
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+%end;
+
+%mend mp_assertscope;

base/mp_base64copy.sas

@@ -76,7 +76,7 @@ run;
 
 %if &outfound=0 %then %do;
   filename &outref temp lrecl=2097088;
-%%end;
+%end;
 
 %if &action=ENCODE %then %do;
   data _null_;
@@ -84,12 +84,12 @@ run;
     retain line "";
     infile &inref recfm=F lrecl= 1 end=eof;
     input @1 stream $char1.;
-    file &outref lrecl=76;
+    file &outref recfm=N;
     substr(line,(_N_-(CEIL(_N_/57)-1)*57),1) = byte(rank(stream));
     if mod(_N_,57)=0 or EOF then do;
       if eof then b64=put(trim(line),$base64X76.);
       else b64=put(line, $base64X76.);
-      put b64;
+      put b64 + (-1) @;
       line="";
     end;
   run;
@@ -101,19 +101,13 @@ run;
     fileout = fopen("&outref",'O',3,'B');
     char= '20'x;
     do while(fread(filein)=0);
-      raw="1234";
+      length raw $4;
       do i=1 to 4;
         rc=fget(filein,char,1);
         substr(raw,i,1)=char;
       end;
-      val="123";
-      val=input(raw,$base64X4.);
-      do i=1 to 3;
-        length byte $1;
-        byte=byte(rank(substr(val,i,1)));
-        rc = fput(fileout, byte);
-      end;
-      rc =fwrite(fileout);
+      rc = fput(fileout,input(raw,$base64X4.));
+      rc = fwrite(fileout);
     end;
     rc = fclose(filein);
     rc = fclose(fileout);

base/mp_binarycopy.sas

@@ -9,10 +9,29 @@
 
         %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
 
-  @param inloc full, quoted "path/and/filename.ext" of the object to be copied
-  @param outloc full, quoted "path/and/filename.ext" of object to be created
-  @param inref can override default input fileref to avoid naming clash
-  @param outref an override default output fileref to avoid naming clash
+  To append to a file, use the mode option, eg:
+
+      filename tmp1 temp;
+      filename tmp2 temp;
+      data _null_;
+        file tmp1;
+        put 'stacking';
+      run;
+
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+
+
+  @param [in] inloc quoted "path/and/filename.ext" of the file to be copied
+  @param [out] outloc quoted "path/and/filename.ext" of the file to be created
+  @param [in] inref (____in) If provided, this fileref will take precedence over
+    the `inloc` parameter
+  @param [out] outref (____in) If provided, this fileref will take precedence
+    over the `outloc` parameter.  It must already exist!
+  @param [in] mode (CREATE) Valid values:
+    @li CREATE - Create the file (even if it already exists)
+    @li APPEND - Append to the file (don't overwrite)
+
   @returns nothing
 
   @version 9.2
@@ -24,20 +43,29 @@
     ,outloc=          /* full path and filename of object to be created */
     ,inref=____in   /* override default to use own filerefs */
     ,outref=____out /* override default to use own filerefs */
+    ,mode=CREATE
 )/*/STORE SOURCE*/;
+  %local mod outmode;
+  %if &mode=APPEND %then %do;
+    %let mod=mod;
+    %let outmode='a';
+  %end;
+  %else %do;
+    %let outmode='o';
+  %end;
   /* these IN and OUT filerefs can point to anything */
   %if &inref = ____in %then %do;
     filename &inref &inloc lrecl=1048576 ;
   %end;
   %if &outref=____out %then %do;
-    filename &outref &outloc lrecl=1048576 ;
+    filename &outref &outloc lrecl=1048576 &mod;
   %end;
 
   /* copy the file byte-for-byte  */
   data _null_;
     length filein 8 fileid 8;
     filein = fopen("&inref",'I',1,'B');
-    fileid = fopen("&outref",'O',1,'B');
+    fileid = fopen("&outref",&outmode,1,'B');
     rec = '20'x;
     do while(fread(filein)=0);
       rc = fget(filein,rec,1);

base/mp_copyfolder.sas

@@ -0,0 +1,81 @@
+/**
+  @file
+  @brief A macro to recursively copy a directory
+  @details Performs a recursive directory listing then works from top to bottom
+    copying files and creating subdirectories.
+
+  Usage:
+
+      %let rootdir=%sysfunc(pathname(work))/demo;
+      %let copydir=%sysfunc(pathname(work))/demo_copy;
+      %mf_mkdir(&rootdir)
+      %mf_mkdir(&rootdir/subdir)
+      %mf_mkdir(&rootdir/subdir/subsubdir)
+      data "&rootdir/subdir/example.sas7bdat";
+      run;
+
+      %mp_copyfolder(&rootdir,&copydir)
+
+  @param source Unquoted path to the folder to copy from.
+  @param target Unquoted path to the folder to copy to.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_isdir.sas
+  @li mf_mkdir.sas
+  @li mp_abort.sas
+  @li mp_dirlist.sas
+
+  <h4> Related Macros </h4>
+  @li mp_copyfolder.test.sas
+
+**/
+
+%macro mp_copyfolder(source,target);
+
+  %mp_abort(iftrue=(%mf_isdir(&source)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Source dir does not exist (&source))
+  )
+
+  %mf_mkdir(&target)
+
+  %mp_abort(iftrue=(%mf_isdir(&target)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Target dir could not be created (&target))
+  )
+
+  /* prep temp table */
+  %local tempds;
+  %let tempds=%mf_getuniquename();
+
+  /* recursive directory listing */
+  %mp_dirlist(path=&source,outds=work.&tempds, maxdepth=MAX)
+
+  /* create folders and copy content */
+  data _null_;
+    length msg $200;
+    call missing(msg);
+    set work.&tempds;
+    if _n_ = 1 then dpos+sum(length(directory),2);
+    filepath2="&target/"!!substr(filepath,dpos);
+    if file_or_folder='folder' then call execute('%mf_mkdir('!!filepath2!!')');
+    else do;
+      length fref1 fref2 $8;
+      rc1=filename(fref1,filepath,'disk','recfm=n');
+      rc2=filename(fref2,filepath2,'disk','recfm=n');
+      if fcopy(fref1,fref2) ne 0 then do;
+        msg=sysmsg();
+        putlog "%str(ERR)OR: Unable to copy " filepath " to " filepath2;
+        putlog msg=;
+      end;
+    end;
+    rc=filename(fref1);
+    rc=filename(fref2);
+  run;
+
+  /* tidy up */
+  proc sql;
+  drop table work.&tempds;
+
+%mend mp_copyfolder;

base/mp_coretable.sas

@@ -0,0 +1,118 @@
+/**
+  @file
+  @brief Create the permanent Core tables
+  @details Several macros in the [core](https://github.com/sasjs/core) library
+    make use of permanent tables.  To avoid duplication in definitions, this
+    macro provides a central location for managing the corresponding DDL.
+
+  Example usage:
+
+      %mp_coretable(LOCKTABLE,libds=work.locktable)
+
+  @param [in] table_ref The type of table to create.  Example values:
+    @li DIFFTABLE - Used to store changes to tables.  Used by mp_storediffs.sas
+      and mp_stackdiffs.sas
+    @li FILTER_DETAIL - For storing detailed filter values.  Used by
+      mp_filterstore.sas.
+    @li FILTER_SUMMARY - For storing summary filter values.  Used by
+      mp_filterstore.sas.
+    @li LOCKANYTABLE - For "locking" tables prior to multipass loads. Used by
+      mp_lockanytable.sas
+    @li MAXKEYTABLE - For storing the maximum retained key information.  Used
+      by mp_retainedkey.sas
+  @param [in] libds= (0) The library.dataset reference used to create the table.
+    If not provided, then the DDL is simply printed to the log.
+
+  <h4> Related Macros </h4>
+  @li mp_filterstore.sas
+  @li mp_lockanytable.sas
+  @li mp_retainedkey.sas
+  @li mp_storediffs.sas
+  @li mp_stackdiffs.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_coretable(table_ref,libds=0
+)/*/STORE SOURCE*/;
+%local outds ;
+%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
+proc sql;
+%if &table_ref=DIFFTABLE %then %do;
+  create table &outds(
+      load_ref char(36) label='unique load reference',
+      processed_dttm num format=E8601DT26.6 label='Processed at timestamp',
+      libref char(8) label='Library Reference (8 chars)',
+      dsn char(32) label='Dataset Name (32 chars)',
+      key_hash char(32) label=
+        'MD5 Hash of primary key values (pipe seperated)',
+      move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified',
+      is_pk num label='Is Primary Key Field? (1/0)',
+      is_diff num label=
+        'Did value change? (1/0/-1).  Always -1 for appends and deletes.',
+      tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
+      tgtvar_nm char(32) label='Target variable name (32 chars)',
+      oldval_num num format=best32. label='Old (numeric) value',
+      newval_num num format=best32. label='New (numeric) value',
+      oldval_char char(32765) label='Old (character) value',
+      newval_char char(32765) label='New (character) value',
+    constraint pk_mpe_audit
+      primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
+  );
+%end;
+%else %if &table_ref=LOCKTABLE %then %do;
+  create table &outds(
+      lock_lib char(8),
+      lock_ds char(32),
+      lock_status_cd char(10) not null,
+      lock_user_nm char(100) not null ,
+      lock_ref char(200),
+      lock_pid char(10),
+      lock_start_dttm num format=E8601DT26.6,
+      lock_end_dttm num format=E8601DT26.6,
+    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+%end;
+%else %if &table_ref=FILTER_SUMMARY %then %do;
+  create table &outds(
+      filter_rk num not null,
+      filter_hash char(32) not null,
+      filter_table char(41) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_rk));
+%end;
+%else %if &table_ref=FILTER_DETAIL %then %do;
+  create table &outds(
+      filter_hash char(32) not null,
+      filter_line num not null,
+      group_logic char(3) not null,
+      subgroup_logic char(3) not null,
+      subgroup_id num not null,
+      variable_nm varchar(32) not null,
+      operator_nm varchar(12) not null,
+      raw_value varchar(4000) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_hash,filter_line));
+%end;
+%else %if &table_ref=MAXKEYTABLE %then %do;
+  create table &outds(
+      keytable varchar(41) label='Base table in libref.dataset format',
+      keycolumn char(32) format=$32.
+        label='The Retained key field containing the key values.',
+      max_key num label=
+        'Integer representing current max RK or SK value in the KEYTABLE',
+      processed_dttm num format=E8601DT26.6
+        label='Datetime this value was last updated',
+    constraint pk_mpe_maxkeyvalues
+        primary key(keytable));
+%end;
+
+
+%if &libds=0 %then %do;
+  describe table &syslast;
+  drop table &syslast;
+%end;
+%mend mp_coretable;

base/mp_createwebservice.sas

@@ -24,20 +24,22 @@ Usage:
         %webout(OBJ,example2) * Object format, easier to work with ;
         %webout(CLOSE)
     ;;;;
-    %mp_createwebservice(path=/Public/app/common,name=appInit,code=ft15f001,replace=YES)
+    %mp_createwebservice(path=/Public/app/common,name=appInit,replace=YES)
 
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
   @li mm_createwebservice.sas
   @li mv_createwebservice.sas
 
-  @param path= The full folder path where the service will be created
-  @param name= Service name.  Avoid spaces.
-  @param desc= The description of the service (optional)
-  @param precode= Space separated list of filerefs, pointing to the code that
-    needs to be attached to the beginning of the service (optional)
-  @param code= Space seperated fileref(s) of the actual code to be added
-  @param replace= select YES to replace any existing service in that location
+  @param [in,out] path= The full folder path where the service will be created
+  @param [in,out] name= Service name.  Avoid spaces.
+  @param [in] desc= The description of the service (optional)
+  @param [in] precode= Space separated list of filerefs, pointing to the code
+    that needs to be attached to the beginning of the service (optional)
+  @param [in] code= (ft15f001) Space seperated fileref(s) of the actual code to
+    be added
+  @param [in] replace= (YES) Select YES to replace any existing service in that
+    location
 
 
   @version 9.2

base/mp_csv2ds.sas

@@ -49,10 +49,6 @@
   ,mac=&sysmacroname
   ,msg=%str(the BASEDS variable must be provided)
 )
-%mp_abort(iftrue=( &baseds=0 )
-  ,mac=&sysmacroname
-  ,msg=%str(the BASEDS variable must be provided)
-)
 %mp_abort(iftrue=( %mf_existds(&baseds)=0 )
   ,mac=&sysmacroname
   ,msg=%str(the BASEDS dataset (&baseds) needs to be assigned, and to exist)

base/mp_deletefolder.sas

@@ -0,0 +1,70 @@
+/**
+  @file
+  @brief A macro to delete a directory
+  @details Will delete all folder content (including subfolder content) and
+    finally, the folder itself.
+
+  Usage:
+
+      %let rootdir=%sysfunc(pathname(work))/demo;
+      %mf_mkdir(&rootdir)
+      %mf_mkdir(&rootdir/subdir)
+      %mf_mkdir(&rootdir/subdir/subsubdir)
+      data "&rootdir/subdir/example.sas7bdat";
+      run;
+
+      %mp_deletefolder(&rootdir)
+
+  @param path Unquoted path to the folder to delete.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_isdir.sas
+  @li mp_dirlist.sas
+
+  <h4> Related Macros </h4>
+  @li mp_deletefolder.test.sas
+
+**/
+
+%macro mp_deletefolder(folder);
+  /* proceed if valid directory */
+  %if %mf_isdir(&folder)=1 %then %do;
+
+    /* prep temp table */
+    %local tempds;
+    %let tempds=%mf_getuniquename();
+
+    /* recursive directory listing */
+    %mp_dirlist(path=&folder,outds=work.&tempds, maxdepth=MAX)
+
+    /* sort descending level so can delete folder contents before folders */
+    proc sort data=work.&tempds;
+      by descending level;
+    run;
+
+    /* ensure top level folder is removed at the end */
+    proc sql;
+    insert into work.&tempds set filepath="&folder";
+
+    /* delete everything */
+    data _null_;
+      set work.&tempds end=last;
+      length fref $8;
+      fref='';
+      rc=filename(fref,filepath);
+      rc=fdelete(fref);
+      if rc then do;
+        msg=sysmsg();
+        put "&sysmacroname:" / rc= / msg= / filepath=;
+      end;
+      rc=filename(fref);
+    run;
+
+    /* tidy up */
+    proc sql;
+    drop table work.&tempds;
+
+  %end;
+  %else %put &sysmacroname: &folder: is not a valid / accessible folder. ;
+%mend mp_deletefolder;

base/mp_dirlist.sas

@@ -3,20 +3,12 @@
   @brief Returns all files and subdirectories within a specified parent
   @details When used with getattrs=NO, is not OS specific (uses dopen / dread).
 
-  If getattrs=YES then the doptname / foptname functions are used to scan all
-  properties - any characters that are not valid in a SAS name (v7) are simply
-  stripped, and the table is transposed so theat each property is a column
-  and there is one file per row.  An attempt is made to get all properties
-  whether a file or folder, but some files/folders cannot be accessed, and so
-  not all properties can / will be populated.
-
   Credit for the rename approach:
   https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003
 
+  Usage:
 
-  usage:
-
-      %mp_dirlist(path=/some/location,outds=myTable)
+      %mp_dirlist(path=/some/location, outds=myTable, maxdepth=MAX)
 
       %mp_dirlist(outds=cwdfileprops, getattrs=YES)
 
@@ -30,11 +22,19 @@
   X CMD) do please raise an issue!
 
 
-  @param path= for which to return contents
-  @param fref= Provide a DISK engine fileref as an alternative to PATH
-  @param outds= the output dataset to create
-  @param getattrs= YES/NO (default=NO).  Uses doptname and foptname to return
-  all attributes for each file / folder.
+  @param [in] path= (%sysfunc(pathname(work))) Path for which to return contents
+  @param [in] fref= (0) Provide a DISK engine fileref as an alternative to PATH
+  @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
+    subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
+  @param [out] outds= (work.mp_dirlist) The output dataset to create
+  @param [out] getattrs= (NO)  If getattrs=YES then the doptname / foptname
+    functions are used to scan all properties - any characters that are not
+    valid in a SAS name (v7) are simply stripped, and the table is transposed
+    so theat each property is a column and there is one file per row.  An
+    attempt is made to get all properties whether a file or folder, but some
+    files/folders cannot be accessed, and so not all properties can / will be
+    populated.
 
 
   @returns outds contains the following variables:
@@ -44,24 +44,47 @@
     - filename (just the file name)
     - ext (.extension)
     - msg (system message if any issues)
+    - level (depth of folder)
     - OS SPECIFIC variables, if <code>getattrs=</code> is used.
 
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_getvarlist.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_dropmembers.sas
+
+  <h4> Related Macros </h4>
+  @li mp_dirlist.test.sas
+
   @version 9.2
-  @author Allan Bowe
 **/
 
 %macro mp_dirlist(path=%sysfunc(pathname(work))
     , fref=0
     , outds=work.mp_dirlist
     , getattrs=NO
+    , maxdepth=0
+    , level=0 /* The level of recursion to perform.  For internal use only. */
 )/*/STORE SOURCE*/;
 %let getattrs=%upcase(&getattrs)XX;
 
-data &outds(compress=no
-    keep=file_or_folder filepath filename ext msg directory
+/* temp table */
+%local out_ds;
+data;run;
+%let out_ds=%str(&syslast);
+
+/* drop main (top) table if it exists */
+%if &level=0 %then %do;
+  %mp_dropmembers(%scan(&outds,-1,.), libref=WORK)
+%end;
+
+data &out_ds(compress=no
+    keep=file_or_folder filepath filename ext msg directory level
   );
   length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
-    ext $20 msg $200;
+    ext $20 msg $200 foption $16;
+  if _n_=1 then call missing(of _all_);
+  retain level &level;
   %if &fref=0 %then %do;
     rc = filename(fref, "&path");
   %end;
@@ -71,7 +94,13 @@ data &outds(compress=no
   %end;
   if rc = 0 then do;
     did = dopen(fref);
-    directory=dinfo(did,'Directory');
+    /* attribute is OS-dependent - could be "Directory" or "Directory Name" */
+    numopts=doptnum(did);
+    do i=1 to numopts;
+      foption=doptname(did,i);
+      if foption=:'Directory' then i=numopts;
+    end;
+    directory=dinfo(did,foption);
     if did=0 then do;
       putlog "NOTE: This directory is empty - " directory;
       msg=sysmsg();
@@ -119,8 +148,8 @@ data &outds(compress=no
 run;
 
 %if %substr(&getattrs,1,1)=Y %then %do;
-  data &outds;
-    set &outds;
+  data &out_ds;
+    set &out_ds;
     length infoname infoval $60 fref $8;
     rc=filename(fref,filepath);
     drop rc infoname fid i close fref;
@@ -161,10 +190,56 @@ run;
   run;
   proc sort;
     by filepath sasname;
-  proc transpose data=&outds out=&outds(drop=_:);
+  proc transpose data=&out_ds out=&out_ds(drop=_:);
     id sasname;
     var infoval;
     by filepath file_or_folder filename ext ;
   run;
 %end;
+
+data &out_ds;
+  set &out_ds(where=(filepath ne ''));
+run;
+
+/**
+  * The above transpose can mean that some updates create additional columns.
+  * This necessitates the occasional use of datastep over proc append.
+  */
+%if %mf_existds(&outds) %then %do;
+  %local basevars appvars newvars;
+  %let basevars=%mf_getvarlist(&outds);
+  %let appvars=%mf_getvarlist(&out_ds);
+  %let newvars=%length(%mf_wordsinstr1butnotstr2(Str1=&appvars,Str2=&basevars));
+  %if &newvars>0 %then %do;
+    data &outds;
+      set &outds &out_ds;
+    run;
+  %end;
+  %else %do;
+    proc append base=&outds data=&out_ds force nowarn;
+    run;
+  %end;
+%end;
+%else %do;
+  proc append base=&outds data=&out_ds;
+  run;
+%end;
+
+/* recursive call */
+%if &maxdepth>&level or &maxdepth=MAX %then %do;
+  data _null_;
+    set &out_ds;
+    where file_or_folder='folder';
+    length code $10000;
+    code=cats('%nrstr(%mp_dirlist(path=',filepath,",outds=&outds"
+      ,",getattrs=&getattrs,level=%eval(&level+1),maxdepth=&maxdepth))");
+    put code=;
+    call execute(code);
+  run;
+%end;
+
+/* tidy up */
+proc sql;
+drop table &out_ds;
+
 %mend mp_dirlist;

base/mp_dropmembers.sas

@@ -17,8 +17,9 @@
   @li mf_isblank.sas
 
 
-  @param list space separated list of datasets / views, WITHOUT libref
-  @param libref= (WORK) Note - you can only drop from a single library at a time
+  @param [in] list space separated list of datasets / views, WITHOUT libref
+  @param [in] libref= (WORK) Note - you can only drop from one library at a time
+  @param [in] iftrue= (1=1) Conditionally drop tables, eg if &debug=N
 
   @version 9.2
   @author Allan Bowe
@@ -28,8 +29,11 @@
 %macro mp_dropmembers(
     list /* space separated list of datasets / views */
     ,libref=WORK  /* can only drop from a single library at a time */
+    ,iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
   %if %mf_isblank(&list) %then %do;
     %put NOTE: nothing to drop!;
     %return;

base/mp_ds2cards.sas

@@ -2,12 +2,21 @@
   @file
   @brief Create a CARDS file from a SAS dataset.
   @details Uses dataset attributes to convert all data into datalines.
-    Running the generated file will rebuild the original dataset.
+    Running the generated file will rebuild the original dataset.  Includes
+    support for large decimals, binary data, PROCESSED_DTTM columns, and
+    alternative encoding.  If the input dataset is empty, the cards file will
+    still be created.
+
+    Additional support to generate a random sample and max rows.
+
   Usage:
 
       %mp_ds2cards(base_ds=sashelp.class
+        , tgt_ds=work.class
         , cards_file= "C:\temp\class.sas"
-        , maxobs=5)
+        , showlog=NO
+        , maxobs=5
+      )
 
   TODO:
     - labelling the dataset
@@ -18,15 +27,24 @@
                   that is converted to a cards file.
   @param [in] tgt_ds= Table that the generated cards file would create.
     Optional - if omitted, will be same as BASE_DS.
-  @param [out] cards_file= Location in which to write the (.sas) cards file
-  @param [in] maxobs= to limit output to the first <code>maxobs</code>
-    observations
-  @param [in] showlog= whether to show generated cards file in the SAS log
-    (YES/NO)
-  @param [in] outencoding= provide encoding value for file statement (eg utf-8)
-  @param [in] append= If NO then will rebuild the cards file if it already
+  @param [out] cards_file= ("%sysfunc(pathname(work))/cardgen.sas") Location in
+    which to write the (.sas) cards file
+  @param [in] maxobs= (max) To limit output to the first <code>maxobs</code>
+    observations, enter an integer here.
+  @param [in] random_sample= (NO) Set to YES to generate a random sample of
+    data.  Can be quite slow.
+  @param [in] showlog= (YES) Whether to show generated cards file in the SAS
+    log.  Valid values:
+    @li YES
+    @li NO
+  @param [in] outencoding= Provide encoding value for file statement (eg utf-8)
+  @param [in] append= (NO) If NO then will rebuild the cards file if it already
     exists, otherwise will append to it.  Used by the mp_lib2cards.sas macro.
 
+  <h4> Related Macros </h4>
+  @li mp_lib2cards.sas
+  @li mp_ds2inserts.sas
+  @li mp_mdtablewrite.sas
 
   @version 9.2
   @author Allan Bowe
@@ -51,15 +69,15 @@
 %if (&tgt_ds = ) %then %let tgt_ds=&base_ds;
 %if %index(&tgt_ds,.)=0 %then %let tgt_ds=WORK.%scan(&base_ds,2,.);
 %if ("&outencoding" ne "") %then %let outencoding=encoding="&outencoding";
-%if ("&append" = "") %then %let append=;
+%if ("&append" = "" or "&append" = "NO") %then %let append=;
 %else %let append=mod;
 
 /* get varcount */
 %let nvars=0;
 proc sql noprint;
 select count(*) into: nvars from dictionary.columns
-  where libname="%scan(%upcase(&base_ds),1)"
-    and memname="%scan(%upcase(&base_ds),2)";
+  where upcase(libname)="%scan(%upcase(&base_ds),1)"
+    and upcase(memname)="%scan(%upcase(&base_ds),2)";
 %if &nvars=0 %then %do;
   %put %str(WARN)ING: Dataset &base_ds has no variables, will not be converted.;
   %return;
@@ -115,14 +133,15 @@ proc sql
 reset outobs=max;
 create table datalines1 as
   select name,type,length,varnum,format,label from dictionary.columns
-  where libname="%upcase(%scan(&base_ds,1))"
-    and memname="%upcase(%scan(&base_ds,2))";
+  where upcase(libname)="%upcase(%scan(&base_ds,1))"
+    and upcase(memname)="%upcase(%scan(&base_ds,2))";
 
 /**
   Due to long decimals cannot use best. format
   So - use bestd. format and then use character functions to strip trailing
-    zeros, if NOT an integer!!
-  resolved code = ifc(int(VARIABLE)=VARIABLE
+    zeros, if NOT an integer or missing!!  Cannot use int() as it upsets
+    note2err when there are missings.
+  resolved code = ifc( mod(coalesce(VARIABLE,0),1)=0
     ,put(VARIABLE,best32.)
     ,substrn(put(VARIABLE,bestd32.),1
     ,findc(put(VARIABLE,bestd32.),'0','TBK')));
@@ -133,11 +152,22 @@ data datalines_2;
   set datalines1 (where=(upcase(name) not in
     ('PROCESSED_DTTM','VALID_FROM_DTTM','VALID_TO_DTTM')));
   if type='num' then dataline=
-    cats('ifc(int(',name,')=',name,'
+    cats('ifc(mod(coalesce(',name,',0),1)=0
       ,put(',name,',best32.-l)
       ,substrn(put(',name,',bestd32.-l),1
       ,findc(put(',name,',bestd32.-l),"0","TBK")))');
-  else dataline=name;
+  /**
+    * binary data must be converted, to store in text format.  It is identified
+    * by the presence of the $HEX keyword in the format.
+    */
+  else if upcase(format)=:'$HEX' then
+    dataline=cats('put(trim(',name,'),',format,')');
+  /**
+    * There is no easy way to store line breaks in a cards file.
+    * To discuss this, use: https://github.com/sasjs/core/issues/80
+    * Removing all nonprintables with kw (keep writeable)
+    */
+  else dataline=cats('compress(',name,', ,"kw")');
 run;
 
 proc sql noprint;
@@ -162,7 +192,8 @@ data _null_;
 
 
 /* Build input statement */
-  if type='char' then type3=':$char.';
+  if upcase(format)=:'$HEX' then type3=':'!!format;
+  else if type='char' then type3=':$char.';
   str2=put(name,$33.)||type3;
 
 
@@ -184,11 +215,12 @@ data _null_;
   file &cards_file. &outencoding lrecl=32767 termstr=nl &append;
   length __attrib $32767;
   if _n_=1 then do;
-    put '/*******************************************************************';
-    put " Datalines for %upcase(%scan(&base_ds,2)) dataset ";
-    put " Generated by %nrstr(%%)mp_ds2cards()";
-    put " Available on github.com/sasjs/core";
-    put '********************************************************************/';
+    put '/**';
+    put '  @file';
+    put "  @brief Datalines for %upcase(%scan(&base_ds,2)) dataset";
+    put "  @details Generated by %nrstr(%%)mp_ds2cards()";
+    put "  Available on github.com/sasjs/core";
+    put '**/';
     put "data &tgt_ds &indexes;";
     put "attrib ";
     %do i = 1 %to &nvars;
@@ -212,11 +244,11 @@ data _null_;
       put 'run;';
     end;
     else do;
-      put "infile cards dsd delimiter=',';";
+      put "infile cards dsd;";
       put "input ";
       %do i = 1 %to &nvars.;
         %if(%length(&&input_stmt_&i..)) %then
-          put "   &&input_stmt_&i..";
+          put "  &&input_stmt_&i..";
         ;
       %end;
       put ";";

base/mp_ds2csv.sas

@@ -1,23 +1,82 @@
 /**
   @file
-  @brief Export a dataset to a CSV file
-  @details Export to a file or a fileref
+  @brief Export a dataset to a CSV file WITH leading blanks
+  @details Export a dataset to a file or fileref, retaining leading blanks.
+
+  When using SASJS headerformat, the input statement is provided in the first
+  row of the CSV.
+
   Usage:
 
       %mp_ds2csv(sashelp.class,outref="%sysfunc(pathname(work))/file.csv")
 
-  @param ds The dataset to be exported
-  @param outfile= The output filename - should be quoted.
-  @param outref= The output fileref (takes precedence if provided)
-  @param outencoding= The output encoding to use (unquoted)
+      filename example temp;
+      %mp_ds2csv(sashelp.air,outref=example,headerformat=SASJS)
+      data; infile example; input;put _infile_; if _n_>5 then stop;run;
+
+      data _null_;
+        infile example;
+        input;
+        call symputx('stmnt',_infile_);
+        stop;
+      run;
+      data work.want;
+        infile example dsd firstobs=2;
+        input &stmnt;
+      run;
+
+  Why use mp_ds2csv over, say, proc export?
+
+  1. Ability to retain leading blanks (this is a major one)
+  2. Control the header format
+  3. Simple one-liner
+
+  @param [in] ds The dataset to be exported
+  @param [in] dlm= (COMMA) The delimeter to apply.  For SASJS, will always be
+    COMMA. Supported values:
+    @li COMMA
+    @li SEMICOLON
+  @param [in] headerformat= (LABEL) The format to use for the header section.
+    Valid values:
+    @li LABEL - Use the variable label (or name, if blank)
+    @li NAME - Use the variable name
+    @li SASJS - Used to create sasjs-formatted input CSVs, eg for use in
+      mp_testservice.sas.  This format will supply an input statement in the
+      first row, making ingestion by datastep a breeze.  Special misisng values
+      will be prefixed with a period (eg `.A`) to enable ingestion on both SAS 9
+      and Viya.  Dates / Datetimes etc are identified by the format type (lookup
+      with mcf_getfmttype.sas) and converted to human readable formats (not
+      numbers).
+  @param [out] outfile= The output filename - should be quoted.
+  @param [out] outref= (0) The output fileref (takes precedence if provided)
+  @param [in] outencoding= (0) The (quoted) output encoding to use, eg `"UTF-8"`
+  @param [in] termstr= (CRLF) The line seperator to use.  For SASJS, will
+    always be CRLF.  Valid values:
+    @li CRLF
+    @li LF
+
+  <h4> SAS Macros </h4>
+  @li mcf_getfmttype.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarformat.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
 
   @version 9.2
   @author Allan Bowe (credit mjsq)
 **/
 
-%macro mp_ds2csv(ds, outref=0, outfile=, outencoding=0
+%macro mp_ds2csv(ds
+  ,dlm=COMMA
+  ,outref=0
+  ,outfile=
+  ,outencoding=0
+  ,headerformat=LABEL
+  ,termstr=CRLF
 )/*/STORE SOURCE*/;
 
+%local outloc delim i varlist var vcnt vat dsv vcom vmiss fmttype vfmt;
+
 %if not %sysfunc(exist(&ds)) %then %do;
   %put %str(WARN)ING:  &ds does not exist;
   %return;
@@ -26,33 +85,128 @@
 %if %index(&ds,.)=0 %then %let ds=WORK.&ds;
 
 %if &outencoding=0 %then %let outencoding=;
-%else %let outencoding=encoding="&outencoding";
+%else %let outencoding=encoding=&outencoding;
 
-%local outloc;
 %if &outref=0 %then %let outloc=&outfile;
 %else %let outloc=&outref;
 
+%if &headerformat=SASJS %then %do;
+  %let delim=",";
+  %let termstr=CRLF;
+  %mcf_getfmttype(wrap=YES)
+%end;
+%else %if &dlm=COMMA %then %let delim=",";
+%else %let delim=";";
+
 /* credit to mjsq - https://stackoverflow.com/a/55642267 */
 
 /* first get headers */
 data _null_;
-  file &outloc dlm=',' dsd &outencoding lrecl=32767;
-  length header $ 2000;
+  file &outloc &outencoding lrecl=32767 termstr=&termstr;
+  length header $ 2000 varnm vfmt $32 dlm $1 fmttype $8;
+  call missing(of _all_);
   dsid=open("&ds.","i");
   num=attrn(dsid,"nvars");
+  dlm=&delim;
   do i=1 to num;
-    header = trim(left(coalescec(varlabel(dsid,i),varname(dsid,i))));
+    varnm=upcase(varname(dsid,i));
+    if i=num then dlm='';
+  %if &headerformat=NAME %then %do;
+    header=cats(varnm,dlm);
+  %end;
+  %else %if &headerformat=LABEL %then %do;
+    header = cats(coalescec(varlabel(dsid,i),varnm),dlm);
+  %end;
+  %else %if &headerformat=SASJS %then %do;
+    if vartype(dsid,i)='C' then header=cats(varnm,':$char',varlen(dsid,i),'.');
+    else do;
+      vfmt=coalescec(varfmt(dsid,i),'0');
+      fmttype=mcf_getfmttype(vfmt);
+      if fmttype='DATE' then header=cats(varnm,':date9.');
+      else if fmttype='DATETIME' then header=cats(varnm,':E8601DT26.6');
+      else if fmttype='TIME' then header=cats(varnm,':TIME12.');
+      else header=cats(varnm,':best.');
+    end;
+  %end;
+  %else %do;
+    %put &sysmacroname: Invalid headerformat value (&headerformat);
+    %return;
+  %end;
     put header @;
   end;
   rc=close(dsid);
 run;
 
+%let varlist=%mf_getvarlist(&ds);
+%let vcnt=%sysfunc(countw(&varlist));
+
+/**
+  * The $quote modifier (without a width) will take the length from the variable
+  * and increase by two.  However this will lead to truncation where the value
+  * contains double quotes (which are doubled up).  To get around this, scan the
+  * data to see the max number of double quotes, so that the appropriate width
+  * can be applied in the subsequent step.
+  */
+data _null_;
+  set &ds end=last;
+%do i=1 %to &vcnt;
+  %let var=%scan(&varlist,&i);
+  %if %mf_getvartype(&ds,&var)=C %then %do;
+    %let dsv1=%mf_getuniquename(prefix=csvcol1_);
+    %let dsv2=%mf_getuniquename(prefix=csvcol2_);
+    retain &dsv1 0;
+    &dsv2=length(&var)+countc(&var,'"');
+    if &dsv2>&dsv1 then &dsv1=&dsv2;
+    if last then call symputx(
+      "vlen&i"
+      /* should be no shorter than varlen, and no longer than 32767 */
+      ,cats('$quote',min(&dsv1+2,32767),'.')
+      ,'l'
+    );
+  %end;
+%end;
+
+%let vat=@;
+%let vcom=&delim;
+%let vmiss=%mf_getuniquename(prefix=csvcol3_);
 /* next, export data */
 data _null_;
   set &ds.;
-  file &outloc mod dlm=',' dsd &outencoding lrecl=32767;
-  put (_all_) (+0);
+  file &outloc mod dlm=&delim dsd &outencoding lrecl=32767 termstr=&termstr;
+  if _n_=1 then &vmiss='  ';
+  %do i=1 %to &vcnt;
+    %let var=%scan(&varlist,&i);
+    %if &i=&vcnt %then %do;
+      %let vat=;
+      %let vcom=;
+    %end;
+    %if %mf_getvartype(&ds,&var)=N %then %do;
+      %if &headerformat = SASJS %then %do;
+        %let vcom=&delim;
+        %let fmttype=%sysfunc(mcf_getfmttype(%mf_getvarformat(&ds,&var)0));
+        %if &fmttype=DATE %then %let vfmt=DATE9.;
+        %else %if &fmttype=DATETIME %then %let vfmt=E8601DT26.6;
+        %else %if &fmttype=TIME %then %let vfmt=TIME12.;
+        %else %do;
+          %let vfmt=;
+          %let vcom=;
+        %end;
+      %end;
+      %else %let vcom=;
+
+      /* must use period - in order to work in both 9.4 and Viya 3.5 */
+      if missing(&var) and &var ne %sysfunc(getoption(MISSING)) then do;
+        &vmiss=cats('.',&var);
+        put &vmiss &vat;
+      end;
+      else put &var &vfmt &vcom &vat;
+
+    %end;
+    %else %do;
+      %if &i ne &vcnt %then %let vcom=&delim;
+      put &var &&vlen&i &vcom &vat;
+    %end;
+  %end;
 run;
 
-
 %mend mp_ds2csv;

base/mp_ds2fmtds.sas

@@ -1,16 +1,22 @@
 /**
   @file
-  @brief Converts every value in a dataset to it's formatted value
+  @brief Converts every value in a dataset to formatted value
   @details Converts every value to it's formatted value.  All variables will
-  become character, and will be in the same order.
+  become character, and will be in the same order as the original dataset.
+
+  Lengths will be adjusted according to the format lengths, where applicable.
 
   Usage:
 
       %mp_ds2fmtds(sashelp.cars,work.cars)
+      %mp_ds2fmtds(sashelp.vallopt,vw_vallopt)
 
   @param [in] libds The library.dataset to be converted
   @param [out] outds The dataset to create.
 
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+
   <h4> Related Macros <h4>
   @li mp_jsonout.sas
 
@@ -22,8 +28,9 @@
 )/*/STORE SOURCE*/;
 
 /* validations */
-%if not %sysfunc(exist(&libds)) %then %do;
-  %put %str(WARN)ING:  &libds does not exist;
+
+%if not %mf_existds(libds=&libds) %then %do;
+  %put %str(WARN)ING:  &libds does not exist as either a VIEW or DATASET;
   %return;
 %end;
 %if %index(&libds,.)=0 %then %let libds=WORK.&libds;

base/mp_ds2inserts.sas

@@ -0,0 +1,180 @@
+/**
+  @file
+  @brief Export a dataset to SQL insert statements
+  @details Converts dataset values to SQL insert statements for use across
+  multiple database types.
+
+  Usage:
+
+      %mp_ds2inserts(sashelp.class,outref=myref,outds=class)
+      data class;
+        set sashelp.class;
+        stop;
+      proc sql;
+      %inc myref;
+
+  @param [in] ds The dataset to be exported
+  @param [in] maxobs= (max) The max number of inserts to create
+  @param [out] outref= (0) The output fileref.  If it does not exist, it is
+    created. If it does exist, new records are APPENDED.
+  @param [out] schema= (0) The library (or schema) in which the target table is
+    located.  If not provided, is ignored.
+  @param [out] outds= (0) The output table to load.  If not provided, will
+    default to the table in the &ds parameter.
+  @param [in] flavour= (SAS) The SQL flavour to be applied to the output. Valid
+    options:
+    @li SAS (default) - suitable for regular proc sql
+    @li PGSQL - Used for Postgres databases
+  @param [in] applydttm= (YES) If YES, any columns using datetime formats will
+    be converted to native DB datetime literals
+
+  <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mf_getvarcount.sas
+  @li mf_getvarformat.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
+
+  @version 9.2
+  @author Allan Bowe (credit mjsq)
+**/
+
+%macro mp_ds2inserts(ds, outref=0,schema=0,outds=0,flavour=SAS,maxobs=max
+  ,applydttm=YES
+)/*/STORE SOURCE*/;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if %index(&ds,.)=0 %then %let ds=WORK.&ds;
+
+%let flavour=%upcase(&flavour);
+%if &flavour ne SAS and &flavour ne PGSQL %then %do;
+  %put %str(WAR)NING:  &flavour is not supported;
+  %return;
+%end;
+
+%if &outref=0 %then %do;
+  %put %str(WAR)NING:  Please provide a fileref;
+  %return;
+%end;
+%if %mf_existfileref(&outref)=0 %then %do;
+  filename &outref temp lrecl=66000;
+%end;
+
+%if &schema=0 %then %let schema=;
+%else %let schema=&schema..;
+
+%if &outds=0 %then %let outds=%scan(&ds,2,.);
+
+%local nobs;
+proc sql noprint;
+select count(*) into: nobs TRIMMED from &ds;
+%if &nobs=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No rows found in &ds */";
+  run;
+%end;
+
+%local vars;
+%let vars=%mf_getvarcount(&ds);
+%if &vars=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No columns found in &schema.&ds */";
+  run;
+  %return;
+%end;
+%else %if &vars>1600 and &flavour=PGSQL %then %do;
+  data _null_;
+    file &fref mod;
+    put "/* &schema.&ds contains &vars vars */";
+    put "/* Postgres cannot handle tables with over 1600 vars */";
+    put "/* No inserts will be generated for this table */";
+  run;
+  %return;
+%end;
+
+%local varlist varlistcomma;
+%let varlist=%mf_getvarlist(&ds);
+%let varlistcomma=%mf_getvarlist(&ds,dlm=%str(,),quote=double);
+
+/* next, export data */
+data _null_;
+  file &outref mod ;
+  if _n_=1 then put "/* &schema.&outds (&nobs rows, &vars columns) */";
+  set &ds;
+  %if &maxobs ne max %then %do;
+    if _n_>&maxobs then stop;
+  %end;
+  length _____str $32767;
+  call missing(_____str);
+  format _numeric_ best.;
+  format _character_ ;
+  %local i comma var vtype vfmt;
+  %do i=1 %to %sysfunc(countw(&varlist));
+    %let var=%scan(&varlist,&i);
+    %let vtype=%mf_getvartype(&ds,&var);
+    %let vfmt=%upcase(%mf_getvarformat(&ds,&var,force=1));
+    %if &i=1 %then %do;
+      %if &flavour=SAS %then %do;
+        put "insert into &schema.&outds set ";
+        put "  &var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        _____str=cats(
+          "INSERT INTO &schema.&outds ("
+          ,symget('varlistcomma')
+          ,") VALUES ("
+        );
+        put _____str;
+        put "  "@;
+      %end;
+    %end;
+    %else %do;
+      %if &flavour=SAS %then %do;
+        put "  ,&var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        put "  ,"@;
+      %end;
+    %end;
+    %if &vtype=N %then %do;
+      %if &flavour=SAS %then %do;
+        put &var;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        if missing(&var) then put 'NULL';
+        %if &applydttm=YES and "%substr(&vfmt.xxxxxxxx,1,8)"="DATETIME"
+        %then %do;
+          else put "TIMESTAMP '" &var E8601DT25.6 "'";
+        %end;
+        %else %do;
+          else put &var;
+        %end;
+      %end;
+    %end;
+    %else %do;
+      _____str="'"!!trim(tranwrd(&var,"'","''"))!!"'";
+      put _____str;
+    %end;
+  %end;
+  %if &flavour=SAS %then %do;
+    put ';';
+  %end;
+  %else %if &flavour=PGSQL %then %do;
+    put ');';
+  %end;
+
+  if _n_=&nobs then put /;
+run;
+
+%mend mp_ds2inserts;

base/mp_ds2md.sas

@@ -14,14 +14,14 @@
   We take the standard definition one step further by embedding the informat
   in the table header row, like so:
 
-      |var1:$|var2:best.|var3:date9.|
+      |var1:$32|var2:best.|var3:date9.|
       |---|---|---|
       |some text|42|01JAN1960|
       |blah|1|31DEC1999|
 
   Which resolves to:
 
-  |var1:$|var2:best.|var3:date9.|
+  |var1:$32|var2:best.|var3:date9.|
   |---|---|---|
   |some text|42|01JAN1960|
   |blah|1|31DEC1999|
@@ -29,44 +29,44 @@
 
   Usage:
 
-      %mp_mdtablewrite(libds=sashelp.class,showlog=YES)
+      %mp_ds2md(sashelp.class)
 
+  @param [in] libds the library / dataset to create or read from.
+  @param [out] outref= (mdtable) Fileref to contain the markdown
+  @param [out] showlog= (YES) Set to NO to avoid printing markdown to the log
 
   <h4> SAS Macros </h4>
   @li mf_getvarlist.sas
   @li mf_getvarformat.sas
 
-  @param [in] libds= the library / dataset to create or read from.
-  @param [out] fref= Fileref to contain the markdown. Default=mdtable.
-  @param [out] showlog= set to YES to show the markdown in the log. Default=NO.
-
   @version 9.3
   @author Allan Bowe
 **/
 
-%macro mp_mdtablewrite(
-  libds=,
-  fref=mdtable,
-  showlog=NO
+%macro mp_ds2md(
+  libds,
+  outref=mdtable,
+  showlog=YES
 )/*/STORE SOURCE*/;
 
 /* check fileref is assigned */
-%if %sysfunc(fileref(&fref)) > 0 %then %do;
-  filename &fref temp;
+%if %sysfunc(fileref(&outref)) > 0 %then %do;
+  filename &outref temp;
 %end;
 
 %local vars;
-%let vars=%mf_getvarlist(&libds);
+%let vars=%upcase(%mf_getvarlist(&libds));
 
 /* create the header row */
 data _null_;
-  file &fref;
+  file &outref;
   length line $32767;
+  call missing(line);
   put '|'
 %local i var fmt;
 %do i=1 %to %sysfunc(countw(&vars));
   %let var=%scan(&vars,&i);
-  %let fmt=%mf_getvarformat(&libds,&var,force=1);
+  %let fmt=%lowcase(%mf_getvarformat(&libds,&var,force=1));
   "&var:&fmt|"
 %end;
   ;
@@ -79,20 +79,20 @@ run;
 
 /* write out the data */
 data _null_;
-  file &fref mod dlm='|' lrecl=32767;
+  file &outref mod dlm='|' lrecl=32767;
   set &libds ;
   length line $32767;
-  line=cats('|',%mf_getvarlist(&libds,dlm=%str(,'|',)),'|');
+  line='|`'!!cats(%mf_getvarlist(&libds,dlm=%str(%)!!' `|`'!!cats%()))!!' `|';
   put line;
 run;
 
 %if %upcase(&showlog)=YES %then %do;
   options ps=max;
   data _null_;
-    infile &fref;
+    infile &outref;
     input;
     putlog _infile_;
   run;
 %end;
 
-%mend mp_mdtablewrite;
+%mend mp_ds2md;

base/mp_ds2squeeze.sas

@@ -0,0 +1,120 @@
+/**
+  @file
+  @brief Create a smaller version of a dataset, without data loss
+  @details This macro will scan the input dataset and create a new one, that
+  has the minimum variable lengths needed to store the data without data loss.
+
+  Inspiration was taken from [How to Reduce the Disk Space Required by a
+  SAS® Data Set](https://www.lexjansen.com/nesug/nesug06/io/io18.pdf) by
+  Selvaratnam Sridharma.  The end of the referenced paper presents a macro named
+  "squeeze", hence the nomenclature.
+
+  Usage:
+
+      data big;
+        length my big $32000;
+        do i=1 to 1e4;
+          my=repeat('oh my',100);
+          big='dawg';
+          special=._;
+          output;
+        end;
+      run;
+
+      %mp_ds2squeeze(work.big,outds=work.smaller)
+
+  The following will also be printed to the log (exact values may differ
+  depending on your OS and COMPRESS settings):
+
+  > MP_DS2SQUEEZE: work.big was  625MB
+
+  > MP_DS2SQUEEZE: work.smaller is    5MB
+
+  @param [in] libds The library.dataset to be squeezed
+  @param [out] outds= (work.mp_ds2squeeze) The squeezed dataset to create
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> SAS Macros </h4>
+  @li mf_getfilesize.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_getmaxvarlengths.sas
+
+  <h4> Related Programs </h4>
+  @li mp_ds2squeeze.test.sas
+
+  @version 9.3
+  @author Allan Bowe
+**/
+
+%macro mp_ds2squeeze(
+  libds,
+  outds=work.mp_ds2squeeze,
+  mdebug=0
+)/*/STORE SOURCE*/;
+%local dbg source;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %do;
+  %let dbg=*;
+  %let source=/source2;
+%end;
+
+%local optval ds fref startsize;
+%let ds=%mf_getuniquename();
+%let fref=%mf_getuniquefileref();
+%let startsize=%mf_getfilesize(libds=&libds,format=yes);
+
+%mp_getmaxvarlengths(&libds,outds=&ds)
+
+data _null_;
+  set &ds end=last;
+  file &fref;
+  /* grab the types */
+  retain dsid;
+  if _n_=1 then dsid=open("&libds",'is');
+  if dsid le 0 then do;
+    msg=sysmsg();
+    put msg=;
+    stop;
+  end;
+  type=vartype(dsid,varnum(dsid, name));
+  if last then rc=close(dsid);
+  /* write out the length statement */
+  if _n_=1 then put 'length ';
+  length len $6;
+  if type='C' then do;
+    if maxlen=0 then len='$1';
+    else len=cats('$',maxlen);
+  end;
+  else do;
+    if maxlen=0 then len='3';
+    else len=cats(maxlen);
+  end;
+  put '  ' name ' ' len;
+  if last then put ';';
+run;
+
+/* configure varlenchk - as we are explicitly shortening the variables */
+%let optval=%sysfunc(getoption(varlenchk));
+options varlenchk=NOWARN;
+
+data &outds;
+  %inc &fref &source;
+  set &libds;
+run;
+
+options varlenchk=&optval;
+
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &ds;
+  filename &fref clear;
+%end;
+
+%put &sysmacroname: &libds was &startsize;
+%put &sysmacroname: &outds is %mf_getfilesize(libds=&outds,format=yes);
+
+%mend mp_ds2squeeze;

base/mp_filtercheck.sas

@@ -109,7 +109,7 @@ data &outds;
     output;
   end;
   if mod(SUBGROUP_ID,1) ne 0 then do;
-    REASON_CD='SUBGROUP_ID should be integer, not '!!left(subgroup_id);
+    REASON_CD='SUBGROUP_ID should be integer, not '!!cats(subgroup_id);
     putlog REASON_CD= SUBGROUP_ID=;
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');
@@ -127,7 +127,7 @@ data &outds;
   if OPERATOR_NM not in
   ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
   then do;
-    REASON_CD='Invalid OPERATOR_NM: '!!left(OPERATOR_NM);
+    REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');

base/mp_filterstore.sas

@@ -0,0 +1,230 @@
+/**
+  @file
+  @brief Checks & Stores an input filter table and returns the Filter Key
+  @details Used to generate a FILTER_RK from an input query dataset.  This
+  process requires several permanent tables (names are configurable).  The
+  benefit of storing query values at backend is to enable stored 'views' of
+  filtered tables at frontend (ie, when building [SAS-Powered Apps](
+  https://sasapps.io)).  This macro is also used in [Data Controller for SAS](
+  https://datacontroller.io).
+
+
+  @param [in] libds= The target dataset to be filtered (lib should be assigned)
+  @param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to
+    be validated.  Has the following format:
+|GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
+|---|---|---|---|---|---|
+|AND|AND|1|SOME_BESTNUM|>|1|
+|AND|AND|1|SOME_TIME|=|77333|
+  @param [in] filter_summary= (PERM.FILTER_SUMMARY) Permanent table containing
+    summary filter values.  The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(FILTER_SUMMARY)`. Example
+    values:
+|FILTER_RK:best.|FILTER_HASH:$32.|FILTER_TABLE:$41.|PROCESSED_DTTM:datetime19.|
+|---|---|---|---|
+|`1 `|`540E96F566D194AB58DD4C413C99C9DB `|`VIYA6014.MPE_TABLES `|`1956084246 `|
+|`2 `|`87737DB9EEE2650F5C89956CEAD0A14F `|`VIYA6014.MPE_X_TEST `|`1956084452.1`|
+|`3 `|`8048BD908DBBD83D013560734E90D394 `|`VIYA6014.MPE_TABLES `|`1956093620.6`|
+  @param [in] filter_detail= (PERM.FILTER_DETAIL) Permanent table containing
+    detailed (raw) filter values. The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(FILTER_DETAIL)`. Example
+    values:
+|FILTER_HASH:$32.|FILTER_LINE:best.|GROUP_LOGIC:$3.|SUBGROUP_LOGIC:$3.|SUBGROUP_ID:best.|VARIABLE_NM:$32.|OPERATOR_NM:$12.|RAW_VALUE:$4000.|PROCESSED_DTTM:datetime19.|
+|---|---|---|---|---|---|---|---|---|
+|`540E96F566D194AB58DD4C413C99C9DB `|`1 `|`AND `|`AND `|`1 `|`LIBREF `|`CONTAINS `|`DC`|`1956084245.8 `|
+|`540E96F566D194AB58DD4C413C99C9DB `|`2 `|`AND `|`OR `|`2 `|`DSN `|`= `|` MPE_LOCK_ANYTABLE `|`1956084245.8 `|
+|`87737DB9EEE2650F5C89956CEAD0A14F `|`1 `|`AND `|`AND `|`1 `|`PRIMARY_KEY_FIELD `|`IN `|`(1,2,3) `|`1956084451.9 `|
+  @param [in] lock_table= (PERM.LOCK_TABLE) Permanent locking table.  Used to
+    manage concurrent access.  The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(LOCKTABLE)`.
+  @param [in] maxkeytable= (0) Optional permanent reference table used for
+    retained key tracking.  Described in mp_retainedkey.sas.
+  @param [in] mdebug= set to 1 to enable DEBUG messages
+  @param [out] outresult= The result table with the FILTER_RK
+  @param [out] outquery= The original query, taken as extract after table load
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_getvalue.sas
+  @li mf_islibds.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_filtercheck.sas
+  @li mp_hashdataset.sas
+  @li mp_retainedkey.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filtercheck.sas
+  @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
+  @li mp_filterstore.test.sas
+
+  @version 9.2
+  @author [Allan Bowe](https://www.linkedin.com/in/allanbowe)
+
+**/
+
+%macro mp_filterstore(libds=,
+  queryds=work.filterquery,
+  filter_summary=PERM.FILTER_SUMMARY,
+  filter_detail=PERM.FILTER_DETAIL,
+  lock_table=PERM.LOCK_TABLE,
+  maxkeytable=PERM.MAXKEYTABLE,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+);
+%put &sysmacroname entry vars:;
+%put _local_;
+
+%local ds1 ds2 ds3 ds4 filter_hash;
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+%mp_abort(iftrue= (%mf_islibds(&filter_summary)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid filter_summary value: &filter_summary)
+)
+%mp_abort(iftrue= (%mf_islibds(&filter_detail)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid filter_detail value: &filter_detail)
+)
+%mp_abort(iftrue= (%mf_islibds(&lock_table)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid lock_table value: &lock_table)
+)
+
+/* validate query */
+%mp_filtercheck(&queryds,targetds=&libds,abort=YES)
+
+/* hash the result */
+%let ds1=%mf_getuniquename(prefix=hashds);
+%mp_hashdataset(&queryds,outds=&ds1,salt=&libds)
+%let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
+%if &mdebug=1 %then %do;
+  data _null_;
+    putlog "filter_hash=&filter_hash";
+    set &ds1;
+    putlog (_all_)(=);
+  run;
+%end;
+
+/* check if data already exists for this hash */
+data &outresult;
+  set &filter_summary;
+  where filter_hash="&filter_hash";
+run;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc after hash check)
+)
+%mp_abort(iftrue= ("&filter_hash "=" ")
+  ,mac=mp_filterstore
+  ,msg=%str(problem with filter_hash generation)
+)
+
+%if %mf_nobs(&outresult)=0 %then %do;
+
+  /* first update summary table */
+  %let ds3=%mf_getuniquename(prefix=filtersum);
+  data work.&ds3;
+    if 0 then set &filter_summary;
+    filter_table=symget('libds');
+    filter_hash="&filter_hash";
+    PROCESSED_DTTM=%sysfunc(datetime());
+    output;
+    stop;
+  run;
+
+  %mp_lockanytable(LOCK,
+    lib=%scan(&filter_summary,1,.)
+    ,ds=%scan(&filter_summary,2,.)
+    ,ref=MP_FILTERSTORE summary update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  %let ds4=%mf_getuniquename(prefix=filtersumappend);
+  %mp_retainedkey(
+    base_lib=%scan(&filter_summary,1,.)
+    ,base_dsn=%scan(&filter_summary,2,.)
+    ,append_lib=work
+    ,append_dsn=&ds3
+    ,retained_key=filter_rk
+    ,business_key=filter_hash
+    ,maxkeytable=&maxkeytable
+    ,locktable=&lock_table
+    ,outds=work.&ds4
+  )
+  proc append base=&filter_summary data=&ds4;
+  run;
+
+  %mp_lockanytable(UNLOCK,
+    lib=%scan(&filter_summary,1,.)
+    ,ds=%scan(&filter_summary,2,.)
+    ,ref=MP_FILTERSTORE summary update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  %if &syscc ne 0 %then %do;
+    data _null_;
+      set &ds4;
+      putlog (_all_)(=);
+    run;
+    %goto err;
+  %end;
+
+  data &outresult;
+    set &filter_summary;
+    where filter_hash="&filter_hash";
+  run;
+
+  /* Next, update detail table */
+  %let ds2=%mf_getuniquename(prefix=filterdetail);
+  data &ds2;
+    if 0 then set &filter_detail;
+    set &queryds;
+    format filter_hash $hex32. filter_line 8.;
+    filter_hash="&filter_hash";
+    filter_line=_n_;
+    PROCESSED_DTTM=%sysfunc(datetime());
+  run;
+  %mp_lockanytable(LOCK,
+    lib=%scan(&filter_detail,1,.)
+    ,ds=%scan(&filter_detail,2,.)
+    ,ref=MP_FILTERSTORE update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+  proc append base=&filter_detail data=&ds2;
+  run;
+
+  %mp_lockanytable(UNLOCK,
+    lib=%scan(&filter_detail,1,.)
+    ,ds=%scan(&filter_detail,2,.)
+    ,ref=MP_FILTERSTORE detail update &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  %if &syscc ne 0 %then %do;
+    data _null_;
+      set &ds2;
+      putlog (_all_)(=);
+    run;
+    %goto err;
+  %end;
+
+%end;
+
+proc sort data=&filter_detail(where=(filter_hash="&filter_hash")) out=&outquery;
+  by filter_line;
+run;
+
+%err:
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc on macro exit)
+)
+
+%mend mp_filterstore;

base/mp_filtervalidate.sas

@@ -33,8 +33,8 @@
   @param [in] targetds The target dataset against which to verify the query
   @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
   @param [out] outds= (work.mp_filtervalidate) Output dataset containing the
-  error / warning message, if one exists.  If this table contains any rows,
-  there are problems!
+    err / warning message, if one exists.  If this table contains any rows,
+    there are problems!
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
@@ -78,7 +78,7 @@ run;
 
 data &outds;
   if &sqlrc or &syscc or &syserr then do;
-    REASON_CD='VALIDATION_ERROR: '!!
+    REASON_CD='VALIDATION_ERR'!!'OR: '!!
       coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
     output;
   end;
@@ -96,8 +96,7 @@ filename &fref1 clear;
     run;
     %mp_abort(
       mac=&sysmacroname,
-      msg=%str(Filter validation issues. ERR=%superq(SYSERRORTEXT)
-        , WARN=%superq(SYSWARNINGTEXT) )
+      msg=%str(Filter validation issues.)
     )
   %end;
   %let syscc=1008;

base/mp_getcols.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Creates a dataset with column metadata.
+  @details This macro takes the `proc contents` output and "tidies it up" in the
+  following ways:
+
+    @li Blank labels are filled in with column names
+    @li Formats are reconstructed with default values
+    @li Types such as DATE / TIME / DATETIME are inferred from the formats
+
+  Example usage:
+
+      %mp_getcols(sashelp.airline,outds=work.myds)
+
+  @param ds The dataset from which to obtain column metadata
+  @param outds= (work.cols) The output dataset to create. Sample data:
+|NAME:$32.|LENGTH:best.|VARNUM:best.|LABEL:$256.|FMTNAME:$32.|FORMAT:$49.|TYPE:$1.|DDTYPE:$9.|
+|---|---|---|---|---|---|---|---|
+|`AIR `|`8 `|`2 `|`international airline travel (thousands) `|` `|`8. `|`N `|`NUMERIC `|
+|`DATE `|`8 `|`1 `|`DATE `|`MONYY `|`MONYY. `|`N `|`DATE `|
+|`REGION `|`3 `|`3 `|`REGION `|` `|`$3. `|`C `|`CHARACTER `|
+
+  <h4> Related Macros </h4>
+  @li mf_getvarlist.sas
+  @li mm_getcols.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_getcols(ds, outds=work.cols);
+%local dropds;
+proc contents noprint data=&ds
+  out=_data_ (keep=name type length label varnum format:);
+run;
+%let dropds=&syslast;
+data &outds(keep=name type length varnum format label ddtype fmtname);
+  set &dropds(rename=(format=fmtname type=type2));
+  name=upcase(name);
+  if type2=2 then do;
+    length format $49.;
+    if fmtname='' then format=cats('$',length,'.');
+    else if formatl=0 then format=cats(fmtname,'.');
+    else format=cats(fmtname,formatl,'.');
+    type='C';
+    ddtype='CHARACTER';
+  end;
+  else do;
+    if fmtname='' then format=cats(length,'.');
+    else if formatl=0 then format=cats(fmtname,'.');
+    else if formatd=0 then format=cats(fmtname,formatl,'.');
+    else format=cats(fmtname,formatl,'.',formatd);
+    type='N';
+    if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
+    else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
+      or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
+      or format=:'MONYY'
+    then ddtype='DATE';
+    else if format=:'TIME' then ddtype='TIME';
+    else ddtype='NUMERIC';
+  end;
+  if label='' then label=name;
+run;
+proc sql;
+drop table &dropds;
+%mend mp_getcols;

base/mp_getconstraints.sas

@@ -4,24 +4,27 @@
   @details Useful for capturing constraints before they are dropped / reapplied
   during an update.
 
-        proc sql;
-        create table work.example(
-          TX_FROM float format=datetime19.,
-          DD_TYPE char(16),
-          DD_SOURCE char(2048),
-          DD_SHORTDESC char(256),
-          constraint pk primary key(tx_from, dd_type,dd_source),
-          constraint unq unique(tx_from, dd_type),
-          constraint nnn not null(DD_SHORTDESC)
-        );
+      proc sql;
+      create table work.example(
+        TX_FROM float format=datetime19.,
+        DD_TYPE char(16),
+        DD_SOURCE char(2048),
+        DD_SHORTDESC char(256),
+        constraint pk primary key(tx_from, dd_type,dd_source),
+        constraint unq unique(tx_from, dd_type),
+        constraint nnn not null(DD_SHORTDESC)
+      );
 
       %mp_getconstraints(lib=work,ds=example,outds=work.constraints)
 
-  @param lib= The target library (default=WORK)
-  @param ds= The target dataset.  Leave blank (default) for all datasets.
-  @param outds the output dataset
+  @param [in] lib= (WORK) The target library
+  @param [in] ds= The target dataset.  Leave blank (default) for all datasets.
+  @param [in] mdebug= (0) Set to 1 to preserve temp tables, print var values etc
+  @param [out] outds= (mp_getconstraints) the output dataset
 
   <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mp_dropmembers.sas
 
   @version 9.2
   @author Allan Bowe
@@ -31,30 +34,63 @@
 %macro mp_getconstraints(lib=WORK
   ,ds=
   ,outds=mp_getconstraints
+  ,mdebug=0
 )/*/STORE SOURCE*/;
 
 %let lib=%upcase(&lib);
 %let ds=%upcase(&ds);
 
+/**
+  * Neither dictionary tables nor sashelp provides a constraint order column,
+  * however they DO arrive in the correct order.  So, create the col.
+  **/
+%local vw;
+%let vw=%mf_getuniquename(prefix=mp_getconstraints_vw_);
+data &vw /view=&vw;
+  set sashelp.vcncolu;
+  where table_catalog="&lib";
+
+  /* use retain approach to reset the constraint order with each constraint */
+  length tmp $1000;
+  retain tmp;
+  drop tmp;
+  if tmp ne catx('|',table_catalog,table_name,constraint_name) then do;
+    constraint_order=1;
+  end;
+  else constraint_order+1;
+  tmp=catx('|',table_catalog, table_name,constraint_name);
+run;
+
 /* must use SQL as proc datasets does not support length changes */
 proc sql noprint;
 create table &outds as
-  select a.TABLE_CATALOG as libref
-    ,a.TABLE_NAME
+  select upcase(a.TABLE_CATALOG) as libref
+    ,upcase(a.TABLE_NAME) as TABLE_NAME
     ,a.constraint_type
     ,a.constraint_name
     ,b.column_name
+    ,b.constraint_order
   from dictionary.TABLE_CONSTRAINTS a
-  left join dictionary.constraint_column_usage  b
-  on a.TABLE_CATALOG=b.TABLE_CATALOG
-    and a.TABLE_NAME=b.TABLE_NAME
+  left join &vw  b
+  on upcase(a.TABLE_CATALOG)=upcase(b.TABLE_CATALOG)
+    and upcase(a.TABLE_NAME)=upcase(b.TABLE_NAME)
     and a.constraint_name=b.constraint_name
-  where a.TABLE_CATALOG="&lib"
-    and b.TABLE_CATALOG="&lib"
+/**
+  * We cannot apply this clause to the underlying dictionary table.  See:
+  * https://communities.sas.com/t5/SAS-Programming/Unexpected-Where-Clause-behaviour-in-dictionary-TABLE/m-p/771554#M244867
+  */
+  where calculated libref="&lib"
   %if "&ds" ne "" %then %do;
-    and a.TABLE_NAME="&ds"
-    and b.TABLE_NAME="&ds"
+    and upcase(a.TABLE_NAME)="&ds"
+    and upcase(b.TABLE_NAME)="&ds"
   %end;
+  order by libref, table_name, constraint_name, constraint_order
   ;
 
+/* tidy up */
+%mp_dropmembers(
+  &vw,
+  iftrue=(&mdebug=0)
+)
+
 %mend mp_getconstraints;

base/mp_getddl.sas

@@ -5,6 +5,8 @@
     to create tables in SAS or a database.  The macro can be used at table or
     library level.  The default behaviour is to create DDL in SAS format.
 
+    Note - views are not currently supported.
+
   Usage:
 
       data test(index=(pk=(x y)/unique /nomiss));
@@ -16,12 +18,14 @@
       %mp_getddl(work,test,flavour=tsql,showlog=YES)
 
   <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mf_getvarcount.sas
   @li mp_getconstraints.sas
 
   @param lib libref of the library to create DDL for.  Should be assigned.
   @param ds dataset to create ddl for (optional)
-  @param fref= the fileref to which to write the DDL.  If not preassigned, will
-    be assigned to TEMP.
+  @param fref= the fileref to which to _append_ the DDL.  If it does not exist,
+    it will be created.
   @param flavour= The type of DDL to create (default=SAS). Supported=TSQL
   @param showlog= Set to YES to show the DDL in the log
   @param schema= Choose a preferred schema name (default is to use actual schema
@@ -37,9 +41,10 @@
 )/*/STORE SOURCE*/;
 
 /* check fileref is assigned */
-%if %sysfunc(fileref(&fref)) > 0 %then %do;
-  filename &fref temp;
+%if %mf_existfileref(&fref)=0 %then %do;
+  filename &fref temp ;
 %end;
+
 %if %length(&libref)=0 %then %let libref=WORK;
 %let flavour=%upcase(&flavour);
 
@@ -47,6 +52,7 @@ proc sql noprint;
 create table _data_ as
   select * from dictionary.tables
   where upcase(libname)="%upcase(&libref)"
+    and memtype='DATA' /* views not currently supported */
   %if %length(&ds)>0 %then %do;
     and upcase(memname)="%upcase(&ds)"
   %end;
@@ -118,7 +124,7 @@ create table _data_ as
 %mend addConst;
 
 data _null_;
-  file &fref;
+  file &fref mod;
   put "/* DDL generated by &sysuserid on %sysfunc(datetime(),datetime19.) */";
 run;
 
@@ -133,7 +139,7 @@ run;
     %let curds=%scan(&dsnlist,&x);
     data _null_;
       file &fref mod;
-      length nm lab $1024 typ $20;
+      length lab $1024 typ $20;
       set &colinfo (where=(upcase(memname)="&curds")) end=last;
 
       if _n_=1 then do;
@@ -141,16 +147,18 @@ run;
           put "create table &libref..&curds(";
         end;
         else do;
+          /* just a placeholder - we filter out views at the top */
           put "create view &libref..&curds(";
         end;
         put "    "@@;
       end;
       else put "   ,"@@;
       if length(format)>1 then fmt=" format="!!cats(format);
-      if length(label)>1 then lab=" label="!!quote(trim(label));
+      if length(label)>1 then
+        lab=" label="!!cats("'",tranwrd(label,"'","''"),"'");
       if notnull='yes' then notnul=' not null';
       if type='char' then typ=cats('char(',length,')');
-      else if length ne 8 then typ='num length='!!left(length);
+      else if length ne 8 then typ='num length='!!cats(length);
       else typ='num';
       put name typ fmt notnul lab;
     run;
@@ -203,7 +211,7 @@ run;
   proc sql noprint;
   select sysvalue into: schemaactual
     from dictionary.libnames
-    where libname="&libref" and engine='SQLSVR';
+    where upcase(libname)="&libref" and engine='SQLSVR';
   %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
 
   %do x=1 %to %sysfunc(countw(&dsnlist));
@@ -219,6 +227,7 @@ run;
           put "create table [&schema].[&curds](";
         end;
         else do;
+          /* just a placeholder - we filter out views at the top */
           put "create view [&schema].[&curds](";
         end;
         put "    "@@;
@@ -295,78 +304,88 @@ run;
   proc sql noprint;
   select sysvalue into: schemaactual
     from dictionary.libnames
-    where libname="&libref" and engine='POSTGRES';
+    where upcase(libname)="&libref" and engine='POSTGRES';
   %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
   data _null_;
     file &fref mod;
     put "CREATE SCHEMA &schema;";
   %do x=1 %to %sysfunc(countw(&dsnlist));
     %let curds=%scan(&dsnlist,&x);
-    data _null_;
-      file &fref mod;
-      put "/* Postgres Flavour DDL for &schema..&curds */";
-    data _null_;
-      file &fref mod;
-      set &colinfo (where=(upcase(memname)="&curds")) end=last;
-      length fmt $32;
-      if _n_=1 then do;
-        if memtype='DATA' then do;
-          put "CREATE TABLE &schema..&curds (";
+    %local curdsvarcount;
+    %let curdsvarcount=%mf_getvarcount(&libref..&curds);
+    %if &curdsvarcount>1600 %then %do;
+      data _null_;
+        file &fref mod;
+        put "/* &libref..&curds contains &curdsvarcount vars */";
+        put "/* Postgres cannot create tables with over 1600 vars */";
+        put "/* No DDL will be generated for this table";
+      run;
+    %end;
+    %else %do;
+      data _null_;
+        file &fref mod;
+        put "/* Postgres Flavour DDL for &schema..&curds */";
+      data _null_;
+        file &fref mod;
+        set &colinfo (where=(upcase(memname)="&curds")) end=last;
+        length fmt $32;
+        if _n_=1 then do;
+          if memtype='DATA' then do;
+            put "CREATE TABLE &schema..&curds (";
+          end;
+          else do;
+            /* just a placeholder - we filter out views at the top */
+            put "CREATE VIEW &schema..&curds (";
+          end;
+          put "    "@@;
         end;
-        else do;
-          put "CREATE VIEW &schema..&curds (";
-        end;
-        put "    "@@;
-      end;
-      else put "   ,"@@;
-      format=upcase(format);
-      if 1=0 then; /* dummy if */
-      %if &applydttm=YES %then %do;
-        else if format=:'DATETIME' then fmt=' TIMESTAMP ';
-      %end;
-      else if type='num' then fmt=' DOUBLE PRECISION';
-      else fmt='VARCHAR('!!cats(length)!!')';
-      if notnull='yes' then notnul=' NOT NULL';
-      /* quote column names in case they represent reserved words */
-      name2=quote(trim(name));
-      put name2 fmt notnul;
-    run;
+        else put "   ,"@@;
+        format=upcase(format);
+        if 1=0 then; /* dummy if */
+        %if &applydttm=YES %then %do;
+          else if format=:'DATETIME' then fmt=' TIMESTAMP ';
+        %end;
+        else if type='num' then fmt=' DOUBLE PRECISION';
+        else fmt='VARCHAR('!!cats(length)!!')';
+        if notnull='yes' then notnul=' NOT NULL';
+        /* quote column names in case they represent reserved words */
+        name2=quote(trim(name));
+        put name2 fmt notnul;
+      run;
 
-    /* Extra step for data constraints */
-    %addConst()
+      /* Extra step for data constraints */
+      %addConst()
 
-    data _null_;
-      file &fref mod;
-      put ');';
-    run;
+      data _null_;
+        file &fref mod;
+        put ');';
+      run;
 
-    /* Create Unique Indexes, but only if they were not already defined within
-      the Constraints section. */
-    data _null_;
-      *length ds $128;
-      set &idxinfo(
-        where=(
-          memname="&curds"
-          and unique='yes'
-          and indxname not in (
-            %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+      /* Create Unique Indexes, but only if they were not already defined within
+        the Constraints section. */
+      data _null_;
+        *length ds $128;
+        set &idxinfo(
+          where=(
+            memname="&curds"
+            and unique='yes'
+            and indxname not in (
+              %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+            )
           )
-        )
-      );
-      file &fref mod;
-      by idxusage indxname;
-/*       ds=cats(libname,'.',memname); */
-      if first.indxname then do;
+        );
+        file &fref mod;
+        by idxusage indxname;
+        if first.indxname then do;
           put 'CREATE UNIQUE INDEX "' indxname +(-1) '" ' "ON &schema..&curds(";
           put '  "' name +(-1) '"' ;
-      end;
-      else put '  ,"' name +(-1) '"';
-      *else put '    ,' name ;
-      if last.indxname then do;
-        put ');';
-      end;
-    run;
-
+        end;
+        else put '  ,"' name +(-1) '"';
+        if last.indxname then do;
+          put ');';
+        end;
+      run;
+    %end;
   %end;
 %end;
 %if %upcase(&showlog)=YES %then %do;

base/mp_getformats.sas

@@ -0,0 +1,142 @@
+/**
+  @file
+  @brief Export format definitions
+  @details Formats are exported from the first (if any) catalog entry in the
+  FMTSEARCH path.
+
+  Formats are taken from the library / dataset reference and / or a static
+  format list.
+
+  Example usage:
+
+      %mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable)
+
+  @param [in] lib= (0) The libref for which to return formats.
+  @todo Enable exporting of formats for an entire library
+  @param [in] ds= (0) The dataset from which to obtain format definitions
+  @param [in] fmtlist= (0) A list of additional format names
+  @param [out] outsummary= (work.mp_getformats_summary) Output dataset
+    containing summary definitions - structure taken from dictionary.formats as
+    follows:
+
+  |libname:$8.|memname:$32.|path:$1024.|objname:$32.|fmtname:$32.|fmttype:$1.|source:$1.|minw:best.|mind:best.|maxw:best.|maxd:best.|defw:best.|defd:best.|
+  |---|---|---|---|---|---|---|---|---|---|---|---|---|
+  | | | | |$|F|B|1|0|32767|0|1|0|
+  | | | | |$|I|B|1|0|32767|0|1|0|
+  |` `|` `|/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWIANYDT|$ANYDTIF|I|U|1|0|60|0|19|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWFASCII|$ASCII|F|U|1|0|32767|0|1|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWIASCII|$ASCII|I|U|1|0|32767|0|1|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWFBASE6|$BASE64X|F|U|1|0|32767|0|1|0|
+
+
+  @param [out] outdetail= (0) Provide an output dataset in which to export all
+    the custom format definitions (from proc format CNTLOUT).  Definitions:
+https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#a002473477.htm
+    Sample data:
+
+  |FMTNAME:$32.|START:$16.|END:$16.|LABEL:$256.|MIN:best.|MAX:best.|DEFAULT:best.|LENGTH:best.|FUZZ:best.|PREFIX:$2.|MULT:best.|FILL:$1.|NOEDIT:best.|TYPE:$1.|SEXCL:$1.|EEXCL:$1.|HLO:$13.|DECSEP:$1.|DIG3SEP:$1.|DATATYPE:$8.|LANGUAGE:$8.|
+  |---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
+  |`WHICHPATH `|`0 `|`0 `|`path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|` `|` `|` `|` `|` `|
+  |`WHICHPATH `|`**OTHER** `|`**OTHER** `|`big fat problem if not path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|`O `|` `|` `|` `|` `|
+
+  <h4> SAS Macros </h4>
+  @li mf_dedup.sas
+  @li mf_getfmtlist.sas
+  @li mf_getfmtname.sas
+  @li mf_getquotedstr.sas
+  @li mf_getuniquename.sas
+
+
+  <h4> Related Macros </h4>
+  @li mp_applyformats.sas
+  @li mp_getformats.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_getformats(lib=0
+  ,ds=0
+  ,fmtlist=0
+  ,outsummary=work.mp_getformats_summary
+  ,outdetail=0
+);
+
+%local i fmt allfmts tempds fmtcnt;
+
+%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( )));
+  /* ensure format list contains format _name_ only */
+  %let fmt=%scan(&fmtlist,&i,%str( ));
+  %let fmt=%mf_getfmtname(&fmt);
+  %let allfmts=&allfmts &fmt;
+%end;
+
+%if &ds=0 and &lib ne 0 %then %do;
+  /* grab formats from library */
+  /* to do */
+%end;
+%else %if &ds ne 0 and &lib ne 0 %then %do;
+  /* grab formats from dataset */
+  %let allfmts=%mf_getfmtlist(&lib..&ds) &allfmts;
+%end;
+
+/* ensure list is unique */
+%let allfmts=%mf_dedup(%upcase(&allfmts));
+
+/* create summary table */
+%if %index(&outsummary,.)=0 %then %let outsummary=WORK.&outsummary;
+proc sql;
+create table &outsummary as
+  select * from dictionary.formats
+  where fmtname in (%mf_getquotedstr(&allfmts,quote=D))
+    and fmttype='F';
+
+%if "&outdetail" ne "0" %then %do;
+  /* ensure base table always exists */
+  proc sql;
+  create table &outdetail(
+      FMTNAME char(32)     label='Format name'
+      ,START char(16)     label='Starting value for format'
+      ,END char(16)     label='Ending value for format'
+      ,LABEL char(256)     label='Format value label'
+      ,MIN num length=3     label='Minimum length'
+      ,MAX num length=3     label='Maximum length'
+      ,DEFAULT num length=3     label='Default length'
+      ,LENGTH num length=3     label='Format length'
+      ,FUZZ num     label='Fuzz value'
+      ,PREFIX char(2)     label='Prefix characters'
+      ,MULT num     label='Multiplier'
+      ,FILL char(1)     label='Fill character'
+      ,NOEDIT num length=3     label='Is picture string noedit?'
+      ,TYPE char(1)     label='Type of format'
+      ,SEXCL char(1)     label='Start exclusion'
+      ,EEXCL char(1)     label='End exclusion'
+      ,HLO char(13)     label='Additional information'
+      ,DECSEP char(1)     label='Decimal separator'
+      ,DIG3SEP char(1)     label='Three-digit separator'
+      ,DATATYPE char(8)     label='Date/time/datetime?'
+      ,LANGUAGE char(8)     label='Language for date strings'
+  );
+  /* grab the location of each format */
+  %let fmtcnt=0;
+  data _null_;
+    set &outsummary;
+    if not missing(libname);
+    x+1;
+    call symputx(cats('fmtloc',x),cats(libname,'.',memname),'l');
+    call symputx(cats('fmtname',x),fmtname,'l');
+    call symputx('fmtcnt',x,'l');
+  run;
+  /* export each format and append to the output table */
+  %let tempds=%mf_getuniquename(prefix=mp_getformats);
+  %do i=1 %to &fmtcnt;
+    proc format library=&&fmtloc&i CNTLOUT=&tempds;
+      select &&fmtname&i;
+    run;
+    proc append base=&outdetail data=&tempds;
+    run;
+  %end;
+%end;
+
+%mend mp_getformats;

base/mp_getmaxvarlengths.sas

@@ -1,28 +1,46 @@
 /**
-  @file mp_getmaxvarlengths.sas
+  @file
   @brief Scans a dataset to find the max length of the variable values
   @details
   This macro will scan a base dataset and produce an output dataset with two
   columns:
 
   - NAME    Name of the base dataset column
-  - MAXLEN Maximum length of the data contained therein.
+  - MAXLEN  Maximum length of the data contained therein.
 
-  Character fields may be allocated very large widths (eg 32000) of which the
-  maximum  value is likely to be much narrower.  This macro was designed to
-  enable a HTML table to be appropriately sized however this could be used as
-  part of a data audit to ensure we aren't over-sizing our tables in relation to
-  the data therein.
+  Character fields are often allocated very large widths (eg 32000) of which the
+  maximum  value is likely to be much narrower.  Identifying such cases can be
+  helpful in the following scenarios:
+
+  @li Enabling a HTML table to be appropriately sized (`num2char=YES`)
+  @li Reducing the size of a dataset to save on storage (mp_ds2squeeze.sas)
+  @li Identifying columns containing nothing but missing values (`MAXLEN=0` in
+    the output table)
+
+  If the entire column is made up of (non-special) missing values then a value
+  of 0 is returned.
 
-  Numeric fields are converted using the relevant format to determine the width.
   Usage:
 
       %mp_getmaxvarlengths(sashelp.class,outds=work.myds)
 
-  @param libds Two part dataset (or view) reference.
-  @param outds= The output dataset to create
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] num2char= (NO) When set to NO, numeric fields are sized according
+    to the number of bytes used (or set to zero in the case of non-special
+    missings). When YES, the numeric field is converted to character (using the
+    format, if available), and that is sized instead, using `lengthn()`.
+  @param [out] outds= The output dataset to create, eg:
+  |NAME:$8.|MAXLEN:best.|
+  |---|---|
+  |`Name `|`7 `|
+  |`Sex `|`1 `|
+  |`Age `|`3 `|
+  |`Height `|`8 `|
+  |`Weight `|`3 `|
 
   <h4> SAS Macros </h4>
+  @li mcf_length.sas
+  @li mf_getuniquename.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
   @li mf_getvarformat.sas
@@ -30,20 +48,32 @@
   @version 9.2
   @author Allan Bowe
 
+  <h4> Related Macros </h4>
+  @li mp_ds2squeeze.sas
+  @li mp_getmaxvarlengths.test.sas
+
 **/
 
 %macro mp_getmaxvarlengths(
-    libds      /* libref.dataset to analyse */
-    ,outds=work.mp_getmaxvarlengths /* name of output dataset to create */
+  libds
+  ,num2char=NO
+  ,outds=work.mp_getmaxvarlengths
 )/*/STORE SOURCE*/;
 
-%local vars x var fmt;
+%local vars prefix x var fmt;
 %let vars=%mf_getvarlist(libds=&libds);
+%let prefix=%substr(%mf_getuniquename(),1,25);
+%let num2char=%upcase(&num2char);
+
+%if &num2char=NO %then %do;
+  /* compile length function for numeric fields */
+  %mcf_length(wrap=YES, insert_cmplib=YES)
+%end;
 
 proc sql;
 create table &outds (rename=(
     %do x=1 %to %sysfunc(countw(&vars,%str( )));
-      ________&x=%scan(&vars,&x)
+      &prefix.&x=%scan(&vars,&x)
     %end;
     ))
   as select
@@ -51,18 +81,21 @@ create table &outds (rename=(
       %let var=%scan(&vars,&x);
       %if &x>1 %then ,;
       %if %mf_getvartype(&libds,&var)=C %then %do;
-        max(length(&var)) as ________&x
+        max(lengthn(&var)) as &prefix.&x
       %end;
-      %else %do;
+      %else %if &num2char=YES %then %do;
         %let fmt=%mf_getvarformat(&libds,&var);
         %put fmt=&fmt;
         %if %str(&fmt)=%str() %then %do;
-          max(length(cats(&var))) as ________&x
+          max(lengthn(cats(&var))) as &prefix.&x
         %end;
         %else %do;
-          max(length(put(&var,&fmt))) as ________&x
+          max(lengthn(put(&var,&fmt))) as &prefix.&x
         %end;
       %end;
+      %else %do;
+        max(mcf_length(&var)) as &prefix.&x
+      %end;
     %end;
   from &libds;
 

base/mp_getpk.sas

@@ -0,0 +1,259 @@
+/**
+  @file
+  @brief Extract the primary key fields from a table or library
+  @details Examines the constraints to identify primary key fields - indicated
+  by an explicit PK constraint, or a unique index that is also NOT NULL.
+
+  Can be executed at both table and library level.  Supports both BASE engine
+  libraries and SQL Server.
+
+  Usage:
+
+      proc sql;
+      create table work.example(
+        TX_FROM float format=datetime19.,
+        DD_TYPE char(16),
+        DD_SOURCE char(2048),
+        DD_SHORTDESC char(256),
+        constraint pk primary key(tx_from, dd_type,dd_source),
+        constraint unq unique(tx_from, dd_type),
+        constraint nnn not null(DD_SHORTDESC)
+      );
+      %mp_getpk(work,ds=example)
+
+  Returns:
+
+|libref:$8.|dsn:$32.|memtype:$8.|dbms_memtype:$32.|typemem:$8.|memlabel:$256.|nvar:best.|compress:$8.|pk_fields:$512.|
+|---|---|---|---|---|---|---|---|---|
+|WORK|EXAMPLE|DATA| |DATA| |4|NO|TX_FROM DD_TYPE DD_SOURCE|
+
+
+  @param [in] lib The libref to examine
+  @param [in] ds= (0) Select the dataset to examine, else use 0 for all tables
+  @param [in] mdebug= (0) Set to 1 to preserve temp tables, print var values etc
+  @param [out] outds= (work.mp_getpk) The name of the output table to create.
+
+  <h4> SAS Macros </h4>
+  @li mf_getengine.sas
+  @li mf_getschema.sas
+  @li mp_dropmembers.sas
+  @li mp_getconstraints.sas
+
+  <h4> Related Macros </h4>
+  @li mp_getpk.test.sas
+  @li mp_guesspk.sas
+
+  @version 9.3
+  @author Macro People Ltd
+**/
+
+%macro mp_getpk(
+  lib,
+  ds=0,
+  outds=work.mp_getpk,
+  mdebug=0
+)/*/STORE SOURCE*/;
+
+
+%local engine schema ds1 ds2 ds3 dsn tabs1 tabs2 sum pk4sure pkdefault finalpks;
+
+%let lib=%upcase(&lib);
+%let ds=%upcase(&ds);
+%let engine=%mf_getengine(&lib);
+%let schema=%mf_getschema(&lib);
+
+%let ds1=%mf_getuniquename(prefix=getpk_ds1);
+%let ds2=%mf_getuniquename(prefix=getpk_ds2);
+%let ds3=%mf_getuniquename(prefix=getpk_ds3);
+%let tabs1=%mf_getuniquename(prefix=getpk_tabs1);
+%let tabs2=%mf_getuniquename(prefix=getpk_tabs2);
+%let sum=%mf_getuniquename(prefix=getpk_sum);
+%let pk4sure=%mf_getuniquename(prefix=getpk_pk4sure);
+%let pkdefault=%mf_getuniquename(prefix=getpk_pkdefault);
+%let finalpks=%mf_getuniquename(prefix=getpk_finalpks);
+
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+proc sql;
+create table &ds1 as
+  select  libname as libref
+    ,upcase(memname) as dsn
+    ,memtype
+    ,upcase(name) as name
+    ,type
+    ,length
+    ,varnum
+    ,label
+    ,format
+    ,idxusage
+    ,notnull
+  from dictionary.columns
+  where upcase(libname)="&lib"
+%if &ds ne 0 %then %do;
+    and upcase(memname)="&ds"
+%end;
+  ;
+
+
+%if &engine=SQLSVR %then %do;
+  proc sql;
+  connect using &lib;
+  create table work.&ds2 as
+  select * from connection to &lib(
+  select
+      s.name as SchemaName,
+      t.name as memname,
+      tc.name as name,
+      ic.key_ordinal as KeyOrderNr
+  from
+      sys.schemas s
+      inner join sys.tables t   on s.schema_id=t.schema_id
+      inner join sys.indexes i  on t.object_id=i.object_id
+      inner join sys.index_columns ic on i.object_id=ic.object_id
+                                    and i.index_id=ic.index_id
+      inner join sys.columns tc on ic.object_id=tc.object_id
+                              and ic.column_id=tc.column_id
+  where i.is_primary_key=1
+    and s.name=%str(%')&schema%str(%')
+  order by t.name, ic.key_ordinal ;
+  );disconnect from &lib;
+  create table &ds3 as
+    select a.*
+      ,case when b.name is not null then 1 else 0 end as pk_ind
+    from work.&ds1 a
+    left join work.&ds2 b
+    on a.dsn=b.memname
+      and upcase(a.name)=upcase(b.name)
+    order by libref,dsn;
+%end;
+%else %do;
+
+  %if &ds = 0 %then %let dsn=;
+
+  /* get all constraints, in constraint order*/
+  %mp_getconstraints(lib=&lib,ds=&dsn,outds=work.&ds2)
+
+  /* extract cols that are clearly primary keys */
+  proc sql;
+  create table &pk4sure as
+    select libref
+      ,table_name
+      ,constraint_name
+      ,constraint_order
+      ,column_name as name
+    from work.&ds2
+    where constraint_type='PRIMARY'
+    order by 1,2,3,4;
+
+  /* extract unique constraints where every col is also NOT NULL */
+  proc sql;
+  create table &sum as
+    select a.libref
+      ,a.table_name
+      ,a.constraint_name
+      ,count(a.column_name) as unq_cnt
+      ,count(b.column_name) as nul_cnt
+    from work.&ds2(where=(constraint_type ='UNIQUE')) a
+    left join work.&ds2(where=(constraint_type ='NOT NULL')) b
+    on a.libref=b.libref
+      and a.table_name=b.table_name
+      and a.column_name=b.column_name
+    group by 1,2,3
+    having unq_cnt=nul_cnt;
+
+  /* extract cols from the relevant unique constraints */
+  create table &pkdefault as
+    select a.libref
+      ,a.table_name
+      ,a.constraint_name
+      ,b.constraint_order
+      ,b.column_name as name
+    from &sum a
+    left join &ds2(where=(constraint_type ='UNIQUE')) b
+    on a.libref=b.libref
+      and a.table_name=b.table_name
+      and a.constraint_name=b.constraint_name
+    order by 1,2,3,4;
+
+  /* create one table */
+  data &finalpks;
+    set &pkdefault &pk4sure ;
+    pk_ind=1;
+    /* if there are multiple unique constraints, take the first */
+    by libref table_name constraint_name;
+    retain keepme;
+    if first.table_name then keepme=1;
+    if first.constraint_name and not first.table_name then keepme=0;
+    if keepme=1;
+  run;
+
+  /* join back to starting table */
+  proc sql;
+  create table &ds3 as
+    select a.*
+      ,b.constraint_order
+      ,case when b.pk_ind=1 then 1 else 0 end as pk_ind
+    from work.&ds1 a
+    left join work.&finalpks b
+    on a.libref=b.libref
+      and a.dsn=b.table_name
+      and upcase(a.name)=upcase(b.name)
+    order by libref,dsn,constraint_order;
+%end;
+
+
+/* prepare tables */
+proc sql;
+create table work.&tabs1 as select
+  libname as libref
+  ,upcase(memname) as dsn
+  ,memtype
+  ,dbms_memtype
+  ,typemem
+  ,memlabel
+  ,nvar
+  ,compress
+from dictionary.tables
+  where upcase(libname)="&lib"
+%if &ds ne 0 %then %do;
+    and upcase(memname)="&ds"
+%end;
+  ;
+data &tabs2;
+  set &ds3;
+  length pk_fields $512;
+  retain pk_fields;
+  by libref dsn constraint_order;
+  if first.dsn then pk_fields='';
+  if pk_ind=1 then pk_fields=catx(' ',pk_fields,name);
+  if last.dsn then output;
+run;
+
+proc sql;
+create table &outds as
+  select a.libref
+    ,a.dsn
+    ,a.memtype
+    ,a.dbms_memtype
+    ,a.typemem
+    ,a.memlabel
+    ,a.nvar
+    ,a.compress
+    ,b.pk_fields
+  from work.&tabs1 a
+  left join work.&tabs2 b
+  on a.libref=b.libref
+    and a.dsn=b.dsn;
+
+/* tidy up */
+%mp_dropmembers(
+  &ds1 &ds2 &ds3 &dsn &tabs1 &tabs2 &sum &pk4sure &pkdefault &finalpks,
+  iftrue=(&mdebug=0)
+)
+
+%mend mp_getpk;

base/mp_gsubfile.sas

@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Performs a text substitution on a file
+  @details Makes use of the GSUB function in LUA to perform a text substitution
+  in a file - either in-place, or writing to a new location.  The benefit of
+  using LUA is that the entire file can be loaded into a single variable,
+  thereby side stepping the 32767 character limit in a data step.
+
+  Usage:
+
+      %let file=%sysfunc(pathname(work))/file.txt;
+      %let str=replace/me;
+      %let rep=with/this;
+      data _null_;
+        file "&file";
+        put "&str";
+      run;
+      %mp_gsubfile(file=&file, patternvar=str, replacevar=rep)
+      data _null_;
+        infile "&file";
+        input;
+        list;
+      run;
+
+  @param file= (0) The file to perform the substitution on
+  @param patternvar= A macro variable containing the Lua
+    [pattern](https://www.lua.org/pil/20.2.html) to search for.  Due to the use
+    of special (magic) characters in Lua patterns, it is safer to pass the NAME
+    of the macro variable containing the string, rather than the value itself.
+  @param replacevar= The name of the macro variable containing the replacement
+    _string_.
+  @param outfile= (0) The file to write the output to. If zero, then the file
+    is overwritten in-place.
+
+  <h4> SAS Macros </h4>
+  @li ml_gsubfile.sas
+
+  <h4> Related Macros </h4>
+  @li mp_gsubfile.test.sas
+
+  @version 9.4
+  @author Allan Bowe
+**/
+
+%macro mp_gsubfile(file=0,
+  patternvar=,
+  replacevar=,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+  %ml_gsubfile()
+
+%mend mp_gsubfile;

base/mp_guesspk.sas

@@ -1,210 +1,268 @@
 /**
-  @file mp_guesspk.sas
+  @file
   @brief Guess the primary key of a table
-  @details Tries to guess the primary key of a table based on the following logic:
+  @details Tries to guess the primary key of a table based on the following
+  logic:
 
       * Columns with nulls are ignored
       * Return only column combinations that provide unique results
-      * Start from one column, then move out to include composite keys of 2 to 6 columns
+      * Start from one column, then move out to composite keys of 2 to 6 columns
 
   The library of the target should be assigned before using this macro.
 
   Usage:
 
-      filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      filename mc url
+        "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
       %mp_guesspk(sashelp.class,outds=classpks)
 
-  @param baseds The dataset to analyse
-  @param outds= The output dataset to contain the possible PKs
-  @param max_guesses= The total number of possible primary keys to generate.  A
-    table is likely to have multiple unlikely PKs, so no need to list them all. Default=3.
-  @param min_rows= The minimum number of rows a table should have in order to try
-    and guess the PK.  Default=5.
+  @param [in] baseds The dataset to analyse
+  @param [out] outds= The output dataset to contain the possible PKs
+  @param [in] max_guesses= (3) The total number of possible primary keys to
+    generate. A table may have multiple (unlikely) PKs, so no need to list them
+    all.
+  @param [in] min_rows= (5) The minimum number of rows a table should have in
+    order to try and guess the PK.
+  @param [in] ignore_cols (0) Space seperated list of columns which you are
+    sure are not part of the primary key (helps to avoid false positives)
+  @param [in] mdebug= Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
   @li mf_getvarlist.sas
   @li mf_getuniquename.sas
+  @li mf_wordsInstr1butnotstr2.sas
   @li mf_nobs.sas
 
+  <h4> Related Macros </h4>
+  @li mp_getpk.sas
+
   @version 9.3
   @author Allan Bowe
 
 **/
 
 %macro mp_guesspk(baseds
-      ,outds=mp_guesspk
-      ,max_guesses=3
-      ,min_rows=5
+  ,outds=mp_guesspk
+  ,max_guesses=3
+  ,min_rows=5
+  ,ignore_cols=0
+  ,mdebug=0
 )/*/STORE SOURCE*/;
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
 
-  /* declare local vars */
-  %local var vars vcnt i j k l tmpvar tmpds rows posspks ppkcnt;
-  %let vars=%mf_getvarlist(&baseds);
-  %let vcnt=%sysfunc(countw(&vars));
+/* declare local vars */
+%local var vars vcnt i j k l tmpvar tmpds rows posspks ppkcnt;
+%let vars=%upcase(%mf_getvarlist(&baseds));
+%let vars=%mf_wordsInStr1ButNotStr2(str1=&vars,str2=%upcase(&ignore_cols));
+%let vcnt=%sysfunc(countw(&vars));
 
-  %if &vcnt=0 %then %do;
-    %put &sysmacroname: &baseds has no variables!  Exiting.;
-    %return;
+%if &vcnt=0 %then %do;
+  %put &sysmacroname: &baseds has no variables!  Exiting.;
+  %return;
+%end;
+
+/* get null count and row count */
+%let tmpvar=%mf_getuniquename();
+proc sql noprint;
+create table _data_ as select
+  count(*) as &tmpvar
+%do i=1 %to &vcnt;
+  %let var=%scan(&vars,&i);
+  ,sum(case when &var is missing then 1 else 0 end) as &var
+%end;
+  from &baseds;
+
+/* transpose table and scan for not null cols */
+proc transpose;
+data _null_;
+  set &syslast end=last;
+  length vars $32767;
+  retain vars ;
+  if _name_="&tmpvar" then call symputx('rows',col1,'l');
+  else if col1=0 then vars=catx(' ',vars,_name_);
+  if last then call symputx('posspks',vars,'l');
+run;
+
+%let ppkcnt=%sysfunc(countw(&posspks));
+%if &ppkcnt=0 %then %do;
+  %put &sysmacroname: &baseds has no non-missing variables!  Exiting.;
+  %return;
+%end;
+
+proc sort data=&baseds(keep=&posspks) out=_data_ noduprec;
+  by _all_;
+run;
+%local pkds; %let pkds=&syslast;
+
+%if &rows > %mf_nobs(&pkds) %then %do;
+  %put &sysmacroname: &baseds has no combination of unique records! Exiting.;
+  %return;
+%end;
+
+/* now check cardinality */
+proc sql noprint;
+create table _data_ as select
+%do i=1 %to &ppkcnt;
+  %let var=%scan(&posspks,&i);
+  count(distinct &var) as &var
+  %if &i<&ppkcnt %then ,;
+%end;
+  from &pkds;
+
+/* transpose and sort by cardinality */
+proc transpose;
+proc sort; by descending col1;
+run;
+
+/* create initial PK list and re-order posspks list */
+data &outds(keep=pkguesses);
+  length pkguesses $5000 vars $5000;
+  set &syslast end=last;
+  retain vars ;
+  vars=catx(' ',vars,_name_);
+  if col1=&rows then do;
+    pkguesses=_name_;
+    output;
+  end;
+  if last then call symputx('posspks',vars,'l');
+run;
+
+%if %mf_nobs(&outds) ge &max_guesses %then %do;
+  %put &sysmacroname: %mf_nobs(&outds) possible primary key values found;
+  %return;
+%end;
+
+%if &ppkcnt=1 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %return;
+%end;
+
+/* begin scanning for uniques on pairs of PKs */
+%let tmpds=%mf_getuniquename();
+%local lev1 lev2;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do;
+      /* check for two level uniqueness */
+      proc sort data=&pkds(keep=&lev1 &lev2) out=&tmpds noduprec;
+        by _all_;
+      run;
+      %if %mf_nobs(&tmpds)=&rows %then %do;
+        proc sql;
+        insert into &outds values("&lev1 &lev2");
+        %if %mf_nobs(&outds) ge &max_guesses %then %do;
+          %put &sysmacroname: Max PKs reached at Level 2 for &baseds;
+          %goto exit;
+        %end;
+      %end;
+    %end;
   %end;
+%end;
 
-  /* get null count and row count */
-  %let tmpvar=%mf_getuniquename();
-  proc sql noprint;
-  create table _data_ as select
-    count(*) as &tmpvar
-  %do i=1 %to &vcnt;
-    %let var=%scan(&vars,&i);
-    ,sum(case when &var is missing then 1 else 0 end) as &var
-  %end;
-    from &baseds;
+%if &ppkcnt=2 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* transpose table and scan for not null cols */
-  proc transpose;
-  data _null_;
-    set &syslast end=last;
-    length vars $32767;
-    retain vars ;
-    if _name_="&tmpvar" then call symputx('rows',col1,'l');
-    else if col1=0 then vars=catx(' ',vars,_name_);
-    if last then call symputx('posspks',vars,'l');
-  run;
-
-  %let ppkcnt=%sysfunc(countw(&posspks));
-  %if &ppkcnt=0 %then %do;
-    %put &sysmacroname: &baseds has no non-missing variables!  Exiting.;
-    %return;
-  %end;
-
-  proc sort data=&baseds(keep=&posspks) out=_data_ noduprec;
-    by _all_;
-  run;
-  %local pkds; %let pkds=&syslast;
-
-  %if &rows > %mf_nobs(&pkds) %then %do;
-    %put &sysmacroname: &baseds has no combination of unique records! Exiting.;
-    %return;
-  %end;
-
-  /* now check cardinality */
-  proc sql noprint;
-  create table _data_ as select
-  %do i=1 %to &ppkcnt;
-    %let var=%scan(&posspks,&i);
-    count(distinct &var) as &var
-    %if &i<&ppkcnt %then ,;
-  %end;
-    from &pkds;
-
-  /* transpose and sort by cardinality */
-  proc transpose;
-  proc sort; by descending col1;
-  run;
-
-  /* create initial PK list and re-order posspks list */
-  data &outds(keep=pkguesses);
-    length pkguesses $5000 vars $5000;
-    set &syslast end=last;
-    retain vars ;
-    vars=catx(' ',vars,_name_);
-    if col1=&rows then do;
-      pkguesses=_name_;
-      output;
-    end;
-    if last then call symputx('posspks',vars,'l');
-  run;
-
-  %if %mf_nobs(&outds) ge &max_guesses %then %do;
-    %put &sysmacroname: %mf_nobs(&outds) possible primary key values found;
-    %return;
-  %end;
-
-  %if &ppkcnt=1 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
-
-  /* begin scanning for uniques on pairs of PKs */
-  %let tmpds=%mf_getuniquename();
-  %local lev1 lev2;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do;
-        /* check for two level uniqueness */
-        proc sort data=&pkds(keep=&lev1 &lev2) out=&tmpds noduprec;
+/* begin scanning for uniques on PK triplets */
+%local lev3;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
+      %let lev3=%scan(&posspks,&k);
+      %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do;
+        /* check for three level uniqueness */
+        proc sort data=&pkds(keep=&lev1 &lev2 &lev3) out=&tmpds noduprec;
           by _all_;
         run;
         %if %mf_nobs(&tmpds)=&rows %then %do;
           proc sql;
-          insert into &outds values("&lev1 &lev2");
+          insert into &outds values("&lev1 &lev2 &lev3");
           %if %mf_nobs(&outds) ge &max_guesses %then %do;
-            %put &sysmacroname: Max PKs reached at Level 2 for &baseds;
-            %return;
+            %put &sysmacroname: Max PKs reached at Level 3 for &baseds;
+            %goto exit;
           %end;
         %end;
       %end;
     %end;
   %end;
+%end;
 
-  %if &ppkcnt=2 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%if &ppkcnt=3 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* begin scanning for uniques on PK triplets */
-  %local lev3;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
-        %let lev3=%scan(&posspks,&k);
-        %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do;
-          /* check for three level uniqueness */
-          proc sort data=&pkds(keep=&lev1 &lev2 &lev3) out=&tmpds noduprec;
+/* scan for uniques on up to 4 PK fields */
+%local lev4;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
+      %let lev3=%scan(&posspks,&k);
+      %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
+        %let lev4=%scan(&posspks,&l);
+        %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then %do;
+          /* check for four level uniqueness */
+          proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4)
+              out=&tmpds noduprec;
             by _all_;
           run;
           %if %mf_nobs(&tmpds)=&rows %then %do;
             proc sql;
-            insert into &outds values("&lev1 &lev2 &lev3");
+            insert into &outds values("&lev1 &lev2 &lev3 &lev4");
             %if %mf_nobs(&outds) ge &max_guesses %then %do;
-              %put &sysmacroname: Max PKs reached at Level 3 for &baseds;
-              %return;
+              %put &sysmacroname: Max PKs reached at Level 4 for &baseds;
+              %goto exit;
             %end;
           %end;
         %end;
       %end;
     %end;
   %end;
+%end;
 
-  %if &ppkcnt=3 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%if &ppkcnt=4 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* scan for uniques on up to 4 PK fields */
-  %local lev4;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
-        %let lev3=%scan(&posspks,&k);
-        %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
-          %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then %do;
+/* scan for uniques on up to 4 PK fields */
+%local lev5 m;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
+      %let lev3=%scan(&posspks,&k);
+      %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
+        %let lev4=%scan(&posspks,&l);
+        %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
+        %do m=5 %to &ppkcnt;
+          %let lev5=%scan(&posspks,&m);
+          %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then %do;
             /* check for four level uniqueness */
-            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4) out=&tmpds noduprec;
+            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5)
+                out=&tmpds noduprec;
               by _all_;
             run;
             %if %mf_nobs(&tmpds)=&rows %then %do;
               proc sql;
-              insert into &outds values("&lev1 &lev2 &lev3 &lev4");
+              insert into &outds values("&lev1 &lev2 &lev3 &lev4 &lev5");
               %if %mf_nobs(&outds) ge &max_guesses %then %do;
-                %put &sysmacroname: Max PKs reached at Level 4 for &baseds;
-                %return;
+                %put &sysmacroname: Max PKs reached at Level 5 for &baseds;
+                %goto exit;
               %end;
             %end;
           %end;
@@ -212,36 +270,44 @@
       %end;
     %end;
   %end;
+%end;
 
-  %if &ppkcnt=4 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%if &ppkcnt=5 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* scan for uniques on up to 4 PK fields */
-  %local lev5 m;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
-        %let lev3=%scan(&posspks,&k);
-        %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
-          %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
-          %do m=5 %to &ppkcnt;
-            %let lev5=%scan(&posspks,&m);
-            %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then %do;
+/* scan for uniques on up to 4 PK fields */
+%local lev6 n;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
+      %let lev3=%scan(&posspks,&k);
+      %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
+        %let lev4=%scan(&posspks,&l);
+        %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
+        %do m=5 %to &ppkcnt;
+          %let lev5=%scan(&posspks,&m);
+          %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5
+          %then %do n=6 %to &ppkcnt;
+            %let lev6=%scan(&posspks,&n);
+            %if &lev1 ne &lev6 & &lev2 ne &lev6 & &lev3 ne &lev6
+            & &lev4 ne &lev6 & &lev5 ne &lev6 %then
+            %do;
               /* check for four level uniqueness */
-              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5) out=&tmpds noduprec;
+              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5 &lev6)
+                out=&tmpds noduprec;
                 by _all_;
               run;
               %if %mf_nobs(&tmpds)=&rows %then %do;
                 proc sql;
-                insert into &outds values("&lev1 &lev2 &lev3 &lev4 &lev5");
+                insert into &outds
+                  values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
                 %if %mf_nobs(&outds) ge &max_guesses %then %do;
-                  %put &sysmacroname: Max PKs reached at Level 5 for &baseds;
-                  %return;
+                  %put &sysmacroname: Max PKs reached at Level 6 for &baseds;
+                  %goto exit;
                 %end;
               %end;
             %end;
@@ -250,55 +316,17 @@
       %end;
     %end;
   %end;
+%end;
 
-  %if &ppkcnt=5 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%if &ppkcnt=6 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* scan for uniques on up to 4 PK fields */
-  %local lev6 n;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
-        %let lev3=%scan(&posspks,&k);
-        %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
-          %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
-          %do m=5 %to &ppkcnt;
-            %let lev5=%scan(&posspks,&m);
-            %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then
-            %do n=6 %to &ppkcnt;
-              %let lev6=%scan(&posspks,&n);
-              %if &lev1 ne &lev6 & &lev2 ne &lev6 & &lev3 ne &lev6
-              & &lev4 ne &lev6 & &lev5 ne &lev6 %then
-              %do;
-                /* check for four level uniqueness */
-                proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5 &lev6)
-                  out=&tmpds noduprec;
-                  by _all_;
-                run;
-                %if %mf_nobs(&tmpds)=&rows %then %do;
-                  proc sql;
-                  insert into &outds values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
-                  %if %mf_nobs(&outds) ge &max_guesses %then %do;
-                    %put &sysmacroname: Max PKs reached at Level 6 for &baseds;
-                    %return;
-                  %end;
-                %end;
-              %end;
-            %end;
-          %end;
-        %end;
-      %end;
-    %end;
-  %end;
-
-  %if &ppkcnt=6 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%exit:
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &tmpds;
+%end;
 
 %mend mp_guesspk;

base/mp_hashdataset.sas

@@ -20,6 +20,8 @@
   @li mf_getvartype.sas
 
   @param [in] libds dataset to hash
+  @param [in] salt= Provide a salt (could be, for instance, the dataset name)
+  @param [in] iftrue= A condition under which the macro should be executed.
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
   will contain one column (hashkey) with one observation (a hex32.
   representation of the input hash)
@@ -33,10 +35,15 @@
 
 %macro mp_hashdataset(
   libds,
-  outds=
+  outds=,
+  salt=,
+  iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
+
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
   %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
-    %put %str(WARN)ING: Dataset &libds is empty;, or is not a dataset;
+    %put %str(WARN)ING: Dataset &libds is empty, or is not a dataset;
   %end;
   %else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
     %put %str(ERR)OR: Dataset &libds is not a dataset;
@@ -53,7 +60,7 @@
     %let varlist=%mf_getvarlist(&libds);
     data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
       length &prevkeyvar &keyvar $32;
-      retain &prevkeyvar;
+      retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
       set &libds end=&lastvar;
       /* hash should include previous row */
       &keyvar=put(md5(&prevkeyvar

base/mp_include.sas

@@ -0,0 +1,106 @@
+/**
+  @file
+  @brief Performs a wrapped \%include
+  @details This macro wrapper is necessary if you need your included code to
+  know that it is being \%included.
+
+  If you are using %include in a regular program, you could make use of the
+  following macro variables:
+
+  @li SYSINCLUDEFILEDEVICE
+  @li SYSINCLUDEFILEDIR
+  @li SYSINCLUDEFILEFILEREF
+  @li SYSINCLUDEFILENAME
+
+  However these variables are NOT available inside a macro, as documented here:
+https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/mcrolref/n1j5tcc0n2xczyn1kg1o0606gsv9.htm
+
+  This macro can be used in place of the %include statement, and will insert
+  the following (equivalent) global variables:
+
+  @li _SYSINCLUDEFILEDEVICE
+  @li _SYSINCLUDEFILEDIR
+  @li _SYSINCLUDEFILEFILEREF
+  @li _SYSINCLUDEFILENAME
+
+  These can be used whenever testing _within a macro_.  Outside of the macro,
+  the regular automatic variables will still be available (thanks to a
+  concatenated file list in the include statement).
+
+  Example usage:
+
+      filename example temp;
+      data _null_;
+        file example;
+        put '%macro test();';
+        put '%put &=_SYSINCLUDEFILEFILEREF;';
+        put '%put &=SYSINCLUDEFILEFILEREF;';
+        put '%mend; %test()';
+        put '%put &=SYSINCLUDEFILEFILEREF;';
+      run;
+      %mp_include(example)
+
+  @param [in] fileref The fileref of the file to be included. Must be provided.
+  @param [in] prefix= (_) The prefix to apply to the global variables.
+  @param [in] opts= (SOURCE2) The options to apply to the %inc statement
+  @param [in] errds= (work.mp_abort_errds) There is no clean way to end a
+    process within a %include called within a macro.  Furthermore, there is no
+    way to test if a macro is called within a %include.  To handle this
+    particular scenario, the %mp_abort() macro will test for the existence of
+    the `_SYSINCLUDEFILEDEVICE` variable and return the outputs (msg,mac) inside
+    this dataset.
+    It will then run an abort cancel FILE to stop the include running, and pass
+    the dataset back.
+
+    IMPORTANT NOTE - it is NOT possible to read this dataset as part of _this_
+    macro! When running abort cancel FILE, ALL macros are closed, so instead it
+    is necessary to invoke "%mp_abort(mode=INCLUDE)" OUTSIDE of macro wrappers.
+
+
+  @version 9.4
+  @author Allan Bowe
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_abort.sas
+
+**/
+
+%macro mp_include(fileref
+  ,prefix=_
+  ,opts=SOURCE2
+  ,errds=work.mp_abort_errds
+)/*/STORE SOURCE*/;
+
+/* prepare precode */
+%local tempref;
+%let tempref=%mf_getuniquefileref();
+data _null_;
+  file &tempref;
+  set sashelp.vextfl(where=(fileref="%upcase(&fileref)"));
+  put '%let _SYSINCLUDEFILEDEVICE=' xengine ';';
+  name=scan(xpath,-1,'/\');
+  put '%let _SYSINCLUDEFILENAME=' name ';';
+  path=subpad(xpath,1,length(xpath)-length(name)-1);
+  put '%let _SYSINCLUDEFILEDIR=' path ';';
+  put '%let _SYSINCLUDEFILEFILEREF=' "&fileref;";
+run;
+
+/* prepare the errds */
+data &errds;
+  length msg mac $1000;
+  call missing(msg,mac);
+  iftrue='1=0';
+run;
+
+/* include the include */
+%inc &tempref &fileref/&opts;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=%str(&_SYSINCLUDEFILEDIR/&_SYSINCLUDEFILENAME)
+  ,msg=%str(syscc=&syscc after executing &_SYSINCLUDEFILENAME)
+)
+
+filename &tempref clear;
+
+%mend mp_include;

base/mp_init.sas

@@ -0,0 +1,76 @@
+/**
+  @file
+  @brief Initialise session with useful settings and variables
+  @details Implements a "strict" set of SAS options for use in defensive
+    programming.  Highly recommended, if you want your code to run on some
+    other machine.
+
+    This macro is recommended to be compiled and invoked in the `initProgram`
+    for SASjs [Jobs](https://cli.sasjs.io/sasjsconfig.html#jobConfig_initProgram
+    ), [Services](
+    https://cli.sasjs.io/sasjsconfig.html#serviceConfig_initProgram) and [Tests]
+    (https://cli.sasjs.io/sasjsconfig.html#testConfig_initProgram).
+
+    For non SASjs projects, you could invoke in the autoexec, or in your own
+    solution initialisation macro.
+
+
+    If you have a good idea for another useful option, setting, or global
+    variable - feel free to [raise an issue](
+    https://github.com/sasjs/core/issues/new)!
+
+    All global variables are prefixed with "SASJS" (unless modified with the
+    prefix parameter).
+
+  @param [in] prefix= (SASJS) The prefix to apply to the global macro variables
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_init(prefix=SASJS
+)/*/STORE SOURCE*/;
+
+%if %symexist(SASJS_PREFIX) %then %return;  /* only run once */
+
+%global
+  SASJS_PREFIX       /* the ONLY hard-coded global macro variable in SASjs    */
+  &prefix._FUNCTIONS /* used in mcf_init() to track core function compilation */
+  &prefix._INIT_NUM  /* initialisation time as numeric                        */
+  &prefix._INIT_DTTM /* initialisation time in E8601DT26.6 format             */
+  &prefix.WORK       /* avoid typing %sysfunc(pathname(work)) every time      */
+;
+
+%let sasjs_prefix=&prefix;
+
+data _null_;
+  dttm=datetime();
+  call symputx("&sasjs_prefix._init_num",dttm,'g');
+  call symputx("&sasjs_prefix._init_dttm",put(dttm,E8601DT26.6),'g');
+  call symputx("&sasjs_prefix.work",pathname('WORK'),'g');
+run;
+
+options
+  compress=CHAR           /* default is none so ensure we have something!   */
+  datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
+  errorcheck=STRICT       /* catch errs in libname/filename statements      */
+  fmterr                  /* ensure err when a format cannot be found       */
+  mergenoby=%str(ERR)OR   /* throw err when a merge has no BY variables     */
+  missing=.               /* changing this can cause hard to detect errs    */
+  noquotelenmax           /* avoid warnings for long strings                */
+  noreplace               /* avoid overwriting permanent datasets           */
+  ps=max                  /* reduce log size slightly                       */
+  ls=max                  /* reduce log even more and avoid word truncation */
+  validmemname=COMPATIBLE /* avoid special characters etc in table names    */
+  validvarname=V7         /* avoid special characters etc in variable names */
+  varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
+  varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
+%if %substr(&sysver,1,1) ne 4 %then %do;
+  noautocorrect           /* disallow misspelled procedure names            */
+  dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
+%end;
+;
+
+%mend mp_init;

base/mp_jsonout.sas

@@ -4,7 +4,7 @@
   @details PROC JSON is faster but will produce errs like the ones below if
   special chars are encountered.
 
-  > ERROR: Some code points did not transcode.
+  > (ERR)OR: Some code points did not transcode.
 
   > An object or array close is not valid at this point in the JSON text.
 
@@ -19,11 +19,12 @@
 
         %mp_jsonout(OPEN,jref=tmp)
         %mp_jsonout(OBJ,class,jref=tmp)
+        %mp_jsonout(OBJ,class,dslabel=class2,jref=tmp,showmeta=YES)
         %mp_jsonout(CLOSE,jref=tmp)
 
         data _null_;
         infile tmp;
-        input;list;
+        input;putlog _infile_;
         run;
 
   If you are building web apps with SAS then you are strongly encouraged to use
@@ -31,22 +32,24 @@
   [sasjs adapter](https://github.com/sasjs/adapter).
   For more information see https://sasjs.io
 
-  @param action Valid values:
+  @param [in] action Valid values:
     @li OPEN - opens the JSON
     @li OBJ - sends a table with each row as an object
     @li ARR - sends a table with each row in an array
     @li CLOSE - closes the JSON
-
-  @param ds the dataset to send.  Must be a work table.
-  @param jref= the fileref to which to send the JSON
-  @param dslabel= the name to give the table in the exported JSON
-  @param fmt= Whether to keep or strip formats from the table
-  @param engine= Which engine to use to send the JSON, valid options are:
+  @param [in] ds The dataset to send.  Must be a work table.
+  @param [out] jref= (_webout) The fileref to which to send the JSON
+  @param [out] dslabel= The name to give the table in the exported JSON
+  @param [in] fmt= (Y) Whether to keep (Y) or strip (N) formats from the table
+  @param [in] engine= (DATASTEP) Which engine to use to send the JSON. Options:
     @li PROCJSON (default)
     @li DATASTEP (more reliable when data has non standard characters)
-
-  @param dbg= DEPRECATED - was used to conditionally add PRETTY to
-    proc json but this can cause line truncation in large files.
+  @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
+    (eg `null`) or as STRING values (eg `".a"` or `".b"`)
+  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+    such as the column formats and types.  The metadata is contained inside an
+    object with the same name as the table but prefixed with a dollar sign - ie,
+    `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
 
   <h4> Related Macros <h4>
   @li mp_ds2fmtds.sas
@@ -57,129 +60,145 @@
 
 **/
 
-%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0
+%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y
+  ,engine=DATASTEP
+  ,missing=NULL
+  ,showmeta=NO
 )/*/STORE SOURCE*/;
-%put output location=&jref;
+%local tempds colinfo fmtds i numcols;
+%let numcols=0;
+
 %if &action=OPEN %then %do;
   options nobomfile;
-  data _null_;file &jref encoding='utf-8';
-    put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
+  data _null_;file &jref encoding='utf-8' ;
+    put '{"PROCESSED_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '"';
   run;
 %end;
 %else %if (&action=ARR or &action=OBJ) %then %do;
   options validvarname=upcase;
-  data _null_;file &jref mod encoding='utf-8';
+  data _null_; file &jref encoding='utf-8' mod;
     put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":";
 
+  /* grab col defs */
+  proc contents noprint data=&ds
+    out=_data_(keep=name type length format formatl formatd varnum label);
+  run;
+  %let colinfo=%scan(&syslast,2,.);
+  proc sort data=&colinfo;
+    by varnum;
+  run;
+  /* move meta to mac vars */
+  data _null_;
+    if _n_=1 then call symputx('numcols',nobs,'l');
+    set &colinfo end=last nobs=nobs;
+    name=upcase(name);
+    /* fix formats */
+    if type=2 or type=6 then do;
+      typelong='char';
+      length fmt $49.;
+      if format='' then fmt=cats('$',length,'.');
+      else if formatl=0 then fmt=cats(format,'.');
+      else fmt=cats(format,formatl,'.');
+      newlen=max(formatl,length);
+    end;
+    else do;
+      typelong='num';
+      if format='' then fmt='best.';
+      else if formatl=0 then fmt=cats(format,'.');
+      else if formatd=0 then fmt=cats(format,formatl,'.');
+      else fmt=cats(format,formatl,'.',formatd);
+      /* needs to be wide, for datetimes etc */
+      newlen=max(length,formatl,24);
+    end;
+    /* 32 char unique name */
+    newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
+
+    call symputx(cats('name',_n_),name,'l');
+    call symputx(cats('newname',_n_),newname,'l');
+    call symputx(cats('len',_n_),newlen,'l');
+    call symputx(cats('length',_n_),length,'l');
+    call symputx(cats('fmt',_n_),fmt,'l');
+    call symputx(cats('type',_n_),type,'l');
+    call symputx(cats('typelong',_n_),typelong,'l');
+    call symputx(cats('label',_n_),coalescec(label,name),'l');
+  run;
+
+  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+
   %if &engine=PROCJSON %then %do;
-    data;run;%let tempds=&syslast;
-    proc sql;drop table &tempds;
-    data &tempds /view=&tempds;set &ds;
+    %if &missing=STRING %then %do;
+      %put &sysmacroname: Special Missings not supported in proc json.;
+      %put &sysmacroname: Switching to DATASTEP engine;
+      %goto datastep;
+    %end;
+    data &tempds;set &ds;
     %if &fmt=N %then format _numeric_ best32.;;
+    /* PRETTY is necessary to avoid line truncation in large files */
     proc json out=&jref pretty
         %if &action=ARR %then nokeys ;
         ;export &tempds / nosastags fmtnumeric;
     run;
-    proc sql;drop view &tempds;
   %end;
   %else %if &engine=DATASTEP %then %do;
-    %local cols i tempds;
-    %let cols=0;
-    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 %then %do;
+    %datastep:
+    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1
+    %then %do;
       %put &sysmacroname:  &ds NOT FOUND!!!;
       %return;
     %end;
-    %if &fmt=Y %then %do;
-      %put converting every variable to a formatted variable;
-      /* see mp_ds2fmtds.sas for source */
-      proc contents noprint data=&ds
-        out=_data_(keep=name type length format formatl formatd varnum);
-      run;
-      proc sort;
-        by varnum;
-      run;
-      %local fmtds;
-      %let fmtds=%scan(&syslast,2,.);
-      /* prepare formats and varnames */
-      data _null_;
-        if _n_=1 then call symputx('nobs',nobs,'l');
-        set &fmtds end=last nobs=nobs;
-        name=upcase(name);
-        /* fix formats */
-        if type=2 or type=6 then do;
-          length fmt $49.;
-          if format='' then fmt=cats('$',length,'.');
-          else if formatl=0 then fmt=cats(format,'.');
-          else fmt=cats(format,formatl,'.');
-          newlen=max(formatl,length);
-        end;
-        else do;
-          if format='' then fmt='best.';
-          else if formatl=0 then fmt=cats(format,'.');
-          else if formatd=0 then fmt=cats(format,formatl,'.');
-          else fmt=cats(format,formatl,'.',formatd);
-          /* needs to be wide, for datetimes etc */
-          newlen=max(length,formatl,24);
-        end;
-        /* 32 char unique name */
-        newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
 
-        call symputx(cats('name',_n_),name,'l');
-        call symputx(cats('newname',_n_),newname,'l');
-        call symputx(cats('len',_n_),newlen,'l');
-        call symputx(cats('fmt',_n_),fmt,'l');
-        call symputx(cats('type',_n_),type,'l');
-      run;
-      data &fmtds;
+    %if &fmt=Y %then %do;
+      data _data_;
         /* rename on entry */
         set &ds(rename=(
-      %local i;
-      %do i=1 %to &nobs;
+      %do i=1 %to &numcols;
         &&name&i=&&newname&i
       %end;
         ));
-      %do i=1 %to &nobs;
+      %do i=1 %to &numcols;
         length &&name&i $&&len&i;
-        &&name&i=left(put(&&newname&i,&&fmt&i));
+        %if &&typelong&i=num %then %do;
+          &&name&i=left(put(&&newname&i,&&fmt&i));
+        %end;
+        %else %do;
+          &&name&i=put(&&newname&i,&&fmt&i);
+        %end;
         drop &&newname&i;
       %end;
         if _error_ then call symputx('syscc',1012);
       run;
-      %let ds=&fmtds;
-    %end; /* &fmt=Y */
-    data _null_;file &jref mod encoding='utf-8';
-      put "["; call symputx('cols',0,'l');
-    proc sort
-      data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
-      out=_data_;
-      by varnum;
-
-    data _null_;
-      set _last_ end=last;
-      call symputx(cats('name',_n_),name,'l');
-      call symputx(cats('type',_n_),type,'l');
-      call symputx(cats('len',_n_),length,'l');
-      if last then call symputx('cols',_n_,'l');
-    run;
+      %let fmtds=&syslast;
+    %end;
 
     proc format; /* credit yabwon for special null removal */
-      value bart ._ - .z = null
+    value bart (default=40)
+    %if &missing=NULL %then %do;
+      ._ - .z = null
+    %end;
+    %else %do;
+      ._ = [quote()]
+      . = null
+      .a - .z = [quote()]
+    %end;
       other = [best.];
 
-    data;run; %let tempds=&syslast; /* temp table for spesh char management */
-    proc sql; drop table &tempds;
-    data &tempds/view=&tempds;
+    data &tempds;
       attrib _all_ label='';
-      %do i=1 %to &cols;
-        %if &&type&i=char %then %do;
+      %do i=1 %to &numcols;
+        %if &&typelong&i=char or &fmt=Y %then %do;
           length &&name&i $32767;
           format &&name&i $32767.;
         %end;
       %end;
-      set &ds;
+      %if &fmt=Y %then %do;
+        set &fmtds;
+      %end;
+      %else %do;
+        set &ds;
+      %end;
       format _numeric_ bart.;
-    %do i=1 %to &cols;
-      %if &&type&i=char %then %do;
+    %do i=1 %to &numcols;
+      %if &&typelong&i=char or &fmt=Y %then %do;
         &&name&i='"'!!trim(prxchange('s/"/\"/',-1,
                     prxchange('s/'!!'0A'x!!'/\n/',-1,
                     prxchange('s/'!!'0D'x!!'/\r/',-1,
@@ -189,44 +208,64 @@
       %end;
     %end;
     run;
+
     /* write to temp loc to avoid _webout truncation
       - https://support.sas.com/kb/49/325.html */
     filename _sjs temp lrecl=131068 encoding='utf-8';
-    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod;
+    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod ;
+      if _n_=1 then put "[";
       set &tempds;
       if _n_>1 then put "," @; put
       %if &action=ARR %then "[" ; %else "{" ;
-      %do i=1 %to &cols;
+      %do i=1 %to &numcols;
         %if &i>1 %then  "," ;
         %if &action=OBJ %then """&&name&i"":" ;
         &&name&i
       %end;
       %if &action=ARR %then "]" ; %else "}" ; ;
-    proc sql;
-    drop view &tempds;
     /* now write the long strings to _webout 1 byte at a time */
     data _null_;
       length filein 8 fileid 8;
-      filein = fopen("_sjs",'I',1,'B');
-      fileid = fopen("&jref",'A',1,'B');
-      rec = '20'x;
+      filein=fopen("_sjs",'I',1,'B');
+      fileid=fopen("&jref",'A',1,'B');
+      rec='20'x;
       do while(fread(filein)=0);
-        rc = fget(filein,rec,1);
-        rc = fput(fileid, rec);
-        rc =fwrite(fileid);
+        rc=fget(filein,rec,1);
+        rc=fput(fileid, rec);
+        rc=fwrite(fileid);
       end;
-      rc = fclose(filein);
-      rc = fclose(fileid);
+      /* close out the table */
+      rc=fput(fileid, "]");
+      rc=fwrite(fileid);
+      rc=fclose(filein);
+      rc=fclose(fileid);
     run;
     filename _sjs clear;
-    data _null_; file &jref mod encoding='utf-8';
-      put "]";
+  %end;
+
+  proc sql;
+  drop table &colinfo, &tempds;
+
+  %if &showmeta=YES %then %do;
+    data _null_; file &jref encoding='utf-8' mod;
+      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{";
+      do i=1 to &numcols;
+        name=quote(trim(symget(cats('name',i))));
+        format=quote(trim(symget(cats('fmt',i))));
+        label=quote(trim(symget(cats('label',i))));
+        length=quote(trim(symget(cats('length',i))));
+        type=quote(trim(symget(cats('typelong',i))));
+        if i>1 then put "," @@;
+        put name ':{"format":' format ',"label":' label
+          ',"length":' length ',"type":' type '}';
+      end;
+      put '}}';
     run;
   %end;
 %end;
 
 %else %if &action=CLOSE %then %do;
-  data _null_;file &jref encoding='utf-8' mod;
+  data _null_; file &jref encoding='utf-8' mod ;
     put "}";
   run;
 %end;

base/mp_lib2inserts.sas

@@ -0,0 +1,77 @@
+/**
+  @file
+  @brief Convert all data in a library to SQL insert statements
+  @details Gets list of members then calls the <code>%mp_ds2inserts()</code>
+  macro.
+  Usage:
+
+      %mp_getddl(sashelp, schema=work, fref=tempref)
+
+      %mp_lib2inserts(sashelp, schema=work, outref=tempref)
+
+      %inc tempref;
+
+
+  The output will be one file in the outref fileref.
+
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2inserts.sas
+
+
+  @param [in] lib Library in which to convert all datasets to inserts
+  @param [in] flavour= (SAS) The SQL flavour to be applied to the output. Valid
+    options:
+    @li SAS (default) - suitable for regular proc sql
+    @li PGSQL - Used for Postgres databases
+  @param [in] maxobs= (max) The max number of observations (per table) to create
+  @param [out] outref= Output fileref in which to create the insert statements.
+    If it exists, it will be appended to, otherwise it will be created.
+  @param [out] schema= (0) The schema of the target database, or the libref.
+  @param [in] applydttm= (YES) If YES, any columns using datetime formats will
+    be converted to native DB datetime literals
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_lib2inserts(lib
+    ,flavour=SAS
+    ,outref=0
+    ,schema=0
+    ,maxobs=max
+    ,applydttm=YES
+)/*/STORE SOURCE*/;
+
+/* Find the tables */
+%local x ds memlist;
+proc sql noprint;
+select distinct lowcase(memname)
+  into: memlist
+  separated by ' '
+  from dictionary.tables
+  where upcase(libname)="%upcase(&lib)"
+    and memtype='DATA'; /* exclude views */
+
+
+%let flavour=%upcase(&flavour);
+%if &flavour ne SAS and &flavour ne PGSQL %then %do;
+  %put %str(WAR)NING:  &flavour is not supported;
+  %return;
+%end;
+
+
+/* create the inserts */
+%do x=1 %to %sysfunc(countw(&memlist));
+  %let ds=%scan(&memlist,&x);
+  %mp_ds2inserts(&lib..&ds
+    ,outref=&outref
+    ,schema=&schema
+    ,outds=&ds
+    ,flavour=&flavour
+    ,maxobs=&maxobs
+    ,applydttm=&applydttm
+  )
+%end;
+
+%mend mp_lib2inserts;

base/mp_lockanytable.sas

@@ -0,0 +1,241 @@
+/**
+  @file
+  @brief Mechanism for locking tables to prevent parallel modifications
+  @details Uses a control table to enable ANY table to be locked for updates.
+  Only useful if every update uses the macro!   Used heavily within
+  [Data Controller for SAS](https://datacontroller.io).
+
+  @param [in] action The action to be performed.  Valid values:
+    @li LOCK - Sets the lock flag, also confirms if a SAS lock is available
+    @li UNLOCK - Unlocks the table
+  @param [in] lib= (WORK) The libref of the table to lock.  Should already be
+    assigned.
+  @param [in] ds= The dataset to lock
+  @param [in] ref= A meaningful reference to enable the lock to be traced. Max
+    length is 200 characters.
+  @param [out] ctl_ds= (0) The control table which controls the actual locking.
+    Should already be assigned and available.  The definition is available by
+    running mp_coretable.sas as follows:  `%mp_coretable(LOCKTABLE)`.
+
+  @param [in] loops= (25) Number of times to check for a lock.
+  @param [in] loop_secs= (1) Seconds to wait between each lock attempt
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_lockfilecheck.sas
+  @li mf_getuser.sas
+
+  <h4> Related Macros </h4>
+  @li mp_lockanytable.test.sas
+
+  @version 9.2
+
+**/
+
+%macro mp_lockanytable(
+  action
+  ,lib= WORK
+  ,ds=0
+  ,ref=
+  ,ctl_ds=0
+  ,loops=25
+  ,loop_secs=1
+  );
+data _null_;
+  if _n_=1 then putlog "&sysmacroname entry vars:";
+  set sashelp.vmacro;
+  where scope="&sysmacroname";
+  put name '=' value;
+run;
+
+%mp_abort(iftrue= (&ds=0 and &action ne MAKETABLE)
+  ,mac=&sysmacroname
+  ,msg=%str(dataset was not provided)
+)
+%mp_abort(iftrue= (&ctl_ds=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Control dataset was not provided)
+)
+
+/* set up lib & mac vars */
+%let lib=%upcase(&lib);
+%let ds=%upcase(&ds);
+%let action=%upcase(&action);
+%local user x trans msg abortme;
+%let user=%mf_getuser();
+%let abortme=0;
+
+%mp_abort(iftrue= (&action ne LOCK & &action ne UNLOCK & &action ne MAKETABLE)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid action (&action) provided)
+)
+
+/* if an err condition exists, exit before we even begin */
+%mp_abort(iftrue= (&syscc>0 and &action=LOCK)
+  ,mac=&sysmacroname
+  ,msg=%str(aborting due to syscc=&syscc on LOCK entry)
+)
+
+/* do not bother locking work tables (else may affect all WORK libraries) */
+%if (%upcase(&lib)=WORK or %str(&lib)=%str()) & &action ne MAKETABLE %then %do;
+  %put NOTE: WORK libraries will not be registered in the locking system.;
+  %return;
+%end;
+
+/* do not proceed if no observations can be processed */
+%mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)
+  ,mac=&sysmacroname
+  ,msg=%str(cannot continue when options obs = 0)
+)
+
+%if &ACTION=LOCK %then %do;
+
+  /* abort if a SAS lock is already in place, or cannot be applied */
+  %mp_lockfilecheck(&lib..&ds)
+
+  /* next, check there is a record for this table */
+  %local record_exists_check;
+  proc sql noprint;
+  select count(*) into: record_exists_check from &ctl_ds
+    where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+  quit;
+  %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
+  %if &record_exists_check=0 %then %do;
+    data _null_;
+      putlog "&sysmacroname: adding record to lock table..";
+    run;
+
+    data ;
+      if 0 then set &ctl_ds;
+      LOCK_LIB ="&lib";
+      LOCK_DS="&ds";
+      LOCK_STATUS_CD='LOCKED';
+      LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt;
+      LOCK_USER_NM="&user";
+      LOCK_PID="&sysjobid";
+      LOCK_REF="&ref";
+      output;stop;
+    run;
+    %let trans=&syslast;
+    proc append base=&ctl_ds data=&trans;
+    run;
+  %end;
+  /* if record does exist, perform lock attempts */
+  %else %do x=1 %to &loops;
+    data _null_;
+      putlog "&sysmacroname: attempting lock (iteration &x) "@;
+      putlog "at %sysfunc(datetime(),datetime19.) ..";
+    run;
+
+    proc sql;
+    update &ctl_ds
+      set LOCK_STATUS_CD='LOCKED'
+        , LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_USER_NM="&user"
+        , LOCK_PID="&sysjobid"
+        , LOCK_REF="&ref"
+      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+    quit;
+    /**
+      * NOTE - occasionally SQL server will return an err code (deadlocked
+      * transaction).  If so, ignore it, keep calm, and carry on..
+      */
+    %if &syscc>0 %then %do;
+      data _null_;
+        putlog 'NOTE-' / 'NOTE-';
+        putlog "NOTE- &sysmacroname: Update failed. "@;
+        putlog "Resetting err conditions and re-attempting.";
+        putlog "NOTE- syscc=&syscc syserr=&syserr sqlrc=&sqlrc";
+        putlog 'NOTE-' / 'NOTE-';
+      run;
+      %let syscc=0;
+      %let sqlrc=0;
+    %end;
+
+    /* now check if the record was successfully updated */
+    %local success_check;
+    proc sql noprint;
+    select count(*) into: success_check from &ctl_ds
+      where LOCK_LIB ="&lib" and LOCK_DS="&ds"
+        and LOCK_PID="&sysjobid" and LOCK_STATUS_CD='LOCKED';
+    quit;
+    %if &success_check=0 %then %do;
+      %if &x < &loops %then %do;
+        /* pause before next check */
+        data _null_;
+          putlog 'NOTE-' / 'NOTE-';
+          putlog "NOTE- &sysmacroname: table locked, waiting "@;
+          putlog "%sysfunc(sleep(&loop_inc)) seconds.. ";
+          putlog "NOTE- (iteration &x of &loops)";
+          putlog 'NOTE-' / 'NOTE-';
+        run;
+      %end;
+      %else %do;
+        %let msg=Unable to lock &lib..&ds via &ctl_ds after &loops attempts.\n
+            Please ask your administrator to investigate!;
+        %let abortme=1;
+      %end;
+    %end;
+    %else %do;
+      data _null_;
+        putlog 'NOTE-' / 'NOTE-';
+        putlog "NOTE- &sysmacroname: Table &lib..&ds locked at "@;
+        putlog " %sysfunc(datetime(),datetime19.) (iteration &x)"@;
+        putlog 'NOTE-' / 'NOTE-';
+      run;
+      %if &syscc>0 %then %do;
+        %put setting syscc(&syscc) back to 0;
+        %let syscc=0;
+      %end;
+      %let x=&loops;  /* no more iterations needed */
+    %end;
+  %end;
+%end;
+%else %if &ACTION=UNLOCK %then %do;
+  %local status;
+  proc sql noprint;
+  select LOCK_STATUS_CD into: status from &ctl_ds
+    where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+  quit;
+  %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
+  %if &status=LOCKED %then %do;
+    data _null_;
+      putlog "&sysmacroname: unlocking &lib..&ds:";
+    run;
+    proc sql;
+    update &ctl_ds
+      set LOCK_STATUS_CD='UNLOCKED'
+        , LOCK_END_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_USER_NM="&user"
+        , LOCK_PID="&sysjobid"
+        , LOCK_REF="&ref"
+      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+    quit;
+  %end;
+  %else %if &status=UNLOCKED %then %do;
+    %put %str(WAR)NING: &lib..&ds is already unlocked!;
+  %end;
+  %else %do;
+    %put NOTE: Unrecognised STATUS_CD (&status) in &ctl_ds;
+    %let abortme=1;
+  %end;
+%end;
+%else %do;
+  %let msg=lock_anytable given unsupported action (&action);
+  %let abortme=1;
+%end;
+
+/* catch errs - mp_abort must be called outside of a logic block */
+%mp_abort(iftrue=(&abortme=1),
+  msg=%superq(msg),
+  mac=&sysmacroname
+)
+
+%exit_macro:
+data _null_;
+  put "&sysmacroname: Exit vars: action=&action lib=&lib ds=&ds";
+  put " syscc=&syscc sqlrc=&sqlrc syserr=&syserr";
+run;
+%mend mp_lockanytable;
+
+

base/mp_lockfilecheck.sas

@@ -0,0 +1,97 @@
+/**
+  @file
+  @brief Aborts if a SAS lock file is in place, or if one cannot be applied.
+  @details Used in conjuction with the mp_lockanytable macro.
+  More info here: https://sasensei.com/flash/24
+
+  Usage:
+
+      data work.test; a=1;run;
+      %mp_lockfilecheck(work.test)
+
+  @param [in] libds The libref.dataset for which to check the lock status
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_getattrc.sas
+
+  <h4> Related Macros </h4>
+  @li mp_lockanytable.sas
+  @li mp_lockfilecheck.test.sas
+
+  @version 9.2
+**/
+
+%macro mp_lockfilecheck(
+  libds
+)/*/STORE SOURCE*/;
+
+data _null_;
+  if _n_=1 then putlog "&sysmacroname entry vars:";
+  set sashelp.vmacro;
+  where scope="&sysmacroname";
+  put name '=' value;
+run;
+
+%mp_abort(iftrue= (&syscc>0)
+  ,mac=checklock.sas
+  ,msg=Aborting with syscc=&syscc on entry.
+)
+%mp_abort(iftrue= (&libds=0)
+  ,mac=&sysmacroname
+  ,msg=%str(libds not provided)
+)
+
+%local msg lib ds;
+%let lib=%upcase(%scan(&libds,1,.));
+%let ds=%upcase(%scan(&libds,2,.));
+
+/* do not proceed if no observations can be processed */
+%let msg=options obs = 0. syserrortext=%superq(syserrortext);
+%mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)
+  ,mac=checklock.sas
+  ,msg=%superq(msg)
+)
+
+data _null_;
+  putlog "Checking engine & member type";
+run;
+%local engine memtype;
+%let memtype=%mf_getattrc(&libds,MTYPE);
+%let engine=%mf_getattrc(&libds,ENGINE);
+
+%if &engine ne V9 and &engine ne BASE %then %do;
+  data _null_;
+    putlog "Lib &lib  is not assigned using BASE engine - uses &engine instead";
+    putlog "SAS lock check will not be performed";
+  run;
+  %return;
+%end;
+%else %if &memtype ne DATA %then %do;
+  %put NOTE: Cannot lock a VIEW!! Memtype=&memtype;
+  %return;
+%end;
+
+data _null_;
+  putlog "Engine = &engine, memtype=&memtype";
+  putlog "Attempting lock statement";
+run;
+
+lock &libds;
+
+%local abortme;
+%let abortme=0;
+%if &syscc>0 or &SYSLCKRC ne 0 %then %do;
+  %let msg=Unable to apply lock on &libds (SYSLCKRC=&SYSLCKRC syscc=&syscc);
+  %put %str(ERR)OR: &sysmacroname: &msg;
+  %let abortme=1;
+%end;
+
+lock &libds clear;
+
+%mp_abort(iftrue= (&abortme=1)
+  ,mac=&sysmacroname
+  ,msg=%superq(msg)
+)
+
+%mend mp_lockfilecheck;

base/mp_makedata.sas

@@ -0,0 +1,107 @@
+/**
+  @file
+  @brief Create sample data based on the structure of an empty table
+  @details Many SAS projects involve sensitive datasets.  One way to _ensure_
+    the data is anonymised, is never to receive it in the first place!  Often
+    consultants are provided with empty tables, and expected to create complex
+    ETL flows.
+
+    This macro can help by taking an empty table, and populating it with data
+    according to the variable types and formats.
+
+    TODO:
+      @li Consider dates, datetimes, times, integers etc
+
+  Usage:
+
+      proc sql;
+      create table work.example(
+        TX_FROM float format=datetime19.,
+        DD_TYPE char(16),
+        DD_SOURCE char(2048),
+        DD_SHORTDESC char(256),
+        constraint pk primary key(tx_from, dd_type,dd_source),
+        constraint nnn not null(DD_SHORTDESC)
+      );
+      %mp_makedata(work.example)
+
+  @param [in] libds The empty table (libref.dataset) in which to create data
+  @param [out] obs= (500) The maximum number of records to create.  The table
+    is sorted with nodup on the primary key, so the actual number of records may
+    be lower than this.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_getvarlen.sas
+  @li mf_getvarlist.sas
+  @li mf_islibds.sas
+  @li mf_nobs.sas
+  @li mp_getcols.sas
+  @li mp_getpk.sas
+
+  <h4> Related Macros </h4>
+  @li mp_makedata.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_makedata(libds
+  ,obs=500
+  ,seed=1
+)/*/STORE SOURCE*/;
+
+%local ds1 ds2 lib ds pk_fields i col charvars numvars ispk;
+
+%if %mf_islibds(&libds)=0 %then %do;
+  %put &sysmacroname: Invalid libds (&libds) - should be library.dataset format;
+  %return;
+%end;
+%else %if %mf_nobs(&libds)>0 %then %do;
+  %put &sysmacroname: &libds has data, it will not be recreated;
+  %return;
+%end;
+
+/* set up temporary vars */
+%let ds1=%mf_getuniquename(prefix=mp_makedatads1);
+%let ds2=%mf_getuniquename(prefix=mp_makedatads2);
+%let lib=%scan(&libds,1,.);
+%let ds=%scan(&libds,2,.);
+
+/* grab the primary key vars */
+%mp_getpk(&lib,ds=&ds,outds=&ds1)
+
+proc sql noprint;
+select coalescec(pk_fields,'_all_') into: pk_fields from &ds1;
+
+data &ds2;
+  if 0 then set &libds;
+  do _n_=1 to &obs;
+    %let charvars=%mf_getvarlist(&libds,typefilter=C);
+    %if &charvars ^= %then %do i=1 %to %sysfunc(countw(&charvars));
+      %let col=%scan(&charvars,&i);
+      /* create random value based on observation number and colum length */
+      &col=repeat(put(md5(cats(_n_)),$hex32.),%mf_getvarlen(&libds,&col)/32);
+    %end;
+
+    %let numvars=%mf_getvarlist(&libds,typefilter=N);
+    %if &numvars ^= %then %do i=1 %to %sysfunc(countw(&numvars));
+      %let col=%scan(&numvars,&i);
+      &col=_n_;
+    %end;
+    output;
+  end;
+  stop;
+run;
+proc sort data=&ds2 nodupkey;
+  by &pk_fields;
+run;
+
+proc append base=&libds data=&ds2;
+run;
+
+proc sql;
+drop table &ds1, &ds2;
+
+%mend mp_makedata;

base/mp_reseterror.sas

@@ -0,0 +1,27 @@
+/**
+  @file
+  @brief Reset when an err condition occurs
+  @details When building apps, sometimes an operation must be attempted that
+  can cause an err condition.  There is no try catch in SAS! So the err state
+  must be caught and reset.
+
+  This macro attempts to do that reset.
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_reseterror(
+)/*/STORE SOURCE*/;
+
+options obs=max replace nosyntaxcheck;
+%let syscc=0;
+
+%if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
+  data _null_;
+    rc=stpsrvset('program error', 0);
+  run;
+%end;
+
+%mend mp_reseterror;

base/mp_resetoption.sas

@@ -3,13 +3,15 @@
   @brief Reset an option to original value
   @details Inspired by the SAS Jedi -
 https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options
-    Called as follows:
 
-    options obs=30;
-    %mp_resetoption(OBS)
+  Called as follows:
+
+      options obs=30 ps=max;
+      %mp_resetoption(OBS)
+      %mp_resetoption(PS)
 
 
-  @param option the option to reset
+  @param [in] option the option to reset
 
   @version 9.2
   @author Allan Bowe

base/mp_retainedkey.sas

@@ -0,0 +1,246 @@
+/**
+  @file
+  @brief Generate and apply retained key values to a staging table
+  @details This macro will populate a staging table with a Retained Key based on
+  a business key and a base (target) table.
+
+  Definition of retained key ([source](
+    http://bukhantsov.org/2012/04/what-is-data-vault/)):
+
+  > The retained key is a key which is mapped to business key one-to-one. In
+  > comparison,  the surrogate key includes time and there can be many surrogate
+  > keys corresponding to one business key. This explains the name of the key,
+  > it is retained with insertion of a new version of a row while surrogate key
+  > is increasing.
+
+  This macro is designed to be used as part of a wider load / ETL process (such
+  as the one in [Data Controller for SAS](https://datacontroller.io)).
+
+  Specifically, the macro assumes that the base table has already been 'locked'
+  (eg with the mp_lockanytable.sas macro) prior to invocation.  Also, several
+  tables are assumed to exist (names are configurable):
+
+  @li work.staging_table - the staged data, minus the retained key element
+  @li permlib.base_table - the target table to be loaded (**not** loaded by this
+    macro)
+  @li permlib.maxkeytable - optional, used to store load metaadata.
+    The definition is available by running mp_coretable.sas as follows:
+    `mp_coretable(MAXKEYTABLE)`.
+  @li permlib.locktable - Necessary if maxkeytable is being populated. The
+    definition is available by running mp_coretable.sas as follows:
+    `mp_coretable(LOCKTABLE)`.
+
+
+  @param [in] base_lib= (WORK) Libref of the base (target) table.
+  @param [in] base_dsn= (BASETABLE) Name of the base (target) table.
+  @param [in] append_lib= (WORK) Libref of the staging table
+  @param [in] append_dsn= (APPENDTABLE) Name of the staging table
+  @param [in] retained_key= (DEFAULT_RK) Name of RK to generate (should exist on
+    base table)
+  @param [in] business_key= (PK1 PK2) Business key against which to generate
+    RK values.  Should be unique and not null on the staging table.
+  @param [in] check_uniqueness=(NO) Set to yes to perform a uniqueness check.
+    Recommended if there is a chance that the staging data is not unique on the
+    business key.
+  @param [in] maxkeytable= (0) Provide a maxkeytable libds reference here, to
+    store load metadata (maxkey val, load time).  Set to zero if metadata is not
+    required, eg, when preparing a 'dummy' load. Structure is described above.
+    See below for sample data.
+    |KEYTABLE:$32.|KEYCOLUMN:$32.|MAX_KEY:best.|PROCESSED_DTTM:E8601DT26.6|
+    |---|---|---|---|
+    |`DC487173.MPE_SELECTBOX `|`SELECTBOX_RK `|`55 `|`1950427787.8 `|
+    |`DC487173.MPE_FILTERANYTABLE `|`filter_rk `|`14 `|`1951053886.8 `|
+  @param [in] locktable= (0) If updating the maxkeytable, provide the libds
+    reference to the lock table (per mp_lockanytable.sas macro)
+  @param [in] filter_str= Apply a filter - useful for SCD2 or BITEMPORAL loads.
+    Example: `filter_str=%str( (where=( &now < &tech_to)) )`
+  @param [out] outds= (WORK.APPEND) Output table (staging table + retained key)
+
+  <h4> SAS Macros </h4>
+  @li mf_existvar.sas
+  @li mf_getquotedstr.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_lockanytable.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filterstore.sas
+  @li mp_retainedkey.test.sas
+
+  @version 9.2
+
+**/
+
+%macro mp_retainedkey(
+  base_lib=WORK
+  ,base_dsn=BASETABLE
+  ,append_lib=WORK
+  ,append_dsn=APPENDTABLE
+  ,retained_key=DEFAULT_RK
+  ,business_key= PK1 PK2
+  ,check_uniqueness=NO
+  ,maxkeytable=0
+  ,locktable=0
+  ,outds=WORK.APPEND
+  ,filter_str=
+);
+%put &sysmacroname entry vars:;
+%put _local_;
+
+%local base_libds app_libds key_field check maxkey idx_pk newkey_cnt iserr
+  msg x tempds1 tempds2 comma_pk appnobs checknobs dropvar tempvar idx_val;
+%let base_libds=%upcase(&base_lib..&base_dsn);
+%let app_libds=%upcase(&append_lib..&append_dsn);
+%let tempds1=%mf_getuniquename();
+%let tempds2=%mf_getuniquename();
+%let comma_pk=%mf_getquotedstr(in_str=%str(&business_key),dlm=%str(,),quote=);
+%let outds=%sysfunc(ifc(%index(&outds,.)=0,work.&outds,&outds));
+/* validation checks */
+%let iserr=0;
+%if &syscc>0 %then %do;
+  %let iserr=1;
+  %let msg=%str(SYSCC=&syscc on macro entry);
+%end;
+%else %if %sysfunc(exist(&base_libds))=0 %then %do;
+  %let iserr=1;
+  %let msg=%str(Base LIBDS (&base_libds) expected but NOT FOUND);
+%end;
+%else %if %sysfunc(exist(&app_libds))=0 %then %do;
+  %let iserr=1;
+  %let msg=%str(Append LIBDS (&app_libds) expected but NOT FOUND);
+%end;
+%else %if &maxkeytable ne 0 and %sysfunc(exist(&maxkeytable))=0  %then %do;
+  %let iserr=1;
+  %let msg=%str(Maxkeytable (&maxkeytable) expected but NOT FOUND);
+%end;
+%else %if &maxkeytable ne 0 and %sysfunc(exist(&locktable))=0  %then %do;
+  %let iserr=1;
+  %let msg=%str(Locktable (&locktable) expected but NOT FOUND);
+%end;
+%else %if %length(&business_key)=0 %then %do;
+  %let iserr=1;
+  %let msg=%str(Business key (&business_key) expected but NOT FOUND);
+%end;
+
+%do x=1 %to %sysfunc(countw(&business_key));
+  /* check business key values exist */
+  %let key_field=%scan(&business_key,&x,%str( ));
+  %if not %mf_existvar(&app_libds,&key_field) %then %do;
+    %let iserr=1;
+    %let msg=Business key (&key_field) not found on &app_libds!;
+    %goto err;
+  %end;
+  %else %if not %mf_existvar(&base_libds,&key_field) %then %do;
+    %let iserr=1;
+    %let msg=Business key (&key_field) not found on &base_libds!;
+    %goto err;
+  %end;
+%end;
+%err:
+%if &iserr=1 %then %do;
+  /* err case so first perform an unlock of the base table before exiting */
+  %mp_lockanytable(
+    UNLOCK,lib=&base_lib,ds=&base_dsn,ref=%superq(msg),ctl_ds=&locktable
+  )
+%end;
+%mp_abort(iftrue=(&iserr=1),mac=mp_retainedkey,msg=%superq(msg))
+
+proc sql noprint;
+select sum(max(&retained_key),0) into: maxkey from &base_libds;
+
+/**
+  * get base table RK and bus field values for lookup
+  */
+proc sql noprint;
+create table &tempds1 as
+  select distinct &comma_pk,&retained_key
+  from &base_libds &filter_str
+  order by &comma_pk,&retained_key;
+
+%if &check_uniqueness=YES %then %do;
+  select count(*) into:checknobs
+    from (select distinct &comma_pk from &app_libds);
+  select count(*) into: appnobs from &app_libds; /* might be view */
+  %if &checknobs ne &appnobs %then %do;
+    %let msg=Source table &app_libds is not unique on (&business_key);
+    %let iserr=1;
+  %end;
+%end;
+%if &iserr=1 %then %do;
+  /* err case so first perform an unlock of the base table before exiting */
+  %mp_lockanytable(
+    UNLOCK,lib=&base_lib,ds=&base_dsn,ref=%superq(msg),ctl_ds=&locktable
+  )
+%end;
+%mp_abort(iftrue= (&iserr=1),mac=mp_retainedkey,msg=%superq(msg))
+
+%if %mf_existvar(&app_libds,&retained_key)
+%then %let dropvar=(drop=&retained_key);
+
+/* prepare interim table with retained key populated for matching keys */
+proc sql noprint;
+create table &tempds2 as
+  select b.&retained_key, a.*
+  from &app_libds &dropvar a
+  left join &tempds1 b
+  on 1
+  %do idx_pk=1 %to %sysfunc(countw(&business_key));
+    %let idx_val=%scan(&business_key,&idx_pk);
+    and a.&idx_val=b.&idx_val
+  %end;
+  order by &retained_key;
+
+/* identify the number of entries without retained keys (new records) */
+select count(*) into: newkey_cnt
+  from &tempds2
+  where missing(&retained_key);
+quit;
+
+/**
+  * Update maxkey table if link provided
+  */
+%if &maxkeytable ne 0 %then %do;
+  proc sql noprint;
+  select count(*) into: check from &maxkeytable
+    where upcase(keytable)="&base_libds";
+
+  %mp_lockanytable(LOCK
+    ,lib=%scan(&maxkeytable,1,.)
+    ,ds=%scan(&maxkeytable,2,.)
+    ,ref=Updating maxkeyvalues with mp_retainedkey
+    ,ctl_ds=&locktable
+  )
+  proc sql;
+  %if &check=0 %then %do;
+  insert into &maxkeytable
+    set keytable="&base_libds"
+      ,keycolumn="&retained_key"
+      ,max_key=%eval(&maxkey+&newkey_cnt)
+      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt;
+  %end;
+  %else %do;
+  update &maxkeytable
+    set max_key=%eval(&maxkey+&newkey_cnt)
+      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt
+    where keytable="&base_libds";
+  %end;
+  %mp_lockanytable(UNLOCK
+    ,lib=%scan(&maxkeytable,1,.)
+    ,ds=%scan(&maxkeytable,2,.)
+    ,ref=Updating maxkeyvalues with maxkey=%eval(&maxkey+&newkey_cnt)
+    ,ctl_ds=&locktable
+  )
+%end;
+
+/* fill in the missing retained key values */
+%let tempvar=%mf_getuniquename();
+data &outds(drop=&tempvar);
+  retain &tempvar %eval(&maxkey+1);
+  set &tempds2;
+  if &retained_key =. then &retained_key=&tempvar;
+  &tempvar=&tempvar+1;
+run;
+
+%mend mp_retainedkey;
+

base/mp_searchdata.sas

@@ -11,22 +11,28 @@
   Usage:
 
       %mp_searchdata(lib=sashelp, string=Jan)
-      %mp_searchdata(lib=sashelp, numval=1)
+      %mp_searchdata(lib=sashelp, ds=bird, numval=1)
+      %mp_searchdata(lib=sashelp, ds=class, string=l,outobs=5)
 
 
   Outputs zero or more tables to an MPSEARCH library with specific records.
 
-  @param lib=  the libref to search (should be already assigned)
-  @param ds= the dataset to search (leave blank to search entire library)
-  @param string= the string value to search
-  @param numval= the numeric value to search (must be exact)
-  @param outloc= the directory in which to create the output datasets with
-    matching rows.  Will default to a subfolder in the WORK library.
-  @param outobs= set to a positive integer to restrict the number of
+  @param [in] lib=  The libref to search (should be already assigned)
+  @param [in] ds= The dataset to search (leave blank to search entire library)
+  @param [in] string= String value to search (case sensitive, can be partial)
+  @param [in] numval= Numeric value to search (must be exact)
+  @param [out] outloc= (0) Optionally specify the directory in which to
+    create the the output datasets with matching rows.  By default it will
+    write them to a temporary subdirectory within the WORK folder.
+  @param [out] outlib= (MPSEARCH) Assign a different libref to the output
+    library containing the matching datasets / records
+  @param [in] outobs= set to a positive integer to restrict the number of
     observations
-  @param filter_text= add a (valid) filter clause to further filter the results
+  @param [in] filter_text= (1=1) Add a (valid) filter clause to further filter
+    the results.
 
   <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
   @li mf_mkdir.sas
@@ -36,11 +42,12 @@
   @author Allan Bowe
 **/
 
-%macro mp_searchdata(lib=sashelp
+%macro mp_searchdata(lib=
   ,ds=
   ,string= /* the query will use a contains (?) operator */
   ,numval= /* numeric must match exactly */
-  ,outloc=%sysfunc(pathname(work))/mpsearch
+  ,outloc=0
+  ,outlib=MPSEARCH
   ,outobs=-1
   ,filter_text=%str(1=1)
 )/*/STORE SOURCE*/;
@@ -57,8 +64,12 @@
 %if &string = %then %let type=N;
 %else %let type=C;
 
+%if "&outloc"="0" %then %do;
+  %let outloc=%sysfunc(pathname(work))/%mf_getuniquename();
+%end;
+
 %mf_mkdir(&outloc)
-libname mpsearch "&outloc";
+libname &outlib "&outloc";
 
 /* get the list of tables in the library */
 proc sql noprint;
@@ -70,11 +81,6 @@ select distinct memname into: table_list separated by ' '
 %end;
   ;
 /* check that we have something to check */
-proc sql
-%if &outobs>-1 %then %do;
-  outobs=&outobs
-%end;
-;
 %if %length(&table_list)=0 %then %put library &lib contains no tables!;
 /* loop through each table */
 %else %do table_num=1 %to %sysfunc(countw(&table_list,%str( )));
@@ -85,10 +91,10 @@ proc sql
   %end;
   %else %do;
     %let check_tm=%sysfunc(datetime());
-    /* build sql statement */
-    create table mpsearch.&table as select * from &lib..&table
-      where %unquote(&filter_text) and
-    (0
+    /* prep input */
+    data &outlib..&table;
+      set &lib..&table;
+      where %unquote(&filter_text) and ( 0
     /* loop through columns */
     %do colnum=1 %to %sysfunc(countw(&vars,%str( )));
       %let col=%scan(&vars,&colnum,%str( ));
@@ -102,15 +108,20 @@ proc sql
         or ("&col"n = &numval)
       %end;
     %end;
-    );
+      );
+    %if &outobs>-1 %then %do;
+      if _n_ > &outobs then stop;
+    %end;
+    run;
     %put Search query for &table took
       %sysevalf(%sysfunc(datetime())-&check_tm) seconds;
-    %if &sqlrc ne 0 %then %do;
-      %put %str(WAR)NING: SQLRC=&sqlrc when processing &table;
+    %if &syscc ne 0 %then %do;
+      %put %str(ERR)ROR: SYSCC=&syscc when processing &lib..&table;
       %return;
     %end;
-    %if %mf_nobs(mpsearch.&table)=0 %then %do;
-      drop table mpsearch.&table;
+    %if %mf_nobs(&outlib..&table)=0 %then %do;
+      proc sql;
+      drop table &outlib..&table;
     %end;
   %end;
 %end;

base/mp_setkeyvalue.sas

@@ -9,11 +9,14 @@
   <h4> SAS Macros </h4>
   @li mf_existds.sas
 
-  @param key Provide a key on which to perform the lookup
-  @param value Provide a value
-  @param type= either C or N will populate valc and valn respectively.  C is
-              default.
-  @param libds= define the target table to hold the parameters
+  <h4> Related Macros </h4>
+  @li mf_getvalue.sas
+
+  @param [in] key Provide a key on which to perform the lookup
+  @param [in] value Provide a value
+  @param [in] type= either C or N will populate valc and valn respectively.
+    C is default.
+  @param [out] libds= define the target table to hold the parameters
 
   @version 9.2
   @author Allan Bowe

base/mp_sortinplace.sas

@@ -0,0 +1,122 @@
+/**
+  @file
+  @brief Sorts a SAS dataset in place, preserving constraints
+  @details Generally if a dataset contains indexes, then it is not necessary to
+  sort it before performing operations such as merges / joins etc.
+  That said, there are a few edge cases where it can be desirable:
+
+    @li To allow adjacent records to be viewed directly in the dataset
+    @li To apply compression, or to remove deleted records
+    @li To improve performance for specific queries
+
+  This macro will only work for BASE (V9) engine libraries.  It works by
+  creating a copy of the dataset (without data, WITH constraints) in the same
+  library, appending a sorted view into it, and finally - renaming it.
+
+  Example usage:
+
+      proc sql;
+      create table work.example as
+        select * from sashelp.class;
+      alter table work.example
+        add constraint pk primary key(name);
+      %mp_sortinplace(work.example)
+
+  @param [in] libds The libref.datasetname that needs to be sorted
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_getengine.sas
+  @li mf_getquotedstr.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_getpk.sas
+
+  <h4> Related Macros </h4>
+  @li mp_sortinplace.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_sortinplace(libds
+)/*/STORE SOURCE*/;
+
+%local lib ds tempds1 tempds2 tempvw sortkey;
+
+/* perform validations */
+%mp_abort(iftrue=(%sysfunc(countc(&libds,.)) ne 1)
+  ,mac=mp_sortinplace
+  ,msg=%str(LIBDS (&libds) should have LIBREF.DATASET format)
+)
+%mp_abort(iftrue=(%mf_existds(&libds)=0)
+  ,mac=mp_sortinplace
+  ,msg=%str(&libds does not exist)
+)
+
+%let lib=%scan(&libds,1,.);
+%let ds=%scan(&libds,2,.);
+%mp_abort(iftrue=(%mf_getengine(&lib) ne V9)
+  ,mac=mp_sortinplace
+  ,msg=%str(&lib is not a BASE engine library)
+)
+
+/* grab a copy of the constraints so we know what to sort by */
+%let tempds1=%mf_getuniquename(prefix=&sysmacroname);
+%mp_getpk(lib=&lib,ds=&ds,outds=work.&tempds1)
+
+%if %mf_nobs(work.&tempds1)=0 %then %do;
+  %put &sysmacroname: No PK found in &lib..&ds;
+  %put Sorting will not take place;
+  %return;
+%end;
+
+/* fallback sortkey is all fields */
+%let sortkey=%mf_getvarlist(&libds);
+
+/* overlay actual sort key if it exists */
+data _null_;
+  set work.&tempds1;
+  call symputx('sortkey',coalescec(pk_fields,symget('sortkey')));
+run;
+
+
+/* create empty copy, with ALL constraints, in the same library */
+%let tempds2=%mf_getuniquename(prefix=&sysmacroname);
+proc append base=&lib..&tempds2 data=&libds(obs=0);
+run;
+
+/* create sorted view */
+%let tempvw=%mf_getuniquename(prefix=&sysmacroname);
+proc sql;
+create view work.&tempvw as select * from &lib..&ds
+order by %mf_getquotedstr(&sortkey,quote=N);
+
+/* append sorted data */
+proc append base=&lib..&tempds2 data=work.&tempvw;
+run;
+
+/* do validations */
+%mp_abort(iftrue=(&syscc ne 0)
+  ,mac=mp_sortinplace
+  ,msg=%str(syscc=&syscc prior to replace operation)
+)
+%mp_abort(iftrue=(%mf_nobs(&lib..&tempds2) ne %mf_nobs(&lib..&ds))
+  ,mac=mp_sortinplace
+  ,msg=%str(new dataset has a different number of logical obs to the old)
+)
+
+/* drop old dataset */
+proc sql;
+drop table &lib..&ds;
+
+/* rename the new dataset */
+proc datasets library=&lib;
+  change &tempds2=&ds;
+run;
+
+
+%mend mp_sortinplace;

base/mp_stackdiffs.sas

@@ -0,0 +1,592 @@
+/**
+  @file
+  @brief Prepares an audit table for stacking (re-applying) the changes.
+  @details When the underlying data from a Base Table is refreshed, it can be
+    helpful to have any previously-applied changes, re-applied.
+
+    Such situation might arise if you are applying those changes using a tool
+    like [Data Controller for SAS®](https://datacontroller.io) - which records
+    all such changes in an audit table.
+    It may also apply if you are preparing a series of specific cell-level
+    transactions, that you would like to apply to multiple sets of (similarly
+    structured) Base Tables.
+
+    In both cases, it is necessary that the transactions are stored using
+    the mp_storediffs.sas macro, or at least that the underlying table is
+    structured as per the definition in mp_coretable.sas (DIFFTABLE entry)
+
+    <b>This</b> macro is used to convert the stored changes (tall format) into
+    staged changes (wide format), with base table values incorporated (in the
+    case of modified rows), ready for the subsequent load process.
+
+    Essentially then, what this macro does, is turn a table like this:
+
+|KEY_HASH:$32.|MOVE_TYPE:$1.|TGTVAR_NM:$32.|IS_PK:best.|IS_DIFF:best.|TGTVAR_TYPE:$1.|OLDVAL_NUM:best32.|NEWVAL_NUM:best32.|OLDVAL_CHAR:$32765.|NEWVAL_CHAR:$32765.|
+|---|---|---|---|---|---|---|---|---|---|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`NAME `|`1 `|`-1 `|`C `|`. `|`. `|` `|`Newbie `|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`AGE `|`0 `|`-1 `|`N `|`. `|`13 `|` `|` `|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`HEIGHT `|`0 `|`-1 `|`N `|`. `|`65.3 `|` `|` `|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`SEX `|`0 `|`-1 `|`C `|`. `|`. `|` `|`F `|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`WEIGHT `|`0 `|`-1 `|`N `|`. `|`98 `|` `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`NAME `|`1 `|`-1 `|`C `|`. `|`. `|`Alfred `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`AGE `|`0 `|`-1 `|`N `|`14 `|`. `|` `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`HEIGHT `|`0 `|`-1 `|`N `|`69 `|`. `|` `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`SEX `|`0 `|`-1 `|`C `|`. `|`. `|`M `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`WEIGHT `|`0 `|`-1 `|`N `|`112.5 `|`. `|` `|` `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`NAME `|`1 `|`0 `|`C `|`. `|`. `|`Alice `|`Alice `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`AGE `|`0 `|`1 `|`N `|`13 `|`99 `|` `|` `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`HEIGHT `|`0 `|`0 `|`N `|`56.5 `|`56.5 `|` `|` `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`SEX `|`0 `|`0 `|`C `|`. `|`. `|`F `|`F `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`WEIGHT `|`0 `|`0 `|`N `|`84 `|`84 `|` `|` `|
+
+    Into three tables like this:
+
+  <b> `work.outmod`: </b>
+  |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.|
+  |---|---|---|---|---|
+  |`Alice `|`F `|`99 `|`56.5 `|`84 `|
+
+  <b> `work.outadd`: </b>
+  |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.|
+  |---|---|---|---|---|
+  |`Newbie `|`F `|`13 `|`65.3 `|`98 `|
+
+  <b> `work.outdel`: </b>
+  |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.|
+  |---|---|---|---|---|
+  |`Alfred `|`M `|`14 `|`69 `|`112.5 `|
+
+    As you might expect, there are a bunch of extra features and checks.
+
+    The macro supports both SCD2 (TXTEMPORAL) and UPDATE loadtypes. If the
+    base table contains a PROCESSED_DTTM column (or similar), this can be
+    ignored by declaring it in the `processed_dttm_var` parameter.
+
+    The macro is also flexible where columns have been added or removed from
+    the base table UNLESS there is a change to the primary key.
+
+    Changes to the primary key fields are NOT supported, and are likely to cause
+    unexpected results.
+
+    The following pre-flight checks are made:
+
+    @li All primary key columns exist on the base table
+    @li There is no change in variable TYPE for any of the columns
+    @li There is no reduction in variable LENGTH below the max-length of the
+      supplied values
+
+    Rules for stacking changes are as follows:
+
+    <table>
+    <tr>
+      <th>Transaction Type</th><th>Key Behaviour</th><th>Column Behaviour</th>
+    </tr>
+    <tr>
+      <td>Deletes</td>
+      <td>
+        The row is added to `&outDEL.` UNLESS it no longer exists
+        in the base table, in which case it is added to `&errDS.` instead.
+      </td>
+      <td>
+        Deletes are unaffected by the addition or removal of non Primary-Key
+        columns.
+      </td>
+    </tr>
+    <tr>
+      <td>Inserts</td>
+      <td>
+        Previously newly added rows are added to the `outADD` table UNLESS they
+        are present in the Base table.<br>In this case they are added to the
+        `&errDS.` table instead.
+      </td>
+      <td>
+        Inserts are unaffected by the addition of columns in the Base Table
+        (they are padded with blanks).  Deleted columns are only a problem if
+        they appear on the previous insert - in which case the record is added
+        to `&errDS.`.
+      </td>
+    </tr>
+    <tr>
+      <td>Updates</td>
+      <td>
+        Previously modified rows are merged with base table values such that
+        only the individual cells that were _previously_ changed are re-applied.
+        Where the row contains cells that were not marked as having changed in
+        the prior transaction, the 'blanks' are filled with base table values in
+        the `outMOD` table.<br>
+        If the row no longer exists on the base table, then the row is added to
+        the `errDS` table instead.
+      </td>
+      <td>
+        Updates are unaffected by the addition of columns in the Base Table -
+        the new cells are simply populated with Base Table values.  Deleted
+        columns are only a problem if they relate to a modified cell
+        (`is_diff=1`) - in which case the record is added to `&errDS.`.
+      </td>
+    </tr>
+    </table>
+
+    To illustrate the above with a diagram:
+
+  @dot
+    digraph {
+      rankdir="TB"
+      start[label="Transaction Type?" shape=Mdiamond]
+      del[label="Does Base Row exist?" shape=rectangle]
+      add [label="Does Base Row exist?" shape=rectangle]
+      mod [label="Does Base Row exist?" shape=rectangle]
+      chkmod [label="Do all modified\n(is_diff=1) cells exist?" shape=rectangle]
+      chkadd [label="Do all inserted cells exist?" shape=rectangle]
+      outmod [label="outMOD\nTable" shape=Msquare style=filled]
+      outadd [label="outADD\nTable" shape=Msquare style=filled]
+      outdel [label="outDEL\nTable" shape=Msquare style=filled]
+      outerr [label="ErrDS Table" shape=Msquare fillcolor=Orange style=filled]
+      start -> del [label="Delete"]
+      start -> add [label="Insert"]
+      start -> mod [label="Update"]
+
+      del -> outdel [label="Yes"]
+      del -> outerr [label="No" color="Red" fontcolor="Red"]
+      add -> chkadd [label="No"]
+      add -> outerr [label="Yes" color="Red" fontcolor="Red"]
+      mod -> outerr [label="No" color="Red" fontcolor="Red"]
+      mod -> chkmod [label="Yes"]
+      chkmod -> outerr [label="No" color="Red" fontcolor="Red"]
+      chkmod -> outmod [label="Yes"]
+      chkadd -> outerr [label="No" color="Red" fontcolor="Red"]
+      chkadd -> outadd [label="Yes"]
+
+    }
+  @enddot
+
+    For examples of usage, check out the mp_stackdiffs.test.sas program.
+
+
+  @param [in] baselibds Base Table against which the changes will be applied,
+    in libref.dataset format.
+  @param [in] auditlibds Dataset with previously applied transactions, to be
+    re-applied. Use libref.dataset format.
+    DDL as follows:  %mp_coretable(DIFFTABLE)
+  @param [in] key Space seperated list of key variables
+  @param [in] mdebug= Set to 1 to enable DEBUG messages and preserve outputs
+  @param [in] processed_dttm_var= (0) If a variable is being used to mark
+    the processed datetime, put the name of the variable here.  It will NOT
+    be included in the staged dataset (the load process is expected to
+    provide this)
+  @param [out] errds= (work.errds) Output table containing problematic records.
+    The columns of this table are:
+    @li PK_VARS - Space separated list of primary key variable names
+    @li PK_VALS - Slash separted list of PK variable values
+    @li ERR_MSG - Explanation of why this record is problematic
+  @param [out] outmod= (work.outmod) Output table containing modified records
+  @param [out] outadd= (work.outadd) Output table containing additional records
+  @param [out] outdel= (work.outdel) Output table containing deleted records
+
+  <h4> SAS Macros </h4>
+  @li mf_existvarlist.sas
+  @li mf_getquotedstr.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_islibds.sas
+  @li mf_nobs.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_abort.sas
+  @li mp_ds2squeeze.sas
+
+  <h4> Related Macros </h4>
+  @li mp_coretable.sas
+  @li mp_stackdiffs.test.sas
+  @li mp_storediffs.sas
+
+  @todo The current approach assumes that a variable called KEY_HASH is not on
+    the base table.  This part will need to be refactored (eg using
+    mf_getuniquename.sas) when such a use case arises.
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mp_stackdiffs(baselibds
+  ,auditlibds
+  ,key
+  ,mdebug=0
+  ,processed_dttm_var=0
+  ,errds=work.errds
+  ,outmod=work.outmod
+  ,outadd=work.outadd
+  ,outdel=work.outdel
+)/*/STORE SOURCE*/;
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+/* input parameter validations */
+%mp_abort(iftrue= (%mf_islibds(&baselibds) ne 1)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid baselibds: &baselibds)
+)
+%mp_abort(iftrue= (%mf_islibds(&auditlibds) ne 1)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid auditlibds: &auditlibds)
+)
+%mp_abort(iftrue= (%length(&key)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Missing key variables!)
+)
+%mp_abort(iftrue= (
+    %mf_existVarList(&auditlibds,LIBREF DSN MOVE_TYPE KEY_HASH TGTVAR_NM IS_PK
+      IS_DIFF TGTVAR_TYPE OLDVAL_NUM NEWVAL_NUM OLDVAL_CHAR NEWVAL_CHAR)=0
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Input &auditlibds is missing required columns!)
+)
+
+
+/* set up macro vars */
+%local prefix dslist x var keyjoin commakey keepvars missvars fref;
+%let prefix=%substr(%mf_getuniquename(),1,25);
+%let dslist=ds1d ds2d ds3d ds1a ds2a ds3a ds1m ds2m ds3m pks dups base
+  delrec delerr addrec adderr modrec moderr;
+%do x=1 %to %sysfunc(countw(&dslist));
+  %let var=%scan(&dslist,&x);
+  %local &var;
+  %let &var=%upcase(&prefix._&var);
+%end;
+
+%let key=%upcase(&key);
+%let commakey=%mf_getquotedstr(&key,quote=N);
+
+%let keyjoin=1=1;
+%do x=1 %to %sysfunc(countw(&key));
+  %let var=%scan(&key,&x);
+  %let keyjoin=&keyjoin and a.&var=b.&var;
+%end;
+
+data &errds;
+  length pk_vars $256 pk_vals $4098 err_msg $512;
+  call missing (of _all_);
+  stop;
+run;
+
+/**
+  * Prepare raw DELETE table
+  * Records are in the OLDVAL_xxx columns
+  */
+%let keepvars=MOVE_TYPE KEY_HASH TGTVAR_NM TGTVAR_TYPE IS_PK
+              OLDVAL_NUM OLDVAL_CHAR
+              NEWVAL_NUM NEWVAL_CHAR;
+proc sort data=&auditlibds(where=(move_type='D') keep=&keepvars)
+  out=&ds1d(drop=move_type);
+by KEY_HASH TGTVAR_NM;
+run;
+proc transpose data=&ds1d(where=(tgtvar_type='N'))
+    out=&ds2d(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var OLDVAL_NUM;
+run;
+proc transpose data=&ds1d(where=(tgtvar_type='C'))
+    out=&ds3d(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var OLDVAL_CHAR;
+run;
+%mp_ds2squeeze(&ds2d,outds=&ds2d)
+%mp_ds2squeeze(&ds3d,outds=&ds3d)
+data &outdel;
+  if 0 then set &baselibds;
+  set &ds2d;
+  set &ds3d;
+  drop key_hash;
+  if not missing(%scan(&key,1));
+run;
+proc sort;
+  by &key;
+run;
+
+/**
+  * Prepare raw APPEND table
+  * Records are in the NEWVAL_xxx columns
+  */
+proc sort data=&auditlibds(where=(move_type='A') keep=&keepvars)
+    out=&ds1a(drop=move_type);
+  by KEY_HASH TGTVAR_NM;
+run;
+proc transpose data=&ds1a(where=(tgtvar_type='N'))
+    out=&ds2a(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var NEWVAL_NUM;
+run;
+proc transpose data=&ds1a(where=(tgtvar_type='C'))
+    out=&ds3a(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var NEWVAL_CHAR;
+run;
+%mp_ds2squeeze(&ds2a,outds=&ds2a)
+%mp_ds2squeeze(&ds3a,outds=&ds3a)
+data &outadd;
+  if 0 then set &baselibds;
+  set &ds2a;
+  set &ds3a;
+  drop key_hash;
+  if not missing(%scan(&key,1));
+run;
+proc sort;
+  by &key;
+run;
+
+/**
+  * Prepare raw MODIFY table
+  * Keep only primary key - will add modified values later
+  */
+proc sort data=&auditlibds(
+      where=(move_type='M' and is_pk=1) keep=&keepvars
+    ) out=&ds1m(drop=move_type);
+  by KEY_HASH TGTVAR_NM;
+run;
+proc transpose data=&ds1m(where=(tgtvar_type='N'))
+    out=&ds2m(drop=_name_);
+  by KEY_HASH ;
+  id TGTVAR_NM;
+  var NEWVAL_NUM;
+run;
+proc transpose data=&ds1m(where=(tgtvar_type='C'))
+    out=&ds3m(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var NEWVAL_CHAR;
+run;
+%mp_ds2squeeze(&ds2m,outds=&ds2m)
+%mp_ds2squeeze(&ds3m,outds=&ds3m)
+data &outmod;
+  if 0 then set &baselibds;
+  set &ds2m;
+  set &ds3m;
+  if not missing(%scan(&key,1));
+run;
+proc sort;
+  by &key;
+run;
+
+/**
+  * Extract matching records from the base table
+  * Do this in one join for efficiency.
+  * At a later date, this should be optimised for large database tables by using
+  * passthrough and a temporary table.
+  */
+data &pks;
+  if 0 then set &baselibds;
+  set &outadd &outmod &outdel;
+  keep &key;
+run;
+
+proc sort noduprec dupout=&dups;
+by &key;
+run;
+data _null_;
+  set &dups;
+  putlog (_all_)(=);
+run;
+%mp_abort(iftrue= (%mf_nobs(&dups) ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(duplicates (%mf_nobs(&dups)) found on &auditlibds!)
+)
+
+proc sql;
+create table &base as
+  select a.*
+  from &baselibds a, &pks b
+  where &keyjoin;
+
+/**
+  * delete check
+  * This is straightforward as it relates to records only
+  */
+proc sql;
+create table &delrec as
+  select a.*
+  from &outdel a
+  left join &base b
+  on &keyjoin
+  where b.%scan(&key,1) is null
+  order by &commakey;
+
+data &delerr;
+  if 0 then set &errds;
+  set &delrec;
+  PK_VARS="&key";
+  PK_VALS=catx('/',&commakey);
+  ERR_MSG="Rows cannot be deleted as they do not exist on the Base dataset";
+  keep PK_VARS PK_VALS ERR_MSG;
+run;
+proc append base=&errds data=&delerr;
+run;
+
+data &outdel;
+  merge &outdel (in=a) &delrec (in=b);
+  by &key;
+  if not b;
+run;
+
+/**
+  * add check
+  * Problems - where record already exists, or base table has columns missing
+  */
+%let missvars=%mf_wordsinstr1butnotstr2(
+  Str1=%upcase(%mf_getvarlist(&outadd)),
+  Str2=%upcase(%mf_getvarlist(&baselibds))
+);
+%if %length(&missvars)>0 %then %do;
+    /* add them to the err table */
+  data &adderr;
+    if 0 then set &errds;
+    set &outadd;
+    PK_VARS="&key";
+    PK_VALS=catx('/',&commakey);
+    ERR_MSG="Rows cannot be added due to missing base vars: &missvars";
+    keep PK_VARS PK_VALS ERR_MSG;
+  run;
+  proc append base=&errds data=&adderr;
+  run;
+  proc sql;
+  delete * from &outadd;
+%end;
+%else %do;
+  proc sql;
+  /* find records that already exist on base table */
+  create table &addrec as
+    select a.*
+    from &outadd a
+    inner join &base b
+    on &keyjoin
+    order by &commakey;
+
+  /* add them to the err table */
+  data &adderr;
+    if 0 then set &errds;
+    set &addrec;
+    PK_VARS="&key";
+    PK_VALS=catx('/',&commakey);
+    ERR_MSG="Rows cannot be added as they already exist on the Base dataset";
+    keep PK_VARS PK_VALS ERR_MSG;
+  run;
+  proc append base=&errds data=&adderr;
+  run;
+
+  /* remove invalid rows from the outadd table */
+  data &outadd;
+    merge &outadd (in=a) &addrec (in=b);
+    by &key;
+    if not b;
+  run;
+%end;
+
+/**
+  * mod check
+  * Problems - where record does not exist or baseds has modified cols missing
+  */
+proc sql noprint;
+select distinct tgtvar_nm into: missvars separated by ' '
+  from &auditlibds
+  where move_type='M' and is_diff=1;
+%let missvars=%mf_wordsinstr1butnotstr2(
+  Str1=&missvars,
+  Str2=%upcase(%mf_getvarlist(&baselibds))
+);
+%if %length(&missvars)>0 %then %do;
+    /* add them to the err table */
+  data &moderr;
+    if 0 then set &errds;
+    set &outmod;
+    PK_VARS="&key";
+    PK_VALS=catx('/',&commakey);
+    ERR_MSG="Rows cannot be modified due to missing base vars: &missvars";
+    keep PK_VARS PK_VALS ERR_MSG;
+  run;
+  proc append base=&errds data=&moderr;
+  run;
+  proc sql;
+  delete * from &outmod;
+%end;
+%else %do;
+  /* now check for records that do not exist (therefore cannot be modified) */
+  proc sql;
+  create table &modrec as
+    select a.*
+    from &outmod a
+    left join &base b
+    on &keyjoin
+    where b.%scan(&key,1) is null
+    order by &commakey;
+  data &moderr;
+    if 0 then set &errds;
+    set &modrec;
+    PK_VARS="&key";
+    PK_VALS=catx('/',&commakey);
+    ERR_MSG="Rows cannot be modified as they do not exist on the Base dataset";
+    keep PK_VARS PK_VALS ERR_MSG;
+  run;
+  proc append base=&errds data=&moderr;
+  run;
+  /* delete the above records from the outmod table */
+  data &outmod;
+    merge &outmod (in=a) &modrec (in=b);
+    by &key;
+    if not b;
+  run;
+  /* now - we can prepare the final MOD table (which is currently PK only) */
+  proc sql undo_policy=none;
+  create table &outmod as
+    select a.key_hash
+      ,b.*
+    from &outmod a
+    inner join &base b
+    on &keyjoin
+    order by &commakey;
+  /* now - to update outmod with modified (is_diff=1) values */
+  %let fref=%mf_getuniquefileref();
+  data _null_;
+    file &fref;
+    set &auditlibds(where=(move_type='M')) end=lastobs;
+    by key_hash;
+    retain comma 'N';
+    if _n_=1 then put 'proc sql;';
+    if first.key_hash then do;
+      comma='N';
+      put "update &outmod set " @@;
+    end;
+    if is_diff=1 then do;
+      if comma='N' then do;
+        put '  '@@;
+        comma='Y';
+      end;
+      else put '  ,'@@;
+      if tgtvar_type='C' then do;
+        length qstr $32767;
+        qstr=quote(trim(NEWVAL_CHAR));
+        put tgtvar_nm '=' qstr;
+      end;
+      else put tgtvar_nm '=' newval_num;
+      if comma='  ' then comma='  ,';
+    end;
+    if last.key_hash then put '  where key_hash=trim("' key_hash '");';
+    if lastobs then put "alter table &outmod drop key_hash;";
+  run;
+  %inc &fref/source2;
+%end;
+
+%if &mdebug=0 %then %do;
+  proc datasets lib=work;
+    delete &prefix:;
+  run;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%mend mp_stackdiffs;
+/** @endcond */

base/mp_storediffs.sas

@@ -0,0 +1,219 @@
+/**
+  @file
+  @brief Converts deletes/changes/appends into a single audit table.
+  @details When tracking changes to data over time, it can be helpful to have
+    a single base table to track ALL modifications - enabling audit trail,
+    data recovery, and change re-application.  This macro is one of many
+    data management utilities used in [Data Controller for SAS](
+    https:datacontroller.io) - a comprehensive data ingestion solution, which
+    works on any SAS platform (Viya, SAS 9, Foundation) and is free for up to 5
+    users.
+
+    NOTE - this macro does not validate the inputs. It is assumed that the
+    datasets containing the new / changed / deleted rows are CORRECT, contain
+    no additional (or missing columns), and that the originals dataset contains
+    all relevant base records (and no additionals).
+
+    Usage:
+
+        data work.orig work.deleted work.changed work.appended;
+          set sashelp.class;
+          if _n_=1 then do;
+            output work.orig work.deleted;
+          end;
+          else if _n_=2 then do;
+            output work.orig;
+            age=99;
+            output work.changed;
+          end;
+          else do;
+            name='Newbie';
+            output work.appended;
+            stop;
+          end;
+        run;
+
+        %mp_storediffs(sashelp.class,work.orig,NAME
+          ,delds=work.deleted
+          ,modds=work.changed
+          ,appds=work.appended
+          ,outds=work.final
+          ,mdebug=1
+        )
+
+  @param [in] libds Target table against which the changes were applied
+  @param [in] origds Dataset with original (unchanged) records.  Can be empty if
+    only appending.
+  @param [in] key Space seperated list of key variables
+  @param [in] delds= (0) Dataset with deleted records
+  @param [in] appds= (0) Dataset with appended records
+  @param [in] modds= (0) Dataset with modified records
+  @param [out] outds= (work.mp_storediffs) Output table containing stored data.
+    DDL as follows:  %mp_coretable(DIFFTABLE)
+
+  @param [in] processed_dttm= (0) Provide a datetime constant in relation to
+    the actual load time.  If not provided, current timestamp is used.
+  @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
+  @param [out] loadref= (0) Provide a unique key to reference the load,
+    otherwise a UUID will be generated.
+
+  <h4> SAS Macros </h4>
+  @li mf_getquotedstr.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
+
+  <h4> Related Macros </h4>
+  @li mp_stackdiffs.sas
+  @li mp_storediffs.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mp_storediffs(libds
+  ,origds
+  ,key
+  ,delds=0
+  ,appds=0
+  ,modds=0
+  ,outds=work.mp_storediffs
+  ,loadref=0
+  ,processed_dttm=0
+  ,mdebug=0
+)/*/STORE SOURCE*/;
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+/* set up unique and temporary vars */
+%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist;
+%let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_ds1));
+%let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_ds2));
+%let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_ds3));
+%let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_ds4));
+%let hashkey=%upcase(%mf_getuniquename(prefix=mpsd_hashkey));
+%let inds_auto=%upcase(%mf_getuniquename(prefix=mpsd_inds_auto));
+%let inds_keep=%upcase(%mf_getuniquename(prefix=mpsd_inds_keep));
+
+%let dslist=&origds;
+%if &delds ne 0 %then %do;
+  %let delds=%upcase(&delds);
+  %if %scan(&delds,-1,.)=&delds %then %let delds=WORK.&delds;
+  %let dslist=&dslist &delds;
+%end;
+%if &appds ne 0 %then %do;
+  %let appds=%upcase(&appds);
+  %if %scan(&appds,-1,.)=&appds %then %let appds=WORK.&appds;
+  %let dslist=&dslist &appds;
+%end;
+%if &modds ne 0 %then %do;
+  %let modds=%upcase(&modds);
+  %if %scan(&modds,-1,.)=&modds %then %let modds=WORK.&modds;
+  %let dslist=&dslist &modds;
+%end;
+
+%let origds=%upcase(&origds);
+%if %scan(&origds,-1,.)=&origds %then %let origds=WORK.&origds;
+
+%let key=%upcase(&key);
+
+/* hash the key and append all the tables (marking the source) */
+data &ds1;
+  set &dslist indsname=&inds_auto;
+  &hashkey=put(md5(catx('|',%mf_getquotedstr(&key,quote=N))),$hex32.);
+  &inds_keep=&inds_auto;
+proc sort;
+  by &inds_keep &hashkey;
+run;
+
+/* transpose numeric & char vars */
+proc transpose data=&ds1
+    out=&ds2(rename=(&hashkey=key_hash _name_=tgtvar_nm col1=newval_num));
+  by &inds_keep &hashkey;
+  var _numeric_;
+run;
+proc transpose data=&ds1
+    out=&ds3(
+      rename=(&hashkey=key_hash _name_=tgtvar_nm col1=newval_char)
+      where=(tgtvar_nm not in ("&hashkey","&inds_keep"))
+    );
+  by &inds_keep &hashkey;
+  var _character_;
+run;
+data &ds4;
+  length &inds_keep $41 tgtvar_nm $32;
+  set &ds2 &ds3 indsname=&inds_auto;
+
+  tgtvar_nm=upcase(tgtvar_nm);
+  if tgtvar_nm in (%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE)));
+
+  if &inds_auto="&ds2" then tgtvar_type='N';
+  else if &inds_auto="&ds3" then tgtvar_type='C';
+  else do;
+    putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
+    call symputx('syscc',98);
+  end;
+
+  if &inds_keep="&appds" then move_type='A';
+  else if &inds_keep="&delds" then move_type='D';
+  else if &inds_keep="&modds" then move_type='M';
+  else if &inds_keep="&origds" then move_type='O';
+  else do;
+    putlog "%str(ERR)OR: unidentified movetype input!" &inds_keep;
+    call symputx('syscc',99);
+  end;
+  tgtvar_nm=upcase(tgtvar_nm);
+  if tgtvar_nm in (%mf_getquotedstr(&key)) then is_pk=1;
+  else is_pk=0;
+  drop &inds_keep;
+run;
+
+%if "&loadref"="0" %then %let loadref=%sysfunc(uuidgen());
+%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime());
+%let libds=%upcase(&libds);
+
+/* join orig vals for modified & deleted */
+proc sql;
+create table &outds as
+  select "&loadref" as load_ref length=36
+    ,&processed_dttm as processed_dttm format=E8601DT26.6
+    ,"%scan(&libds,1,.)" as libref length=8
+    ,"%scan(&libds,2,.)" as dsn length=32
+    ,b.key_hash length=32
+    ,b.move_type length=1
+    ,b.tgtvar_nm length=32
+    ,b.is_pk
+    ,case when b.move_type ne 'M' then -1
+      when a.newval_num=b.newval_num and a.newval_char=b.newval_char then 0
+      else 1
+      end as is_diff
+    ,b.tgtvar_type length=1
+    ,case when b.move_type='D' then b.newval_num
+      else a.newval_num
+      end as oldval_num format=best32.
+    ,case when b.move_type='D' then .
+      else b.newval_num
+      end as newval_num format=best32.
+    ,case when b.move_type='D' then b.newval_char
+      else a.newval_char
+      end as oldval_char length=32765
+    ,case when b.move_type='D' then ''
+      else b.newval_char
+      end as newval_char length=32765
+  from &ds4(where=(move_type='O')) as a
+  right join &ds4(where=(move_type ne 'O')) as b
+  on a.tgtvar_nm=b.tgtvar_nm
+  and a.key_hash=b.key_hash
+  order by move_type, key_hash,is_pk desc, tgtvar_nm;
+
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &ds1, &ds2, &ds3, &ds4;
+%end;
+
+%mend mp_storediffs;
+/** @endcond */

base/mp_streamfile.sas

@@ -12,17 +12,19 @@
 
       %mp_streamfile(contenttype=csv,inloc=/some/where.txt,outname=myfile.txt)
 
+  @param [in] contenttype= (TEXTS) Either TEXT, ZIP, CSV, EXCEL
+  @param [in] inloc= /path/to/file.ext to be sent
+  @param [in] inref= fileref of file to be sent (if provided, overrides `inloc`)
+  @param [in] iftrue= (1=1) Provide a condition under which to execute.
+  @param [out] outname= the name of the file, as downloaded by the browser
+  @param [out] outref= (_webout) The destination where the file should be
+    streamed.
+
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
   @li mp_binarycopy.sas
 
-  @param contenttype= Either TEXT, ZIP, CSV, EXCEL (default TEXT)
-  @param inloc= /path/to/file.ext to be sent
-  @param inref= fileref of file to be sent (if provided, overrides `inloc`)
-  @param outname= the name of the file, as downloaded by the browser
-
   @author Allan Bowe
-  @source https://github.com/sasjs/core
 
 **/
 
@@ -30,12 +32,16 @@
   contenttype=TEXT
   ,inloc=
   ,inref=0
+  ,iftrue=%str(1=1)
   ,outname=
+  ,outref=_webout
 )/*/STORE SOURCE*/;
 
-%let contentype=%upcase(&contenttype);
-%local platform; %let platform=%mf_getplatform();
+%if not(%eval(%unquote(&iftrue))) %then %return;
 
+%let contentype=%upcase(&contenttype);
+%let outref=%upcase(&outref);
+%local platform; %let platform=%mf_getplatform();
 
 /**
   * check engine type to avoid the below err message:
@@ -44,20 +50,20 @@
 %local streamweb;
 %let streamweb=0;
 data _null_;
-  set sashelp.vextfl(where=(upcase(fileref)="_WEBOUT"));
+  set sashelp.vextfl(where=(upcase(fileref)="&outref"));
   if xengine='STREAM' then call symputx('streamweb',1,'l');
 run;
 
-%if &contentype=ZIP %then %do;
+%if &contentype=CSV %then %do;
   %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/zip');
+      rc=stpsrv_header('Content-type','application/csv');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.zip'
-      contenttype='application/zip'
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
+      contenttype='application/csv'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
@@ -70,11 +76,41 @@ run;
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
       contenttype='application/vnd.ms-excel'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
+%else %if &contentype=HTML %then %do;
+  %if &platform=SASVIYA %then %do;
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
+      contenttype="text/html";
+  %end;
+%end;
+%else %if &contentype=TEXT %then %do;
+  %if &platform=SASMETA and &streamweb=1 %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type','application/text');
+      rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
+      contenttype='application/text'
+      contentdisp="attachment; filename=&outname";
+  %end;
+%end;
+%else %if &contentype=WOFF or &contentype=WOFF2 or &contentype=TTF %then %do;
+  %if &platform=SASMETA and &streamweb=1 %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type',"font/%lowcase(&contenttype)");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI"
+      contenttype="font/%lowcase(&contenttype)";
+  %end;
+%end;
 %else %if &contentype=XLSX %then %do;
   %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
@@ -84,54 +120,34 @@ run;
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
       contenttype=
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
-%else %if &contentype=TEXT %then %do;
+%else %if &contentype=ZIP %then %do;
   %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/text');
+      rc=stpsrv_header('Content-type','application/zip');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
-      contenttype='application/text'
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.zip'
+      contenttype='application/zip'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
-%else %if &contentype=CSV %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
-    data _null_;
-      rc=stpsrv_header('Content-type','application/csv');
-      rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
-    run;
-  %end;
-  %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
-      contenttype='application/csv'
-      contentdisp="attachment; filename=&outname";
-  %end;
-%end;
-%else %if &contentype=HTML %then %do;
-  %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
-      contenttype="text/html";
-  %end;
-%end;
 %else %do;
   %put %str(ERR)OR: Content Type &contenttype NOT SUPPORTED by &sysmacroname!;
-  %return;
 %end;
 
 %if &inref ne 0 %then %do;
-  %mp_binarycopy(inref=&inref,outref=_webout)
+  %mp_binarycopy(inref=&inref,outref=&outref)
 %end;
 %else %do;
-  %mp_binarycopy(inloc="&inloc",outref=_webout)
+  %mp_binarycopy(inloc="&inloc",outref=&outref)
 %end;
 
 %mend mp_streamfile;

base/mp_testservice.sas

@@ -1,12 +1,10 @@
 /**
-  @file mp_testservice.sas
-  @brief Will execute a test against a SASjs web service on SAS 9 or Viya
+  @file
+  @brief Will execute a SASjs web service on SAS 9 or Viya
   @details Prepares the input files and retrieves the resulting datasets from
   the response JSON.
 
-      %mp_testjob(
-        duration=60*5
-      )
+
 
   Note - the _webout fileref should NOT be assigned prior to running this macro.
 
@@ -14,6 +12,10 @@
   @param [in] inputfiles=(0) A list of space seperated fileref:filename pairs as
     follows:
         inputfiles=inref:filename inref2:filename2
+  @param [in] inputdatasets= (0) All datasets in this space seperated list are
+    converted into SASJS-formatted CSVs (see mp_ds2csv.sas) files and added to
+    the list of `inputfiles` for ingestion.  The dataset will be sent with the
+    same name (no need for a colon modifier).
   @param [in] inputparams=(0) A dataset containing name/value pairs in the
     following format:
     |name:$32|value:$1000|
@@ -38,9 +40,13 @@
   @li mf_getuniquename.sas
   @li mp_abort.sas
   @li mp_binarycopy.sas
+  @li mp_ds2csv.sas
   @li mv_getjobresult.sas
   @li mv_jobflow.sas
 
+  <h4> Related Programs </h4>
+  @li mp_testservice.test.sas
+
   @version 9.4
   @author Allan Bowe
 
@@ -48,6 +54,7 @@
 
 %macro mp_testservice(program,
   inputfiles=0,
+  inputdatasets=0,
   inputparams=0,
   debug=log,
   mdebug=0,
@@ -56,7 +63,7 @@
   viyaresult=WEBOUT_JSON,
   viyacontext=SAS Job Execution compute context
 )/*/STORE SOURCE*/;
-%local dbg;
+%local dbg pcnt fref1 webref i webcount var platform;
 %if &mdebug=1 %then %do;
   %put &sysmacroname entry vars:;
   %put _local_;
@@ -64,7 +71,6 @@
 %else %let dbg=*;
 
 /* sanitise inputparams */
-%local pcnt;
 %let pcnt=0;
 %if &inputparams ne 0 %then %do;
   data _null_;
@@ -76,7 +82,7 @@
     else do;
       x+1;
       call symputx(name,quote(cats(value)),'l');
-      call symputx('pval'!!left(x),name,'l');
+      call symputx(cats('pval',x),name,'l');
       call symputx('pcnt',x,'l');
     end;
   run;
@@ -86,17 +92,25 @@
   )
 %end;
 
+/* convert inputdatasets to filerefs */
+%if "&inputdatasets" ne "0" %then %do;
+  %if %quote(&inputfiles)=0 %then %let inputfiles=;
+  %do i=1 %to %sysfunc(countw(&inputdatasets,%str( )));
+    %let var=%scan(&inputdatasets,&i,%str( ));
+    %local dsref&i;
+    %let dsref&i=%mf_getuniquefileref();
+    %mp_ds2csv(&var,outref=&&dsref&i,headerformat=SASJS)
+    %let inputfiles=&inputfiles &&dsref&i:%scan(&var,-1,.);
+  %end;
+%end;
+
 
-%local fref1 webref;
 %let fref1=%mf_getuniquefileref();
 %let webref=%mf_getuniquefileref();
-
-%local platform;
 %let platform=%mf_getplatform();
 %if &platform=SASMETA %then %do;
 
   /* parse the input files */
-  %local webcount i var;
   %if %quote(&inputfiles) ne 0 %then %do;
     %let webcount=%sysfunc(countw(&inputfiles));
     %put &=webcount;

base/mp_unzip.sas

@@ -2,22 +2,27 @@
   @file mp_unzip.sas
   @brief Unzips a zip file
   @details Opens the zip file and copies all the contents to another directory.
-    It is not possible to retain permissions / timestamps, also the BOF marker
-    is lost so it cannot extract binary files.
+  It is not possible to retain permissions / timestamps, also the BOF marker
+  is lost so it cannot extract binary files.
 
-    Usage:
+  Usage:
 
-      filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      filename mc url
+        "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
 
       %mp_unzip(ziploc="/some/file.zip",outdir=/some/folder)
 
+  More info:  https://blogs.sas.com/content/sasdummy/2015/05/11/using-filename-zip-to-unzip-and-read-data-files-in-sas/
+
+  @param ziploc= Fileref or quoted full path to zip file ("/path/to/file.zip")
+  @param outdir= (%sysfunc(pathname(work))) Directory in which to write the
+    outputs (created if non existant)
+
   <h4> SAS Macros </h4>
   @li mf_mkdir.sas
   @li mf_getuniquefileref.sas
-
-  @param ziploc= fileref or quoted full path to zip file ("/path/to/file.zip")
-  @param outdir= directory in which to write the outputs (created if non existant)
+  @li mp_binarycopy.sas
 
   @version 9.4
   @author Allan Bowe
@@ -30,17 +35,20 @@
   ,outdir=%sysfunc(pathname(work))
 )/*/STORE SOURCE*/;
 
-%local fname1 fname2 fname3;
-%let fname1=%mf_getuniquefileref();
-%let fname2=%mf_getuniquefileref();
-%let fname3=%mf_getuniquefileref();
+%local f1 f2 ;
+%let f1=%mf_getuniquefileref();
+%let f2=%mf_getuniquefileref();
 
-filename &fname1 ZIP &ziploc; * Macro variable &datazip would be read from the file*;
+/* Macro variable &datazip would be read from the file */
+filename &f1 ZIP &ziploc;
+
+/* create target folder */
+%mf_mkdir(&outdir)
 
 /* Read the "members" (files) from the ZIP file */
 data _data_(keep=memname isFolder);
   length memname $200 isFolder 8;
-  fid=dopen("&fname1");
+  fid=dopen("&f1");
   if fid=0 then stop;
   memcount=dnum(fid);
   do i=1 to memcount;
@@ -51,16 +59,32 @@ data _data_(keep=memname isFolder);
   end;
   rc=dclose(fid);
 run;
-filename &fname1 clear;
+
+filename &f2 temp;
 
 /* loop through each entry and either create the subfolder or extract member */
 data _null_;
   set &syslast;
+  file &f2;
   if isFolder then call execute('%mf_mkdir(&outdir/'!!memname!!')');
-  else call execute('filename &fname2 zip &ziploc member='
-    !!quote(trim(memname))!!';filename &fname3 "&outdir/'
-    !!trim(memname)!!'" recfm=n;data _null_; rc=fcopy("&fname2","&fname3");run;'
-    !!'filename &fname2 clear; filename &fname3 clear;');
+  else do;
+    qname=quote(cats("&outdir/",memname));
+    bname=cats('(',memname,')');
+    put '/* hat tip: "data _null_" on SAS-L */';
+    put 'data _null_;';
+    put '  infile &f1 ' bname ' lrecl=256 recfm=F length=length eof=eof unbuf;';
+    put '  file ' qname ' lrecl=256 recfm=N;';
+    put '  input;';
+    put '  put _infile_ $varying256. length;';
+    put '  return;';
+    put 'eof:';
+    put '  stop;';
+    put 'run;';
+  end;
 run;
 
-%mend mp_unzip;
+%inc &f2/source2;
+
+filename &f2 clear;
+
+%mend mp_unzip;

base/mp_validatecol.sas

@@ -20,15 +20,24 @@
       ;;;;
       run;
 
+  For more examples, see mp_validatecol.test.sas
+
+  Tip - when contributing, use https://regex101.com to test the regex validity!
+
   @param [in] incol The column to be validated
   @param [in] rule The rule to apply.  Current rules:
+    @li ISINT - checks if the variable is an integer
     @li ISNUM - checks if the variable is numeric
     @li LIBDS - matches LIBREF.DATASET format
+    @li FORMAT - checks if the provided format is syntactically valid
   @param [out] outcol The variable to create, with the results of the match
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas
 
+  <h4> Related Macros </h4>
+  @li mp_validatecol.test.sas
+
   @version 9.3
 **/
 
@@ -38,7 +47,15 @@
 %local tempcol;
 %let tempcol=%mf_getuniquename();
 
-%if &rule=ISNUM %then %do;
+%if &rule=ISINT %then %do;
+  &outcol=0;
+  if not missing(&incol) then do;
+    &tempcol=input(&incol,?? best32.);
+    if not missing(&tempcol) then if mod(&tempcol,1)=0 then &outcol=1;
+  end;
+  drop &tempcol;
+%end;
+%else %if &rule=ISNUM %then %do;
   /*
     credit SØREN LASSEN
     https://sasmacro.blogspot.com/2009/06/welcome-isnum-macro.html
@@ -62,5 +79,19 @@
   if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
   else &outcol=0;
 %end;
+%else %if &rule=FORMAT %then %do;
+  /* match valid format - regex could probably be improved */
+  if _n_=1 then do;
+    retain &tempcol;
+    &tempcol=prxparse('/^[_a-z\$]\w{0,31}\.[0-9]*$/i');
+    if missing(&tempcol) then do;
+      putlog "%str(ERR)OR: Invalid expression for FORMAT";
+      stop;
+    end;
+    drop &tempcol;
+  end;
+  if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
+  else &outcol=0;
+%end;
 
 %mend mp_validatecol;

base/mp_wait4file.sas

@@ -0,0 +1,37 @@
+/**
+  @file
+  @brief Wait until a file arrives before continuing execution
+  @details Loops with a `sleep()` command until a file arrives or the max wait
+  period expires.
+
+  @example
+
+  Wait 3 minutes OR for /tmp/flag.txt to appear
+
+    %mp_wait4file(/tmp/flag.txt , maxwait=60*3)
+
+  @param [in] file The file to wait for.  Must be provided.
+  @param [in] maxwait= (0) Number of seconds to wait.  If set to zero, will
+    loop indefinitely (to a maximum of 46 days, per SAS [documentation](
+      https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a001418809.htm
+    )).  Otherwise, execution will proceed upon sleep expiry.
+  @param [in] interval= (1) The wait period between sleeps, in seconds
+
+
+**/
+
+%macro mp_wait4file(file, maxwait=0, interval=1);
+
+%if %str(&file)=%str() %then %do;
+  %put %str(ERR)OR: file not provided;
+%end;
+
+data _null_;
+  maxwait=&maxwait;
+  if maxwait=0 then maxwait=60*60*24*46;
+  do until (fileexist("&file") or slept>maxwait );
+    slept=sum(slept,sleep(&interval,1));
+  end;
+run;
+
+%mend mp_wait4file;

base/mp_webin.sas

@@ -0,0 +1,59 @@
+/**
+  @file
+  @brief Fix the `_WEBIN` variables provided to SAS web services
+  @details When uploading files to SAS Stored Processes or Viya Jobs a number
+  of global macro variables are automatically created - however there are some
+  differences in behaviour both between SAS 9 and Viya, and also between a
+  single file upload and a multi-file upload.
+
+  This macro "straightens" up the global macro variables to make it easier /
+  simpler to write code that works in both environments and with a variable
+  number of file inputs.
+
+  After running this macro, the following global variables will *always* exist:
+  @li `_WEBIN_FILE_COUNT`
+  @li `_WEBIN_FILENAME1`
+  @li `_WEBIN_FILEREF1`
+  @li `_WEBIN_NAME1`
+
+  Usage:
+
+    %mp_webin()
+
+  This was created as a macro procedure (over a macro function) as it will also
+  use the filename statement in Viya environments (where `_webin_fileuri` is
+  provided).
+
+  <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+
+**/
+
+%macro mp_webin();
+
+/* prepare global variables */
+%global _webin_file_count
+  _webin_filename _webin_filename1
+  _webin_fileref _webin_fileref1
+  _webin_fileuri _webin_fileuri1
+  _webin_name _webin_name1
+  ;
+
+/* create initial versions */
+%let _webin_file_count=%eval(&_webin_file_count+0);
+%let _webin_filename1=%sysfunc(coalescec(&_webin_filename1,&_webin_filename));
+%let _webin_fileref1=%sysfunc(coalescec(&_webin_fileref1,&_webin_fileref));
+%let _webin_fileuri1=%sysfunc(coalescec(&_webin_fileuri1,&_webin_fileuri));
+%let _webin_name1=%sysfunc(coalescec(&_webin_name1,&_webin_name));
+
+
+/* If Viya, create temporary fileref(s) */
+%local i;
+%if %mf_getplatform()=SASVIYA %then %do i=1 %to &_webin_file_count;
+  %let _webin_fileref&i=%mf_getuniquefileref();
+  filename &&_webin_fileref&i filesrvc "&&_webin_fileuri&i";
+%end;
+
+
+%mend mp_webin;

base/mp_zip.sas

@@ -16,11 +16,18 @@
   @li mp_dirlist.sas
 
   @param in= unquoted filepath, dataset of files or directory to zip
-  @param type= FILE, DATASET, DIRECTORY. (FILE / DATASET not ready yet)
-  @param outname= output file to create, without .zip extension
-  @param outpath= location for output zip file
+  @param type= (FILE) Valid values:
+    @li FILE - /full/path/and/filename.extension to a particular file
+    @li DATASET - a dataset containing a list of files to zip (see `incol`)
+    @li DIRECTORY - a directory to zip
+  @param outname= (FILE) Output file to create, _without_ .zip extension
+  @param outpath= (%sysfunc(pathname(WORK))) Parent folder for output zip file
   @param incol= if DATASET input, say which column contains the filepath
 
+  <h4> Related Macros </h4>
+  @li mp_unzip.sas
+  @li mp_zip.test.sas
+
   @version 9.2
   @author Allan Bowe
   @source https://github.com/sasjs/core
@@ -51,9 +58,9 @@ ods package open nopf;
     set &ds;
     length __command $4000;
     if file_or_folder='file';
-    command=cats('ods package add file="',filepath
+    __command=cats('ods package add file="',filepath
       ,'" mimetype="application/x-compress";');
-    call execute(command);
+    call execute(__command);
   run;
   /* tidy up */
   %if &debug=NO %then %do;
@@ -64,11 +71,10 @@ ods package open nopf;
   data _null_;
     set &in;
     length __command $4000;
-    command=cats('ods package add file="',&incol
+    __command=cats('ods package add file="',&incol
       ,'" mimetype="application/x-compress";');
-    call execute(command);
+    call execute(__command);
   run;
-  ods package add file="&in" mimetype="application/x-compress";
 %end;