Merge pull request #128 from sasjs/writefile

Writefile

.all-contributorsrc

@@ -0,0 +1,124 @@
+{
+  "projectName": "core",
+  "projectOwner": "sasjs",
+  "repoType": "github",
+  "repoHost": "https://github.com",
+  "files": [
+    "README.md"
+  ],
+  "imageSize": 100,
+  "commit": false,
+  "commitConvention": "angular",
+  "contributors": [
+    {
+      "login": "allanbowe",
+      "name": "Allan Bowe",
+      "avatar_url": "https://avatars.githubusercontent.com/u/4420615?v=4",
+      "profile": "https://github.com/allanbowe",
+      "contributions": [
+        "business",
+        "code",
+        "content",
+        "doc",
+        "infra",
+        "maintenance",
+        "mentoring",
+        "question",
+        "review",
+        "test"
+      ]
+    },
+    {
+      "login": "rafgag",
+      "name": "rafgag",
+      "avatar_url": "https://avatars.githubusercontent.com/u/69139928?v=4",
+      "profile": "https://github.com/rafgag",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "tmoody",
+      "name": "Trevor Moody",
+      "avatar_url": "https://avatars.githubusercontent.com/u/79837106?v=4",
+      "profile": "https://github.com/tmoody",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "krishna-acondy",
+      "name": "Krishna Acondy",
+      "avatar_url": "https://avatars.githubusercontent.com/u/2980428?v=4",
+      "profile": "https://krishna-acondy.io/",
+      "contributions": [
+        "code",
+        "infra",
+        "blog",
+        "content",
+        "ideas",
+        "video"
+      ]
+    },
+    {
+      "login": "saadjutt01",
+      "name": "Muhammad Saad ",
+      "avatar_url": "https://avatars.githubusercontent.com/u/8914650?v=4",
+      "profile": "https://github.com/saadjutt01",
+      "contributions": [
+        "code",
+        "ideas"
+      ]
+    },
+    {
+      "login": "YuryShkoda",
+      "name": "Yury Shkoda",
+      "avatar_url": "https://avatars.githubusercontent.com/u/25773492?v=4",
+      "profile": "https://www.erudicat.com/",
+      "contributions": [
+        "code",
+        "infra",
+        "video"
+      ]
+    },
+    {
+      "login": "medjedovicm",
+      "name": "Mihajlo Medjedovic",
+      "avatar_url": "https://avatars.githubusercontent.com/u/18329105?v=4",
+      "profile": "https://github.com/medjedovicm",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "kkchandok",
+      "name": "kkchandok",
+      "avatar_url": "https://avatars.githubusercontent.com/u/46090627?v=4",
+      "profile": "https://github.com/kkchandok",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "VladislavParhomchik",
+      "name": "Vladislav Parhomchik",
+      "avatar_url": "https://avatars.githubusercontent.com/u/83717836?v=4",
+      "profile": "https://github.com/VladislavParhomchik",
+      "contributions": [
+        "test",
+        "review"
+      ]
+    },
+    {
+      "login": "vznesh",
+      "name": "Vignesh T.",
+      "avatar_url": "https://avatars.githubusercontent.com/u/28916792?v=4",
+      "profile": "https://github.com/vznesh",
+      "contributions": [
+        "bug"
+      ]
+    }
+  ],
+  "contributorsPerLine": 7,
+  "skipCi": true
+}

.editorconfig

@@ -1,12 +0,0 @@
-# http://editorconfig.org
-root = true
-
-[*]
-charset = utf-8
-indent_style = space
-indent_size = 2
-end_of_line = lf
-insert_final_newline = false
-trim_trailing_whitespace = true
-
-

.git-hooks/commit-msg

@@ -0,0 +1,18 @@
+#!/bin/sh
+RED="\033[1;31m"
+GREEN="\033[1;32m"
+
+# Get the commit message (the parameter we're given is just the path to the
+# temporary file which holds the message).
+commit_message=$(cat "$1")
+
+if (echo "$commit_message" | grep -Eq "^(build|chore|ci|docs|feat|fix|perf|refactor|revert|style|test)(\([a-z \-]+\))?!?: .+$") then
+   echo "${GREEN} ✔ Commit message meets Conventional Commit standards"
+   exit 0
+fi
+
+echo "${RED}❌ Commit message does not meet the Conventional Commit standard!"
+echo "An example of a valid message is:"
+echo "  feat(login): add the 'remember me' button"
+echo "ℹ More details at: https://www.conventionalcommits.org/en/v1.0.0/#summary"
+exit 1

.git-hooks/pre-commit

@@ -0,0 +1,11 @@
+#!/bin/bash
+sasjs lint
+
+# Avoid commits to the master branch
+BRANCH=`git rev-parse --abbrev-ref HEAD`
+
+if [[ "$BRANCH" =~ ^(master|main|develop)$ ]]; then
+  echo "You are on branch $BRANCH. Are you sure you want to commit to this branch?"
+  echo "If so, commit with -n to bypass the pre-commit hook."
+  exit 1
+fi

.github/CHANGELOG.md

@@ -0,0 +1,5 @@
+# CHANGELOG
+
+As the changes are managed automatically in github, we don't generate an additional changelog.  To view the fixes/features in each release, check out the releases page below:
+
+[https://github.com/sasjs/core/releases](https://github.com/sasjs/core/releases)

.github/CODE_OF_CONDUCT.md

@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+https://sasapps.io/contact-us.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

.github/SECURITY.md

@@ -0,0 +1,18 @@
+# Security Policy
+
+Security is an extremely high priority when it comes to the SASjs product suite.  We take a number of steps across all repositories to minimise risk, such as:
+
+* Regular dependabot updates
+* Snyk reports
+* Minimising dependencies, especially production dependencies (sasjs/core has NONE)
+* Testing & Code review process
+
+## Supported Versions
+
+We support only the latest version
+
+## Reporting a Vulnerability
+
+We welcome disclosures of all kinds in relation to all the SASjs libraries.  You can submit them here: [https://sasapps.io/contact-us](https://sasapps.io/contact-us)
+
+

.github/vpn/config.ovpn

@@ -0,0 +1,30 @@
+cipher AES-256-CBC
+setenv FORWARD_COMPATIBLE 1
+client
+server-poll-timeout 4
+nobind
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 443 tcp
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 1194 udp
+dev tun
+dev-type tun
+ns-cert-type server
+setenv opt tls-version-min 1.0 or-highest
+reneg-sec 604800
+sndbuf 0
+rcvbuf 0
+# NOTE: LZO commands are pushed by the Access Server at connect time.
+# NOTE: The below line doesn't disable LZO.
+comp-lzo no
+verb 3
+setenv PUSH_PEER_INFO
+
+ca ca.crt
+cert user.crt
+key user.key
+tls-auth tls.key 1

.github/workflows/run-tests.yml

@@ -0,0 +1,84 @@
+# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+
+name: Node.js CI
+
+on:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [lts/fermium]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Write VPN Files
+        run: |
+          echo "$CA_CRT" > .github/vpn/ca.crt
+          echo "$USER_CRT" > .github/vpn/user.crt
+          echo "$USER_KEY" > .github/vpn/user.key
+          echo "$TLS_KEY" > .github/vpn/tls.key
+        shell: bash
+        env:
+          CA_CRT: ${{ secrets.CA_CRT}}
+          USER_CRT: ${{ secrets.USER_CRT }}
+          USER_KEY: ${{ secrets.USER_KEY }}
+          TLS_KEY: ${{ secrets.TLS_KEY }}
+
+      - name: Install Open VPN
+        run: |
+          sudo apt install apt-transport-https
+          sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
+          sudo apt-key add openvpn-repo-pkg-key.pub
+          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-focal.list
+          sudo apt update
+          sudo apt install openvpn3
+
+      - name: Start Open VPN 3
+        run: openvpn3 session-start --config .github/vpn/config.ovpn
+
+      - name: Install Doxygen
+        run: sudo apt-get install doxygen
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Check code style
+        run: npm run lint
+
+      - name: Add client
+        run: echo "CLIENT=${{secrets.CLIENT}}"> .env.viya
+
+      - name: Add secret
+        run: echo "SECRET=${{secrets.SECRET}}" >> .env.viya
+
+      - name: Add access token
+        run: echo "ACCESS_TOKEN=${{secrets.ACCESS_TOKEN}}" >> .env.viya
+
+      - name: Add refresh token
+        run: echo "REFRESH_TOKEN=${{secrets.REFRESH_TOKEN}}" >> .env.viya
+
+      - name: Build Project
+        run: npm run build
+
+      - name: Run SASjs  tests
+        run: npm run test
+        env:
+          CI: true
+          CLIENT: ${{secrets.CLIENT}}
+          SECRET: ${{secrets.SECRET}}
+          SAS_USERNAME: ${{secrets.SAS_USERNAME}}
+          SAS_PASSWORD: ${{secrets.SAS_PASSWORD}}
+          SERVER_URL: ${{secrets.SERVER_URL}}
+          SERVER_TYPE: ${{secrets.SERVER_TYPE}}
+          ACCESS_TOKEN: ${{secrets.ACCESS_TOKEN}}
+          REFRESH_TOKEN: ${{secrets.REFRESH_TOKEN}}

.gitpod.yml

@@ -1,8 +1,25 @@
 tasks:
-  - init: npm i -g @sasjs/cli
+  - init: nvm install --lts && npm i -g @sasjs/cli
 
 image:
   file: .gitpod.dockerfile
 vscode:
   extensions:
     - sasjs.sasjs-for-vscode
+
+github:
+  prebuilds:
+    # enable for the master/default branch (defaults to true)
+    master: true
+    # enable for all branches in this repo (defaults to false)
+    branches: false
+    # enable for pull requests coming from this repo (defaults to true)
+    pullRequests: true
+    # enable for pull requests coming from forks (defaults to false)
+    pullRequestsFromForks: true
+    # add a "Review in Gitpod" button as a comment to pull requests (defaults to true)
+    addComment: true
+    # add a "Review in Gitpod" button to pull requests (defaults to false)
+    addBadge: false
+    # add a label once the prebuild is ready to pull requests (defaults to false)
+    addLabel: prebuilt-in-gitpod

.npmignore

@@ -0,0 +1,12 @@
+all.sas
+build.py
+.gitpod*
+tests/
+sasjs/
+.github/
+.git-hooks/
+.vscode/
+main.dox
+make_singlefile.sh
+*.md
+.all-contributorsrc

.sasjslint

@@ -2,11 +2,11 @@
     "noTrailingSpaces": true,
     "noEncodedPasswords": true,
     "hasDoxygenHeader": true,
-    "hasMacroNameInMend": false,
+    "hasMacroNameInMend": true,
     "hasMacroParentheses": true,
     "noNestedMacros": false,
     "noSpacesInFileNames": true,
-    "maxLineLength": 135,
+    "maxLineLength": 300,
     "lowerCaseFileNames": true,
     "noTabIndentation": true,
     "indentationMultiple": 2

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2020 (Allan Bowe)
+Copyright 2021 (Allan Bowe)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

README.md

@@ -1,4 +1,22 @@
 # Macro Core
+[![npm package][npm-image]][npm-url]
+[![Github Workflow][githubworkflow-image]][githubworkflow-url]
+![npm](https://img.shields.io/npm/dt/@sasjs/core)
+![Snyk Vulnerabilities for npm package](https://img.shields.io/snyk/vulnerabilities/npm/@sasjs/core)
+[![License](https://img.shields.io/apm/l/atomic-design-ui.svg)](/LICENSE)
+![GitHub top language](https://img.shields.io/github/languages/top/sasjs/core)
+[![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sasjs/core)](https://github.com/sasjs/core/issues?q=is%3Aissue+is%3Aclosed)
+[![GitHub issues](https://img.shields.io/github/issues-raw/sasjs/core)](https://github.com/sasjs/core/issues)
+![total lines](https://tokei.rs/b1/github/sasjs/core)
+[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-908a85?logo=gitpod)](https://gitpod.io/#https://github.com/sasjs/core)
+
+
+[npm-image]:https://img.shields.io/npm/v/@sasjs/core.svg
+[npm-url]:http://npmjs.org/package/@sasjs/core
+[githubworkflow-image]:https://github.com/sasjs/core/actions/workflows/main.yml/badge.svg
+[githubworkflow-url]:https://github.com/sasjs/core/blob/main/.github/workflows/main.yml
+[dependency-url]:https://github.com/sasjs/core/blob/main/package.json
+
 
 Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
 
@@ -11,42 +29,61 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 
 Documentation: https://core.sasjs.io
 
-# Components
+## Components
 
-**base** library (SAS9/Viya)
+### BASE library (SAS9/Viya)
 
 - OS independent
 - Not metadata aware
 - No X command
 - Prefixes: _mf_, _mp_
 
-**meta** library (SAS9 only)
+#### FCMP library (SAS9/Viya)
+- Function and macro names are identical, except for special cases
+- Prefixes: _mcf_
+
+The fcmp macros are used to generate fcmp functions, and can be used with or
+without the `proc fcmp` wrapper.
+
+### META library (SAS9 only)
+
+Macros used in SAS EBI, which connect to the metadata server.
 
 - OS independent
 - Metadata aware
 - No X command
 - Prefixes: _mm_
 
-**viya** library (Viya only)
+### SERVER library (@sasjs/server only)
+These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
+
+- OS independent
+- @sasjs/server aware
+- No X command
+- Prefixes: _ms_
+
+### VIYA library (Viya only)
+
+Macros used for interfacing with SAS Viya.
 
 - OS independent
 - No X command
-- Prefixes: _mv_
+- Prefixes: _mv_, _mvf_
 
-**metax** library (SAS9 only)
+### METAX library (SAS9 only)
 
 - OS specific
 - Metadata aware
 - X command enabled
 - Prefixes: _mmw_,_mmu_,_mmx_
 
-**lua** library
+### LUA library
 
 Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
 
 To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert your LUA into a data step with put statements, and create the macro wrapper with a `ml_` prefix.  You can then use your module in any program by running:
 
-```
+```sas
 /* compile the lua module */
 %ml_yourmodule()
 
@@ -61,9 +98,9 @@ run;
 - X command enabled
 - Prefixes: _mmw_,_mmu_,_mmx_
 
-# Installation
+## Installation
 
-First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available,eg:
+First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available, eg:
 
 ```sas
 options insert=(sasautos="/your/path/macrocore/base");
@@ -79,9 +116,9 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 %inc mc;
 ```
 
-# Standards
+## Standards
 
-## File Properties
+### File Properties
 
 - filenames much match macro names
 - filenames must be lowercase, without spaces
@@ -89,19 +126,20 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 - one macro per file
 - prefixes:
   - _mf_ for macro functions (can be used in open code).
-  - _mp_ for macro procedures (which generate sas code)
+  - _ml_ for macros that are used to compile LUA modules
   - _mm_ for metadata macros (interface with the metadata server).
   - _mmx_ for macros that use metadata and are XCMD enabled
+  - _mp_ for macro procedures (which generate sas code)
+  - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
+  - _mv_ for macro procedures that will only work in Viya
   - _mx_ for macros that are XCMD enabled
-  - _ml_ for macros that are used to compile LUA modules
-  - _mv_ for macros that will only work in Viya
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
 - UTF-8
 
 
-## Header Properties
+### Header Properties
 
 The **Macro Core** documentation is created using [doxygen](http://www.doxygen.nl). A full list of attributes can be found [here](http://www.doxygen.nl/manual/commands.html) but the following are most relevant:
 
@@ -115,25 +153,25 @@ The **Macro Core** documentation is created using [doxygen](http://www.doxygen.n
 
 All macros must be commented in the doxygen format, to enable the [online documentation](https://core.sasjs.io).
 
-### Dependencies
-SAS code can contain one of two types of dependency - SAS Macros, and SAS Programs.  When compiling projects using the [SASjs CLI](https://cli.sasjs.io) the doxygen header is scanned for `  @li` items under the following headers:
+#### Dependencies
+SAS code can contain one of two types of dependency - SAS Macros, and SAS Includes.  When compiling projects using the [SASjs CLI](https://cli.sasjs.io) the doxygen header is scanned for `  @li` items under the following headers:
 
 ```sas
   <h4> SAS Macros </h4>
   @li mf_nobs.sas
   @li mm_assignlib.sas
 
-  <h4> SAS Programs </h4>
+  <h4> SAS Includes </h4>
   @li somefile.ddl SOMEFREF
   @li someprogram.sas FREFTWO
 ```
 
-The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Programs) when creating SAS Jobs and Services.
+The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Includes) when creating SAS Jobs and Services.
 
 When contributing to this library, it is therefore important to ensure that all dependencies are listed in the header in this format.
 
 
-## Coding Standards
+### Coding Standards
 
 - Indentation = 2 spaces. No tabs!
 - no trailing white space
@@ -146,6 +184,47 @@ When contributing to this library, it is therefore important to ensure that all
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
 - The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
 
-# General Notes
+## General Notes
 
 - All macros should be compatible with SAS versions from support level B and above (so currently 9.2 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully (eg `%if %sysevalf(&sysver<9.3) %then %return`).
+
+## Star Gazing
+
+If you find this library useful, please leave a [star](https://github.com/sasjs/core/stargazers) and help us grow our star graph!
+
+![](https://starchart.cc/sasjs/core.svg)
+
+
+
+## Contributors ✨
+<!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
+[![All Contributors](https://img.shields.io/badge/all_contributors-10-orange.svg?style=flat-square)](#contributors-)
+<!-- ALL-CONTRIBUTORS-BADGE:END -->
+Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
+
+<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-disable -->
+<table>
+  <tr>
+    <td align="center"><a href="https://github.com/allanbowe"><img src="https://avatars.githubusercontent.com/u/4420615?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Allan Bowe</b></sub></a><br /><a href="#business-allanbowe" title="Business development">💼</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Code">💻</a> <a href="#content-allanbowe" title="Content">🖋</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Documentation">📖</a> <a href="#infra-allanbowe" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#maintenance-allanbowe" title="Maintenance">🚧</a> <a href="#mentoring-allanbowe" title="Mentoring">🧑‍🏫</a> <a href="#question-allanbowe" title="Answering Questions">💬</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3Aallanbowe" title="Reviewed Pull Requests">👀</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Tests">⚠️</a></td>
+    <td align="center"><a href="https://github.com/rafgag"><img src="https://avatars.githubusercontent.com/u/69139928?v=4?s=100" width="100px;" alt=""/><br /><sub><b>rafgag</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rafgag" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/tmoody"><img src="https://avatars.githubusercontent.com/u/79837106?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Trevor Moody</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=tmoody" title="Code">💻</a></td>
+    <td align="center"><a href="https://krishna-acondy.io/"><img src="https://avatars.githubusercontent.com/u/2980428?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Krishna Acondy</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=krishna-acondy" title="Code">💻</a> <a href="#infra-krishna-acondy" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#blog-krishna-acondy" title="Blogposts">📝</a> <a href="#content-krishna-acondy" title="Content">🖋</a> <a href="#ideas-krishna-acondy" title="Ideas, Planning, & Feedback">🤔</a> <a href="#video-krishna-acondy" title="Videos">📹</a></td>
+    <td align="center"><a href="https://github.com/saadjutt01"><img src="https://avatars.githubusercontent.com/u/8914650?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Muhammad Saad </b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=saadjutt01" title="Code">💻</a> <a href="#ideas-saadjutt01" title="Ideas, Planning, & Feedback">🤔</a></td>
+    <td align="center"><a href="https://www.erudicat.com/"><img src="https://avatars.githubusercontent.com/u/25773492?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Yury Shkoda</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=YuryShkoda" title="Code">💻</a> <a href="#infra-YuryShkoda" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#video-YuryShkoda" title="Videos">📹</a></td>
+    <td align="center"><a href="https://github.com/medjedovicm"><img src="https://avatars.githubusercontent.com/u/18329105?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Mihajlo Medjedovic</b></sub></a><br /><a href="#infra-medjedovicm" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+  </tr>
+  <tr>
+    <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
+    <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
+    <td align="center"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
+  </tr>
+</table>
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+
+<!-- ALL-CONTRIBUTORS-LIST:END -->
+
+This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!

base/mf_abort.sas

@@ -1,30 +1,17 @@
 /**
   @file
-  @brief abort gracefully according to context
-  @details Do not use directly!  See bottom of explanation for details.
+  @brief Abort, ungracefully
+  @details Will abort with a straightforward %abort if the condition is true.
 
-  Configures an abort mechanism according to site specific policies or the
-    particulars of an environment.  For instance, can stream custom
-    results back to the client in an STP Web App context, or completely stop
-    in the case of a batch run.
-
-  For the sharp eyed readers - this is no longer a macro function!! It became
-  a macro procedure during a project and now it's kinda stuck that way until
-  that project is updated (if it's ever updated).  In the meantime we created
-  `mp_abort` which is just a wrapper for this one, and so we recomend you use
-  that for forwards compatibility reasons.
-
-  @param mac= to contain the name of the calling macro
-  @param type= deprecated.  Not used.
-  @param msg= message to be returned
-  @param iftrue= supply a condition under which the macro should be executed.
+  <h4> Related Macros </h4>
+  @li mp_abort.sas
 
   @version 9.2
   @author Allan Bowe
   @cond
 **/
 
-%macro mf_abort(mac=mf_abort.sas, type=, msg=, iftrue=%str(1=1)
+%macro mf_abort(mac=mf_abort.sas, type=deprecated, msg=, iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
   %if not(%eval(%unquote(&iftrue))) %then %return;
@@ -33,116 +20,8 @@
   %if %length(&mac)>0 %then %put NOTE- called by &mac;
   %put NOTE - &msg;
 
-  /* Stored Process Server web app context */
-  %if %symexist(_metaperson) or "&SYSPROCESSNAME"="Compute Server" %then %do;
-    options obs=max replace nosyntaxcheck mprint;
-    /* extract log err / warn, if exist */
-    %local logloc logline;
-    %global logmsg; /* capture global messages */
-    %if %symexist(SYSPRINTTOLOG) %then %let logloc=&SYSPRINTTOLOG;
-    %else %let logloc=%qsysfunc(getoption(LOG));
-    proc printto log=log;run;
-    %if %length(&logloc)>0 %then %do;
-      %let logline=0;
-      data _null_;
-        infile &logloc lrecl=5000;
-        input; putlog _infile_;
-        i=1;
-        retain logonce 0;
-        if (
-            _infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR"
-          ) and logonce=0
-        then do;
-          call symputx('logline',_n_);
-          logonce+1;
-        end;
-      run;
-      /* capture log including lines BEFORE the err */
-      %if &logline>0 %then %do;
-        data _null_;
-          infile &logloc lrecl=5000;
-          input;
-          i=1;
-          stoploop=0;
-          if _n_ ge &logline-5 and stoploop=0 then do until (i>12);
-            call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
-            input;
-            i+1;
-            stoploop=1;
-          end;
-          if stoploop=1 then stop;
-        run;
-      %end;
-    %end;
+  %abort;
 
-    /* send response in SASjs JSON format */
-    data _null_;
-      file _webout mod lrecl=32000;
-      length msg $32767;
-      sasdatetime=datetime();
-      msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
-      /* escape the quotes */
-      msg=tranwrd(msg,'"','\"');
-      /* ditch the CRLFs as chrome complains */
-      msg=compress(msg,,'kw');
-      /* quote without quoting the quotes (which are escaped instead) */
-      msg=cats('"',msg,'"');
-      if symexist('_debug') then debug=symget('_debug');
-      if debug ge 131 then put '>>weboutBEGIN<<';
-      put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
-      put ',"sasjsAbort" : [{';
-      put ' "MSG":' msg ;
-      put ' ,"MAC": "' "&mac" '"}]';
-      put ",""SYSUSERID"" : ""&sysuserid"" ";
-      if symexist('_metauser') then do;
-        _METAUSER=quote(trim(symget('_METAUSER')));
-        put ",""_METAUSER"": " _METAUSER;
-        _METAPERSON=quote(trim(symget('_METAPERSON')));
-        put ',"_METAPERSON": ' _METAPERSON;
-      end;
-      _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
-      put ',"_PROGRAM" : ' _PROGRAM ;
-      put ",""SYSCC"" : ""&syscc"" ";
-      put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
-      put ",""SYSJOBID"" : ""&sysjobid"" ";
-      put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
-      put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
-      put "}" @;
-      %if &_debug ge 131 %then %do;
-        put '>>weboutEND<<';
-      %end;
-    run;
-    %let syscc=0;
-    %if %symexist(SYS_JES_JOB_URI) %then %do;
-      /* refer web service output to file service in one hit */
-      filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
-        name="_webout.json";
-      %let rc=%sysfunc(fcopy(_web,_webout));
-    %end;
-    %else %do;
-      data _null_;
-        if symexist('sysprocessmode')
-          then if symget("sysprocessmode")="SAS Stored Process Server"
-            then rc=stpsrvset('program error', 0);
-      run;
-    %end;
-    /**
-      * endsas is reliable but kills some deployments.
-      * Abort variants are ungraceful (non zero return code)
-      * This approach lets SAS run silently until the end :-)
-      */
-    %put _all_;
-    filename skip temp;
-    data _null_;
-      file skip;
-      put '%macro skip(); %macro skippy();';
-    run;
-    %inc skip;
-  %end;
-  %else %do;
-    %put _all_;
-    %abort cancel;
-  %end;
-%mend;
+%mend mf_abort;
 
 /** @endcond */

base/mf_dedup.sas

@@ -0,0 +1,54 @@
+/**
+  @file
+  @brief de-duplicates a macro string
+  @details Removes all duplicates from a string of words.  A delimeter can be
+  chosen.  Is inspired heavily by this excellent [macro](
+  https://github.com/scottbass/SAS/blob/master/Macro/dedup_mstring.sas) from
+  [Scott Base](https://www.linkedin.com/in/scottbass).  Case sensitive.
+
+  Usage:
+
+      %let str=One two one two and through and through;
+      %put %mf_dedup(&str);
+      %put %mf_dedup(&str,outdlm=%str(,));
+
+  Which returns:
+
+      > One two one and through
+      > One,two,one,and,through
+
+  @param [in] str String to be deduplicated
+  @param [in] indlm= ( ) Delimeter of the input string
+  @param [out] outdlm= ( ) Delimiter of the output string
+
+  <h4> Related Macros </h4>
+  @li mf_trimstr.sas
+  @li mf_wordsinstr1butnotstr2.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_dedup(str
+  ,indlm=%str( )
+  ,outdlm=%str( )
+)/*/STORE SOURCE*/;
+
+%local num word i pos out;
+
+%* loop over each token, searching the target for that token ;
+%let num=%sysfunc(countc(%superq(str),%str(&indlm)));
+%do i=1 %to %eval(&num+1);
+  %let word=%scan(%superq(str),&i,%str(&indlm));
+  %let pos=%sysfunc(indexw(&out,&word,%str(&outdlm)));
+  %if (&pos eq 0) %then %do;
+    %if (&i gt 1) %then %let out=&out%str(&outdlm);
+    %let out=&out&word;
+  %end;
+%end;
+
+%unquote(&out)
+
+%mend mf_dedup;
+
+

base/mf_existds.sas

@@ -12,6 +12,10 @@
 
   @param libds library.dataset
   @return output returns 1 or 0
+
+  <h4> Related Macros </h4>
+  @li mf_existds.test.sas
+
   @warning Untested on tables registered in metadata but not physically present
   @version 9.2
   @author Allan Bowe
@@ -23,4 +27,4 @@
   %if %sysfunc(exist(&libds)) ne 1 & %sysfunc(exist(&libds,VIEW)) ne 1 %then 0;
   %else 1;
 
-%mend;
+%mend mf_existds;

base/mf_existfeature.sas

@@ -5,7 +5,7 @@
     Run without arguments to see a list of detectable features.
     Note - this list is based on known versions of SAS rather than
     actual feature detection, as that is tricky / impossible to do
-    without generating errors in most cases.
+    without generating errs in most cases.
 
         %put %mf_existfeature(PROCLUA);
 
@@ -42,6 +42,6 @@
     -1
     %put &sysmacroname: &feature not found;
   %end;
-%mend;
+%mend mf_existfeature;
 
 /** @endcond */

base/mf_existfileref.sas

@@ -17,11 +17,17 @@
 %macro mf_existfileref(fref
 )/*/STORE SOURCE*/;
 
-  %if %sysfunc(fileref(&fref))=0 %then %do;
+  %local rc;
+  %let rc=%sysfunc(fileref(&fref));
+  %if &rc=0 %then %do;
+    1
+  %end;
+  %else %if &rc<0 %then %do;
+    %put &sysmacroname: Fileref &fref exists but the underlying file does not;
     1
   %end;
   %else %do;
     0
   %end;
 
-%mend;
+%mend mf_existfileref;

base/mf_existfunction.sas

@@ -0,0 +1,37 @@
+/**
+  @file
+  @brief Checks if a function exists
+  @details Returns 1 if the function exists, else 0.  Note that this function
+  can be slow as it needs to open the sashelp.vfuncs table.
+
+  Usage:
+
+      %put %mf_existfunction(CAT);
+      %put %mf_existfunction(DOG);
+
+  Full credit to [Bart](https://sasensei.com/user/305) for the vfunc pointer
+  and the tidy approach for pure macro data set filtering.
+  Check out his [SAS Packages](https://github.com/yabwon/SAS_PACKAGES)
+  framework!  Where you can find the same [function](
+https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionexists-macro
+  ).
+
+  @param [in] name (positional) - function name
+
+  @author Allan Bowe
+**/
+/** @cond */
+%macro mf_existfunction(name
+)/*/STORE SOURCE*/;
+
+  %local dsid rc exist;
+  %let dsid=%sysfunc(open(sashelp.vfunc(where=(fncname="%upcase(&name)"))));
+  %let exist=1;
+  %let exist=%sysfunc(fetch(&dsid, NOSET));
+  %let rc=%sysfunc(close(&dsid));
+
+  %sysevalf(0 = &exist)
+
+%mend mf_existfunction;
+
+/** @endcond */

base/mf_existvar.sas

@@ -7,8 +7,12 @@
 
         %put %mf_existvar(work.someds, somevar)
 
-  @param libds (positional) - 2 part dataset or view reference
-  @param var (positional) - variable name
+  @param [in] libds 2 part dataset or view reference
+  @param [in] var variable name
+
+  <h4> Related Macros </h4>
+  @li mf_existvar.test.sas
+
   @version 9.2
   @author Allan Bowe
 **/
@@ -30,6 +34,6 @@
       %let rc=%sysfunc(close(&dsid));
   %end;
 
-%mend;
+%mend mf_existvar;
 
 /** @endcond */

base/mf_existvarlist.sas

@@ -54,6 +54,6 @@
     0
     %put Vars not found: &found;
   %end;
-%mend;
+%mend mf_existvarlist;
 
 /** @endcond */

base/mf_getapploc.sas

@@ -0,0 +1,73 @@
+/**
+  @file
+  @brief Returns the appLoc from the _program variable
+  @details When working with SASjs apps, web services / tests / jobs are always
+  deployed to a root (app) location in the SAS logical folder tree.
+
+  When building apps for use in other environments, you do not necessarily know
+  where the backend services will be deployed.  Therefore a function like this
+  is handy in order to dynamically figure out the appLoc, and enable other
+  services to be connected by a relative reference.
+
+  SASjs apps always have the same immediate substructure (one or more of the
+  following):
+
+  @li /data
+  @li /jobs
+  @li /services
+  @li /tests/jobs
+  @li /tests/services
+  @li /tests/macros
+
+  This function works by testing for the existence of any of the above in the
+  automatic _program variable, and returning the part to the left of it.
+
+  Usage:
+
+      %put %mf_getapploc(&_program)
+
+      %put %mf_getapploc(/some/location/services/admin/myservice);
+      %put %mf_getapploc(/some/location/jobs/extract/somejob/);
+      %put %mf_getapploc(/some/location/tests/jobs/somejob/);
+
+
+  @author Allan Bowe
+**/
+
+%macro mf_getapploc(pgm);
+%if "&pgm"="" %then %do;
+  %if %symexist(_program) %then %let pgm=&_program;
+  %else %do;
+    %put &sysmacroname: No value provided and no _program variable available;
+    %return;
+  %end;
+%end;
+%local root;
+
+/**
+  * First check we are not in the tests/macros folder (which has no subfolders)
+  */
+%if %index(&pgm,/tests/macros/) %then %do;
+  %let root=%substr(&pgm,1,%index(&pgm,/tests/macros)-1);
+  &root
+  %return;
+%end;
+
+/**
+  * Next, move up two levels to avoid matches on subfolder or service name
+  */
+%let root=%substr(&pgm,1,%length(&pgm)-%length(%scan(&pgm,-1,/))-1);
+%let root=%substr(&root,1,%length(&root)-%length(%scan(&root,-1,/))-1);
+
+%if %index(&root,/tests/) %then %do;
+  %let root=%substr(&root,1,%index(&root,/tests/)-1);
+%end;
+%else %if %index(&root,/services) %then %do;
+  %let root=%substr(&root,1,%index(&root,/services)-1);
+%end;
+%else %if %index(&root,/jobs) %then %do;
+  %let root=%substr(&root,1,%index(&root,/jobs)-1);
+%end;
+%else %put &sysmacroname: Could not find an app location from &pgm;
+  &root
+%mend mf_getapploc ;

base/mf_getattrc.sas

@@ -10,7 +10,7 @@
   @param attr full list in [documentation](
     https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000147794.htm)
   @return output returns result of the attrc value supplied, or -1 and log
-    message if error.
+    message if err.
 
   @version 9.2
   @author Allan Bowe
@@ -31,4 +31,4 @@
     %sysfunc(attrc(&dsid,&attr))
     %let rc=%sysfunc(close(&dsid));
   %end;
-%mend;
+%mend mf_getattrc;

base/mf_getattrn.sas

@@ -10,7 +10,7 @@
   @param attr Common values are NLOBS and NVARS, full list in [documentation](
   http://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000212040.htm)
   @return output returns result of the attrn value supplied, or -1 and log
-    message if error.
+    message if err.
 
   @version 9.2
   @author Allan Bowe
@@ -31,4 +31,4 @@
     %sysfunc(attrn(&dsid,&attr))
     %let rc=%sysfunc(close(&dsid));
   %end;
-%mend;
+%mend mf_getattrn;

base/mf_getengine.sas

@@ -12,9 +12,10 @@
   contributors of Chris Hemedingers blog [post](
   http://blogs.sas.com/content/sasdummy/2013/06/04/find-a-sas-library-engine/)
 
-  @param libref Library reference (also accepts a 2 level libds ref).
+  @param [in] libref Library reference (also accepts a 2 level libds ref).
 
-  @return output returns the library engine for the FIRST library encountered.
+  @return output returns the library engine (uppercase) for the FIRST library
+    encountered.
 
   @warning will only return the FIRST library engine - for concatenated
     libraries, with different engines, inconsistent results may be encountered.
@@ -46,8 +47,8 @@
     %let rc= %sysfunc(close(&dsid));
   %end;
 
-  &engine
+  %upcase(&engine)
 
-%mend;
+%mend mf_getengine;
 
 /** @endcond */

base/mf_getfilesize.sas

@@ -44,4 +44,4 @@
     %sysfunc(INPUTN(&bytes, best.),sizekmg.)
   %end;
 
-%mend ;
+%mend mf_getfilesize ;

base/mf_getfmtlist.sas

@@ -0,0 +1,61 @@
+/**
+  @file
+  @brief Returns a distinct list of formats from a table
+  @details Reads the dataset header and returns a distinct list of formats
+  applied.
+
+        %put NOTE- %mf_getfmtlist(sashelp.prdsale);
+        %put NOTE- %mf_getfmtlist(sashelp.shoes);
+        %put NOTE- %mf_getfmtlist(sashelp.demographics);
+
+  returns:
+
+  > DOLLAR $CHAR W MONNAME
+  > $CHAR BEST DOLLAR
+  > BEST Z $CHAR COMMA PERCENTN
+
+
+  @param [in] libds Two part library.dataset reference.
+
+  <h4> SAS Macros </h4>
+  @li mf_getfmtname.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_getfmtlist(libds
+)/*/STORE SOURCE*/;
+/* declare local vars */
+%local out dsid nvars x rc fmt;
+
+/* open dataset in macro */
+%let dsid=%sysfunc(open(&libds));
+
+/* continue if dataset exists */
+%if &dsid %then %do;
+  /* loop each variable in the dataset */
+  %let nvars=%sysfunc(attrn(&dsid,NVARS));
+  %do x=1 %to &nvars;
+    /* grab format and check it exists */
+    %let fmt=%sysfunc(varfmt(&dsid,&x));
+    %if %quote(&fmt) ne %quote() %then %let fmt=%mf_getfmtname(&fmt);
+    %else %do;
+      /* assign default format depending on variable type */
+      %if %sysfunc(vartype(&dsid, &x))=C %then %let fmt=$CHAR;
+      %else %let fmt=BEST;
+    %end;
+    /* concatenate unique list of formats */
+    %if %sysfunc(indexw(&out,&fmt,%str( )))=0 %then %let out=&out &fmt;
+  %end;
+  %let rc=%sysfunc(close(&dsid));
+%end;
+%else %do;
+  %put &sysmacroname: Unable to open &libds (rc=&dsid);
+  %put &sysmacroname: SYSMSG= %sysfunc(sysmsg());
+  %let rc=%sysfunc(close(&dsid));
+%end;
+/* send them out without spaces or quote markers */
+%do;%unquote(&out)%end;
+%mend mf_getfmtlist;

base/mf_getfmtname.sas

@@ -0,0 +1,44 @@
+/**
+  @file
+  @brief Extracts a format name from a fully defined format
+  @details Converts formats in like $thi3. and th13.2 $THI and TH.
+  Usage:
+
+      %put %mf_getfmtname(8.);
+      %put %mf_getfmtname($4.);
+      %put %mf_getfmtname(comma14.10);
+
+  Returns:
+
+  > W
+  > $CHAR
+  > COMMA
+
+  Note that system defaults are inferred from the values provided.
+
+  @param [in] fmt The fully defined format. If left blank, nothing is returned.
+
+  @returns The name (without width or decimal) of the format.
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_getfmtname(fmt
+)/*/STORE SOURCE*/ /minoperator mindelimiter=' ';
+
+%local out dsid nvars x rc fmt;
+
+/* extract actual format name from the format definition */
+%let fmt=%scan(&fmt,1,.);
+%do %while(%substr(&fmt,%length(&fmt),1) in 1 2 3 4 5 6 7 8 9 0);
+  %if %length(&fmt)=1 %then %let fmt=W;
+  %else %let fmt=%substr(&fmt,1,%length(&fmt)-1);
+%end;
+
+%if &fmt=$ %then %let fmt=$CHAR;
+
+/* send them out without spaces or quote markers */
+%do;%unquote(%upcase(&fmt))%end;
+%mend mf_getfmtname;

base/mf_getkeyvalue.sas

@@ -29,4 +29,4 @@
   &valc
 %end;
 %else %put %str(ERR)OR: Unable to find key &key in ds &libds;
-%mend;
+%mend mf_getkeyvalue;

base/mf_getplatform.sas

@@ -62,4 +62,4 @@
 %else %if &switch=VIYARESTAPI %then %do;
   %mf_trimstr(%sysfunc(getoption(servicesbaseurl)),/)
 %end;
-%mend;
+%mend mf_getplatform;

base/mf_getquotedstr.sas

@@ -15,12 +15,12 @@
   for:
 > "these","words","are","double","quoted"
 
-  @param in_str the unquoted, spaced delimited string to transform
-  @param dlm= the delimeter to be applied to the output (default comma)
-  @param indlm= the delimeter used for the input (default is space)
-  @param quote= the quote mark to apply (S=Single, D=Double). If any other value
-    than uppercase S or D is supplied, then that value will be used as the
-    quoting character.
+  @param [in] in_str The unquoted, spaced delimited string to transform
+  @param [in] dlm= (,) The delimeter to be applied to the output (default comma)
+  @param [in] indlm= ( ) The delimeter used for the input (default is space)
+  @param [in] quote= (S) The quote mark to apply (S=Single, D=Double, N=None).
+    If any other value than uppercase S or D is supplied, then that value will
+    be used as the quoting character.
   @return output returns a string with the newly quoted / delimited output.
 
   @version 9.2
@@ -28,11 +28,15 @@
 **/
 
 
-%macro mf_getquotedstr(IN_STR,DLM=%str(,),QUOTE=S,indlm=%str( )
+%macro mf_getquotedstr(IN_STR
+  ,DLM=%str(,)
+  ,QUOTE=S
+  ,indlm=%str( )
 )/*/STORE SOURCE*/;
-  %if &quote=S %then %let quote=%str(%');
-  %else %if &quote=D %then %let quote=%str(%");
-  %else %let quote=%str();
+  /* credit Rowland Hale  - byte34 is double quote, 39 is single quote */
+  %if &quote=S %then %let quote=%qsysfunc(byte(39));
+  %else %if &quote=D %then %let quote=%qsysfunc(byte(34));
+  %else %if &quote=N %then %let quote=;
   %local i item buffer;
   %let i=1;
   %do %while (%qscan(&IN_STR,&i,%str(&indlm)) ne %str() ) ;
@@ -50,4 +54,4 @@
 
   &buffer
 
-%mend;
+%mend mf_getquotedstr;

base/mf_getschema.sas

@@ -38,6 +38,6 @@
 
   &schema
 
-%mend;
+%mend mf_getschema;
 
 /** @endcond */

base/mf_getuniquefileref.sas

@@ -1,37 +1,58 @@
 /**
   @file
   @brief Assigns and returns an unused fileref
-  @details
+  @details  Using the native approach for assigning filerefs fails as some
+  procedures (such as proc http) do not recognise the temporary names (starting
+  with a hash), returning a message such as:
+
+  > ERROR 22-322: Expecting a name.
+
+  This macro works by attempting a random fileref (with a prefix), seeing if it
+  is already assigned, and if not - returning the fileref.
+
+  If your process can accept filerefs with the hash (#) prefix, then set
+  `prefix=0` to revert to the native approach - which is significantly faster
+  when there are a lot of filerefs in a session.
+
   Use as follows:
 
       %let fileref1=%mf_getuniquefileref();
-      %let fileref2=%mf_getuniquefileref();
+      %let fileref2=%mf_getuniquefileref(prefix=0);
       %put &fileref1 &fileref2;
 
-  which returns:
+  which returns filerefs similar to:
 
-> mcref0 mcref1
+> _7432233 #LN00070
 
-  @param prefix= first part of fileref. Remember that filerefs can only be 8
-    characters, so a 7 letter prefix would mean that `maxtries` should be 10.
-  @param maxtries= the last part of the libref.  Provide an integer value.
+  @param [in] prefix= (_) first part of fileref. Remember that filerefs can only
+    be 8 characters, so a 7 letter prefix would mean `maxtries` should be 10.
+    if using zero (0) as the prefix, a native assignment is used.
+  @param [in] maxtries= (1000) the last part of the libref. Must be an integer.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-%macro mf_getuniquefileref(prefix=mcref,maxtries=1000);
-  %local x fname;
-  %let x=0;
-  %do x=0 %to &maxtries;
-  %if %sysfunc(fileref(&prefix&x)) > 0 %then %do;
-    %let fname=&prefix&x;
+%macro mf_getuniquefileref(prefix=_,maxtries=1000);
+  %local rc fname;
+  %if &prefix=0 %then %do;
     %let rc=%sysfunc(filename(fname,,temp));
     %if &rc %then %put %sysfunc(sysmsg());
-    &prefix&x
-    %*put &sysmacroname: Fileref &prefix&x was assigned and returned;
-    %return;
+    &fname
   %end;
+  %else %do;
+    %local x len;
+    %let len=%eval(8-%length(&prefix));
+    %let x=0;
+    %do x=0 %to &maxtries;
+      %let fname=&prefix%substr(%sysfunc(ranuni(0)),3,&len);
+      %if %sysfunc(fileref(&fname)) > 0 %then %do;
+        %let rc=%sysfunc(filename(fname,,temp));
+        %if &rc %then %put %sysfunc(sysmsg());
+        &fname
+        %return;
+      %end;
+    %end;
+    %put unable to find available fileref after &maxtries attempts;
   %end;
-  %put unable to find available fileref in range &prefix.0-&maxtries;
-%mend;
+%mend mf_getuniquefileref;

base/mf_getuniquelibref.sas

@@ -37,4 +37,4 @@
   %end;
   %end;
   %put unable to find available libref in range &prefix.0-&maxtries;
-%mend;
+%mend mf_getuniquelibref;

base/mf_getuniquename.sas

@@ -18,5 +18,5 @@
 
 
 %macro mf_getuniquename(prefix=MC);
-  &prefix.%substr(%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32-%length(&prefix))
-%mend;
+&prefix.%substr(%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32-%length(&prefix))
+%mend mf_getuniquename;

base/mf_getuser.sas

@@ -39,4 +39,4 @@
 
   %quote(&user)
 
-%mend;
+%mend mf_getuser;

base/mf_getvalue.sas

@@ -10,6 +10,9 @@
   <h4> SAS Macros </h4>
   @li mf_getattrn.sas
 
+  <h4> Related Macros </h4>
+  @li mp_setkeyvalue.sas
+
   @param libds dataset to query
   @param variable the variable which contains the value to return.
   @param filter contents of where clause
@@ -30,4 +33,4 @@
     %trim(&&&variable)
 
   %end;
-%mend;
+%mend mf_getvalue;

base/mf_getvarcount.sas

@@ -29,4 +29,4 @@
     %let rc=%sysfunc(close(&dsid));
   %end;
   &nvars
-%mend;
+%mend mf_getvarcount;

base/mf_getvarformat.sas

@@ -23,9 +23,9 @@
       8.
       NOTE: Variable renegade does not exist in test
 
-  @param libds Two part dataset (or view) reference.
-  @param var Variable name for which a format should be returned
-  @param force Set to 1 to supply a default if the variable has no format
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] var Variable name for which a format should be returned
+  @param [in] force=(0) Set to 1 to supply a default if the variable has no format
   @returns outputs format
 
   @author Allan Bowe
@@ -51,7 +51,8 @@
     %end;
   %end;
   %else %do;
-    %put dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
     %return;
   %end;
 
@@ -60,7 +61,7 @@
     %let vlen = %sysfunc(varlen(&dsid, &vnum));
     %let vtype = %sysfunc(vartype(&dsid, &vnum.));
     %if &vtype=C %then %let vformat=$&vlen..;
-    %else %let vformat=8.;
+    %else %let vformat=best.;
   %end;
 
 
@@ -68,4 +69,4 @@
   %let rc = %sysfunc(close(&dsid));
   /* Return variable format */
   &vformat
-%mend;
+%mend mf_getVarFormat;

base/mf_getvarlen.sas

@@ -43,10 +43,14 @@
       %let vlen = %str( );
     %end;
   %end;
-  %else %put dataset &libds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));
   /* Return variable format */
   &vlen
-%mend;
+%mend mf_getVarLen;

base/mf_getvarlist.sas

@@ -66,8 +66,9 @@
     %let rc=%sysfunc(close(&dsid));
   %end;
   %else %do;
-    %put unable to open &libds (rc=&dsid);
+    %put &sysmacroname: Unable to open &libds (rc=&dsid);
+    %put &sysmacroname: SYSMSG= %sysfunc(sysmsg());
     %let rc=%sysfunc(close(&dsid));
   %end;
-  &outvar
+  %do;%unquote(&outvar)%end;
 %mend mf_getvarlist;

base/mf_getvarnum.sas

@@ -43,7 +43,11 @@ returns:
       %let vnum = %str( );
     %end;
   %end;
-  %else %put dataset &ds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));
@@ -51,4 +55,4 @@ returns:
   /* Return variable number */
     &vnum.
 
-%mend;
+%mend mf_getVarNum;

base/mf_getvartype.sas

@@ -39,7 +39,11 @@ Usage:
       %let vtype = %str( );
     %end;
   %end;
-  %else %put dataset &libds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));

base/mf_getxengine.sas

@@ -40,4 +40,4 @@
 
   &engine
 
-%mend;
+%mend mf_getxengine;

base/mf_isblank.sas

@@ -24,4 +24,4 @@
 
   %sysevalf(%superq(param)=,boolean)
 
-%mend;
+%mend mf_isblank;

base/mf_isdir.sas

@@ -31,4 +31,4 @@
 
   &is_directory
 
-%mend;
+%mend mf_isdir;

base/mf_isint.sas

@@ -0,0 +1,33 @@
+/**
+  @file
+  @brief Returns 1 if the variable contains only digits 0-9, else 0
+  @details Note that numerics containing any punctuation (including decimals
+    or exponents) will be flagged zero.
+
+  If you'd like support for this, then do raise an issue (or even better, a
+  pull request!)
+
+  Usage:
+
+      %put %mf_isint(1) returns 1;
+      %put %mf_isint(1.1) returns 0;
+      %put %mf_isint(%str(1,1)) returns 0;
+
+  @param [in] arg input value to check
+
+  @version 9.2
+**/
+
+%macro mf_isint(arg
+)/*/STORE SOURCE*/;
+  /* remove minus sign if exists */
+
+  %local val;
+  %if "%substr(%str(&arg),1,1)"="-" %then %let val=%substr(%str(&arg),2);
+  %else %let val=&arg;
+
+  /* check remaining chars */
+  %if %sysfunc(findc(%str(&val),,kd)) %then %do;0%end;
+  %else %do;1%end;
+
+%mend mf_isint;

base/mf_islibds.sas

@@ -0,0 +1,40 @@
+/**
+  @file
+  @brief Checks whether a string follows correct library.dataset format
+  @details Many macros in the core library accept a library.dataset parameter
+  referred to as 'libds'.  This macro validates the structure of that parameter,
+  eg:
+
+    @li 8 character libref?
+    @li 32 character dataset?
+    @li contains a period?
+
+  It does NOT check whether the dataset exists, or if the library is assigned.
+
+  Usage:
+
+      %put %mf_islibds(work.something)=1;
+      %put %mf_islibds(nolib)=0;
+      %put %mf_islibds(badlibref.ds)=0;
+      %put %mf_islibds(w.t.f)=0;
+
+  @param [in] libds The string to be checked
+
+  @return output Returns 1 if libds is valid, 0 if it is not
+
+  <h4> Related Macros </h4>
+  @li mf_islibds.test.sas
+  @li mp_validatecol.sas
+
+  @version 9.2
+**/
+
+%macro mf_islibds(libds
+)/*/STORE SOURCE*/;
+
+%local regex;
+%let regex=%sysfunc(prxparse(%str(/^[_a-z]\w{0,7}\.[_a-z]\w{0,31}$/i)));
+
+%sysfunc(prxmatch(&regex,&libds))
+
+%mend mf_islibds;

base/mf_loc.sas

@@ -26,4 +26,4 @@
   &root
 %end;
 
-%mend;
+%mend mf_loc;

base/mf_mkdir.sas

@@ -51,7 +51,7 @@ Usage:
     %end;
 
     /*
-      Now create the directory.  Complain loudly of any errors.
+      Now create the directory.  Complain loudly of any errs.
     */
 
     %let dname = %sysfunc(dcreate(&child, &parent));
@@ -64,4 +64,4 @@ Usage:
     %end;
   %end;
   /* exit quietly if directory did exist.*/
-%mend;
+%mend mf_mkdir;

base/mf_mval.sas

@@ -16,4 +16,4 @@
   %if %symexist(&var) %then %do;
     %superq(&var)
   %end;
-%mend;
+%mend mf_mval;

base/mf_nobs.sas

@@ -12,7 +12,7 @@
   @param libds library.dataset
 
   @return output returns result of the attrn value supplied, or log message
-    if error.
+    if err.
 
 
   @version 9.2
@@ -23,4 +23,4 @@
 %macro mf_nobs(libds
 )/*/STORE SOURCE*/;
   %mf_getattrn(&libds,NLOBS)
-%mend;
+%mend mf_nobs;

base/mf_trimstr.sas

@@ -47,4 +47,4 @@
   &basestr
 %end;
 
-%mend;
+%mend mf_trimstr;

base/mf_verifymacvars.sas

@@ -62,4 +62,4 @@
     %else %mf_abort(mac=mf_verifymacvars,type=&mabort,msg=&abortmsg);
   %exit_success:
 
-%mend;
+%mend mf_verifymacvars;

base/mf_wordsinstr1andstr2.sas

@@ -0,0 +1,54 @@
+/**
+  @file
+  @brief Returns words that are in both string 1 and string 2
+  @details  Compares two space separated strings and returns the words that are
+  in both.
+  Usage:
+
+      %put %mf_wordsInStr1andStr2(
+        Str1=blah sss blaaah brah bram boo
+        ,Str2=   blah blaaah brah ssss
+      );
+
+  returns:
+  > blah blaaah brah
+
+  @param str1= string containing words to extract
+  @param str2= used to compare with the extract string
+
+  @warning CASE SENSITIVE!
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_wordsInStr1andStr2(
+  Str1= /* string containing words to extract */
+  ,Str2= /* used to compare with the extract string */
+)/*/STORE SOURCE*/;
+
+%local count_base count_extr i i2 extr_word base_word match outvar;
+%if %length(&str1)=0 or %length(&str2)=0 %then %do;
+  %put %str(WARN)ING: empty string provided!;
+  %put base string (str1)= &str1;
+  %put compare string (str2) = &str2;
+  %return;
+%end;
+%let count_base=%sysfunc(countw(&Str2));
+%let count_extr=%sysfunc(countw(&Str1));
+
+%do i=1 %to &count_extr;
+  %let extr_word=%scan(&Str1,&i,%str( ));
+  %let match=0;
+  %do i2=1 %to &count_base;
+    %let base_word=%scan(&Str2,&i2,%str( ));
+    %if &extr_word=&base_word %then %let match=1;
+  %end;
+  %if &match=1 %then %let outvar=&outvar &extr_word;
+%end;
+
+  &outvar
+
+%mend mf_wordsInStr1andStr2;
+

base/mf_wordsinstr1butnotstr2.sas

@@ -50,5 +50,5 @@
 
   &outvar
 
-%mend;
+%mend mf_wordsInStr1ButNotStr2;
 

base/mf_writefile.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Creates a text file using pure macro
+  @details Creates a text file of up to 10 lines.  If further lines are
+    desired, feel free to [create an issue](
+    https://github.com/sasjs/core/issues/new), or make a pull request!
+
+    The use of PARMBUFF was considered for this macro, but it would have made
+    things problematic for writing lines containing commas.
+
+    Usage:
+
+        %mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=more content)
+        data _null_;
+          infile "&sasjswork/myfile.txt";
+          input;
+          list;
+        run;
+
+  @param [in] fpath Full path to file to be created or appended to
+  @param [in] mode= (O) Available options are A or O as follows:
+    @li A APPEND mode, writes new records after the current end of the file.
+    @li O OUTPUT mode, writes new records from the beginning of the file.
+  @param [in] l1= First line
+  @param [in] l2= Second line (etc through to l10)
+
+  <h4> Related Macros </h4>
+  @li mf_writefile.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mf_writefile(fpath,mode=O,l1=,l2=,l3=,l4=,l5=,l6=,l7=,l8=,l9=,l10=
+)/*/STORE SOURCE*/;
+%local fref rc fid i total_lines;
+
+/* find number of lines by reference to first non-blank param */
+%do i=10 %to 1 %by -1;
+  %if %str(&&l&i) ne %str() %then %goto continue;
+%end;
+%continue:
+%let total_lines=&i;
+
+%if %sysfunc(filename(fref,&fpath)) ne 0 %then %do;
+  %put &=fref &=fpath;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let fid=%sysfunc(fopen(&fref,&mode));
+
+%if &fid=0 %then %do;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%do i=1 %to &total_lines;
+  %let rc=%sysfunc(fput(&fid, &&l&i));
+  %let rc=%sysfunc(fwrite(&fid));
+%end;
+%let rc=%sysfunc(fclose(&fid));
+%let rc=%sysfunc(filename(&fref));
+
+%mend mf_writefile;
+/** @endcond */

base/mp_abort.sas

@@ -4,37 +4,106 @@
   @details Configures an abort mechanism according to site specific policies or
     the particulars of an environment.  For instance, can stream custom
     results back to the client in an STP Web App context, or completely stop
-    in the case of a batch run.
+    in the case of a batch run.  For STP sessions
+
+  The method used varies according to the context.  Important points:
+
+  @li should not use endsas or abort cancel in 9.4m3 environments as this can
+    cause hung multibridge sessions and result in a frozen STP server
+  @li should not use endsas in viya 3.5 as this destroys the session and cannot
+    fetch results (although both mv_getjoblog.sas and the @sasjs/adapter will
+    recognise this and fetch the log of the parent session instead)
+  @li STP environments must finish cleanly to avoid the log being sent to
+    _webout.  To assist with this, we also run stpsrvset('program error', 0)
+    and set SYSCC=0.  We take a unique "soft abort" approach - we open a macro
+    but don't close it!  This works everywhere EXCEPT inside a \%include inside
+    a macro.  For that, we recommend you use mp_include.sas to perform the
+    include, and then call \%mp_abort(mode=INCLUDE) from the source program (ie,
+    OUTSIDE of the top-parent macro).
 
-  Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored
-  Process environments.  This macro takes a unique approach - we set the SAS
-  syscc to 0, run `stpsrvset('program error', 0)` (if SAS 9) and then - we open
-  a macro but don't close it!  This provides a graceful abort for SAS web
-  services in all web enabled environments.
 
   @param mac= to contain the name of the calling macro
   @param msg= message to be returned
   @param iftrue= supply a condition under which the macro should be executed.
+  @param errds= (work.mp_abort_errds) There is no clean way to end a process
+    within a %include called within a macro.  Furthermore, there is no way to
+    test if a macro is called within a %include.  To handle this particular
+    scenario, the %include should be switched for the mp_include.sas macro.
+    This provides an indicator that we are running a macro within a \%include
+    (`_SYSINCLUDEFILEDEVICE`) and allows us to provide a dataset with the abort
+    values (msg, mac).
+    We can then run an abort cancel FILE to stop the include running, and pass
+    the dataset back to the calling program to run a regular \%mp_abort().
+    The dataset will contain the following fields:
+    @li iftrue (1=1)
+    @li msg (the message)
+    @li mac (the mac param)
 
-  @version 9.4M3
+  @param mode= (REGULAR) If mode=INCLUDE then the &errds dataset is checked for
+    an abort status.
+    Valid values:
+    @li REGULAR (default)
+    @li INCLUDE
+
+  <h4> Related Macros </h4>
+  @li mp_include.sas
+
+  @version 9.4
   @author Allan Bowe
   @cond
 **/
 
 %macro mp_abort(mac=mp_abort.sas, type=, msg=, iftrue=%str(1=1)
+  , errds=work.mp_abort_errds
+  , mode=REGULAR
 )/*/STORE SOURCE*/;
 
+  %global sysprocessmode sysprocessname;
+
   %if not(%eval(%unquote(&iftrue))) %then %return;
 
   %put NOTE: ///  mp_abort macro executing //;
   %if %length(&mac)>0 %then %put NOTE- called by &mac;
   %put NOTE - &msg;
 
+  %if %symexist(_SYSINCLUDEFILEDEVICE) %then %do;
+    %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do;
+      data &errds;
+        iftrue='1=1';
+        length mac $100 msg $5000;
+        mac=symget('mac');
+        msg=symget('msg');
+      run;
+      data _null_;
+        abort cancel FILE;
+      run;
+      %return;
+    %end;
+  %end;
+
   /* Stored Process Server web app context */
   %if %symexist(_metaperson)
-  or (%symexist(SYSPROCESSNAME) and "&SYSPROCESSNAME"="Compute Server" )
+    or "&SYSPROCESSNAME "="Compute Server "
+    or &mode=INCLUDE
   %then %do;
     options obs=max replace nosyntaxcheck mprint;
+    %if &mode=INCLUDE %then %do;
+      %if %sysfunc(exist(&errds))=1 %then %do;
+        data _null_;
+          set &errds;
+          call symputx('iftrue',iftrue,'l');
+          call symputx('mac',mac,'l');
+          call symputx('msg',msg,'l');
+          putlog (_all_)(=);
+        run;
+        %if (&iftrue)=0 %then %return;
+      %end;
+      %else %do;
+        %put &sysmacroname: No include errors found;
+        %return;
+      %end;
+    %end;
+
     /* extract log errs / warns, if exist */
     %local logloc logline;
     %global logmsg; /* capture global messages */
@@ -63,7 +132,7 @@
           input;
           i=1;
           stoploop=0;
-          if _n_ ge &logline-5 and stoploop=0 then do until (i>12);
+          if _n_ ge &logline-15 and stoploop=0 then do until (i>22);
             call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
             input;
             i+1;
@@ -88,8 +157,8 @@
 
     /* send response in SASjs JSON format */
     data _null_;
-      file _webout mod lrecl=32000;
-      length msg $32767 debug $8;
+      file _webout mod lrecl=32000 encoding='utf-8';
+      length msg $32767 ;
       sasdatetime=datetime();
       msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
       /* escape the quotes */
@@ -100,7 +169,7 @@
       msg=cats('"',msg,'"');
       if symexist('_debug') then debug=quote(trim(symget('_debug')));
       else debug='""';
-      if debug ge '"131"' then put '>>weboutBEGIN<<';
+      put '>>weboutBEGIN<<';
       put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
       put ',"sasjsAbort" : [{';
       put ' "MSG":' msg ;
@@ -120,39 +189,68 @@
       _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
       put ',"_PROGRAM" : ' _PROGRAM ;
       put ",""SYSCC"" : ""&syscc"" ";
-      put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+      syserrortext=quote(trim(symget('syserrortext')));
+      put ",""SYSERRORTEXT"" : " syserrortext;
+      put ",""SYSHOSTNAME"" : ""&syshostname"" ";
       put ",""SYSJOBID"" : ""&sysjobid"" ";
-      put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+      put ",""SYSSCPL"" : ""&sysscpl"" ";
+      put ",""SYSSITE"" : ""&syssite"" ";
+      sysvlong=quote(trim(symget('sysvlong')));
+      put ',"SYSVLONG" : ' sysvlong;
+      syswarningtext=quote(trim(symget('syswarningtext')));
+      put ",""SYSWARNINGTEXT"" : " syswarningtext;
       put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
       put "}" @;
-      if debug ge '"131"' then put '>>weboutEND<<';
+      put '>>weboutEND<<';
     run;
 
-    %let syscc=0;
-    %if %symexist(_metaport) %then %do;
+    %put _all_;
+
+    %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
       data _null_;
-        if symexist('sysprocessmode')
-          then if symget("sysprocessmode")="SAS Stored Process Server"
-            then rc=stpsrvset('program error', 0);
+        putlog 'stpsrvset program err and syscc';
+        rc=stpsrvset('program error', 0);
+        call symputx("syscc",0,"g");
+      run;
+      /**
+        * endsas kills 9.4m3 deployments by orphaning multibridges.
+        * Abort variants are ungraceful (non zero return code)
+        * This approach lets SAS run silently until the end :-)
+        * Caution - fails when called within a %include within a macro
+        * Use mp_include() to handle this.
+        */
+      filename skip temp;
+      data _null_;
+        file skip;
+        put '%macro skip();';
+        comment '%mend skip; -> fix lint ';
+        put '%macro skippy();';
+        comment '%mend skippy; -> fix lint ';
+      run;
+      %inc skip;
+    %end;
+    %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
+      /* endsas kills the session making it harder to fetch results */
+      data _null_;
+        syswarningtext=symget('syswarningtext');
+        syserrortext=symget('syserrortext');
+        abort_msg=symget('msg');
+        syscc=symget('syscc');
+        sysuserid=symget('sysuserid');
+        iftrue=symget('iftrue');
+        put (_all_)(/=);
+        call symputx('syscc',0);
+        abort cancel nolist;
       run;
     %end;
-    /**
-      * endsas is reliable but kills some deployments.
-      * Abort variants are ungraceful (non zero return code)
-      * This approach lets SAS run silently until the end :-)
-      */
-    %put _all_;
-    filename skip temp;
-    data _null_;
-      file skip;
-      put '%macro skip(); %macro skippy();';
-    run;
-    %inc skip;
+    %else %do;
+      %abort cancel;
+    %end;
   %end;
   %else %do;
     %put _all_;
     %abort cancel;
   %end;
-%mend;
+%mend mp_abort;
 
 /** @endcond */

base/mp_appendfile.sas

@@ -0,0 +1,57 @@
+/**
+  @file
+  @brief Append (concatenate) two or more files.
+  @details Will append one more more `appendrefs` (filerefs) to a `baseref`.
+  Uses a binary mechanism, so will work with any file type.  For that reason -
+  use with care!   And supply your own trailing carriage returns in each file..
+
+  Usage:
+
+        filename tmp1 temp;
+        filename tmp2 temp;
+        filename tmp3 temp;
+        data _null_; file tmp1; put 'base file';
+        data _null_; file tmp2; put 'append1';
+        data _null_; file tmp3; put 'append2';
+        run;
+        %mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3)
+
+
+  @param [in] baseref= Fileref of the base file (should exist)
+  @param [in] appendrefs= One or more filerefs to be appended to the base
+    fileref.  Space separated.
+
+  @version 9.2
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+
+
+**/
+
+%macro mp_appendfile(
+  baseref=0,
+  appendrefs=0
+)/*/STORE SOURCE*/;
+
+%mp_abort(iftrue= (&baseref=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Baseref NOT specified!)
+)
+%mp_abort(iftrue= (&appendrefs=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Appendrefs NOT specified!)
+)
+
+%local i;
+%do i=1 %to %sysfunc(countw(&appendrefs));
+  %mp_abort(iftrue= (&syscc>0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc)
+  )
+  %mp_binarycopy(inref=%scan(&appendrefs,&i), outref=&baseref, mode=APPEND)
+%end;
+
+%mend mp_appendfile;

base/mp_applyformats.sas

@@ -0,0 +1,181 @@
+/**
+  @file
+  @brief Apply a set of formats to a table
+  @details Applies a set of formats to one or more SAS datasets.  Can be used
+    to migrate formats from one table to another.  The input table must contain
+    the following columns:
+
+    @li lib - the libref of the table to be updated
+    @li ds - the dataset to be updated
+    @li var - the variable to be updated
+    @li fmt - the format to apply.  Missing or default ($CHAR, 8.) formats are
+      ignored.
+
+  The macro will abort in the following scenarios:
+
+    @li Libref not assigned
+    @li Dataset does not exist
+    @li Input table contains null or invalid values
+
+  Example usage:
+
+      data work.example;
+        set sashelp.prdsale;
+        format _all_ clear;
+      run;
+
+      %mp_getcols(sashelp.prdsale,outds=work.cols)
+
+      data work.cols2;
+        set work.cols;
+        lib='WORK';
+        ds='EXAMPLE';
+        var=name;
+        fmt=format;
+        keep lib ds var fmt;
+      run;
+
+      %mp_applyformats(work.cols2)
+
+  @param [in] inds The input dataset containing the formats to apply (and where
+    to apply them).  Example structure:
+  |LIB:$8.|DS:$32.|VAR:$32.|FMT:$49.|
+  |---|---|---|---|
+  |`WORK `|`EXAMPLE `|`ACTUAL `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`COUNTRY `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`DIVISION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`MONTH `|`MONNAME3. `|
+  |`WORK `|`EXAMPLE `|`PREDICT `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`PRODTYPE `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`PRODUCT `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`QUARTER `|`8. `|
+  |`WORK `|`EXAMPLE `|`REGION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`YEAR `|`8. `|
+
+  @param [out] errds= (0) Provide a libds reference here to export the
+    error messages to a table.  In this case, they will not be printed to the
+    log.
+
+  <h4> SAS Macros </h4>
+  @li mf_getengine.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_validatecol.sas
+
+
+  <h4> Related Macros </h4>
+  @li mp_getformats.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_applyformats(inds,errds=0
+)/*/STORE SOURCE*/;
+%local outds liblist i engine lib msg ;
+
+/**
+  * Validations
+  */
+proc sort data=&inds;
+  by lib ds var fmt;
+run;
+
+%if &errds=0 %then %let outds=%mf_getuniquename(prefix=mp_applyformats);
+%else %let outds=&errds;
+
+data &outds;
+  set &inds;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  length msg $128;
+  by lib ds var fmt;
+  if libref(lib) ne 0 then do;
+    msg=catx(' ','libref',lib,'is not assigned!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  if exist(cats(lib,'.',ds)) ne 1 then do;
+    msg=catx(' ','libds',lib,'.',ds,'does not exist!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  %mp_validatecol(fmt,FORMAT,is_fmt)
+  if is_fmt=0 then do;
+    msg=catx(' ','format',fmt,'on libds',lib,'.',ds,'.',var,'is not valid!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+
+  if first.ds then do;
+    retain dsid;
+    dsid=open(cats(lib,'.',ds));
+    if dsid=0 then do;
+      msg=catx(' ','libds',lib,'.',ds,' could not be opened!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+      return;
+    end;
+    if varnum(dsid,var)<1 then do;
+      msg=catx(' ','Variable',lib,'.',ds,'.',var,' was not found!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+    end;
+  end;
+  if last.ds then rc=close(dsid);
+run;
+
+proc sql noprint;
+select distinct lib into: liblist separated by ' ' from &inds;
+%put &=liblist;
+%do i=1 %to %sysfunc(countw(&liblist));
+  %let lib=%scan(&liblist,1);
+  %let engine=%mf_getengine(&lib);
+  %if &engine ne V9 and &engine ne BASE %then %do;
+    %let msg=&lib has &engine engine - formats cannot be applied;
+    proc sql;
+    insert into &outds set lib="&lib",ds="_all_",var="_all", msg="&msg" ;
+    %if &errds=0 %then %put %str(ERR)OR: &msg;
+  %end;
+%end;
+
+%if %mf_nobs(&outds)>0 %then %return;
+
+/**
+  * Validations complete - now apply the actual formats!
+  */
+%let fref=%mf_getuniquefileref();
+data _null_;
+  set &inds;
+  by lib ds var fmt;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  file &fref;
+  if first.lib then put 'proc datasets nolist lib=' lib ';';
+  if first.ds then put '  modify ' ds ';';
+  put '    format ' var fmt ';';
+  if last.ds then put '  run;';
+  if last.lib then put 'quit;';
+run;
+
+%inc &fref/source2;
+
+%if &errds=0 %then %do;
+  proc sql;
+  drop table &outds;
+%end;
+
+%mend mp_applyformats;

base/mp_assertcols.sas

@@ -142,4 +142,4 @@
   proc sql;
   drop table &ds;
 
-%mend;
+%mend mp_assertcols;

base/mp_assertcolvals.sas

@@ -30,6 +30,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
+  @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
 
@@ -115,6 +116,26 @@
   select count(*) into: orig from &lib..&ds;
   quit;
 
+  %local notfound tmp1 tmp2;
+  %let tmp1=%mf_getuniquename();
+  %let tmp2=%mf_getuniquename();
+
+  /* this is a bit convoluted - but using sql outobs=10 throws warnings */
+  proc sql noprint;
+  create view &tmp1 as
+    select distinct &col
+    from &lib..&ds
+    where &col not in (
+      select &ccol from &clib..&cds
+    );
+  data &tmp2;
+    set &tmp1;
+    if _n_>10 then stop;
+  run;
+  proc sql;
+  select distinct &col  into: notfound separated by ' ' from &tmp2;
+
+
   %mp_abort(iftrue= (&syscc ne 0)
     ,mac=&sysmacroname
     ,msg=%str(syscc=&syscc after macro query)
@@ -125,7 +146,7 @@
     test_description=symget('desc');
     test_result='FAIL';
     test_comments="&sysmacroname: &lib..&ds..&col has &result values "
-      !!"not in &clib..&cds..&ccol ";
+      !!"not in &clib..&cds..&ccol.. First 10 vals:"!!symget('notfound');
   %if &test=ANYVAL %then %do;
     if &result < &orig then test_result='PASS';
   %end;
@@ -144,4 +165,4 @@
   proc sql;
   drop table &ds;
 
-%mend;
+%mend mp_assertcolvals;

base/mp_assertdsobs.sas

@@ -8,6 +8,10 @@
 
       %mp_assertdsobs(sashelp.class) %* tests if any observations are present;
 
+      %mp_assertdsobs(sashelp.class,test=ATLEAST 10) %* pass if >9 obs present;
+
+      %mp_assertdsobs(sashelp.class,test=ATMOST 20) %* pass if <21 obs present;
+
   <h4> SAS Macros </h4>
   @li mf_nobs.sas
   @li mp_abort.sas
@@ -18,7 +22,11 @@
   @param [in] test= (HASOBS) The test to apply.  Valid values are:
     @li HASOBS - Test is a PASS if the input dataset has any observations
     @li EMPTY - Test is a PASS if input dataset is empty
-    @li EQUALS [integer] - Test passes if obs count matches the provided integer
+    @li EQUALS [integer] - Test passes if row count matches the provided integer
+    @li ATLEAST [integer] - Test passes if row count is more than or equal to
+      the provided integer
+    @li ATMOST [integer] - Test passes if row count is less than or equal to
+      the provided integer
   @param [out] outds= (work.test_results) The output dataset to contain the
   results.  If it does not exist, it will be created, with the following format:
   |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
@@ -27,6 +35,8 @@
 
   <h4> Related Macros </h4>
   @li mp_assertcolvals.sas
+  @li mp_assert.sas
+  @li mp_assertcols.sas
 
   @version 9.2
   @author Allan Bowe
@@ -51,6 +61,22 @@
     )
     %let test=EQUALS;
   %end;
+  %else %if %substr(&test.xxxxxxx,1,7)=ATLEAST %then %do;
+    %let val=%scan(&test,2,%str( ));
+    %mp_abort(iftrue= (%DATATYP(&val)=CHAR)
+      ,mac=&sysmacroname
+      ,msg=%str(Invalid test - &test, expected ATLEAST [integer])
+    )
+    %let test=ATLEAST;
+  %end;
+  %else %if %substr(&test.xxxxxxx,1,7)=ATMOST %then %do;
+    %let val=%scan(&test,2,%str( ));
+    %mp_abort(iftrue= (%DATATYP(&val)=CHAR)
+      ,mac=&sysmacroname
+      ,msg=%str(Invalid test - &test, expected ATMOST [integer])
+    )
+    %let test=ATMOST;
+  %end;
   %else %if &test ne HASOBS and &test ne EMPTY %then %do;
     %mp_abort(
       mac=&sysmacroname,
@@ -62,7 +88,8 @@
     length test_description $256 test_result $4 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
-    test_comments="&sysmacroname: Dataset &inds has &nobs observations";
+    test_comments="&sysmacroname: Dataset &inds has &nobs observations.";
+    test_comments=test_comments!!" Test was "!!symget('test');
   %if &test=HASOBS %then %do;
     if &nobs>0 then test_result='PASS';
   %end;
@@ -72,6 +99,12 @@
   %else %if &test=EQUALS %then %do;
     if &nobs=&val then test_result='PASS';
   %end;
+  %else %if &test=ATLEAST %then %do;
+    if &nobs ge &val then test_result='PASS';
+  %end;
+  %else %if &test=ATMOST %then %do;
+    if &nobs le &val then test_result='PASS';
+  %end;
   %else %do;
     test_comments="&sysmacroname: Unsatisfied test condition - &test";
   %end;
@@ -86,4 +119,4 @@
   proc sql;
   drop table &ds;
 
-%mend;
+%mend mp_assertdsobs;

base/mp_base64copy.sas

@@ -0,0 +1,117 @@
+/**
+  @file
+  @brief Convert a file to/from base64 format
+  @details Creates a new version of a file either encoded or decoded using
+  Base64.  Inspired by this post by Michael Dixon:
+  https://support.selerity.com.au/hc/en-us/articles/223345708-Tip-SAS-and-Base64
+
+  Usage:
+
+        filename tmp temp;
+        data _null_;
+          file tmp;
+          put 'base ik ally';
+        run;
+        %mp_base64copy(inref=tmp, outref=myref, action=ENCODE)
+
+        data _null_;
+          infile myref;
+          input;
+          put _infile_;
+        run;
+
+        %mp_base64copy(inref=myref, outref=mynewref, action=DECODE)
+
+        data _null_;
+          infile mynewref;
+          input;
+          put _infile_;
+        run;
+
+  @param [in] inref= Fileref of the input file (should exist)
+  @param [out] outref= Output fileref. If it does not exist, it is created.
+  @param [in] action= (ENCODE) The action to take. Valid values:
+    @li ENCODE - Convert the file to base64 format
+    @li DECODE - Decode the file from base64 format
+
+  @version 9.2
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+
+
+**/
+
+%macro mp_base64copy(
+  inref=0,
+  outref=0,
+  action=ENCODE
+)/*/STORE SOURCE*/;
+
+%let inref=%upcase(&inref);
+%let outref=%upcase(&outref);
+%let action=%upcase(&action);
+%local infound outfound;
+%let infound=0;
+%let outfound=0;
+data _null_;
+  set sashelp.vextfl(where=(fileref="&inref" or fileref="&outref"));
+  if fileref="&inref" then call symputx('infound',1,'l');
+  if fileref="&outref" then call symputx('outfound',1,'l');
+run;
+
+%mp_abort(iftrue= (&infound=0)
+  ,mac=&sysmacroname
+  ,msg=%str(INREF &inref NOT FOUND!)
+)
+%mp_abort(iftrue= (&outref=0)
+  ,mac=&sysmacroname
+  ,msg=%str(OUTREF NOT PROVIDED!)
+)
+%mp_abort(iftrue= (&action ne ENCODE and &action ne DECODE)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid action! Should be ENCODE OR DECODE)
+)
+
+%if &outfound=0 %then %do;
+  filename &outref temp lrecl=2097088;
+%end;
+
+%if &action=ENCODE %then %do;
+  data _null_;
+    length b64 $ 76 line $ 57;
+    retain line "";
+    infile &inref recfm=F lrecl= 1 end=eof;
+    input @1 stream $char1.;
+    file &outref recfm=N;
+    substr(line,(_N_-(CEIL(_N_/57)-1)*57),1) = byte(rank(stream));
+    if mod(_N_,57)=0 or EOF then do;
+      if eof then b64=put(trim(line),$base64X76.);
+      else b64=put(line, $base64X76.);
+      put b64 + (-1) @;
+      line="";
+    end;
+  run;
+%end;
+%else %if &action=DECODE %then %do;
+  data _null_;
+    length filein 8 fileout 8;
+    filein = fopen("&inref",'I',4,'B');
+    fileout = fopen("&outref",'O',3,'B');
+    char= '20'x;
+    do while(fread(filein)=0);
+      length raw $4;
+      do i=1 to 4;
+        rc=fget(filein,char,1);
+        substr(raw,i,1)=char;
+      end;
+      rc = fput(fileout,input(raw,$base64X4.));
+      rc = fwrite(fileout);
+    end;
+    rc = fclose(filein);
+    rc = fclose(fileout);
+  run;
+%end;
+
+%mend mp_base64copy;

base/mp_binarycopy.sas

@@ -9,10 +9,29 @@
 
         %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
 
-  @param inloc full, quoted "path/and/filename.ext" of the object to be copied
-  @param outloc full, quoted "path/and/filename.ext" of object to be created
-  @param inref can override default input fileref to avoid naming clash
-  @param outref an override default output fileref to avoid naming clash
+  To append to a file, use the mode option, eg:
+
+      filename tmp1 temp;
+      filename tmp2 temp;
+      data _null_;
+        file tmp1;
+        put 'stacking';
+      run;
+
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+
+
+  @param [in] inloc quoted "path/and/filename.ext" of the file to be copied
+  @param [out] outloc quoted "path/and/filename.ext" of the file to be created
+  @param [in] inref (____in) If provided, this fileref will take precedence over
+    the `inloc` parameter
+  @param [out] outref (____in) If provided, this fileref will take precedence
+    over the `outloc` parameter.  It must already exist!
+  @param [in] mode (CREATE) Valid values:
+    @li CREATE - Create the file (even if it already exists)
+    @li APPEND - Append to the file (don't overwrite)
+
   @returns nothing
 
   @version 9.2
@@ -24,20 +43,29 @@
     ,outloc=          /* full path and filename of object to be created */
     ,inref=____in   /* override default to use own filerefs */
     ,outref=____out /* override default to use own filerefs */
+    ,mode=CREATE
 )/*/STORE SOURCE*/;
+  %local mod outmode;
+  %if &mode=APPEND %then %do;
+    %let mod=mod;
+    %let outmode='a';
+  %end;
+  %else %do;
+    %let outmode='o';
+  %end;
   /* these IN and OUT filerefs can point to anything */
   %if &inref = ____in %then %do;
     filename &inref &inloc lrecl=1048576 ;
   %end;
   %if &outref=____out %then %do;
-    filename &outref &outloc lrecl=1048576 ;
+    filename &outref &outloc lrecl=1048576 &mod;
   %end;
 
   /* copy the file byte-for-byte  */
   data _null_;
     length filein 8 fileid 8;
     filein = fopen("&inref",'I',1,'B');
-    fileid = fopen("&outref",'O',1,'B');
+    fileid = fopen("&outref",&outmode,1,'B');
     rec = '20'x;
     do while(fread(filein)=0);
       rc = fget(filein,rec,1);
@@ -53,4 +81,4 @@
   %if &outref=____out %then %do;
     filename &outref clear;
   %end;
-%mend;
+%mend mp_binarycopy;

base/mp_cleancsv.sas

@@ -67,5 +67,5 @@
       else put inchar $char1.;
     end;
   run;
-%mend;
+%mend mp_cleancsv;
 /** @endcond */

base/mp_copyfolder.sas

@@ -0,0 +1,81 @@
+/**
+  @file
+  @brief A macro to recursively copy a directory
+  @details Performs a recursive directory listing then works from top to bottom
+    copying files and creating subdirectories.
+
+  Usage:
+
+      %let rootdir=%sysfunc(pathname(work))/demo;
+      %let copydir=%sysfunc(pathname(work))/demo_copy;
+      %mf_mkdir(&rootdir)
+      %mf_mkdir(&rootdir/subdir)
+      %mf_mkdir(&rootdir/subdir/subsubdir)
+      data "&rootdir/subdir/example.sas7bdat";
+      run;
+
+      %mp_copyfolder(&rootdir,&copydir)
+
+  @param source Unquoted path to the folder to copy from.
+  @param target Unquoted path to the folder to copy to.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_isdir.sas
+  @li mf_mkdir.sas
+  @li mp_abort.sas
+  @li mp_dirlist.sas
+
+  <h4> Related Macros </h4>
+  @li mp_copyfolder.test.sas
+
+**/
+
+%macro mp_copyfolder(source,target);
+
+  %mp_abort(iftrue=(%mf_isdir(&source)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Source dir does not exist (&source))
+  )
+
+  %mf_mkdir(&target)
+
+  %mp_abort(iftrue=(%mf_isdir(&target)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Target dir could not be created (&target))
+  )
+
+  /* prep temp table */
+  %local tempds;
+  %let tempds=%mf_getuniquename();
+
+  /* recursive directory listing */
+  %mp_dirlist(path=&source,outds=work.&tempds, maxdepth=MAX)
+
+  /* create folders and copy content */
+  data _null_;
+    length msg $200;
+    call missing(msg);
+    set work.&tempds;
+    if _n_ = 1 then dpos+sum(length(directory),2);
+    filepath2="&target/"!!substr(filepath,dpos);
+    if file_or_folder='folder' then call execute('%mf_mkdir('!!filepath2!!')');
+    else do;
+      length fref1 fref2 $8;
+      rc1=filename(fref1,filepath,'disk','recfm=n');
+      rc2=filename(fref2,filepath2,'disk','recfm=n');
+      if fcopy(fref1,fref2) ne 0 then do;
+        msg=sysmsg();
+        putlog "%str(ERR)OR: Unable to copy " filepath " to " filepath2;
+        putlog msg=;
+      end;
+    end;
+    rc=filename(fref1);
+    rc=filename(fref2);
+  run;
+
+  /* tidy up */
+  proc sql;
+  drop table work.&tempds;
+
+%mend mp_copyfolder;

base/mp_createconstraints.sas

@@ -64,4 +64,4 @@ data &outds;
   output;
 run;
 
-%mend;
+%mend mp_createconstraints;

base/mp_createwebservice.sas

@@ -80,4 +80,4 @@ Usage:
   )
 %end;
 
-%mend;
+%mend mp_createwebservice;

base/mp_csv2ds.sas

@@ -141,4 +141,4 @@ data &outds
   %end;
 run;
 
-%mend;
+%mend mp_csv2ds;

base/mp_deleteconstraints.sas

@@ -49,4 +49,4 @@ data &outds;
   end;
 run;
 
-%mend;
+%mend mp_deleteconstraints;

base/mp_deletefolder.sas

@@ -0,0 +1,69 @@
+/**
+  @file
+  @brief A macro to delete a directory
+  @details Will delete all folder content (including subfolder content) and
+    finally, the folder itself.
+
+  Usage:
+
+      %let rootdir=%sysfunc(pathname(work))/demo;
+      %mf_mkdir(&rootdir)
+      %mf_mkdir(&rootdir/subdir)
+      %mf_mkdir(&rootdir/subdir/subsubdir)
+      data "&rootdir/subdir/example.sas7bdat";
+      run;
+
+      %mp_deletefolder(&rootdir)
+
+  @param path Unquoted path to the folder to delete.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_isdir.sas
+  @li mp_dirlist.sas
+
+  <h4> Related Macros </h4>
+  @li mp_deletefolder.test.sas
+
+**/
+
+%macro mp_deletefolder(folder);
+  /* proceed if valid directory */
+  %if %mf_isdir(&folder)=1 %then %do;
+
+    /* prep temp table */
+    %local tempds;
+    %let tempds=%mf_getuniquename();
+
+    /* recursive directory listing */
+    %mp_dirlist(path=&folder,outds=work.&tempds, maxdepth=MAX)
+
+    /* sort descending level so can delete folder contents before folders */
+    proc sort data=work.&tempds;
+      by descending level;
+    run;
+
+    /* ensure top level folder is removed at the end */
+    proc sql;
+    insert into work.&tempds set filepath="&folder";
+
+    /* delete everything */
+    data _null_;
+      set work.&tempds end=last;
+      length fref $8;
+      rc=filename(fref,filepath);
+      rc=fdelete(fref);
+      if rc then do;
+        msg=sysmsg();
+        put "&sysmacroname:" / rc= / msg= / filepath=;
+      end;
+      rc=filename(fref);
+    run;
+
+    /* tidy up */
+    proc sql;
+    drop table work.&tempds;
+
+  %end;
+  %else %put &sysmacroname: &folder: is not a valid / accessible folder. ;
+%mend mp_deletefolder;

base/mp_dirlist.sas

@@ -3,20 +3,13 @@
   @brief Returns all files and subdirectories within a specified parent
   @details When used with getattrs=NO, is not OS specific (uses dopen / dread).
 
-  If getattrs=YES then the doptname / foptname functions are used to scan all
-  properties - any characters that are not valid in a SAS name (v7) are simply
-  stripped, and the table is transposed so theat each property is a column
-  and there is one file per row.  An attempt is made to get all properties
-  whether a file or folder, but some files/folders cannot be accessed, and so
-  not all properties can / will be populated.
-
   Credit for the rename approach:
   https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003
 
 
   usage:
 
-      %mp_dirlist(path=/some/location,outds=myTable)
+      %mp_dirlist(path=/some/location, outds=myTable, maxdepth=MAX)
 
       %mp_dirlist(outds=cwdfileprops, getattrs=YES)
 
@@ -30,11 +23,19 @@
   X CMD) do please raise an issue!
 
 
-  @param path= for which to return contents
-  @param fref= Provide a DISK engine fileref as an alternative to PATH
-  @param outds= the output dataset to create
-  @param getattrs= YES/NO (default=NO).  Uses doptname and foptname to return
-  all attributes for each file / folder.
+  @param [in] path= for which to return contents
+  @param [in] fref= Provide a DISK engine fileref as an alternative to PATH
+  @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
+    subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
+  @param [out] outds= the output dataset to create
+  @param [out] getattrs= (NO)  If getattrs=YES then the doptname / foptname
+    functions are used to scan all properties - any characters that are not
+    valid in a SAS name (v7) are simply stripped, and the table is transposed
+    so theat each property is a column and there is one file per row.  An
+    attempt is made to get all properties whether a file or folder, but some
+    files/folders cannot be accessed, and so not all properties can / will be
+    populated.
 
 
   @returns outds contains the following variables:
@@ -44,8 +45,15 @@
     - filename (just the file name)
     - ext (.extension)
     - msg (system message if any issues)
+    - level (depth of folder)
     - OS SPECIFIC variables, if <code>getattrs=</code> is used.
 
+  <h4> SAS Macros </h4>
+  @li mp_dropmembers.sas
+
+  <h4> Related Macros </h4>
+  @li mp_dirlist.test.sas
+
   @version 9.2
   @author Allan Bowe
 **/
@@ -54,14 +62,27 @@
     , fref=0
     , outds=work.mp_dirlist
     , getattrs=NO
+    , maxdepth=0
+    , level=0 /* The level of recursion to perform.  For internal use only. */
 )/*/STORE SOURCE*/;
 %let getattrs=%upcase(&getattrs)XX;
 
-data &outds(compress=no
-    keep=file_or_folder filepath filename ext msg directory
+/* temp table */
+%local out_ds;
+data;run;
+%let out_ds=%str(&syslast);
+
+/* drop main (top) table if it exists */
+%if &level=0 %then %do;
+  %mp_dropmembers(%scan(&outds,-1,.), libref=WORK)
+%end;
+
+data &out_ds(compress=no
+    keep=file_or_folder filepath filename ext msg directory level
   );
   length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
     ext $20 msg $200;
+  retain level &level;
   %if &fref=0 %then %do;
     rc = filename(fref, "&path");
   %end;
@@ -119,8 +140,8 @@ data &outds(compress=no
 run;
 
 %if %substr(&getattrs,1,1)=Y %then %do;
-  data &outds;
-    set &outds;
+  data &out_ds;
+    set &out_ds;
     length infoname infoval $60 fref $8;
     rc=filename(fref,filepath);
     drop rc infoname fid i close fref;
@@ -161,10 +182,36 @@ run;
   run;
   proc sort;
     by filepath sasname;
-  proc transpose data=&outds out=&outds(drop=_:);
+  proc transpose data=&out_ds out=&out_ds(drop=_:);
     id sasname;
     var infoval;
     by filepath file_or_folder filename ext ;
   run;
 %end;
-%mend;
+
+data &out_ds;
+  set &out_ds(where=(filepath ne ''));
+run;
+
+/* update main table */
+proc append base=&outds data=&out_ds;
+run;
+
+/* recursive call */
+%if &maxdepth>&level or &maxdepth=MAX %then %do;
+  data _null_;
+    set &out_ds;
+    where file_or_folder='folder';
+    length code $10000;
+    code=cats('%nrstr(%mp_dirlist(path=',filepath,",outds=&outds"
+      ,",getattrs=&getattrs,level=%eval(&level+1),maxdepth=&maxdepth))");
+    put code=;
+    call execute(code);
+  run;
+%end;
+
+/* tidy up */
+proc sql;
+drop table &out_ds;
+
+%mend mp_dirlist;

base/mp_distinctfmtvalues.sas

@@ -47,4 +47,4 @@
     %end;
         as &outvar length=&varlen
     from &libds;
-%mend;
+%mend mp_distinctfmtvalues;

base/mp_dropmembers.sas

@@ -1,20 +1,25 @@
 /**
   @file
   @brief Drops tables / views (if they exist) without warnings in the log
-  @details
+  @details Useful for dropping tables when you're not sure they exist, or if
+  you are not sure whether they are a dataset or view.  Also efficient for
+  dropping multiple tables / views.
+
   Example usage:
 
       proc sql;
       create table data1 as select * from sashelp.class;
       create view view2 as select * from sashelp.class;
-      %mp_dropmembers(list=data1 view2)
+      %mp_dropmembers(data1 view2, libref=WORK)
+
 
   <h4> SAS Macros </h4>
   @li mf_isblank.sas
 
 
-  @param list space separated list of datasets / views
-  @param libref= can only drop from a single library at a time
+  @param [in] list space separated list of datasets / views, WITHOUT libref
+  @param [in] libref= (WORK) Note - you can only drop from one library at a time
+  @param [in] iftrue= (1=1) Conditionally drop tables, eg if &debug=N
 
   @version 9.2
   @author Allan Bowe
@@ -24,8 +29,11 @@
 %macro mp_dropmembers(
     list /* space separated list of datasets / views */
     ,libref=WORK  /* can only drop from a single library at a time */
+    ,iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
   %if %mf_isblank(&list) %then %do;
     %put NOTE: nothing to drop!;
     %return;
@@ -35,4 +43,4 @@
     delete &list;
     delete &list /mtype=view;
   run;
-%mend;
+%mend mp_dropmembers;

base/mp_ds2cards.sas

@@ -2,12 +2,21 @@
   @file
   @brief Create a CARDS file from a SAS dataset.
   @details Uses dataset attributes to convert all data into datalines.
-    Running the generated file will rebuild the original dataset.
+    Running the generated file will rebuild the original dataset.  Includes
+    support for large decimals, binary data, PROCESSED_DTTM columns, and
+    alternative encoding.  If the input dataset is empty, the cards file will
+    still be created.
+
+    Additional support to generate a random sample and max rows.
+
   Usage:
 
       %mp_ds2cards(base_ds=sashelp.class
+        , tgt_ds=work.class
         , cards_file= "C:\temp\class.sas"
-        , maxobs=5)
+        , showlog=NO
+        , maxobs=5
+      )
 
   TODO:
     - labelling the dataset
@@ -18,15 +27,24 @@
                   that is converted to a cards file.
   @param [in] tgt_ds= Table that the generated cards file would create.
     Optional - if omitted, will be same as BASE_DS.
-  @param [out] cards_file= Location in which to write the (.sas) cards file
-  @param [in] maxobs= to limit output to the first <code>maxobs</code>
-    observations
-  @param [in] showlog= whether to show generated cards file in the SAS log
-    (YES/NO)
-  @param [in] outencoding= provide encoding value for file statement (eg utf-8)
-  @param [in] append= If NO then will rebuild the cards file if it already
+  @param [out] cards_file= ("%sysfunc(pathname(work))/cardgen.sas") Location in
+    which to write the (.sas) cards file
+  @param [in] maxobs= (max) To limit output to the first <code>maxobs</code>
+    observations, enter an integer here.
+  @param [in] random_sample= (NO) Set to YES to generate a random sample of
+    data.  Can be quite slow.
+  @param [in] showlog= (YES) Whether to show generated cards file in the SAS
+    log.  Valid values:
+    @li YES
+    @li NO
+  @param [in] outencoding= Provide encoding value for file statement (eg utf-8)
+  @param [in] append= (NO) If NO then will rebuild the cards file if it already
     exists, otherwise will append to it.  Used by the mp_lib2cards.sas macro.
 
+  <h4> Related Macros </h4>
+  @li mp_lib2cards.sas
+  @li mp_ds2inserts.sas
+  @li mp_mdtablewrite.sas
 
   @version 9.2
   @author Allan Bowe
@@ -51,15 +69,15 @@
 %if (&tgt_ds = ) %then %let tgt_ds=&base_ds;
 %if %index(&tgt_ds,.)=0 %then %let tgt_ds=WORK.%scan(&base_ds,2,.);
 %if ("&outencoding" ne "") %then %let outencoding=encoding="&outencoding";
-%if ("&append" = "") %then %let append=;
+%if ("&append" = "" or "&append" = "NO") %then %let append=;
 %else %let append=mod;
 
 /* get varcount */
 %let nvars=0;
 proc sql noprint;
 select count(*) into: nvars from dictionary.columns
-  where libname="%scan(%upcase(&base_ds),1)"
-    and memname="%scan(%upcase(&base_ds),2)";
+  where upcase(libname)="%scan(%upcase(&base_ds),1)"
+    and upcase(memname)="%scan(%upcase(&base_ds),2)";
 %if &nvars=0 %then %do;
   %put %str(WARN)ING: Dataset &base_ds has no variables, will not be converted.;
   %return;
@@ -115,8 +133,8 @@ proc sql
 reset outobs=max;
 create table datalines1 as
   select name,type,length,varnum,format,label from dictionary.columns
-  where libname="%upcase(%scan(&base_ds,1))"
-    and memname="%upcase(%scan(&base_ds,2))";
+  where upcase(libname)="%upcase(%scan(&base_ds,1))"
+    and upcase(memname)="%upcase(%scan(&base_ds,2))";
 
 /**
   Due to long decimals cannot use best. format
@@ -137,7 +155,18 @@ data datalines_2;
       ,put(',name,',best32.-l)
       ,substrn(put(',name,',bestd32.-l),1
       ,findc(put(',name,',bestd32.-l),"0","TBK")))');
-  else dataline=name;
+  /**
+    * binary data must be converted, to store in text format.  It is identified
+    * by the presence of the $HEX keyword in the format.
+    */
+  else if upcase(format)=:'$HEX' then
+    dataline=cats('put(trim(',name,'),',format,')');
+  /**
+    * There is no easy way to store line breaks in a cards file.
+    * To discuss this, use: https://github.com/sasjs/core/issues/80
+    * Removing all nonprintables with kw (keep writeable)
+    */
+  else dataline=cats('compress(',name,', ,"kw")');
 run;
 
 proc sql noprint;
@@ -162,7 +191,8 @@ data _null_;
 
 
 /* Build input statement */
-  if type='char' then type3=':$char.';
+  if upcase(format)=:'$HEX' then type3=':'!!format;
+  else if type='char' then type3=':$char.';
   str2=put(name,$33.)||type3;
 
 
@@ -184,11 +214,12 @@ data _null_;
   file &cards_file. &outencoding lrecl=32767 termstr=nl &append;
   length __attrib $32767;
   if _n_=1 then do;
-    put '/*******************************************************************';
-    put " Datalines for %upcase(%scan(&base_ds,2)) dataset ";
-    put " Generated by %nrstr(%%)mp_ds2cards()";
-    put " Available on github.com/sasjs/core";
-    put '********************************************************************/';
+    put '/**';
+    put '  @file';
+    put "  @brief Datalines for %upcase(%scan(&base_ds,2)) dataset";
+    put "  @details Generated by %nrstr(%%)mp_ds2cards()";
+    put "  Available on github.com/sasjs/core";
+    put '**/';
     put "data &tgt_ds &indexes;";
     put "attrib ";
     %do i = 1 %to &nvars;
@@ -212,11 +243,11 @@ data _null_;
       put 'run;';
     end;
     else do;
-      put "infile cards dsd delimiter=',';";
+      put "infile cards dsd;";
       put "input ";
       %do i = 1 %to &nvars.;
         %if(%length(&&input_stmt_&i..)) %then
-          put "   &&input_stmt_&i..";
+          put "  &&input_stmt_&i..";
         ;
       %end;
       put ";";
@@ -251,4 +282,4 @@ quit;
 %put NOTE-;%put NOTE-;
 %put NOTE- %sysfunc(dequote(&cards_file.));
 %put NOTE-;%put NOTE-;
-%mend;
+%mend mp_ds2cards;

base/mp_ds2csv.sas

@@ -55,4 +55,4 @@ data _null_;
 run;
 
 
-%mend;
+%mend mp_ds2csv;

base/mp_ds2fmtds.sas

@@ -0,0 +1,105 @@
+/**
+  @file
+  @brief Converts every value in a dataset to formatted value
+  @details Converts every value to it's formatted value.  All variables will
+  become character, and will be in the same order as the original dataset.
+
+  Lengths will be adjusted according to the format lengths, where applicable.
+
+  Usage:
+
+      %mp_ds2fmtds(sashelp.cars,work.cars)
+      %mp_ds2fmtds(sashelp.vallopt,vw_vallopt)
+
+  @param [in] libds The library.dataset to be converted
+  @param [out] outds The dataset to create.
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+
+  <h4> Related Macros <h4>
+  @li mp_jsonout.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_ds2fmtds(libds, outds
+)/*/STORE SOURCE*/;
+
+/* validations */
+
+%if not %mf_existds(libds=&libds) %then %do;
+  %put %str(WARN)ING:  &libds does not exist as either a VIEW or DATASET;
+  %return;
+%end;
+%if %index(&libds,.)=0 %then %let libds=WORK.&libds;
+
+/* grab metadata */
+proc contents noprint data=&libds
+  out=_data_(keep=name type length format formatl formatd varnum);
+run;
+proc sort;
+  by varnum;
+run;
+
+/* prepare formats and varnames */
+data _null_;
+  set &syslast end=last;
+  name=upcase(name);
+  /* fix formats */
+  if type=2 or type=6 then do;
+    length fmt $49.;
+    if format='' then fmt=cats('$',length,'.');
+    else if formatl=0 then fmt=cats(format,'.');
+    else fmt=cats(format,formatl,'.');
+    newlen=max(formatl,length);
+  end;
+  else do;
+    if format='' then fmt='best.';
+    else if formatl=0 then fmt=cats(format,'.');
+    else if formatd=0 then fmt=cats(format,formatl,'.');
+    else fmt=cats(format,formatl,'.',formatd);
+    /* needs to be wide, for datetimes etc */
+    newlen=max(length,formatl,24);
+  end;
+  /* 32 char unique name */
+  newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
+
+  call symputx(cats('name',_n_),name,'l');
+  call symputx(cats('newname',_n_),newname,'l');
+  call symputx(cats('len',_n_),newlen,'l');
+  call symputx(cats('fmt',_n_),fmt,'l');
+  call symputx(cats('type',_n_),type,'l');
+  if last then call symputx('nobs',_n_,'l');
+run;
+
+/* clean up */
+proc sql;
+drop table &syslast;
+
+%if &nobs=0 %then %do;
+  %put Dataset &libds has no columns!
+  data &outds;
+    set &libds;
+  run;
+  %return;
+%end;
+
+data &outds;
+  /* rename on entry */
+  set &libds(rename=(
+%local i;
+%do i=1 %to &nobs;
+  &&name&i=&&newname&i
+%end;
+  ));
+%do i=1 %to &nobs;
+  length &&name&i $&&len&i;
+  &&name&i=left(put(&&newname&i,&&fmt&i));
+  drop &&newname&i;
+%end;
+  if _error_ then call symputx('syscc',1012);
+run;
+
+%mend mp_ds2fmtds;

base/mp_ds2inserts.sas

@@ -0,0 +1,180 @@
+/**
+  @file
+  @brief Export a dataset to SQL insert statements
+  @details Converts dataset values to SQL insert statements for use across
+  multiple database types.
+
+  Usage:
+
+      %mp_ds2inserts(sashelp.class,outref=myref,outds=class)
+      data class;
+        set sashelp.class;
+        stop;
+      proc sql;
+      %inc myref;
+
+  @param [in] ds The dataset to be exported
+  @param [in] maxobs= (max) The max number of inserts to create
+  @param [out] outref= (0) The output fileref.  If it does not exist, it is
+    created. If it does exist, new records are APPENDED.
+  @param [out] schema= (0) The library (or schema) in which the target table is
+    located.  If not provided, is ignored.
+  @param [out] outds= (0) The output table to load.  If not provided, will
+    default to the table in the &ds parameter.
+  @param [in] flavour= (SAS) The SQL flavour to be applied to the output. Valid
+    options:
+    @li SAS (default) - suitable for regular proc sql
+    @li PGSQL - Used for Postgres databases
+  @param [in] applydttm= (YES) If YES, any columns using datetime formats will
+    be converted to native DB datetime literals
+
+  <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mf_getvarcount.sas
+  @li mf_getvarformat.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
+
+  @version 9.2
+  @author Allan Bowe (credit mjsq)
+**/
+
+%macro mp_ds2inserts(ds, outref=0,schema=0,outds=0,flavour=SAS,maxobs=max
+  ,applydttm=YES
+)/*/STORE SOURCE*/;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if %index(&ds,.)=0 %then %let ds=WORK.&ds;
+
+%let flavour=%upcase(&flavour);
+%if &flavour ne SAS and &flavour ne PGSQL %then %do;
+  %put %str(WAR)NING:  &flavour is not supported;
+  %return;
+%end;
+
+%if &outref=0 %then %do;
+  %put %str(WAR)NING:  Please provide a fileref;
+  %return;
+%end;
+%if %mf_existfileref(&outref)=0 %then %do;
+  filename &outref temp lrecl=66000;
+%end;
+
+%if &schema=0 %then %let schema=;
+%else %let schema=&schema..;
+
+%if &outds=0 %then %let outds=%scan(&ds,2,.);
+
+%local nobs;
+proc sql noprint;
+select count(*) into: nobs TRIMMED from &ds;
+%if &nobs=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No rows found in &ds */";
+  run;
+%end;
+
+%local vars;
+%let vars=%mf_getvarcount(&ds);
+%if &vars=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No columns found in &schema.&ds */";
+  run;
+  %return;
+%end;
+%else %if &vars>1600 and &flavour=PGSQL %then %do;
+  data _null_;
+    file &fref mod;
+    put "/* &schema.&ds contains &vars vars */";
+    put "/* Postgres cannot handle tables with over 1600 vars */";
+    put "/* No inserts will be generated for this table */";
+  run;
+  %return;
+%end;
+
+%local varlist varlistcomma;
+%let varlist=%mf_getvarlist(&ds);
+%let varlistcomma=%mf_getvarlist(&ds,dlm=%str(,),quote=double);
+
+/* next, export data */
+data _null_;
+  file &outref mod ;
+  if _n_=1 then put "/* &schema.&outds (&nobs rows, &vars columns) */";
+  set &ds;
+  %if &maxobs ne max %then %do;
+    if _n_>&maxobs then stop;
+  %end;
+  length _____str $32767;
+  call missing(_____str);
+  format _numeric_ best.;
+  format _character_ ;
+  %local i comma var vtype vfmt;
+  %do i=1 %to %sysfunc(countw(&varlist));
+    %let var=%scan(&varlist,&i);
+    %let vtype=%mf_getvartype(&ds,&var);
+    %let vfmt=%upcase(%mf_getvarformat(&ds,&var,force=1));
+    %if &i=1 %then %do;
+      %if &flavour=SAS %then %do;
+        put "insert into &schema.&outds set ";
+        put "  &var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        _____str=cats(
+          "INSERT INTO &schema.&outds ("
+          ,symget('varlistcomma')
+          ,") VALUES ("
+        );
+        put _____str;
+        put "  "@;
+      %end;
+    %end;
+    %else %do;
+      %if &flavour=SAS %then %do;
+        put "  ,&var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        put "  ,"@;
+      %end;
+    %end;
+    %if &vtype=N %then %do;
+      %if &flavour=SAS %then %do;
+        put &var;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        if missing(&var) then put 'NULL';
+        %if &applydttm=YES and "%substr(&vfmt.xxxxxxxx,1,8)"="DATETIME"
+        %then %do;
+          else put "TIMESTAMP '" &var E8601DT25.6 "'";
+        %end;
+        %else %do;
+          else put &var;
+        %end;
+      %end;
+    %end;
+    %else %do;
+      _____str="'"!!trim(tranwrd(&var,"'","''"))!!"'";
+      put _____str;
+    %end;
+  %end;
+  %if &flavour=SAS %then %do;
+    put ';';
+  %end;
+  %else %if &flavour=PGSQL %then %do;
+    put ');';
+  %end;
+
+  if _n_=&nobs then put /;
+run;
+
+%mend mp_ds2inserts;

base/mp_ds2md.sas

@@ -14,14 +14,14 @@
   We take the standard definition one step further by embedding the informat
   in the table header row, like so:
 
-      |var1:$|var2:best.|var3:date9.|
+      |var1:$32|var2:best.|var3:date9.|
       |---|---|---|
       |some text|42|01JAN1960|
       |blah|1|31DEC1999|
 
   Which resolves to:
 
-  |var1:$|var2:best.|var3:date9.|
+  |var1:$32|var2:best.|var3:date9.|
   |---|---|---|
   |some text|42|01JAN1960|
   |blah|1|31DEC1999|
@@ -29,44 +29,44 @@
 
   Usage:
 
-      %mp_mdtablewrite(libds=sashelp.class,showlog=YES)
+      %mp_ds2md(sashelp.class)
 
+  @param [in] libds the library / dataset to create or read from.
+  @param [out] outref= (mdtable) Fileref to contain the markdown
+  @param [out] showlog= (YES) Set to NO to avoid printing markdown to the log
 
   <h4> SAS Macros </h4>
   @li mf_getvarlist.sas
   @li mf_getvarformat.sas
 
-  @param [in] libds= the library / dataset to create or read from.
-  @param [out] fref= Fileref to contain the markdown. Default=mdtable.
-  @param [out] showlog= set to YES to show the markdown in the log. Default=NO.
-
   @version 9.3
   @author Allan Bowe
 **/
 
-%macro mp_mdtablewrite(
-  libds=,
-  fref=mdtable,
-  showlog=NO
+%macro mp_ds2md(
+  libds,
+  outref=mdtable,
+  showlog=YES
 )/*/STORE SOURCE*/;
 
 /* check fileref is assigned */
-%if %sysfunc(fileref(&fref)) > 0 %then %do;
-  filename &fref temp;
+%if %sysfunc(fileref(&outref)) > 0 %then %do;
+  filename &outref temp;
 %end;
 
 %local vars;
-%let vars=%mf_getvarlist(&libds);
+%let vars=%upcase(%mf_getvarlist(&libds));
 
 /* create the header row */
 data _null_;
-  file &fref;
+  file &outref;
   length line $32767;
+  call missing(line);
   put '|'
 %local i var fmt;
 %do i=1 %to %sysfunc(countw(&vars));
   %let var=%scan(&vars,&i);
-  %let fmt=%mf_getvarformat(&libds,&var,force=1);
+  %let fmt=%lowcase(%mf_getvarformat(&libds,&var,force=1));
   "&var:&fmt|"
 %end;
   ;
@@ -79,20 +79,20 @@ run;
 
 /* write out the data */
 data _null_;
-  file &fref mod dlm='|' lrecl=32767;
+  file &outref mod dlm='|' lrecl=32767;
   set &libds ;
   length line $32767;
-  line=cats('|',%mf_getvarlist(&libds,dlm=%str(,'|',)),'|');
+  line='|`'!!cats(%mf_getvarlist(&libds,dlm=%str(%)!!' `|`'!!cats%()))!!' `|';
   put line;
 run;
 
 %if %upcase(&showlog)=YES %then %do;
   options ps=max;
   data _null_;
-    infile &fref;
+    infile &outref;
     input;
     putlog _infile_;
   run;
 %end;
 
-%mend mp_mdtablewrite;
+%mend mp_ds2md;

base/mp_filtercheck.sas

@@ -33,19 +33,19 @@
   @returns The &outds table containing any bad rows, plus a REASON_CD column.
 
   @param [in] inds The table to be checked, with the format above
-  @param [in] targetds= The target dataset against which to verify VARIABLE_NM
+  @param [in] targetds= The target dataset against which to verify VARIABLE_NM.
+    This must be available (ie, the library must be assigned).
   @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
   @param [out] outds= The output table, which is a copy of the &inds. table
-  plus a REASON_CD column, containing only bad records.  If bad records found,
-  the SYSCC value will be set to 1008 (general data problem).  Downstream
-  processes should check this table (and return code) before continuing.
+    plus a REASON_CD column, containing only bad records.  If bad records found,
+    the SYSCC value will be set to 1008 (general data problem).  Downstream
+    processes should check this table (and return code) before continuing.
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas
   @li mf_getuniquefileref.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
-  @li mf_nobs.sas
   @li mp_filtergenerate.sas
   @li mp_filtervalidate.sas
 
@@ -85,41 +85,52 @@
   * quotes, commas, periods and spaces.
   * Only numeric values should remain
   */
-%local reason_cd;
+%local reason_cd nobs;
+%let nobs=0;
 data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
-  length reason_cd $32;
+  length reason_cd $4032;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
-    REASON_CD='GROUP_LOGIC should be either AND or OR';
+    REASON_CD='GROUP_LOGIC should be AND/OR, not:'!!cats(GROUP_LOGIC);
     putlog REASON_CD= GROUP_LOGIC=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if SUBGROUP_LOGIC not in ('AND','OR') then do;
-    REASON_CD='SUBGROUP_LOGIC should be either AND or OR';
+    REASON_CD='SUBGROUP_LOGIC should be AND/OR, not:'!!cats(SUBGROUP_LOGIC);
     putlog REASON_CD= SUBGROUP_LOGIC=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if mod(SUBGROUP_ID,1) ne 0 then do;
-    REASON_CD='SUBGROUP_ID should be integer';
+    REASON_CD='SUBGROUP_ID should be integer, not '!!left(subgroup_id);
     putlog REASON_CD= SUBGROUP_ID=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if upcase(VARIABLE_NM) not in
   (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
   then do;
-    REASON_CD="VARIABLE_NM not in &targetds";
+    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
     putlog REASON_CD= VARIABLE_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if OPERATOR_NM not in
   ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
   then do;
-    REASON_CD='Invalid OPERATOR_NM';
+    REASON_CD='Invalid OPERATOR_NM: '!!left(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
 
@@ -129,8 +140,10 @@ data &outds;
     if substr(raw_value,1,1) ne '('
     or substr(cats(reverse(raw_value)),1,1) ne ')'
     then do;
-      REASON_CD='Missing brackets in RAW_VALUE';
+      REASON_CD='Missing start/end bracket in RAW_VALUE';
       putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
       output;
     end;
     else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
@@ -151,25 +164,27 @@ data &outds;
   /* output records that contain values other than digits and spaces */
   if notdigit(compress(raw_value3,' '))>0 then do;
     putlog raw_value3= $hex32.;
-    REASON_CD='Invalid RAW_VALUE';
+    REASON_CD=cats('Invalid RAW_VALUE:',raw_value);
     putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
 
 run;
 
+
 data _null_;
-  set &outds;
-  call symputx('REASON_CD',reason_cd,'l');
-  stop;
+  set &outds end=last;
+  putlog (_all_)(=);
 run;
 
-%mp_abort(iftrue=(&abort=YES and %mf_nobs(&outds)>0),
+%mp_abort(iftrue=(&abort=YES and &nobs>0),
   mac=&sysmacroname,
-  msg=%str(Filter issues in &inds, reason: &reason_cd, details in &outds)
+  msg=%str(Data issue: %superq(reason_cd))
 )
 
-%if %mf_nobs(&outds)>0 %then %do;
+%if &nobs>0 %then %do;
   %let syscc=1008;
   %return;
 %end;

base/mp_filtergenerate.sas

@@ -99,4 +99,4 @@ filename &outref temp;
   run;
 %end;
 
-%mend;
+%mend mp_filtergenerate;

base/mp_filtervalidate.sas

@@ -33,8 +33,8 @@
   @param [in] targetds The target dataset against which to verify the query
   @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
   @param [out] outds= (work.mp_filtervalidate) Output dataset containing the
-  error / warning message, if one exists.  If this table contains any rows,
-  there are problems!
+    err / warning message, if one exists.  If this table contains any rows,
+    there are problems!
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
@@ -78,7 +78,8 @@ run;
 
 data &outds;
   if &sqlrc or &syscc or &syserr then do;
-    REASON_CD=coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
+    REASON_CD='VALIDATION_ERR'!!'OR: '!!
+      coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
     output;
   end;
   else stop;
@@ -95,10 +96,11 @@ filename &fref1 clear;
     run;
     %mp_abort(
       mac=&sysmacroname,
-      msg=%str(Filter issues in &inref: %quote(&reason_cd))
+      msg=%str(Filter validation issues. ERR=%superq(SYSERRORTEXT)
+        , WARN=%superq(SYSWARNINGTEXT) )
     )
   %end;
   %let syscc=1008;
 %end;
 
-%mend;
+%mend mp_filtervalidate;

base/mp_getcols.sas

@@ -0,0 +1,65 @@
+/**
+  @file
+  @brief Creates a dataset with column metadata.
+  @details This macro takes the `proc contents` output and "tidies it up" in the
+  following ways:
+
+    @li Blank labels are filled in with column names
+    @li Formats are reconstructed with default values
+    @li Types such as DATE / TIME / DATETIME are inferred from the formats
+
+  Example usage:
+
+      %mp_getcols(sashelp.airline,outds=work.myds)
+
+  @param ds The dataset from which to obtain column metadata
+  @param outds= (work.cols) The output dataset to create. Sample data:
+  |NAME $|LENGTH 8|VARNUM 8|LABEL $|FORMAT $49|TYPE $1 |DDTYPE $|
+  |---|---|---|---|---|---|---|
+  |AIR|8|2|international airline travel (thousands)|8.|N|NUMERIC|
+  |DATE|8|1|DATE|MONYY.|N|DATE|
+  |REGION|3|3|REGION|$3.|C|CHARACTER|
+
+  <h4> Related Macros </h4>
+  @li mf_getvarlist.sas
+  @li mm_getcols.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_getcols(ds, outds=work.cols);
+
+proc contents noprint data=&ds
+  out=_data_ (keep=name type length label varnum format:);
+run;
+data &outds(keep=name type length varnum format label ddtype);
+  set &syslast(rename=(format=format2 type=type2));
+  name=upcase(name);
+  if type2=2 then do;
+    length format $49.;
+    if format2='' then format=cats('$',length,'.');
+    else if formatl=0 then format=cats(format2,'.');
+    else format=cats(format2,formatl,'.');
+    type='C';
+    ddtype='CHARACTER';
+  end;
+  else do;
+    if format2='' then format=cats(length,'.');
+    else if formatl=0 then format=cats(format2,'.');
+    else if formatd=0 then format=cats(format2,formatl,'.');
+    else format=cats(format2,formatl,'.',formatd);
+    type='N';
+    if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
+    else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
+      or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
+      or format=:'MONYY'
+    then ddtype='DATE';
+    else if format=:'TIME' then ddtype='TIME';
+    else ddtype='NUMERIC';
+  end;
+  if label='' then label=name;
+run;
+
+%mend mp_getcols;

base/mp_getconstraints.sas

@@ -4,24 +4,27 @@
   @details Useful for capturing constraints before they are dropped / reapplied
   during an update.
 
-        proc sql;
-        create table work.example(
-          TX_FROM float format=datetime19.,
-          DD_TYPE char(16),
-          DD_SOURCE char(2048),
-          DD_SHORTDESC char(256),
-          constraint pk primary key(tx_from, dd_type,dd_source),
-          constraint unq unique(tx_from, dd_type),
-          constraint nnn not null(DD_SHORTDESC)
-        );
+      proc sql;
+      create table work.example(
+        TX_FROM float format=datetime19.,
+        DD_TYPE char(16),
+        DD_SOURCE char(2048),
+        DD_SHORTDESC char(256),
+        constraint pk primary key(tx_from, dd_type,dd_source),
+        constraint unq unique(tx_from, dd_type),
+        constraint nnn not null(DD_SHORTDESC)
+      );
 
       %mp_getconstraints(lib=work,ds=example,outds=work.constraints)
 
-  @param lib= The target library (default=WORK)
-  @param ds= The target dataset.  Leave blank (default) for all datasets.
-  @param outds the output dataset
+  @param [in] lib= (WORK) The target library
+  @param [in] ds= The target dataset.  Leave blank (default) for all datasets.
+  @param [in] mdebug= (0) Set to 1 to preserve temp tables, print var values etc
+  @param [out] outds= (mp_getconstraints) the output dataset
 
   <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mp_dropmembers.sas
 
   @version 9.2
   @author Allan Bowe
@@ -31,30 +34,63 @@
 %macro mp_getconstraints(lib=WORK
   ,ds=
   ,outds=mp_getconstraints
+  ,mdebug=0
 )/*/STORE SOURCE*/;
 
 %let lib=%upcase(&lib);
 %let ds=%upcase(&ds);
 
+/**
+  * Neither dictionary tables nor sashelp provides a constraint order column,
+  * however they DO arrive in the correct order.  So, create the col.
+  **/
+%local vw;
+%let vw=%mf_getuniquename(prefix=mp_getconstraints_vw_);
+data &vw /view=&vw;
+  set sashelp.vcncolu;
+  where table_catalog="&lib";
+
+  /* use retain approach to reset the constraint order with each constraint */
+  length tmp $1000;
+  retain tmp;
+  drop tmp;
+  if tmp ne catx('|',table_catalog,table_name,constraint_name) then do;
+    constraint_order=1;
+  end;
+  else constraint_order+1;
+  tmp=catx('|',table_catalog, table_name,constraint_name);
+run;
+
 /* must use SQL as proc datasets does not support length changes */
 proc sql noprint;
 create table &outds as
-  select a.TABLE_CATALOG as libref
-    ,a.TABLE_NAME
+  select upcase(a.TABLE_CATALOG) as libref
+    ,upcase(a.TABLE_NAME) as TABLE_NAME
     ,a.constraint_type
     ,a.constraint_name
     ,b.column_name
+    ,b.constraint_order
   from dictionary.TABLE_CONSTRAINTS a
-  left join dictionary.constraint_column_usage  b
-  on a.TABLE_CATALOG=b.TABLE_CATALOG
-    and a.TABLE_NAME=b.TABLE_NAME
+  left join &vw  b
+  on upcase(a.TABLE_CATALOG)=upcase(b.TABLE_CATALOG)
+    and upcase(a.TABLE_NAME)=upcase(b.TABLE_NAME)
     and a.constraint_name=b.constraint_name
-  where a.TABLE_CATALOG="&lib"
-    and b.TABLE_CATALOG="&lib"
+/**
+  * We cannot apply this clause to the underlying dictionary table.  See:
+  * https://communities.sas.com/t5/SAS-Programming/Unexpected-Where-Clause-behaviour-in-dictionary-TABLE/m-p/771554#M244867
+  */
+  where calculated libref="&lib"
   %if "&ds" ne "" %then %do;
-    and a.TABLE_NAME="&ds"
-    and b.TABLE_NAME="&ds"
+    and upcase(a.TABLE_NAME)="&ds"
+    and upcase(b.TABLE_NAME)="&ds"
   %end;
+  order by libref, table_name, constraint_name, constraint_order
   ;
 
-%mend;
+/* tidy up */
+%mp_dropmembers(
+  &vw,
+  iftrue=(&mdebug=0)
+)
+
+%mend mp_getconstraints;

base/mp_getdbml.sas

@@ -332,4 +332,4 @@ run;
   run;
 %end;
 
-%mend;
+%mend mp_getdbml;

base/mp_getddl.sas

@@ -5,6 +5,8 @@
     to create tables in SAS or a database.  The macro can be used at table or
     library level.  The default behaviour is to create DDL in SAS format.
 
+    Note - views are not currently supported.
+
   Usage:
 
       data test(index=(pk=(x y)/unique /nomiss));
@@ -16,12 +18,14 @@
       %mp_getddl(work,test,flavour=tsql,showlog=YES)
 
   <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mf_getvarcount.sas
   @li mp_getconstraints.sas
 
   @param lib libref of the library to create DDL for.  Should be assigned.
   @param ds dataset to create ddl for (optional)
-  @param fref= the fileref to which to write the DDL.  If not preassigned, will
-    be assigned to TEMP.
+  @param fref= the fileref to which to _append_ the DDL.  If it does not exist,
+    it will be created.
   @param flavour= The type of DDL to create (default=SAS). Supported=TSQL
   @param showlog= Set to YES to show the DDL in the log
   @param schema= Choose a preferred schema name (default is to use actual schema
@@ -37,9 +41,10 @@
 )/*/STORE SOURCE*/;
 
 /* check fileref is assigned */
-%if %sysfunc(fileref(&fref)) > 0 %then %do;
-  filename &fref temp;
+%if %mf_existfileref(&fref)=0 %then %do;
+  filename &fref temp ;
 %end;
+
 %if %length(&libref)=0 %then %let libref=WORK;
 %let flavour=%upcase(&flavour);
 
@@ -47,6 +52,7 @@ proc sql noprint;
 create table _data_ as
   select * from dictionary.tables
   where upcase(libname)="%upcase(&libref)"
+    and memtype='DATA' /* views not currently supported */
   %if %length(&ds)>0 %then %do;
     and upcase(memname)="%upcase(&ds)"
   %end;
@@ -115,10 +121,10 @@ create table _data_ as
   end;
   run;
   %put &=constraints_used;
-%mend;
+%mend addConst;
 
 data _null_;
-  file &fref;
+  file &fref mod;
   put "/* DDL generated by &sysuserid on %sysfunc(datetime(),datetime19.) */";
 run;
 
@@ -133,7 +139,7 @@ run;
     %let curds=%scan(&dsnlist,&x);
     data _null_;
       file &fref mod;
-      length nm lab $1024 typ $20;
+      length lab $1024 typ $20;
       set &colinfo (where=(upcase(memname)="&curds")) end=last;
 
       if _n_=1 then do;
@@ -141,13 +147,15 @@ run;
           put "create table &libref..&curds(";
         end;
         else do;
+          /* just a placeholder - we filter out views at the top */
           put "create view &libref..&curds(";
         end;
         put "    "@@;
       end;
       else put "   ,"@@;
       if length(format)>1 then fmt=" format="!!cats(format);
-      if length(label)>1 then lab=" label="!!quote(trim(label));
+      if length(label)>1 then
+        lab=" label="!!cats("'",tranwrd(label,"'","''"),"'");
       if notnull='yes' then notnul=' not null';
       if type='char' then typ=cats('char(',length,')');
       else if length ne 8 then typ='num length='!!left(length);
@@ -203,7 +211,7 @@ run;
   proc sql noprint;
   select sysvalue into: schemaactual
     from dictionary.libnames
-    where libname="&libref" and engine='SQLSVR';
+    where upcase(libname)="&libref" and engine='SQLSVR';
   %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
 
   %do x=1 %to %sysfunc(countw(&dsnlist));
@@ -219,6 +227,7 @@ run;
           put "create table [&schema].[&curds](";
         end;
         else do;
+          /* just a placeholder - we filter out views at the top */
           put "create view [&schema].[&curds](";
         end;
         put "    "@@;
@@ -295,78 +304,88 @@ run;
   proc sql noprint;
   select sysvalue into: schemaactual
     from dictionary.libnames
-    where libname="&libref" and engine='POSTGRES';
+    where upcase(libname)="&libref" and engine='POSTGRES';
   %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
   data _null_;
     file &fref mod;
     put "CREATE SCHEMA &schema;";
   %do x=1 %to %sysfunc(countw(&dsnlist));
     %let curds=%scan(&dsnlist,&x);
-    data _null_;
-      file &fref mod;
-      put "/* Postgres Flavour DDL for &schema..&curds */";
-    data _null_;
-      file &fref mod;
-      set &colinfo (where=(upcase(memname)="&curds")) end=last;
-      length fmt $32;
-      if _n_=1 then do;
-        if memtype='DATA' then do;
-          put "CREATE TABLE &schema..&curds (";
+    %local curdsvarcount;
+    %let curdsvarcount=%mf_getvarcount(&libref..&curds);
+    %if &curdsvarcount>1600 %then %do;
+      data _null_;
+        file &fref mod;
+        put "/* &libref..&curds contains &curdsvarcount vars */";
+        put "/* Postgres cannot create tables with over 1600 vars */";
+        put "/* No DDL will be generated for this table";
+      run;
+    %end;
+    %else %do;
+      data _null_;
+        file &fref mod;
+        put "/* Postgres Flavour DDL for &schema..&curds */";
+      data _null_;
+        file &fref mod;
+        set &colinfo (where=(upcase(memname)="&curds")) end=last;
+        length fmt $32;
+        if _n_=1 then do;
+          if memtype='DATA' then do;
+            put "CREATE TABLE &schema..&curds (";
+          end;
+          else do;
+            /* just a placeholder - we filter out views at the top */
+            put "CREATE VIEW &schema..&curds (";
+          end;
+          put "    "@@;
         end;
-        else do;
-          put "CREATE VIEW &schema..&curds (";
-        end;
-        put "    "@@;
-      end;
-      else put "   ,"@@;
-      format=upcase(format);
-      if 1=0 then; /* dummy if */
-      %if &applydttm=YES %then %do;
-        else if format=:'DATETIME' then fmt=' TIMESTAMP ';
-      %end;
-      else if type='num' then fmt=' DOUBLE PRECISION';
-      else fmt='VARCHAR('!!cats(length)!!')';
-      if notnull='yes' then notnul=' NOT NULL';
-      /* quote column names in case they represent reserved words */
-      name2=quote(trim(name));
-      put name2 fmt notnul;
-    run;
+        else put "   ,"@@;
+        format=upcase(format);
+        if 1=0 then; /* dummy if */
+        %if &applydttm=YES %then %do;
+          else if format=:'DATETIME' then fmt=' TIMESTAMP ';
+        %end;
+        else if type='num' then fmt=' DOUBLE PRECISION';
+        else fmt='VARCHAR('!!cats(length)!!')';
+        if notnull='yes' then notnul=' NOT NULL';
+        /* quote column names in case they represent reserved words */
+        name2=quote(trim(name));
+        put name2 fmt notnul;
+      run;
 
-    /* Extra step for data constraints */
-    %addConst()
+      /* Extra step for data constraints */
+      %addConst()
 
-    data _null_;
-      file &fref mod;
-      put ');';
-    run;
+      data _null_;
+        file &fref mod;
+        put ');';
+      run;
 
-    /* Create Unique Indexes, but only if they were not already defined within
-      the Constraints section. */
-    data _null_;
-      *length ds $128;
-      set &idxinfo(
-        where=(
-          memname="&curds"
-          and unique='yes'
-          and indxname not in (
-            %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+      /* Create Unique Indexes, but only if they were not already defined within
+        the Constraints section. */
+      data _null_;
+        *length ds $128;
+        set &idxinfo(
+          where=(
+            memname="&curds"
+            and unique='yes'
+            and indxname not in (
+              %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+            )
           )
-        )
-      );
-      file &fref mod;
-      by idxusage indxname;
-/*       ds=cats(libname,'.',memname); */
-      if first.indxname then do;
+        );
+        file &fref mod;
+        by idxusage indxname;
+        if first.indxname then do;
           put 'CREATE UNIQUE INDEX "' indxname +(-1) '" ' "ON &schema..&curds(";
           put '  "' name +(-1) '"' ;
-      end;
-      else put '  ,"' name +(-1) '"';
-      *else put '    ,' name ;
-      if last.indxname then do;
-        put ');';
-      end;
-    run;
-
+        end;
+        else put '  ,"' name +(-1) '"';
+        if last.indxname then do;
+          put ');';
+        end;
+      run;
+    %end;
   %end;
 %end;
 %if %upcase(&showlog)=YES %then %do;
@@ -378,4 +397,4 @@ run;
   run;
 %end;
 
-%mend;
+%mend mp_getddl;

base/mp_getformats.sas

@@ -0,0 +1,142 @@
+/**
+  @file
+  @brief Export format definitions
+  @details Formats are exported from the first (if any) catalog entry in the
+  FMTSEARCH path.
+
+  Formats are taken from the library / dataset reference and / or a static
+  format list.
+
+  Example usage:
+
+      %mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable)
+
+  @param [in] lib= (0) The libref for which to return formats.
+  @todo Enable exporting of formats for an entire library
+  @param [in] ds= (0) The dataset from which to obtain format definitions
+  @param [in] fmtlist= (0) A list of additional format names
+  @param [out] outsummary= (work.mp_getformats_summary) Output dataset
+    containing summary definitions - structure taken from dictionary.formats as
+    follows:
+
+  |libname:$8.|memname:$32.|path:$1024.|objname:$32.|fmtname:$32.|fmttype:$1.|source:$1.|minw:best.|mind:best.|maxw:best.|maxd:best.|defw:best.|defd:best.|
+  |---|---|---|---|---|---|---|---|---|---|---|---|---|
+  | | | | |$|F|B|1|0|32767|0|1|0|
+  | | | | |$|I|B|1|0|32767|0|1|0|
+  |` `|` `|/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWIANYDT|$ANYDTIF|I|U|1|0|60|0|19|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWFASCII|$ASCII|F|U|1|0|32767|0|1|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWIASCII|$ASCII|I|U|1|0|32767|0|1|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWFBASE6|$BASE64X|F|U|1|0|32767|0|1|0|
+
+
+  @param [out] outdetail= (0) Provide an output dataset in which to export all
+    the custom format definitions (from proc format CNTLOUT).  Definitions:
+https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#a002473477.htm
+    Sample data:
+
+  |FMTNAME:$32.|START:$16.|END:$16.|LABEL:$256.|MIN:best.|MAX:best.|DEFAULT:best.|LENGTH:best.|FUZZ:best.|PREFIX:$2.|MULT:best.|FILL:$1.|NOEDIT:best.|TYPE:$1.|SEXCL:$1.|EEXCL:$1.|HLO:$13.|DECSEP:$1.|DIG3SEP:$1.|DATATYPE:$8.|LANGUAGE:$8.|
+  |---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
+  |`WHICHPATH `|`0 `|`0 `|`path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|` `|` `|` `|` `|` `|
+  |`WHICHPATH `|`**OTHER** `|`**OTHER** `|`big fat problem if not path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|`O `|` `|` `|` `|` `|
+
+  <h4> SAS Macros </h4>
+  @li mf_dedup.sas
+  @li mf_getfmtlist.sas
+  @li mf_getfmtname.sas
+  @li mf_getquotedstr.sas
+  @li mf_getuniquename.sas
+
+
+  <h4> Related Macros </h4>
+  @li mp_applyformats.sas
+  @li mp_getformats.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_getformats(lib=0
+  ,ds=0
+  ,fmtlist=0
+  ,outsummary=work.mp_getformats_summary
+  ,outdetail=0
+);
+
+%local i fmt allfmts tempds fmtcnt;
+
+%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( )));
+  /* ensure format list contains format _name_ only */
+  %let fmt=%scan(&fmtlist,&i,%str( ));
+  %let fmt=%mf_getfmtname(&fmt);
+  %let allfmts=&allfmts &fmt;
+%end;
+
+%if &ds=0 and &lib ne 0 %then %do;
+  /* grab formats from library */
+  /* to do */
+%end;
+%else %if &ds ne 0 and &lib ne 0 %then %do;
+  /* grab formats from dataset */
+  %let allfmts=%mf_getfmtlist(&lib..&ds) &allfmts;
+%end;
+
+/* ensure list is unique */
+%let allfmts=%mf_dedup(%upcase(&allfmts));
+
+/* create summary table */
+%if %index(&outsummary,.)=0 %then %let outsummary=WORK.&outsummary;
+proc sql;
+create table &outsummary as
+  select * from dictionary.formats
+  where fmtname in (%mf_getquotedstr(&allfmts,quote=D))
+    and fmttype='F';
+
+%if "&outdetail" ne "0" %then %do;
+  /* ensure base table always exists */
+  proc sql;
+  create table &outdetail(
+      FMTNAME char(32)     label='Format name'
+      ,START char(16)     label='Starting value for format'
+      ,END char(16)     label='Ending value for format'
+      ,LABEL char(256)     label='Format value label'
+      ,MIN num length=3     label='Minimum length'
+      ,MAX num length=3     label='Maximum length'
+      ,DEFAULT num length=3     label='Default length'
+      ,LENGTH num length=3     label='Format length'
+      ,FUZZ num     label='Fuzz value'
+      ,PREFIX char(2)     label='Prefix characters'
+      ,MULT num     label='Multiplier'
+      ,FILL char(1)     label='Fill character'
+      ,NOEDIT num length=3     label='Is picture string noedit?'
+      ,TYPE char(1)     label='Type of format'
+      ,SEXCL char(1)     label='Start exclusion'
+      ,EEXCL char(1)     label='End exclusion'
+      ,HLO char(13)     label='Additional information'
+      ,DECSEP char(1)     label='Decimal separator'
+      ,DIG3SEP char(1)     label='Three-digit separator'
+      ,DATATYPE char(8)     label='Date/time/datetime?'
+      ,LANGUAGE char(8)     label='Language for date strings'
+  );
+  /* grab the location of each format */
+  %let fmtcnt=0;
+  data _null_;
+    set &outsummary;
+    if not missing(libname);
+    x+1;
+    call symputx(cats('fmtloc',x),cats(libname,'.',memname),'l');
+    call symputx(cats('fmtname',x),fmtname,'l');
+    call symputx('fmtcnt',x,'l');
+  run;
+  /* export each format and append to the output table */
+  %let tempds=%mf_getuniquename(prefix=mp_getformats);
+  %do i=1 %to &fmtcnt;
+    proc format library=&&fmtloc&i CNTLOUT=&tempds;
+      select &&fmtname&i;
+    run;
+    proc append base=&outdetail data=&tempds;
+    run;
+  %end;
+%end;
+
+%mend mp_getformats;

base/mp_getmaxvarlengths.sas

@@ -70,4 +70,4 @@ create table &outds (rename=(
     out=&outds(rename=(_name_=NAME COL1=MAXLEN));
   run;
 
-%mend;
+%mend mp_getmaxvarlengths;

base/mp_getpk.sas

@@ -0,0 +1,259 @@
+/**
+  @file
+  @brief Extract the primary key fields from a table or library
+  @details Examines the constraints to identify primary key fields - indicated
+  by an explicit PK constraint, or a unique index that is also NOT NULL.
+
+  Can be executed at both table and library level.  Supports both BASE engine
+  libraries and SQL Server.
+
+  Usage:
+
+      proc sql;
+      create table work.example(
+        TX_FROM float format=datetime19.,
+        DD_TYPE char(16),
+        DD_SOURCE char(2048),
+        DD_SHORTDESC char(256),
+        constraint pk primary key(tx_from, dd_type,dd_source),
+        constraint unq unique(tx_from, dd_type),
+        constraint nnn not null(DD_SHORTDESC)
+      );
+      %mp_getpk(work,ds=example)
+
+  Returns:
+
+|libref:$8.|dsn:$32.|memtype:$8.|dbms_memtype:$32.|typemem:$8.|memlabel:$256.|nvar:best.|compress:$8.|pk_fields:$512.|
+|---|---|---|---|---|---|---|---|---|
+|WORK|EXAMPLE|DATA| |DATA| |4|NO|TX_FROM DD_TYPE DD_SOURCE|
+
+
+  @param [in] lib The libref to examine
+  @param [in] ds= (0) Select the dataset to examine, else use 0 for all tables
+  @param [in] mdebug= (0) Set to 1 to preserve temp tables, print var values etc
+  @param [out] outds= (work.mp_getpk) The name of the output table to create.
+
+  <h4> SAS Macros </h4>
+  @li mf_getengine.sas
+  @li mf_getschema.sas
+  @li mp_dropmembers.sas
+  @li mp_getconstraints.sas
+
+  <h4> Related Macros </h4>
+  @li mp_getpk.test.sas
+  @li mp_guesspk.sas
+
+  @version 9.3
+  @author Macro People Ltd
+**/
+
+%macro mp_getpk(
+  lib,
+  ds=0,
+  outds=work.mp_getpk,
+  mdebug=0
+)/*/STORE SOURCE*/;
+
+
+%local engine schema ds1 ds2 ds3 dsn tabs1 tabs2 sum pk4sure pkdefault finalpks;
+
+%let lib=%upcase(&lib);
+%let ds=%upcase(&ds);
+%let engine=%mf_getengine(&lib);
+%let schema=%mf_getschema(&lib);
+
+%let ds1=%mf_getuniquename(prefix=getpk_ds1);
+%let ds2=%mf_getuniquename(prefix=getpk_ds2);
+%let ds3=%mf_getuniquename(prefix=getpk_ds3);
+%let tabs1=%mf_getuniquename(prefix=getpk_tabs1);
+%let tabs2=%mf_getuniquename(prefix=getpk_tabs2);
+%let sum=%mf_getuniquename(prefix=getpk_sum);
+%let pk4sure=%mf_getuniquename(prefix=getpk_pk4sure);
+%let pkdefault=%mf_getuniquename(prefix=getpk_pkdefault);
+%let finalpks=%mf_getuniquename(prefix=getpk_finalpks);
+
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+proc sql;
+create table &ds1 as
+  select  libname as libref
+    ,upcase(memname) as dsn
+    ,memtype
+    ,upcase(name) as name
+    ,type
+    ,length
+    ,varnum
+    ,label
+    ,format
+    ,idxusage
+    ,notnull
+  from dictionary.columns
+  where upcase(libname)="&lib"
+%if &ds ne 0 %then %do;
+    and upcase(memname)="&ds"
+%end;
+  ;
+
+
+%if &engine=SQLSVR %then %do;
+  proc sql;
+  connect using &lib;
+  create table work.&ds2 as
+  select * from connection to &lib(
+  select
+      s.name as SchemaName,
+      t.name as memname,
+      tc.name as name,
+      ic.key_ordinal as KeyOrderNr
+  from
+      sys.schemas s
+      inner join sys.tables t   on s.schema_id=t.schema_id
+      inner join sys.indexes i  on t.object_id=i.object_id
+      inner join sys.index_columns ic on i.object_id=ic.object_id
+                                    and i.index_id=ic.index_id
+      inner join sys.columns tc on ic.object_id=tc.object_id
+                              and ic.column_id=tc.column_id
+  where i.is_primary_key=1
+    and s.name=%str(%')&schema%str(%')
+  order by t.name, ic.key_ordinal ;
+  );disconnect from &lib;
+  create table &ds3 as
+    select a.*
+      ,case when b.name is not null then 1 else 0 end as pk_ind
+    from work.&ds1 a
+    left join work.&ds2 b
+    on a.dsn=b.memname
+      and upcase(a.name)=upcase(b.name)
+    order by libref,dsn;
+%end;
+%else %do;
+
+  %if &ds = 0 %then %let dsn=;
+
+  /* get all constraints, in constraint order*/
+  %mp_getconstraints(lib=&lib,ds=&dsn,outds=work.&ds2)
+
+  /* extract cols that are clearly primary keys */
+  proc sql;
+  create table &pk4sure as
+    select libref
+      ,table_name
+      ,constraint_name
+      ,constraint_order
+      ,column_name as name
+    from work.&ds2
+    where constraint_type='PRIMARY'
+    order by 1,2,3,4;
+
+  /* extract unique constraints where every col is also NOT NULL */
+  proc sql;
+  create table &sum as
+    select a.libref
+      ,a.table_name
+      ,a.constraint_name
+      ,count(a.column_name) as unq_cnt
+      ,count(b.column_name) as nul_cnt
+    from work.&ds2(where=(constraint_type ='UNIQUE')) a
+    left join work.&ds2(where=(constraint_type ='NOT NULL')) b
+    on a.libref=b.libref
+      and a.table_name=b.table_name
+      and a.column_name=b.column_name
+    group by 1,2,3
+    having unq_cnt=nul_cnt;
+
+  /* extract cols from the relevant unique constraints */
+  create table &pkdefault as
+    select a.libref
+      ,a.table_name
+      ,a.constraint_name
+      ,b.constraint_order
+      ,b.column_name as name
+    from &sum a
+    left join &ds2(where=(constraint_type ='UNIQUE')) b
+    on a.libref=b.libref
+      and a.table_name=b.table_name
+      and a.constraint_name=b.constraint_name
+    order by 1,2,3,4;
+
+  /* create one table */
+  data &finalpks;
+    set &pkdefault &pk4sure ;
+    pk_ind=1;
+    /* if there are multiple unique constraints, take the first */
+    by libref table_name constraint_name;
+    retain keepme;
+    if first.table_name then keepme=1;
+    if first.constraint_name and not first.table_name then keepme=0;
+    if keepme=1;
+  run;
+
+  /* join back to starting table */
+  proc sql;
+  create table &ds3 as
+    select a.*
+      ,b.constraint_order
+      ,case when b.pk_ind=1 then 1 else 0 end as pk_ind
+    from work.&ds1 a
+    left join work.&finalpks b
+    on a.libref=b.libref
+      and a.dsn=b.table_name
+      and upcase(a.name)=upcase(b.name)
+    order by libref,dsn,constraint_order;
+%end;
+
+
+/* prepare tables */
+proc sql;
+create table work.&tabs1 as select
+  libname as libref
+  ,upcase(memname) as dsn
+  ,memtype
+  ,dbms_memtype
+  ,typemem
+  ,memlabel
+  ,nvar
+  ,compress
+from dictionary.tables
+  where upcase(libname)="&lib"
+%if &ds ne 0 %then %do;
+    and upcase(memname)="&ds"
+%end;
+  ;
+data &tabs2;
+  set &ds3;
+  length pk_fields $512;
+  retain pk_fields;
+  by libref dsn constraint_order;
+  if first.dsn then pk_fields='';
+  if pk_ind=1 then pk_fields=catx(' ',pk_fields,name);
+  if last.dsn then output;
+run;
+
+proc sql;
+create table &outds as
+  select a.libref
+    ,a.dsn
+    ,a.memtype
+    ,a.dbms_memtype
+    ,a.typemem
+    ,a.memlabel
+    ,a.nvar
+    ,a.compress
+    ,b.pk_fields
+  from work.&tabs1 a
+  left join work.&tabs2 b
+  on a.libref=b.libref
+    and a.dsn=b.dsn;
+
+/* tidy up */
+%mp_dropmembers(
+  &ds1 &ds2 &ds3 &dsn &tabs1 &tabs2 &sum &pk4sure &pkdefault &finalpks,
+  iftrue=(&mdebug=0)
+)
+
+%mend mp_getpk;

base/mp_gsubfile.sas

@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Performs a text substitution on a file
+  @details Makes use of the GSUB function in LUA to perform a text substitution
+  in a file - either in-place, or writing to a new location.  The benefit of
+  using LUA is that the entire file can be loaded into a single variable,
+  thereby side stepping the 32767 character limit in a data step.
+
+  Usage:
+
+      %let file=%sysfunc(pathname(work))/file.txt;
+      %let str=replace/me;
+      %let rep=with/this;
+      data _null_;
+        file "&file";
+        put "&str";
+      run;
+      %mp_gsubfile(file=&file, patternvar=str, replacevar=rep)
+      data _null_;
+        infile "&file";
+        input;
+        list;
+      run;
+
+  @param file= (0) The file to perform the substitution on
+  @param patternvar= A macro variable containing the Lua
+    [pattern](https://www.lua.org/pil/20.2.html) to search for.  Due to the use
+    of special (magic) characters in Lua patterns, it is safer to pass the NAME
+    of the macro variable containing the string, rather than the value itself.
+  @param replacevar= The name of the macro variable containing the replacement
+    _string_.
+  @param outfile= (0) The file to write the output to. If zero, then the file
+    is overwritten in-place.
+
+  <h4> SAS Macros </h4>
+  @li ml_gsubfile.sas
+
+  <h4> Related Macros </h4>
+  @li mp_gsubfile.test.sas
+
+  @version 9.4
+  @author Allan Bowe
+**/
+
+%macro mp_gsubfile(file=0,
+  patternvar=,
+  replacevar=,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+  %ml_gsubfile()
+
+%mend mp_gsubfile;

base/mp_guesspk.sas

@@ -1,32 +1,37 @@
 /**
-  @file mp_guesspk.sas
+  @file
   @brief Guess the primary key of a table
-  @details Tries to guess the primary key of a table based on the following logic:
+  @details Tries to guess the primary key of a table based on the following
+  logic:
 
       * Columns with nulls are ignored
       * Return only column combinations that provide unique results
-      * Start from one column, then move out to include composite keys of 2 to 6 columns
+      * Start from one column, then move out to composite keys of 2 to 6 columns
 
   The library of the target should be assigned before using this macro.
 
   Usage:
 
-      filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      filename mc url
+        "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
       %mp_guesspk(sashelp.class,outds=classpks)
 
   @param baseds The dataset to analyse
   @param outds= The output dataset to contain the possible PKs
-  @param max_guesses= The total number of possible primary keys to generate.  A
-    table is likely to have multiple unlikely PKs, so no need to list them all. Default=3.
-  @param min_rows= The minimum number of rows a table should have in order to try
-    and guess the PK.  Default=5.
+  @param max_guesses= (3) The total number of possible primary keys to generate.
+    A table may have multiple unlikely PKs, so no need to list them all.
+  @param min_rows= (5) The minimum number of rows a table should have in order
+    to try and guess the PK.
 
   <h4> SAS Macros </h4>
   @li mf_getvarlist.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
 
+  <h4> Related Macros </h4>
+  @li mp_getpk.sas
+
   @version 9.3
   @author Allan Bowe
 
@@ -196,7 +201,8 @@
           %let lev4=%scan(&posspks,&l);
           %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then %do;
             /* check for four level uniqueness */
-            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4) out=&tmpds noduprec;
+            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4)
+                out=&tmpds noduprec;
               by _all_;
             run;
             %if %mf_nobs(&tmpds)=&rows %then %do;
@@ -233,7 +239,8 @@
             %let lev5=%scan(&posspks,&m);
             %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then %do;
               /* check for four level uniqueness */
-              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5) out=&tmpds noduprec;
+              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5)
+                  out=&tmpds noduprec;
                 by _all_;
               run;
               %if %mf_nobs(&tmpds)=&rows %then %do;
@@ -282,7 +289,8 @@
                 run;
                 %if %mf_nobs(&tmpds)=&rows %then %do;
                   proc sql;
-                  insert into &outds values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
+                  insert into &outds
+                    values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
                   %if %mf_nobs(&outds) ge &max_guesses %then %do;
                     %put &sysmacroname: Max PKs reached at Level 6 for &baseds;
                     %return;
@@ -301,4 +309,4 @@
     %return;
   %end;
 
-%mend;
+%mend mp_guesspk;

base/mp_hashdataset.sas

@@ -20,6 +20,8 @@
   @li mf_getvartype.sas
 
   @param [in] libds dataset to hash
+  @param [in] salt= Provide a salt (could be, for instance, the dataset name)
+  @param [in] iftrue= A condition under which the macro should be executed.
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
   will contain one column (hashkey) with one observation (a hex32.
   representation of the input hash)
@@ -33,10 +35,15 @@
 
 %macro mp_hashdataset(
   libds,
-  outds=
+  outds=,
+  salt=,
+  iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
+
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
   %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
-    %put %str(WARN)ING: Dataset &libds is empty;, or is not a dataset;
+    %put %str(WARN)ING: Dataset &libds is empty, or is not a dataset;
   %end;
   %else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
     %put %str(ERR)OR: Dataset &libds is not a dataset;
@@ -53,7 +60,7 @@
     %let varlist=%mf_getvarlist(&libds);
     data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
       length &prevkeyvar &keyvar $32;
-      retain &prevkeyvar;
+      retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
       set &libds end=&lastvar;
       /* hash should include previous row */
       &keyvar=put(md5(&prevkeyvar
@@ -72,4 +79,4 @@
       if &lastvar then output;
     run;
   %end;
-%mend;
+%mend mp_hashdataset;

base/mp_include.sas

@@ -0,0 +1,105 @@
+/**
+  @file
+  @brief Performs a wrapped \%include
+  @details This macro wrapper is necessary if you need your included code to
+  know that it is being \%included.
+
+  If you are using %include in a regular program, you could make use of the
+  following macro variables:
+
+  @li SYSINCLUDEFILEDEVICE
+  @li SYSINCLUDEFILEDIR
+  @li SYSINCLUDEFILEFILEREF
+  @li SYSINCLUDEFILENAME
+
+  However these variables are NOT available inside a macro, as documented here:
+https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/mcrolref/n1j5tcc0n2xczyn1kg1o0606gsv9.htm
+
+  This macro can be used in place of the %include statement, and will insert
+  the following (equivalent) global variables:
+
+  @li _SYSINCLUDEFILEDEVICE
+  @li _SYSINCLUDEFILEDIR
+  @li _SYSINCLUDEFILEFILEREF
+  @li _SYSINCLUDEFILENAME
+
+  These can be used whenever testing _within a macro_.  Outside of the macro,
+  the regular automatic variables will still be available (thanks to a
+  concatenated file list in the include statement).
+
+  Example usage:
+
+      filename example temp;
+      data _null_;
+        file example;
+        put '%macro test();';
+        put '%put &=_SYSINCLUDEFILEFILEREF;';
+        put '%put &=SYSINCLUDEFILEFILEREF;';
+        put '%mend; %test()';
+        put '%put &=SYSINCLUDEFILEFILEREF;';
+      run;
+      %mp_include(example)
+
+  @param [in] fileref The fileref of the file to be included. Must be provided.
+  @param [in] prefix= (_) The prefix to apply to the global variables.
+  @param [in] opts= (SOURCE2) The options to apply to the %inc statement
+  @param [in] errds= (work.mp_abort_errds) There is no clean way to end a
+    process within a %include called within a macro.  Furthermore, there is no
+    way to test if a macro is called within a %include.  To handle this
+    particular scenario, the %mp_abort() macro will test for the existence of
+    the `_SYSINCLUDEFILEDEVICE` variable and return the outputs (msg,mac) inside
+    this dataset.
+    It will then run an abort cancel FILE to stop the include running, and pass
+    the dataset back.
+    NOTE - it is NOT possible to read this dataset as part of _this_ macro -
+    when running abort cancel FILE, ALL macros are closed, so instead it is
+    necessary to invoke "%mp_abort(mode=INCLUDE)" OUTSIDE of any macro wrappers.
+
+
+  @version 9.4
+  @author Allan Bowe
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_abort.sas
+
+**/
+
+%macro mp_include(fileref
+  ,prefix=_
+  ,opts=SOURCE2
+  ,errds=work.mp_abort_errds
+)/*/STORE SOURCE*/;
+
+/* prepare precode */
+%local tempref;
+%let tempref=%mf_getuniquefileref();
+data _null_;
+  file &tempref;
+  set sashelp.vextfl(where=(fileref="%upcase(&fileref)"));
+  put '%let _SYSINCLUDEFILEDEVICE=' xengine ';';
+  name=scan(xpath,-1,'/\');
+  put '%let _SYSINCLUDEFILENAME=' name ';';
+  path=subpad(xpath,1,length(xpath)-length(name)-1);
+  put '%let _SYSINCLUDEFILEDIR=' path ';';
+  put '%let _SYSINCLUDEFILEFILEREF=' "&fileref;";
+run;
+
+/* prepare the errds */
+data &errds;
+  length msg mac $1000;
+  call missing(msg,mac);
+  iftrue='1=0';
+run;
+
+/* include the include */
+%inc &tempref &fileref/&opts;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=%str(&_SYSINCLUDEFILEDIR/&_SYSINCLUDEFILENAME)
+  ,msg=%str(syscc=&syscc after executing &_SYSINCLUDEFILENAME)
+)
+
+filename &tempref clear;
+
+%mend mp_include;

base/mp_init.sas

@@ -0,0 +1,68 @@
+/**
+  @file
+  @brief Initialise session with useful settings and variables
+  @details Implements a "strict" set of SAS options for use in defensive
+    programming.  Highly recommended, if you want your code to run on some
+    other machine.
+
+    This macro is recommended to be compiled and invoked in the `initProgram`
+    for SASjs [Jobs](https://cli.sasjs.io/sasjsconfig.html#jobConfig_initProgram
+    ), [Services](
+    https://cli.sasjs.io/sasjsconfig.html#serviceConfig_initProgram) and [Tests]
+    (https://cli.sasjs.io/sasjsconfig.html#testConfig_initProgram).
+
+    For non SASjs projects, you could invoke in the autoexec, or in your own
+    solution initialisation macro.
+
+
+    If you have a good idea for another useful option, setting, or global
+    variable - feel free to [raise an issue](
+    https://github.com/sasjs/core/issues/new)!
+
+    All global variables are prefixed with "SASJS" (unless modified with the
+    prefix parameter).
+
+  @param [in] prefix= (SASJS) The prefix to apply to the global macro variables
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_init(prefix=SASJS
+)/*/STORE SOURCE*/;
+
+  %global
+    &prefix._INIT_NUM   /* initialisation time as numeric                   */
+    &prefix._INIT_DTTM  /* initialisation time in E8601DT26.6 format        */
+    &prefix.WORK        /* avoid typing %sysfunc(pathname(work)) every time */
+  ;
+  %if %eval(&&&prefix._INIT_NUM>0) %then %return;  /* only run once */
+
+  data _null_;
+    dttm=datetime();
+    call symputx("&prefix._init_num",dttm,'g');
+    call symputx("&prefix._init_dttm",put(dttm,E8601DT26.6),'g');
+    call symputx("&prefix.work",pathname('WORK'),'g');
+  run;
+
+  options
+    noautocorrect           /* disallow misspelled procedure names            */
+    compress=CHAR           /* default is none so ensure we have something!   */
+    datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
+    %str(err)orcheck=STRICT /* catch errs in libname/filename statements      */
+    fmterr                  /* ensure err when a format cannot be found       */
+    mergenoby=%str(ERR)OR   /* throw err when a merge has no BY variables     */
+    missing=.               /* changing this can cause hard to detect errs    */
+    noquotelenmax           /* avoid warnings for long strings                */
+    noreplace               /* avoid overwriting permanent datasets           */
+    ps=max                  /* reduce log size slightly                       */
+    ls=max                  /* reduce log even more and avoid word truncation */
+    validmemname=COMPATIBLE /* avoid special characters etc in table names    */
+    validvarname=V7         /* avoid special characters etc in variable names */
+    varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
+    varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
+  ;
+
+%mend mp_init;

base/mp_jsonout.sas

@@ -4,7 +4,7 @@
   @details PROC JSON is faster but will produce errs like the ones below if
   special chars are encountered.
 
-  > ERROR: Some code points did not transcode.
+  > (ERR)OR: Some code points did not transcode.
 
   > An object or array close is not valid at this point in the JSON text.
 
@@ -48,6 +48,8 @@
   @param dbg= DEPRECATED - was used to conditionally add PRETTY to
     proc json but this can cause line truncation in large files.
 
+  <h4> Related Macros <h4>
+  @li mp_ds2fmtds.sas
 
   @version 9.2
   @author Allan Bowe
@@ -55,17 +57,18 @@
 
 **/
 
-%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=PROCJSON,dbg=0
+%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0
 )/*/STORE SOURCE*/;
 %put output location=&jref;
 %if &action=OPEN %then %do;
-  data _null_;file &jref encoding='utf-8';
-    put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
+  options nobomfile;
+  data _null_;file &jref encoding='utf-8' ;
+    put '{"PROCESSED_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '"';
   run;
 %end;
 %else %if (&action=ARR or &action=OBJ) %then %do;
   options validvarname=upcase;
-  data _null_;file &jref mod encoding='utf-8';
+  data _null_;file &jref mod encoding='utf-8' ;
     put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":";
 
   %if &engine=PROCJSON %then %do;
@@ -86,7 +89,65 @@
       %put &sysmacroname:  &ds NOT FOUND!!!;
       %return;
     %end;
-    data _null_;file &jref mod ;
+    %if &fmt=Y %then %do;
+      %put converting every variable to a formatted variable;
+      /* see mp_ds2fmtds.sas for source */
+      proc contents noprint data=&ds
+        out=_data_(keep=name type length format formatl formatd varnum);
+      run;
+      proc sort;
+        by varnum;
+      run;
+      %local fmtds;
+      %let fmtds=%scan(&syslast,2,.);
+      /* prepare formats and varnames */
+      data _null_;
+        if _n_=1 then call symputx('nobs',nobs,'l');
+        set &fmtds end=last nobs=nobs;
+        name=upcase(name);
+        /* fix formats */
+        if type=2 or type=6 then do;
+          length fmt $49.;
+          if format='' then fmt=cats('$',length,'.');
+          else if formatl=0 then fmt=cats(format,'.');
+          else fmt=cats(format,formatl,'.');
+          newlen=max(formatl,length);
+        end;
+        else do;
+          if format='' then fmt='best.';
+          else if formatl=0 then fmt=cats(format,'.');
+          else if formatd=0 then fmt=cats(format,formatl,'.');
+          else fmt=cats(format,formatl,'.',formatd);
+          /* needs to be wide, for datetimes etc */
+          newlen=max(length,formatl,24);
+        end;
+        /* 32 char unique name */
+        newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
+
+        call symputx(cats('name',_n_),name,'l');
+        call symputx(cats('newname',_n_),newname,'l');
+        call symputx(cats('len',_n_),newlen,'l');
+        call symputx(cats('fmt',_n_),fmt,'l');
+        call symputx(cats('type',_n_),type,'l');
+      run;
+      data &fmtds;
+        /* rename on entry */
+        set &ds(rename=(
+      %local i;
+      %do i=1 %to &nobs;
+        &&name&i=&&newname&i
+      %end;
+        ));
+      %do i=1 %to &nobs;
+        length &&name&i $&&len&i;
+        &&name&i=left(put(&&newname&i,&&fmt&i));
+        drop &&newname&i;
+      %end;
+        if _error_ then call symputx('syscc',1012);
+      run;
+      %let ds=&fmtds;
+    %end; /* &fmt=Y */
+    data _null_;file &jref mod encoding='utf-8' ;
       put "["; call symputx('cols',0,'l');
     proc sort
       data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
@@ -131,7 +192,7 @@
     /* write to temp loc to avoid _webout truncation
       - https://support.sas.com/kb/49/325.html */
     filename _sjs temp lrecl=131068 encoding='utf-8';
-    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod;
+    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod ;
       set &tempds;
       if _n_>1 then put "," @; put
       %if &action=ARR %then "[" ; %else "{" ;
@@ -158,15 +219,15 @@
       rc = fclose(fileid);
     run;
     filename _sjs clear;
-    data _null_; file &jref mod encoding='utf-8';
+    data _null_; file &jref mod encoding='utf-8' ;
       put "]";
     run;
   %end;
 %end;
 
 %else %if &action=CLOSE %then %do;
-  data _null_;file &jref encoding='utf-8' mod;
+  data _null_;file &jref encoding='utf-8' mod ;
     put "}";
   run;
 %end;
-%mend;
+%mend mp_jsonout;

base/mp_lib2cards.sas

@@ -75,4 +75,4 @@ select distinct lowcase(memname)
   )
 %end;
 
-%mend;
+%mend mp_lib2cards;