feat: new mfv_existsashdat() macro for checking whether a dataset exists in persistent storage

fix: ignoring empty files in mp_hashdirectory. Closes #314

.all-contributorsrc

@@ -0,0 +1,142 @@
+{
+  "projectName": "core",
+  "projectOwner": "sasjs",
+  "repoType": "github",
+  "repoHost": "https://github.com",
+  "files": [
+    "README.md"
+  ],
+  "imageSize": 100,
+  "commit": false,
+  "commitConvention": "angular",
+  "contributors": [
+    {
+      "login": "allanbowe",
+      "name": "Allan Bowe",
+      "avatar_url": "https://avatars.githubusercontent.com/u/4420615?v=4",
+      "profile": "https://github.com/allanbowe",
+      "contributions": [
+        "business",
+        "code",
+        "content",
+        "doc",
+        "infra",
+        "maintenance",
+        "mentoring",
+        "question",
+        "review",
+        "test"
+      ]
+    },
+    {
+      "login": "rafgag",
+      "name": "rafgag",
+      "avatar_url": "https://avatars.githubusercontent.com/u/69139928?v=4",
+      "profile": "https://github.com/rafgag",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "tmoody",
+      "name": "Trevor Moody",
+      "avatar_url": "https://avatars.githubusercontent.com/u/79837106?v=4",
+      "profile": "https://github.com/tmoody",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "krishna-acondy",
+      "name": "Krishna Acondy",
+      "avatar_url": "https://avatars.githubusercontent.com/u/2980428?v=4",
+      "profile": "https://krishna-acondy.io/",
+      "contributions": [
+        "code",
+        "infra",
+        "blog",
+        "content",
+        "ideas",
+        "video"
+      ]
+    },
+    {
+      "login": "saadjutt01",
+      "name": "Muhammad Saad ",
+      "avatar_url": "https://avatars.githubusercontent.com/u/8914650?v=4",
+      "profile": "https://github.com/saadjutt01",
+      "contributions": [
+        "code",
+        "ideas"
+      ]
+    },
+    {
+      "login": "YuryShkoda",
+      "name": "Yury Shkoda",
+      "avatar_url": "https://avatars.githubusercontent.com/u/25773492?v=4",
+      "profile": "https://www.erudicat.com/",
+      "contributions": [
+        "code",
+        "infra",
+        "video"
+      ]
+    },
+    {
+      "login": "medjedovicm",
+      "name": "Mihajlo Medjedovic",
+      "avatar_url": "https://avatars.githubusercontent.com/u/18329105?v=4",
+      "profile": "https://github.com/medjedovicm",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "kkchandok",
+      "name": "kkchandok",
+      "avatar_url": "https://avatars.githubusercontent.com/u/46090627?v=4",
+      "profile": "https://github.com/kkchandok",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "VladislavParhomchik",
+      "name": "Vladislav Parhomchik",
+      "avatar_url": "https://avatars.githubusercontent.com/u/83717836?v=4",
+      "profile": "https://github.com/VladislavParhomchik",
+      "contributions": [
+        "test",
+        "review"
+      ]
+    },
+    {
+      "login": "vznesh",
+      "name": "Vignesh T.",
+      "avatar_url": "https://avatars.githubusercontent.com/u/28916792?v=4",
+      "profile": "https://github.com/vznesh",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "yabwon",
+      "name": "Bart Jablonski",
+      "avatar_url": "https://avatars.githubusercontent.com/u/9314894?v=4",
+      "profile": "https://github.com/yabwon",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "eltociear",
+      "name": "Ikko Ashimine",
+      "avatar_url": "https://avatars.githubusercontent.com/u/22633385?v=4",
+      "profile": "https://bandism.net/",
+      "contributions": [
+        "code"
+      ]
+    }
+  ],
+  "contributorsPerLine": 7,
+  "skipCi": true
+}

.devcontainer/Dockerfile

@@ -0,0 +1,8 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/javascript-node/.devcontainer/base.Dockerfile
+
+# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
+ARG VARIANT="18-bullseye"
+FROM mcr.microsoft.com/vscode/devcontainers/javascript-node:0-${VARIANT}
+
+RUN apt-get update \
+    && apt-get install -y doxygen

.devcontainer/devcontainer.json

@@ -0,0 +1,26 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/typescript-node
+{
+	"name": "Node.js & TypeScript",
+	"build": {
+		"dockerfile": "Dockerfile",
+		// Update 'VARIANT' to pick a Node version: 18, 16, 14.
+		// Append -bullseye or -buster to pin to an OS version.
+		// Use -bullseye variants on local on arm64/Apple Silicon.
+		"args": {
+			"VARIANT": "16-bullseye"
+		}
+	},
+	// Set *default* container specific settings.json values on container create.
+	"settings": {},
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"SASjs.sasjs-for-vscode"
+	],
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "npm i && npm i -g @sasjs/cli",
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "node"
+}

.editorconfig

@@ -1,12 +0,0 @@
-# http://editorconfig.org
-root = true
-
-[*]
-charset = utf-8
-indent_style = space
-indent_size = 2
-end_of_line = lf
-insert_final_newline = false
-trim_trailing_whitespace = true
-
-

.git-hooks/commit-msg

@@ -0,0 +1,18 @@
+#!/bin/sh
+RED="\033[1;31m"
+GREEN="\033[1;32m"
+
+# Get the commit message (the parameter we're given is just the path to the
+# temporary file which holds the message).
+commit_message=$(cat "$1")
+
+if (echo "$commit_message" | grep -Eq "^(build|chore|ci|docs|feat|fix|perf|refactor|revert|style|test)(\([a-z \-]+\))?!?: .+$") then
+   echo "${GREEN} ✔ Commit message meets Conventional Commit standards"
+   exit 0
+fi
+
+echo "${RED}❌ Commit message does not meet the Conventional Commit standard!"
+echo "An example of a valid message is:"
+echo "  feat(login): add the 'remember me' button"
+echo "ℹ More details at: https://www.conventionalcommits.org/en/v1.0.0/#summary"
+exit 1

.git-hooks/pre-commit

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Ensure lint is passing
+LINT=`sasjs lint`
+if [[ "$LINT" != "✔ All matched files use @sasjs/lint code style!" ]]; then
+  echo "$LINT"
+  echo "To commit in spite of these warnings, use the -n parameter."
+  exit 1
+fi
+
+# Avoid commits to the master branch
+BRANCH=`git rev-parse --abbrev-ref HEAD`
+
+if [[ "$BRANCH" =~ ^(master|main|develop)$ ]]; then
+  echo "You are on branch $BRANCH. Are you sure you want to commit to this branch?"
+  echo "If so, commit with -n to bypass the pre-commit hook."
+  exit 1
+fi

.github/CHANGELOG.md

@@ -0,0 +1,5 @@
+# CHANGELOG
+
+As the changes are managed automatically in github, we don't generate an additional changelog.  To view the fixes/features in each release, check out the releases page below:
+
+[https://github.com/sasjs/core/releases](https://github.com/sasjs/core/releases)

.github/CODE_OF_CONDUCT.md

@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+https://sasapps.io/contact-us.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

.github/CONTRIBUTING.md

@@ -0,0 +1,31 @@
+# Contributing
+
+Contributions are warmly welcomed!  To avoid any misunderstandings, do please first discuss the change you wish to make via issue, email, or any other method with the owners of this repository before submitting a PR.
+
+Please note we have a [code of conduct](https://www.contributor-covenant.org/version/2/0/code_of_conduct/), please follow it in all your interactions with the project.
+
+# Environment Setup
+
+This repository makes use of the [SASjs](https://sasjs.io) framework for code organisation, compilation, documentation, and deployment.  The following tools are highly recommended:
+
+* [NPM](https://sasjs.io/windows/#npm) - the runtime and dependency manager for [SASjs CLI](https://cli.sasjs.io) (batteries included)
+* [VSCode](https://sasjs.io/windows/#vscode) - feature packed IDE for code editing (warning - highly effective!)
+* [GIT](https://sasjs.io/windows/#git) - a safety net you cannot (and should not) do without.
+
+For generating the documentation (`sasjs doc`) it is also necessary to install [doxygen](https://www.doxygen.nl/manual/install.html) and GraphViz (`sudo port install graphviz` on mac, or `sudo apt-get install graphviz` on Ubuntu).
+
+
+To get configured:
+
+1.  Clone the repository
+2.  Install local dependencies (`npm install`)
+3.  Install the SASjs CLI globally (`npm install -g @sasjs/cli`)
+4.  Add a target, and authentication (`npm add`).  See [docs](https://cli.sasjs.io/add/).
+
+To contribute:
+
+1.  Create your feature branch (`git checkout -b myfeature`)
+2.  Make your change
+3.  Update the `all.sas` file (`python3 build.py`)
+4.  Push and make a PR
+

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [sasjs]

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,17 @@
+## Issue
+
+Link any related issue(s) in this section.
+
+## Intent
+
+What this PR intends to achieve.
+
+## Implementation
+
+What code changes have been made to achieve the intent.
+
+## Checks
+
+- [ ] Code is formatted correctly (`sasjs lint`).
+- [ ] Any new functionality has been unit tested.
+- [ ] All unit tests are passing (`sasjs test`).

.github/SECURITY.md

@@ -0,0 +1,18 @@
+# Security Policy
+
+Security is an extremely high priority when it comes to the SASjs product suite.  We take a number of steps across all repositories to minimise risk, such as:
+
+* Regular dependabot updates
+* Snyk reports
+* Minimising dependencies, especially production dependencies (sasjs/core has NONE)
+* Testing & Code review process
+
+## Supported Versions
+
+We support only the latest version
+
+## Reporting a Vulnerability
+
+We welcome disclosures of all kinds in relation to all the SASjs libraries.  You can submit them here: [https://sasapps.io/contact-us](https://sasapps.io/contact-us)
+
+

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+  - package-ecosystem: npm
+    directory: '/'
+    schedule:
+      interval: monthly
+    open-pull-requests-limit: 3
+    allow:
+      - dependency-type: "production"

.github/workflows/run-tests.yml

@@ -0,0 +1,57 @@
+# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+
+name: Node.js CI
+
+on:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [lts/fermium]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Install Doxygen
+        run: sudo apt-get install doxygen
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Check code style
+        run: npm run lint
+
+      - name: Add client
+        run: echo "CLIENT=${{secrets.CLIENT}}"> .env.viya
+
+      - name: Add secret
+        run: echo "SECRET=${{secrets.SECRET}}" >> .env.viya
+
+      - name: Add access token
+        run: echo "ACCESS_TOKEN=${{secrets.ACCESS_TOKEN}}" >> .env.viya
+
+      - name: Add refresh token
+        run: echo "REFRESH_TOKEN=${{secrets.REFRESH_TOKEN}}" >> .env.viya
+
+      - name: Build Project
+        run: npm run build
+
+      - name: Run SASjs  tests
+        run: npm run test
+        env:
+          CI: true
+          CLIENT: ${{secrets.CLIENT}}
+          SECRET: ${{secrets.SECRET}}
+          SAS_USERNAME: ${{secrets.SAS_USERNAME}}
+          SAS_PASSWORD: ${{secrets.SAS_PASSWORD}}
+          ACCESS_TOKEN: ${{secrets.ACCESS_TOKEN}}
+          REFRESH_TOKEN: ${{secrets.REFRESH_TOKEN}}

.gitignore

@@ -1,2 +1,13 @@
 node_modules
-.DS_Store
+.DS_Store
+sasjsbuild/
+sasjsresults/
+
+# avoid filenames with spaces being committed to source control
+**\ **
+
+# ignore the mc_* files - containing macros for individual libraries
+mc_*
+
+# ignore .env files as they can contain sasjs access tokens
+*.env*

.gitpod.dockerfile

@@ -0,0 +1,6 @@
+FROM gitpod/workspace-full
+
+RUN sudo apt-get update \
+ && sudo apt-get install -y doxygen \
+ && sudo apt-get install -y graphviz \
+ && sudo rm -rf /var/lib/apt/lists/*

.gitpod.yml

@@ -0,0 +1,27 @@
+tasks:
+  - init: npm install -g npm
+  - command: npm i
+  - command: npm i -g @sasjs/cli
+
+image:
+  file: .gitpod.dockerfile
+vscode:
+  extensions:
+    - sasjs.sasjs-for-vscode
+
+github:
+  prebuilds:
+    # enable for the master/default branch (defaults to true)
+    master: true
+    # enable for all branches in this repo (defaults to false)
+    branches: false
+    # enable for pull requests coming from this repo (defaults to true)
+    pullRequests: true
+    # enable for pull requests coming from forks (defaults to false)
+    pullRequestsFromForks: true
+    # add a "Review in Gitpod" button as a comment to pull requests (defaults to true)
+    addComment: true
+    # add a "Review in Gitpod" button to pull requests (defaults to false)
+    addBadge: false
+    # add a label once the prebuild is ready to pull requests (defaults to false)
+    addLabel: prebuilt-in-gitpod

.npmignore

@@ -0,0 +1,11 @@
+all.sas
+build.py
+.gitpod*
+tests/
+sasjs/
+.github/
+.git-hooks/
+.vscode/
+make_singlefile.sh
+*.md
+.all-contributorsrc

.sasjslint

@@ -0,0 +1,13 @@
+{
+    "noTrailingSpaces": true,
+    "noEncodedPasswords": true,
+    "hasDoxygenHeader": true,
+    "hasMacroNameInMend": true,
+    "hasMacroParentheses": true,
+    "noNestedMacros": false,
+    "noSpacesInFileNames": true,
+    "maxLineLength": 300,
+    "lowerCaseFileNames": true,
+    "noTabIndentation": true,
+    "indentationMultiple": 2
+}

.vscode/.editorconfig

@@ -0,0 +1,9 @@
+{
+  "search.exclude": {
+    "**/sasjsbuild/**": true,
+    "**/dist/**":true
+  },
+  "editor.insertSpaces": true,
+  "editor.tabSize": 2,
+  "trim_trailing_whitespace": true
+}

.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+  "recommendations": [
+    "sasjs.sasjs-for-vscode"
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,10 @@
+{
+  "editor.tabSize": 2,
+  "editor.insertSpaces": true,
+  "editor.detectIndentation": true,
+  "editor.formatOnSave": true,
+  "editor.rulers": [
+    80
+  ],
+  "files.trimTrailingWhitespace": true
+}

CONTRIBUTING.md

@@ -1,83 +0,0 @@
-# Contributing
-
-Contributions to the macrocore library are warmly welcomed!  To avoid any
-misunderstandings, do please first discuss the change you wish to make via issue,
-email, or any other method with the owners of this repository before submitting
-a PR.
-
-Please note we have a code of conduct, please follow it in all your interactions
-with the project.
-
-
-## Code of Conduct
-
-### Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to making participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, gender identity and expression, level of experience,
-nationality, personal appearance, race, religion, or sexual identity and
-orientation.
-
-### Our Standards
-
-Examples of behavior that contributes to creating a positive environment
-include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-### Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-### Scope
-
-This Code of Conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
-address, posting via an official social media account, or acting as an appointed
-representative at an online or offline event. Representation of a project may be
-further defined and clarified by project maintainers.
-
-### Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at support@macropeople.com. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-### Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at [http://contributor-covenant.org/version/1/4][version]
-
-[homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/4/

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2018 (Allan Bowe)
+Copyright 2021 (Allan Bowe)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

README.md

@@ -1,6 +1,24 @@
 # Macro Core
+[![npm package][npm-image]][npm-url]
+[![Github Workflow][githubworkflow-image]][githubworkflow-url]
+![npm](https://img.shields.io/npm/dt/@sasjs/core)
+![Snyk Vulnerabilities for npm package](https://img.shields.io/snyk/vulnerabilities/npm/@sasjs/core)
+[![License](https://img.shields.io/apm/l/atomic-design-ui.svg)](/LICENSE)
+![GitHub top language](https://img.shields.io/github/languages/top/sasjs/core)
+[![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sasjs/core)](https://github.com/sasjs/core/issues?q=is%3Aissue+is%3Aclosed)
+[![GitHub issues](https://img.shields.io/github/issues-raw/sasjs/core)](https://github.com/sasjs/core/issues)
+![total lines](https://tokei.rs/b1/github/sasjs/core)
+[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-908a85?logo=gitpod)](https://gitpod.io/#https://github.com/sasjs/core)
 
-Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of Application Development on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
+
+[npm-image]:https://img.shields.io/npm/v/@sasjs/core.svg
+[npm-url]:http://npmjs.org/package/@sasjs/core
+[githubworkflow-image]:https://github.com/sasjs/core/actions/workflows/main.yml/badge.svg
+[githubworkflow-url]:https://github.com/sasjs/core/blob/main/.github/workflows/main.yml
+[dependency-url]:https://github.com/sasjs/core/blob/main/package.json
+
+
+Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/.github/CONTRIBUTING.md) are welcome.
 
 You can download and compile them all in just two lines of SAS code:
 
@@ -9,44 +27,108 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 %inc mc;
 ```
 
-Documentation: https://sasjs.github.io/core.github.io/files.html
+Documentation: https://core.sasjs.io
 
-# Components
+## Components
 
-**base** library (SAS9/Viya)
+### BASE folder (All Platforms)
 
 - OS independent
-- Not metadata aware
+- Works on all SAS Platforms
 - No X command
-- Prefixes: _mf_, _mp_
+- Prefixes: `mf_`, `mp_`
 
-**meta** library (SAS9 only)
+### DDL folder (All Platforms)
+
+- OS independent
+- Works on all SAS Platforms
+- No X command
+- Prefixes: `mddl_(lib)_` -> where lib can be "SAS" (in relation to a SAS component) or "DC" (in relation to a Data Controller component)
+
+This library will not be used for storing data entries (such as formats or datalines).  Where this becomes necessary in the future, a new repo will be created, in order to keep the NPM bundle size down (for the benefit of those looking to embed purely macros in their applications).
+
+### FCMP folder (All Platforms)
+
+- Function and macro names are identical, except for special cases
+- Prefixes: `mcf_`
+
+The fcmp macros are used to generate fcmp functions, and can be used with or without the `proc fcmp` wrapper.
+
+### LUA folder
+
+Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
+
+To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert all files with a ".lua" extension into a macro wrapper with an `ml_` prefix (embedding the necessary data step put statements).  You can then use your module in any program by running:
+
+```sas
+/* compile the lua module */
+%ml_yourmodule()
+
+/* Execute.  Do not use the restart keyword! */
+proc lua;
+submit;
+  print(yourStuff);
+endsubmit;
+run;
+```
+
+- Prefixes: `ml_`
+
+### META folder (SAS9 only)
+
+Macros used in SAS EBI, which connect to the metadata server.
 
 - OS independent
 - Metadata aware
 - No X command
-- Prefixes: _mm_
+- Prefixes: `mm_`
 
-**viya** library (Viya only)
-
-- OS independent
-- No X command
-- Prefixes: _mv_
-
-**metax** library (SAS9 only)
+### METAX folder (SAS9 only)
 
 - OS specific
 - Metadata aware
 - X command enabled
-- Prefixes: _mmw_,_mmu_,_mmx_
+- Prefixes: `mmx_`
 
-# Installation
+### SERVER folder (@sasjs/server only)
+These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
 
-First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available,eg:
+- OS independent
+- @sasjs/server aware
+- No X command
+- Prefixes: `ms_`
+
+### VIYA folder (Viya only)
+
+Macros used for interfacing with SAS Viya.
+
+- OS independent
+- No X command
+- Prefixes: `mv_`, `mvf_`
+
+### XPLATFORM folder (Viya, Meta, and Server)
+
+Sometimes it is helpful to use a macro that can be used interchangeably regardless of the server type on which is is running (SASVIYA, SAS9, SASJS).
+
+- OS independent
+- No X command
+- Prefixes: `mx_`
+
+## Installation
+
+First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available, eg:
 
 ```sas
-options insert=(sasautos="/your/path/macrocore/base");
-options insert=(sasautos="/your/path/macrocore/meta");
+%let repoloc=/your/path/core;
+options insert=(sasautos="&repoloc/base");
+options insert=(sasautos="&repoloc/ddl");
+options insert=(sasautos="&repoloc/fcmp");
+options insert=(sasautos="&repoloc/lua");
+options insert=(sasautos="&repoloc/meta");
+options insert=(sasautos="&repoloc/metax");
+options insert=(sasautos="&repoloc/server");
+options insert=(sasautos="&repoloc/viya");
+options insert=(sasautos="&repoloc/xplatform");
 ```
 
 The above can be done directly in your sas program, via an autoexec, or an initialisation program.
@@ -58,28 +140,32 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 %inc mc;
 ```
 
-# Standards
+## Standards
 
-## File Properties
+### File Properties
 
 - filenames much match macro names
-- filenames must be lowercase
+- filenames must be lowercase, without spaces
 - macro names must be lowercase
 - one macro per file
 - prefixes:
+  - _mcf_ for macro compiled functions (proc fcmp)
+  - _mddl_ for macros containing DDL (Data Definition Language)
   - _mf_ for macro functions (can be used in open code).
-  - _mp_ for macro procedures (which generate sas code)
+  - _ml_ for macros that are used to compile LUA modules
   - _mm_ for metadata macros (interface with the metadata server).
-  - _mmx_ for macros that use metadata and are XCMD enabled
-  - _mx_ for macros that are XCMD enabled
-  - _mv_ for macros that will only work in Viya
+  - _mmx_ for macros that use metadata and are XCMD enabled (working on both windows and unix)
+  - _mp_ for macro procedures (which generate sas code)
+  - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
+  - _mv_ for macro procedures that will only work in Viya
+  - _mx_ for macros that work on Viya, SAS 9 EBI and SASjs Server
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
 - UTF-8
-- no trailing white space
 
-## Header Properties
+
+### Header Properties
 
 The **Macro Core** documentation is created using [doxygen](http://www.doxygen.nl). A full list of attributes can be found [here](http://www.doxygen.nl/manual/commands.html) but the following are most relevant:
 
@@ -91,18 +177,106 @@ The **Macro Core** documentation is created using [doxygen](http://www.doxygen.n
 - version. The EARLIEST SAS version in which this macro is known to work.
 - author. Author name, contact details optional
 
-All macros must be commented in the doxygen format, to enable the [online documentation](https://sasjs.github.io/core.github.io/).
+All macros must be commented in the doxygen format, to enable the [online documentation](https://core.sasjs.io).
 
-## Coding Standards
+#### Dependencies
+SAS code can contain one of two types of dependency - SAS Macros, and SAS Includes.  When compiling projects using the [SASjs CLI](https://cli.sasjs.io) the doxygen header is scanned for `  @li` items under the following headers:
+
+```sas
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mm_assignlib.sas
+
+  <h4> SAS Includes </h4>
+  @li somefile.ddl SOMEFREF
+  @li someprogram.sas FREFTWO
+```
+
+The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Includes) when creating SAS Jobs and Services (and Tests).
+
+When contributing to this library, it is therefore important to ensure that all dependencies are listed in the header in this format.
+
+
+### Coding Standards
 
 - Indentation = 2 spaces. No tabs!
+- no trailing white space
+- no invisible characters, other than spaces. If invisibles are needed, use hex literals.
 - Macro variables should not have the trailing dot (`&var` not `&var.`) unless necessary to prevent incorrect resolution
-- The closing `%mend;` should not contain the macro name.
+- The closing `%mend;` should **not** contain the macro name.
 - All macros should be defined with brackets, even if no variables are needed - ie `%macro x();` not `%macro x;`
 - Mandatory parameters should be positional, all optional parameters should be keyword (var=) style.
-- All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect.
+- All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect, or the [USER](https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/lrcon/n18m1vkqmeo4esn1moikt23zhp8s.htm) library is active.
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
+- Where global macro variables are absolutely necessary, they should make use of `&sasjs_prefix` - see mp_init.sas
+- The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
+- Use [sasjs lint](https://github.com/sasjs/lint)!
 
-# General Notes
+## General Notes
 
 - All macros should be compatible with SAS versions from support level B and above (so currently 9.2 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully (eg `%if %sysevalf(&sysver<9.3) %then %return`).
+
+## Breaking Changes
+
+We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major/breaking release (v5) becomes necessary:
+
+* mf_getuniquelibref.sas to have the deprecated maxtried parameter removed (no longer needed)
+* mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
+* `insert_cmplib` option of mcf_xxx macros will be deprecated (the option is now checked automatically with value inserted only if needed)
+* mcf_xxx macros to have `wrap=` option defaulted to YES for convenience.  Set this option explicitly to avoid issues.
+* mp_getddl.sas to be renamed to mp_ds2ddl.sas (consistent with other ds2xxx macros).  A wrapper macro is already in place, and you are able to use this immediately.  The default for SHOWLOG will also be YES instead of NO.
+* mp_coretable.sas will be replaced by the standalone macros in the `ddl` folder (which are already available)
+
+## Star Gazing
+
+If you find this library useful, please leave a [star](https://github.com/sasjs/core/stargazers) and help us grow our star graph!
+
+![](https://starchart.cc/sasjs/core.svg)
+
+## Other SAS Repositories
+
+The following repositories are also worth checking out:
+
+* [xieliaing/SAS](https://github.com/xieliaing/SAS)
+* [SASJedi/sas-macros](https://github.com/SASJedi/sas-macros)
+* [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
+* [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)
+* [KatjaGlassConsulting/SMILE-SmartSASMacros](https://github.com/KatjaGlassConsulting/SMILE-SmartSASMacros)
+* [rogerjdeangelis](https://github.com/rogerjdeangelis)
+* [scottbass/sas](https://github.com/scottbass/SAS)
+* [yabwon/sas_packages](https://github.com/yabwon/SAS_PACKAGES)
+
+## Contributors ✨
+<!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
+[![All Contributors](https://img.shields.io/badge/all_contributors-12-orange.svg?style=flat-square)](#contributors-)
+<!-- ALL-CONTRIBUTORS-BADGE:END -->
+Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
+
+<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-disable -->
+<table>
+  <tr>
+    <td align="center"><a href="https://github.com/allanbowe"><img src="https://avatars.githubusercontent.com/u/4420615?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Allan Bowe</b></sub></a><br /><a href="#business-allanbowe" title="Business development">💼</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Code">💻</a> <a href="#content-allanbowe" title="Content">🖋</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Documentation">📖</a> <a href="#infra-allanbowe" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#maintenance-allanbowe" title="Maintenance">🚧</a> <a href="#mentoring-allanbowe" title="Mentoring">🧑‍🏫</a> <a href="#question-allanbowe" title="Answering Questions">💬</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3Aallanbowe" title="Reviewed Pull Requests">👀</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Tests">⚠️</a></td>
+    <td align="center"><a href="https://github.com/rafgag"><img src="https://avatars.githubusercontent.com/u/69139928?v=4?s=100" width="100px;" alt=""/><br /><sub><b>rafgag</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rafgag" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/tmoody"><img src="https://avatars.githubusercontent.com/u/79837106?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Trevor Moody</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=tmoody" title="Code">💻</a></td>
+    <td align="center"><a href="https://krishna-acondy.io/"><img src="https://avatars.githubusercontent.com/u/2980428?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Krishna Acondy</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=krishna-acondy" title="Code">💻</a> <a href="#infra-krishna-acondy" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#blog-krishna-acondy" title="Blogposts">📝</a> <a href="#content-krishna-acondy" title="Content">🖋</a> <a href="#ideas-krishna-acondy" title="Ideas, Planning, & Feedback">🤔</a> <a href="#video-krishna-acondy" title="Videos">📹</a></td>
+    <td align="center"><a href="https://github.com/saadjutt01"><img src="https://avatars.githubusercontent.com/u/8914650?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Muhammad Saad </b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=saadjutt01" title="Code">💻</a> <a href="#ideas-saadjutt01" title="Ideas, Planning, & Feedback">🤔</a></td>
+    <td align="center"><a href="https://www.erudicat.com/"><img src="https://avatars.githubusercontent.com/u/25773492?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Yury Shkoda</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=YuryShkoda" title="Code">💻</a> <a href="#infra-YuryShkoda" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#video-YuryShkoda" title="Videos">📹</a></td>
+    <td align="center"><a href="https://github.com/medjedovicm"><img src="https://avatars.githubusercontent.com/u/18329105?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Mihajlo Medjedovic</b></sub></a><br /><a href="#infra-medjedovicm" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+  </tr>
+  <tr>
+    <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
+    <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
+    <td align="center"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
+    <td align="center"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
+    <td align="center"><a href="https://bandism.net/"><img src="https://avatars.githubusercontent.com/u/22633385?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Ikko Ashimine</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=eltociear" title="Code">💻</a></td>
+  </tr>
+</table>
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+
+<!-- ALL-CONTRIBUTORS-LIST:END -->
+
+This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!

base/mf_abort.sas

@@ -1,29 +1,17 @@
 /**
   @file
-  @brief abort gracefully according to context
-  @details Do not use directly!  See bottom of explanation for details.
+  @brief Abort, ungracefully
+  @details Will abort with a straightforward %abort if the condition is true.
 
-   Configures an abort mechanism according to site specific policies or the
-    particulars of an environment.  For instance, can stream custom
-    results back to the client in an STP Web App context, or completely stop
-    in the case of a batch run.
-
-  For the sharp eyed readers - this is no longer a macro function!! It became
-  a macro procedure during a project and now it's kinda stuck that way until
-  that project is updated (if it's ever updated).  In the meantime we created
-  `mp_abort` which is just a wrapper for this one, and so we recomend you use
-  that for forwards compatibility reasons.
-
-  @param mac= to contain the name of the calling macro
-  @param type= deprecated.  Not used.
-  @param msg= message to be returned
-  @param iftrue= supply a condition under which the macro should be executed.
+  <h4> Related Macros </h4>
+  @li mp_abort.sas
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
-%macro mf_abort(mac=mf_abort.sas, type=, msg=, iftrue=%str(1=1)
+%macro mf_abort(mac=mf_abort.sas, msg=, iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
   %if not(%eval(%unquote(&iftrue))) %then %return;
@@ -32,110 +20,8 @@
   %if %length(&mac)>0 %then %put NOTE- called by &mac;
   %put NOTE - &msg;
 
-  /* Stored Process Server web app context */
-  %if %symexist(_metaperson) or "&SYSPROCESSNAME"="Compute Server" %then %do;
-    options obs=max replace nosyntaxcheck mprint;
-    /* extract log err / warn, if exist */
-    %local logloc logline;
-    %global logmsg; /* capture global messages */
-    %if %symexist(SYSPRINTTOLOG) %then %let logloc=&SYSPRINTTOLOG;
-    %else %let logloc=%qsysfunc(getoption(LOG));
-    proc printto log=log;run;
-    %if %length(&logloc)>0 %then %do;
-      %let logline=0;
-      data _null_;
-        infile &logloc lrecl=5000;
-        input; putlog _infile_;
-        i=1;
-        retain logonce 0;
-        if (_infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR") and logonce=0 then do;
-          call symputx('logline',_n_);
-          logonce+1;
-        end;
-      run;
-      /* capture log including lines BEFORE the err */
-      %if &logline>0 %then %do;
-        data _null_;
-          infile &logloc lrecl=5000;
-          input;
-          i=1;
-          stoploop=0;
-          if _n_ ge &logline-5 and stoploop=0 then do until (i>12);
-            call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
-            input;
-            i+1;
-            stoploop=1;
-          end;
-          if stoploop=1 then stop;
-        run;
-      %end;
-    %end;
+  %abort;
 
-    /* send response in SASjs JSON format */
-    data _null_;
-      file _webout mod lrecl=32000;
-      length msg $32767;
-      sasdatetime=datetime();
-      msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
-      /* escape the quotes */
-      msg=tranwrd(msg,'"','\"');
-      /* ditch the CRLFs as chrome complains */
-      msg=compress(msg,,'kw');
-      /* quote without quoting the quotes (which are escaped instead) */
-      msg=cats('"',msg,'"');
-      if symexist('_debug') then debug=symget('_debug');
-      if debug ge 131 then put '>>weboutBEGIN<<';
-      put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
-      put ',"sasjsAbort" : [{';
-      put ' "MSG":' msg ;
-      put ' ,"MAC": "' "&mac" '"}]';
-      put ",""SYSUSERID"" : ""&sysuserid"" ";
-      if symexist('_metauser') then do;
-        _METAUSER=quote(trim(symget('_METAUSER')));
-        put ",""_METAUSER"": " _METAUSER;
-        _METAPERSON=quote(trim(symget('_METAPERSON')));
-        put ',"_METAPERSON": ' _METAPERSON;
-      end;
-      _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
-      put ',"_PROGRAM" : ' _PROGRAM ;
-      put ",""SYSCC"" : ""&syscc"" ";
-      put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
-      put ",""SYSJOBID"" : ""&sysjobid"" ";
-      put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
-      put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
-      put "}" @;
-      %if &_debug ge 131 %then %do;
-        put '>>weboutEND<<';
-      %end;
-    run;
-    %let syscc=0;
-    %if %symexist(SYS_JES_JOB_URI) %then %do;
-      /* refer web service output to file service in one hit */
-      filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json";
-      %let rc=%sysfunc(fcopy(_web,_webout));
-    %end;
-    %else %do;
-      data _null_;
-        if symexist('sysprocessmode')
-         then if symget("sysprocessmode")="SAS Stored Process Server"
-          then rc=stpsrvset('program error', 0);
-      run;
-    %end;
-    /**
-     * endsas is reliable but kills some deployments.
-     * Abort variants are ungraceful (non zero return code)
-     * This approach lets SAS run silently until the end :-)
-     */
-    %put _all_;
-    filename skip temp;
-    data _null_;
-      file skip;
-      put '%macro skip(); %macro skippy();';
-    run;
-    %inc skip;
-  %end;
-  %else %do;
-    %put _all_;
-    %abort cancel;
-  %end;
-%mend;
+%mend mf_abort;
+
+/** @endcond */

base/mf_dedup.sas

@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief de-duplicates a macro string
+  @details Removes all duplicates from a string of words.  A delimeter can be
+  chosen.  Is inspired heavily by this excellent [macro](
+  https://github.com/scottbass/SAS/blob/master/Macro/dedup_mstring.sas) from
+  [Scott Base](https://www.linkedin.com/in/scottbass).  Case sensitive.
+
+  Usage:
+
+      %let str=One two one two and through and through;
+      %put %mf_dedup(&str);
+      %put %mf_dedup(&str,outdlm=%str(,));
+
+  Which returns:
+
+      > One two one and through
+      > One,two,one,and,through
+
+  @param [in] str String to be deduplicated
+  @param [in] indlm= ( ) Delimeter of the input string
+  @param [out] outdlm= ( ) Delimiter of the output string
+
+  <h4> Related Macros </h4>
+  @li mf_trimstr.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_dedup(str
+  ,indlm=%str( )
+  ,outdlm=%str( )
+)/*/STORE SOURCE*/;
+
+%local num word i pos out;
+
+%* loop over each token, searching the target for that token ;
+%let num=%sysfunc(countc(%superq(str),%str(&indlm)));
+%do i=1 %to %eval(&num+1);
+  %let word=%scan(%superq(str),&i,%str(&indlm));
+  %let pos=%sysfunc(indexw(&out,&word,%str(&outdlm)));
+  %if (&pos eq 0) %then %do;
+    %if (&i gt 1) %then %let out=&out%str(&outdlm);
+    %let out=&out&word;
+  %end;
+%end;
+
+%unquote(&out)
+
+%mend mf_dedup;
+
+

base/mf_deletefile.sas

@@ -0,0 +1,31 @@
+/**
+  @file
+  @brief Deletes a physical file, if it exists
+  @details Usage:
+
+      %mf_writefile(&sasjswork/myfile.txt,l1=some content)
+
+      %mf_deletefile(&sasjswork/myfile.txt)
+
+      %mf_deletefile(&sasjswork/myfile.txt)
+
+
+  @param filepath Full path to the target file
+
+  @returns The return code from the fdelete() invocation
+
+  <h4> Related Macros </h4>
+  @li mf_deletefile.test.sas
+  @li mf_writefile.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_deletefile(file
+)/*/STORE SOURCE*/;
+  %local rc fref;
+  %let rc= %sysfunc(filename(fref,&file));
+  %if %sysfunc(fdelete(&fref)) ne 0 %then %put %sysfunc(sysmsg());
+  %let rc= %sysfunc(filename(fref));
+%mend mf_deletefile;

base/mf_existds.sas

@@ -12,6 +12,10 @@
 
   @param libds library.dataset
   @return output returns 1 or 0
+
+  <h4> Related Macros </h4>
+  @li mf_existds.test.sas
+
   @warning Untested on tables registered in metadata but not physically present
   @version 9.2
   @author Allan Bowe
@@ -23,4 +27,4 @@
   %if %sysfunc(exist(&libds)) ne 1 & %sysfunc(exist(&libds,VIEW)) ne 1 %then 0;
   %else 1;
 
-%mend;
+%mend mf_existds;

base/mf_existfeature.sas

@@ -1,26 +1,25 @@
 /**
-  @file mf_existfeature.sas
+  @file
   @brief Checks whether a feature exists
   @details Check to see if a feature is supported in your environment.
     Run without arguments to see a list of detectable features.
     Note - this list is based on known versions of SAS rather than
     actual feature detection, as that is tricky / impossible to do
-    without generating errors in most cases.
+    without generating errs in most cases.
 
-      %put %mf_existfeature(PROCLUA);
+        %put %mf_existfeature(PROCLUA);
+
+  @param [in] feature The feature to detect.
 
-  @param feature the feature to detect.  Leave blank to list all in log.
-  
   @return output returns 1 or 0 (or -1 if not found)
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_getplatform.sas
 
-
   @version 8
   @author Allan Bowe
 **/
-
+/** @cond */
 %macro mf_existfeature(feature
 )/*/STORE SOURCE*/;
   %let feature=%upcase(&feature);
@@ -28,15 +27,33 @@
   %let platform=%mf_getplatform();
 
   %if &feature= %then %do;
-    %put Supported features:  PROCLUA;
+    %put No feature was requested for detection;
+  %end;
+  %else %if &feature=COLCONSTRAINTS %then %do;
+    %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 0;
+    %else 1;
   %end;
   %else %if &feature=PROCLUA %then %do;
+    /* https://blogs.sas.com/content/sasdummy/2015/08/03/using-lua-within-your-sas-programs */
     %if &platform=SASVIYA %then 1;
-    %else %if "&sysver"="9.3" or "&sysver"="9.4" %then 1;
+    %else %if "&sysver"="9.2" or "&sysver"="9.3" %then 0;
+    %else %if "&SYSVLONG" < "9.04.01M3" %then 0;
+    %else 1;
+  %end;
+  %else %if &feature=DBMS_MEMTYPE %then %do;
+    /* does dbms_memtype exist in dictionary.tables? */
+    %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 0;
+    %else 1;
+  %end;
+  %else %if &feature=EXPORTXLS %then %do;
+    /* is it possible to PROC EXPORT an excel file? */
+    %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 1;
+    %else %if %sysfunc(sysprod(SAS/ACCESS Interface to PC Files)) = 1 %then 1;
     %else 0;
   %end;
   %else %do;
     -1
     %put &sysmacroname: &feature not found;
   %end;
-%mend;
+%mend mf_existfeature;
+/** @endcond */

base/mf_existfileref.sas

@@ -0,0 +1,33 @@
+/**
+  @file
+  @brief Checks whether a fileref exists
+  @details You can probably do without this macro as it is just a one liner.
+  Mainly it is here as a convenient way to remember the syntax!
+
+  @param fref the fileref to detect
+
+  @return output Returns 1 if found and 0 if not found.  Note - it is possible
+  that the fileref is found, but the file does not (yet) exist. If you need
+  to test for this, you may as well use the fileref function directly.
+
+  @version 8
+  @author [Allan Bowe](https://www.linkedin.com/in/allanbowe/)
+**/
+
+%macro mf_existfileref(fref
+)/*/STORE SOURCE*/;
+
+  %local rc;
+  %let rc=%sysfunc(fileref(&fref));
+  %if &rc=0 %then %do;
+    1
+  %end;
+  %else %if &rc<0 %then %do;
+    %put &sysmacroname: Fileref &fref exists but the underlying file does not;
+    1
+  %end;
+  %else %do;
+    0
+  %end;
+
+%mend mf_existfileref;

base/mf_existfunction.sas

@@ -0,0 +1,37 @@
+/**
+  @file
+  @brief Checks if a function exists
+  @details Returns 1 if the function exists, else 0.  Note that this function
+  can be slow as it needs to open the sashelp.vfuncs table.
+
+  Usage:
+
+      %put %mf_existfunction(CAT);
+      %put %mf_existfunction(DOG);
+
+  Full credit to [Bart](https://sasensei.com/user/305) for the vfunc pointer
+  and the tidy approach for pure macro data set filtering.
+  Check out his [SAS Packages](https://github.com/yabwon/SAS_PACKAGES)
+  framework!  Where you can find the same [function](
+https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionexists-macro
+  ).
+
+  @param [in] name (positional) - function name
+
+  @author Allan Bowe
+**/
+/** @cond */
+%macro mf_existfunction(name
+)/*/STORE SOURCE*/;
+
+  %local dsid rc exist;
+  %let dsid=%sysfunc(open(sashelp.vfunc(where=(fncname="%upcase(&name)"))));
+  %let exist=1;
+  %let exist=%sysfunc(fetch(&dsid, NOSET));
+  %let rc=%sysfunc(close(&dsid));
+
+  %sysevalf(0 = &exist)
+
+%mend mf_existfunction;
+
+/** @endcond */

base/mf_existvar.sas

@@ -1,17 +1,22 @@
 /**
   @file
   @brief Checks if a variable exists in a data set.
-  @details Returns 0 if the variable does NOT exist, and return the position of
-    the var if it does.
-    Usage:
+  @details Returns 0 if the variable does NOT exist, and the position of the var
+  if it does.
+  Usage:
 
-        %put %mf_existvar(work.someds, somevar)
+      %put %mf_existvar(work.someds, somevar)
+
+  @param [in] libds 2 part dataset or view reference
+  @param [in] var variable name
+
+  <h4> Related Macros </h4>
+  @li mf_existvar.test.sas
 
-  @param libds (positional) - 2 part dataset or view reference
-  @param var (positional) - variable name
   @version 9.2
   @author Allan Bowe
 **/
+/** @cond */
 
 %macro mf_existvar(libds /* 2 part dataset name */
       , var /* variable name */
@@ -29,4 +34,6 @@
       %let rc=%sysfunc(close(&dsid));
   %end;
 
-%mend;
+%mend mf_existvar;
+
+/** @endcond */

base/mf_existvarlist.sas

@@ -2,18 +2,16 @@
   @file
   @brief Checks if a set of variables ALL exist in a data set.
   @details Returns 0 if ANY of the variables do not exist, or 1 if they ALL do.
-    Usage:
+  Usage:
 
-        %put %mf_existVarList(sashelp.class, age sex name dummyvar)
-
-  <h4> Dependencies </h4>
-  @li mf_abort.sas
+      %put %mf_existVarList(sashelp.class, age sex name dummyvar);
 
   @param libds 2 part dataset or view reference
   @param varlist space separated variable names
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
 %macro mf_existvarlist(libds, varlist
@@ -28,7 +26,7 @@
   %let dsid=%sysfunc(open(&libds,is));
 
   %if &dsid=0 %then %do;
-    %put WARNING:  unable to open &libds in mf_existvarlist (&dsid);
+    %put %str(WARN)ING:  unable to open &libds in mf_existvarlist (&dsid);
   %end;
 
   %if %sysfunc(attrn(&dsid,NVARS))=0 %then %do;
@@ -53,4 +51,6 @@
     0
     %put Vars not found: &found;
   %end;
-%mend;
+%mend mf_existvarlist;
+
+/** @endcond */

base/mf_fmtdttm.sas

@@ -0,0 +1,42 @@
+/**
+  @file
+  @brief Returns E8601DT26.6 if compatible else DATETIME19.3
+  @details From our experience in [Data Controller for SAS]
+  (https://datacontroller.io) deployments, the E8601DT26.6 datetime format has
+  the widest support when it comes to pass-through SQL queries.
+
+  However, it is not supported in WPS or early versions of SAS 9 (M3 and below)
+  when used as a datetime literal, eg:
+
+      data _null_;
+        demo="%sysfunc(datetime(),E8601DT26.6)"dt;
+        demo=;
+      run;
+
+  This macro will therefore return DATEITME19.3 as an alternative format
+  based on the runtime environment so that it can be used in such cases, eg:
+
+      data _null_;
+        demo="%sysfunc(datetime(),%mf_fmtdttm())"dt;
+        demo=;
+      run;
+
+  <h4> Related Macros </h4>
+  @li mf_fmtdttm.test.sas
+
+  @author Allan Bowe
+**/
+
+%macro mf_fmtdttm(
+)/*/STORE SOURCE*/;
+
+%if "&sysver"="9.2" or "&sysver"="9.3"
+  or ("&sysver"="9.4" and "%substr(&SYSVLONG,9,1)" le "3")
+  or "%substr(&sysver,1,1)"="4"
+  or "%substr(&sysver,1,1)"="5"
+%then %do;DATETIME19.3%end;
+%else %do;E8601DT26.6%end;
+
+%mend mf_fmtdttm;
+
+

base/mf_getapploc.sas

@@ -0,0 +1,78 @@
+/**
+  @file
+  @brief Returns the appLoc from the _program variable
+  @details When working with SASjs apps, web services / tests / jobs are always
+  deployed to a root (app) location in the SAS logical folder tree.
+
+  When building apps for use in other environments, you do not necessarily know
+  where the backend services will be deployed.  Therefore a function like this
+  is handy in order to dynamically figure out the appLoc, and enable other
+  services to be connected by a relative reference.
+
+  SASjs apps always have the same immediate substructure (one or more of the
+  following):
+
+  @li /data
+  @li /jobs
+  @li /services
+  @li /tests
+  @li /tests/jobs
+  @li /tests/services
+  @li /tests/macros
+
+  This function works by testing for the existence of any of the above in the
+  automatic _program variable, and returning the part to the left of it.
+
+  Usage:
+
+      %put %mf_getapploc(&_program)
+
+      %put %mf_getapploc(/some/location/services/admin/myservice);
+      %put %mf_getapploc(/some/location/jobs/extract/somejob/);
+      %put %mf_getapploc(/some/location/tests/jobs/somejob/);
+
+
+  @author Allan Bowe
+**/
+
+%macro mf_getapploc(pgm);
+%if "&pgm"="" %then %do;
+  %if %symexist(_program) %then %let pgm=&_program;
+  %else %do;
+    %put &sysmacroname: No value provided and no _program variable available;
+    %return;
+  %end;
+%end;
+%local root;
+
+/**
+  * First check we are not in the tests/macros folder (which has no subfolders)
+  * or specifically in the testsetup or testteardown services
+  */
+%if %index(&pgm,/tests/macros/)
+or %index(&pgm,/tests/testsetup)
+or %index(&pgm,/tests/testteardown)
+%then %do;
+  %let root=%substr(&pgm,1,%index(&pgm,/tests)-1);
+  &root
+  %return;
+%end;
+
+/**
+  * Next, move up two levels to avoid matches on subfolder or service name
+  */
+%let root=%substr(&pgm,1,%length(&pgm)-%length(%scan(&pgm,-1,/))-1);
+%let root=%substr(&root,1,%length(&root)-%length(%scan(&root,-1,/))-1);
+
+%if %index(&root,/tests/) %then %do;
+  %let root=%substr(&root,1,%index(&root,/tests/)-1);
+%end;
+%else %if %index(&root,/services) %then %do;
+  %let root=%substr(&root,1,%index(&root,/services)-1);
+%end;
+%else %if %index(&root,/jobs) %then %do;
+  %let root=%substr(&root,1,%index(&root,/jobs)-1);
+%end;
+%else %put &sysmacroname: Could not find an app location from &pgm;
+  &root
+%mend mf_getapploc ;

base/mf_getattrc.sas

@@ -10,20 +10,20 @@
   @param attr full list in [documentation](
     https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000147794.htm)
   @return output returns result of the attrc value supplied, or -1 and log
-    message if error.
+    message if err.
 
   @version 9.2
   @author Allan Bowe
 **/
 
 %macro mf_getattrc(
-     libds
+    libds
     ,attr
 )/*/STORE SOURCE*/;
   %local dsid rc;
   %let dsid=%sysfunc(open(&libds,is));
   %if &dsid = 0 %then %do;
-    %put WARNING: Cannot open %trim(&libds), system message below;
+    %put %str(WARN)ING: Cannot open %trim(&libds), system message below;
     %put %sysfunc(sysmsg());
     -1
   %end;
@@ -31,4 +31,4 @@
     %sysfunc(attrc(&dsid,&attr))
     %let rc=%sysfunc(close(&dsid));
   %end;
-%mend;
+%mend mf_getattrc;

base/mf_getattrn.sas

@@ -8,22 +8,22 @@
 
   @param libds library.dataset
   @param attr Common values are NLOBS and NVARS, full list in [documentation](
-    http://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000212040.htm)
+  http://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000212040.htm)
   @return output returns result of the attrn value supplied, or -1 and log
-    message if error.
+    message if err.
 
   @version 9.2
   @author Allan Bowe
 **/
 
 %macro mf_getattrn(
-     libds
+    libds
     ,attr
 )/*/STORE SOURCE*/;
   %local dsid rc;
   %let dsid=%sysfunc(open(&libds,is));
   %if &dsid = 0 %then %do;
-    %put WARNING: Cannot open %trim(&libds), system message below;
+    %put %str(WARN)ING: Cannot open %trim(&libds), system message below;
     %put %sysfunc(sysmsg());
     -1
   %end;
@@ -31,4 +31,4 @@
     %sysfunc(attrn(&dsid,&attr))
     %let rc=%sysfunc(close(&dsid));
   %end;
-%mend;
+%mend mf_getattrn;

base/mf_getengine.sas

@@ -3,7 +3,7 @@
   @brief Returns the engine type of a SAS library
   @details Usage:
 
-      %put %mf_getEngine(SASHELP);
+      %put %mf_getengine(SASHELP);
 
   returns:
   > V9
@@ -12,25 +12,33 @@
   contributors of Chris Hemedingers blog [post](
   http://blogs.sas.com/content/sasdummy/2013/06/04/find-a-sas-library-engine/)
 
-  @param libref Library reference (also accepts a 2 level libds ref).
+  @param [in] libref Library reference (also accepts a 2 level libds ref).
 
-  @return output returns the library engine for the FIRST library encountered.
+  @return output returns the library engine (uppercase) for the FIRST library
+    encountered.
 
   @warning will only return the FIRST library engine - for concatenated
     libraries, with different engines, inconsistent results may be encountered.
 
   @version 9.2
   @author Allan Bowe
-**/
 
-%macro mf_getEngine(libref
+  <h4> Related Macros </h4>
+  @li mf_getxengine.sas
+
+**/
+/** @cond */
+
+%macro mf_getengine(libref
 )/*/STORE SOURCE*/;
   %local dsid engnum rc engine;
 
   /* in case the parameter is a libref.tablename, pull off just the libref */
   %let libref = %upcase(%scan(&libref, 1, %str(.)));
 
-  %let dsid=%sysfunc(open(sashelp.vlibnam(where=(libname="%upcase(&libref)")),i));
+  %let dsid=%sysfunc(
+    open(sashelp.vlibnam(where=(libname="%upcase(&libref)")),i)
+  );
   %if (&dsid ^= 0) %then %do;
     %let engnum=%sysfunc(varnum(&dsid,ENGINE));
     %let rc=%sysfunc(fetch(&dsid));
@@ -39,6 +47,8 @@
     %let rc= %sysfunc(close(&dsid));
   %end;
 
- &engine
+  %upcase(&engine)
 
-%mend;
+%mend mf_getengine;
+
+/** @endcond */

base/mf_getfilesize.sas

@@ -5,18 +5,19 @@
 
       %put %mf_getfilesize(fpath=C:\temp\myfile.txt);
 
-      or
+  or, provide a libds value as follows:
 
       data x;do x=1 to 100000;y=x;output;end;run;
       %put %mf_getfilesize(libds=work.x,format=yes);
 
-      gives:
+  Which gives:
 
-      2mb
+  > 2mb
+
+  @param [in] fpath= Full path and filename.  Provide this OR the libds value.
+  @param [in] libds= (0) Library.dataset value (assumes library is BASE engine)
+  @param [in] format= (NO) Set to yes to apply sizekmg. format
 
-  @param fpath= full path and filename.  Provide this OR the libds value.
-  @param libds= library.dataset value (assumes library is BASE engine)
-  @param format=  set to yes to apply sizekmg. format
   @returns bytes
 
   @version 9.2
@@ -26,22 +27,38 @@
 %macro mf_getfilesize(fpath=,libds=0,format=NO
 )/*/STORE SOURCE*/;
 
+  %local rc fid fref bytes dsid lib vnum;
+
   %if &libds ne 0 %then %do;
-    %let fpath=%sysfunc(pathname(%scan(&libds,1,.)))/%scan(&libds,2,.).sas7bdat;
+    %let libds=%upcase(&libds);
+    %if %index(&libds,.)=0 %then %let lib=WORK;
+    %else %let lib=%scan(&libds,1,.);
+    %let dsid=%sysfunc(open(
+      sashelp.vtable(where=(libname="&lib" and memname="%scan(&libds,-1,.)")
+        keep=libname memname filesize
+      )
+    ));
+    %if (&dsid ^= 0) %then %do;
+      %let vnum=%sysfunc(varnum(&dsid,FILESIZE));
+      %let rc=%sysfunc(fetch(&dsid));
+      %let bytes=%sysfunc(getvarn(&dsid,&vnum));
+      %let rc= %sysfunc(close(&dsid));
+    %end;
+    %else %put &sysmacroname: &libds could not be opened! %sysfunc(sysmsg());
+  %end;
+  %else %do;
+    %let rc=%sysfunc(filename(fref,&fpath));
+    %let fid=%sysfunc(fopen(&fref));
+    %let bytes=%sysfunc(finfo(&fid,File Size (bytes)));
+    %let rc=%sysfunc(fclose(&fid));
+    %let rc=%sysfunc(filename(fref));
   %end;
 
-  %local rc fid fref bytes;
-  %let rc=%sysfunc(filename(fref,&fpath));
-  %let fid=%sysfunc(fopen(&fref));
-  %let bytes=%sysfunc(finfo(&fid,File Size (bytes)));
-  %let rc=%sysfunc(fclose(&fid));
-  %let rc=%sysfunc(filename(fref));
-
   %if &format=NO %then %do;
-     &bytes
+    &bytes
   %end;
   %else %do;
     %sysfunc(INPUTN(&bytes, best.),sizekmg.)
   %end;
 
-%mend ;
+%mend mf_getfilesize ;

base/mf_getfmtlist.sas

@@ -0,0 +1,61 @@
+/**
+  @file
+  @brief Returns a distinct list of formats from a table
+  @details Reads the dataset header and returns a distinct list of formats
+  applied.
+
+        %put NOTE- %mf_getfmtlist(sashelp.prdsale);
+        %put NOTE- %mf_getfmtlist(sashelp.shoes);
+        %put NOTE- %mf_getfmtlist(sashelp.demographics);
+
+  returns:
+
+  > DOLLAR $CHAR W MONNAME
+  > $CHAR BEST DOLLAR
+  > BEST Z $CHAR COMMA PERCENTN
+
+
+  @param [in] libds Two part library.dataset reference.
+
+  <h4> SAS Macros </h4>
+  @li mf_getfmtname.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_getfmtlist(libds
+)/*/STORE SOURCE*/;
+/* declare local vars */
+%local out dsid nvars x rc fmt;
+
+/* open dataset in macro */
+%let dsid=%sysfunc(open(&libds));
+
+/* continue if dataset exists */
+%if &dsid %then %do;
+  /* loop each variable in the dataset */
+  %let nvars=%sysfunc(attrn(&dsid,NVARS));
+  %do x=1 %to &nvars;
+    /* grab format and check it exists */
+    %let fmt=%sysfunc(varfmt(&dsid,&x));
+    %if %quote(&fmt) ne %quote() %then %let fmt=%mf_getfmtname(&fmt);
+    %else %do;
+      /* assign default format depending on variable type */
+      %if %sysfunc(vartype(&dsid, &x))=C %then %let fmt=$CHAR;
+      %else %let fmt=BEST;
+    %end;
+    /* concatenate unique list of formats */
+    %if %sysfunc(indexw(&out,&fmt,%str( )))=0 %then %let out=&out &fmt;
+  %end;
+  %let rc=%sysfunc(close(&dsid));
+%end;
+%else %do;
+  %put &sysmacroname: Unable to open &libds (rc=&dsid);
+  %put &sysmacroname: SYSMSG= %sysfunc(sysmsg());
+  %let rc=%sysfunc(close(&dsid));
+%end;
+/* send them out without spaces or quote markers */
+%do;%unquote(&out)%end;
+%mend mf_getfmtlist;

base/mf_getfmtname.sas

@@ -0,0 +1,44 @@
+/**
+  @file
+  @brief Extracts a format name from a fully defined format
+  @details Converts formats in like $thi3. and th13.2 $THI and TH.
+  Usage:
+
+      %put %mf_getfmtname(8.);
+      %put %mf_getfmtname($4.);
+      %put %mf_getfmtname(comma14.10);
+
+  Returns:
+
+  > W
+  > $CHAR
+  > COMMA
+
+  Note that system defaults are inferred from the values provided.
+
+  @param [in] fmt The fully defined format. If left blank, nothing is returned.
+
+  @returns The name (without width or decimal) of the format.
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_getfmtname(fmt
+)/*/STORE SOURCE*/ /minoperator mindelimiter=' ';
+
+%local out dsid nvars x rc fmt;
+
+/* extract actual format name from the format definition */
+%let fmt=%scan(&fmt,1,.);
+%do %while(%substr(&fmt,%length(&fmt),1) in 1 2 3 4 5 6 7 8 9 0);
+  %if %length(&fmt)=1 %then %let fmt=W;
+  %else %let fmt=%substr(&fmt,1,%length(&fmt)-1);
+%end;
+
+%if &fmt=$ %then %let fmt=$CHAR;
+
+/* send them out without spaces or quote markers */
+%do;%unquote(%upcase(&fmt))%end;
+%mend mf_getfmtname;

base/mf_getkeyvalue.sas

@@ -3,8 +3,8 @@
   @brief retrieves a key value pair from a control dataset
   @details By default, control dataset is work.mp_setkeyvalue.  Usage:
 
-    %mp_setkeyvalue(someindex,22,type=N)
-    %put %mf_getkeyvalue(someindex)
+      %mp_setkeyvalue(someindex,22,type=N)
+      %put %mf_getkeyvalue(someindex)
 
 
   @param key Provide a key on which to perform the lookup
@@ -16,7 +16,7 @@
 
 %macro mf_getkeyvalue(key,libds=work.mp_setkeyvalue
 )/*/STORE SOURCE*/;
- %local ds dsid key valc valn type rc;
+%local ds dsid key valc valn type rc;
 %let dsid=%sysfunc(open(&libds(where=(key="&key"))));
 %syscall set(dsid);
 %let rc = %sysfunc(fetch(&dsid));
@@ -29,4 +29,4 @@
   &valc
 %end;
 %else %put %str(ERR)OR: Unable to find key &key in ds &libds;
-%mend;
+%mend mf_getkeyvalue;

base/mf_getplatform.sas

@@ -5,13 +5,18 @@
 
       %put %mf_getplatform();
 
-    returns:
-      SASMETA  (or SASVIYA)
+  returns one of:
+
+  @li SASMETA
+  @li SASVIYA
+  @li SASJS
+  @li BASESAS
 
   @param switch the param for which to return a platform specific variable
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_mval.sas
+  @li mf_trimstr.sas
 
   @version 9.4 / 3.4
   @author Allan Bowe
@@ -21,26 +26,34 @@
 )/*/STORE SOURCE*/;
 %local a b c;
 %if &switch.NONE=NONE %then %do;
+  %if %symexist(sasjsprocessmode) %then %do;
+    %if &sasjsprocessmode=Stored Program %then %do;
+      SASJS
+      %return;
+    %end;
+  %end;
   %if %symexist(sysprocessmode) %then %do;
-    %if "&sysprocessmode"="SAS Object Server" 
+    %if "&sysprocessmode"="SAS Object Server"
     or "&sysprocessmode"= "SAS Compute Server" %then %do;
         SASVIYA
     %end;
-    %else %if "&sysprocessmode"="SAS Stored Process Server" %then %do;
+    %else %if "&sysprocessmode"="SAS Stored Process Server"
+      or "&sysprocessmode"="SAS Workspace Server"
+    %then %do;
       SASMETA
       %return;
     %end;
     %else %do;
-      SAS
+      BASESAS
       %return;
     %end;
   %end;
-  %else %if %symexist(_metaport) %then %do;
+  %else %if %symexist(_metaport) or %symexist(_metauser) %then %do;
     SASMETA
     %return;
   %end;
   %else %do;
-    SAS
+    BASESAS
     %return;
   %end;
 %end;
@@ -57,6 +70,6 @@
   %else 0;
 %end;
 %else %if &switch=VIYARESTAPI %then %do;
-  %sysfunc(getoption(servicesbaseurl))
+  %mf_trimstr(%sysfunc(getoption(servicesbaseurl)),/)
 %end;
-%mend;
+%mend mf_getplatform;

base/mf_getquotedstr.sas

@@ -3,17 +3,24 @@
   @brief Adds custom quotes / delimiters to a  delimited string
   @details Can be used in open code, eg as follows:
 
-    %put %mf_getquotedstr(blah   blah  blah);
+      %put %mf_getquotedstr(blah   blah  blah);
 
   which returns:
 > 'blah','blah','blah'
 
-  @param in_str the unquoted, spaced delimited string to transform
-  @param dlm= the delimeter to be applied to the output (default comma)
-  @param indlm= the delimeter used for the input (default is space)
-  @param quote= the quote mark to apply (S=Single, D=Double). If any other value
-    than uppercase S or D is supplied, then that value will be used as the
-    quoting character.
+  Alternatively:
+
+      %put %mf_getquotedstr(these words are double quoted,quote=D)
+
+  for:
+> "these","words","are","double","quoted"
+
+  @param [in] in_str The unquoted, spaced delimited string to transform
+  @param [in] dlm= (,) The delimeter to be applied to the output (default comma)
+  @param [in] indlm= ( ) The delimeter used for the input (default is space)
+  @param [in] quote= (S) The quote mark to apply (S=Single, D=Double, N=None).
+    If any other value than uppercase S or D is supplied, then that value will
+    be used as the quoting character.
   @return output returns a string with the newly quoted / delimited output.
 
   @version 9.2
@@ -21,11 +28,15 @@
 **/
 
 
-%macro mf_getquotedstr(IN_STR,DLM=%str(,),QUOTE=S,indlm=%str( )
+%macro mf_getquotedstr(IN_STR
+  ,DLM=%str(,)
+  ,QUOTE=S
+  ,indlm=%str( )
 )/*/STORE SOURCE*/;
-  %if &quote=S %then %let quote=%str(%');
-  %else %if &quote=D %then %let quote=%str(%");
-  %else %let quote=%str();
+  /* credit Rowland Hale  - byte34 is double quote, 39 is single quote */
+  %if &quote=S %then %let quote=%qsysfunc(byte(39));
+  %else %if &quote=D %then %let quote=%qsysfunc(byte(34));
+  %else %if &quote=N %then %let quote=;
   %local i item buffer;
   %let i=1;
   %do %while (%qscan(&IN_STR,&i,%str(&indlm)) ne %str() ) ;
@@ -43,4 +54,4 @@
 
   &buffer
 
-%mend;
+%mend mf_getquotedstr;

base/mf_getschema.sas

@@ -17,6 +17,7 @@
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
 %macro mf_getschema(libref
@@ -37,4 +38,6 @@
 
   &schema
 
-%mend;
+%mend mf_getschema;
+
+/** @endcond */

base/mf_getuniquefileref.sas

@@ -1,36 +1,60 @@
 /**
   @file
   @brief Assigns and returns an unused fileref
-  @details Use as follows:
+  @details  Using the native approach for assigning filerefs fails as some
+  procedures (such as proc http) do not recognise the temporary names (starting
+  with a hash), returning a message such as:
 
-    %let fileref1=%mf_getuniquefileref();
-    %let fileref2=%mf_getuniquefileref();
-    %put &fileref1 &fileref2;
+  > ERROR 22-322: Expecting a name.
 
-  which returns:
+  This macro works by attempting a random fileref (with a prefix), seeing if it
+  is already assigned, and if not - returning the fileref.
 
-> mcref0 mcref1
+  If your process can accept filerefs with the hash (#) prefix, then set
+  `prefix=0` to revert to the native approach - which is significantly faster
+  when there are a lot of filerefs in a session.
 
-  @prefix= first part of fileref. Remember that filerefs can only be 8
-    characters, so a 7 letter prefix would mean that `maxtries` should be 10.
-  @param maxtries= the last part of the libref.  Provide an integer value.
+  Use as follows:
+
+      %let fileref1=%mf_getuniquefileref();
+      %let fileref2=%mf_getuniquefileref(prefix=0);
+      %put &fileref1 &fileref2;
+
+  which returns filerefs similar to:
+
+> _7432233 #LN00070
+
+  @param [in] prefix= (_) first part of fileref. Remember that filerefs can only
+    be 8 characters, so a 7 letter prefix would mean `maxtries` should be 10.
+    if using zero (0) as the prefix, a native assignment is used.
+  @param [in] maxtries= (1000) the last part of the libref. Must be an integer.
+  @param [in] lrecl= (32767) Provide a default lrecl with which to initialise
+    the generated fileref.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-%macro mf_getuniquefileref(prefix=mcref,maxtries=1000);
-  %local x fname;
-  %let x=0;
-  %do x=0 %to &maxtries;
-  %if %sysfunc(fileref(&prefix&x)) > 0 %then %do;
-    %let fname=&prefix&x;
-    %let rc=%sysfunc(filename(fname,,temp));
+%macro mf_getuniquefileref(prefix=_,maxtries=1000,lrecl=32767);
+  %local rc fname;
+  %if &prefix=0 %then %do;
+    %let rc=%sysfunc(filename(fname,,temp,lrecl=&lrecl));
     %if &rc %then %put %sysfunc(sysmsg());
-    &prefix&x
-    %*put &sysmacroname: Fileref &prefix&x was assigned and returned;
-    %return;
+    &fname
   %end;
+  %else %do;
+    %local x len;
+    %let len=%eval(8-%length(&prefix));
+    %let x=0;
+    %do x=0 %to &maxtries;
+      %let fname=&prefix%substr(%sysfunc(ranuni(0)),3,&len);
+      %if %sysfunc(fileref(&fname)) > 0 %then %do;
+        %let rc=%sysfunc(filename(fname,,temp,lrecl=&lrecl));
+        %if &rc %then %put %sysfunc(sysmsg());
+        &fname
+        %return;
+      %end;
+    %end;
+    %put unable to find available fileref after &maxtries attempts;
   %end;
-  %put unable to find available fileref in range &prefix.0-&maxtries;
-%mend;
+%mend mf_getuniquefileref;

base/mf_getuniquelibref.sas

@@ -3,38 +3,55 @@
   @brief Returns an unused libref
   @details Use as follows:
 
-    libname mclib0 (work);
-    libname mclib1 (work);
-    libname mclib2 (work);
+      libname mclib0 (work);
+      libname mclib1 (work);
+      libname mclib2 (work);
 
-    %let libref=%mf_getuniquelibref();
-    %put &=libref;
+      %let libref=%mf_getuniquelibref();
+      %put &=libref;
 
   which returns:
 
 > mclib3
 
-  @prefix= first part of libref.  Remember that librefs can only be 8 characters,
-    so a 7 letter prefix would mean that maxtries should be 10.
-  @param maxtries= the last part of the libref.  Provide an integer value.
+  A blank value is returned if no usable libname is determined.
+
+  @param [in] prefix= (mclib) first part of the returned libref. As librefs can
+    be as long as 8 characters, a maximum length of 7 characters is premitted
+    for this prefix.
+  @param [in] maxtries= Deprecated parameter. Remains here to ensure a
+    non-breaking change.  Will be removed in v5.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-
 %macro mf_getuniquelibref(prefix=mclib,maxtries=1000);
-  %local x libref;
-  %let x=0;
-  %do x=0 %to &maxtries;
-  %if %sysfunc(libref(&prefix&x)) ne 0 %then %do;
-    %let libref=&prefix&x;
-    %let rc=%sysfunc(libname(&libref,%sysfunc(pathname(work))));
-    %if &rc %then %put %sysfunc(sysmsg());
-    &prefix&x
-    %*put &sysmacroname: Libref &libref assigned as WORK and returned;
+  %local x;
+
+  %if ( %length(&prefix) gt 7 ) %then %do;
+    %put %str(ERR)OR: The prefix parameter cannot exceed 7 characters.;
+    0
     %return;
   %end;
+  %else %if (%sysfunc(NVALID(&prefix,v7))=0) %then %do;
+    %put %str(ERR)OR: Invalid prefix (&prefix);
+    0
+    %return;
   %end;
-  %put unable to find available libref in range &prefix.0-&maxtries;
-%mend;
+
+  /* Set maxtries equal to '10 to the power of [# unused characters] - 1' */
+  %let maxtries=%eval(10**(8-%length(&prefix))-1);
+
+  %do x = 0 %to &maxtries;
+    %if %sysfunc(libref(&prefix&x)) ne 0 %then %do;
+      &prefix&x
+      %return;
+    %end;
+    %let x = %eval(&x + 1);
+  %end;
+
+  %put %str(ERR)OR: No usable libref in range &prefix.0-&maxtries;
+  %put %str(ERR)OR- Try reducing the prefix or deleting some libraries!;
+  0
+%mend mf_getuniquelibref;

base/mf_getuniquename.sas

@@ -18,5 +18,5 @@
 
 
 %macro mf_getuniquename(prefix=MC);
-  &prefix.%substr(%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32-%length(&prefix))
-%mend;
+&prefix.%substr(%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32-%length(&prefix))
+%mend mf_getuniquename;

base/mf_getuser.sas

@@ -2,7 +2,7 @@
   @file
   @brief Returns a userid according to session context
   @details In a workspace session, a user is generally represented by <code>
-    &sysuserid</code> or <code>SYS_COMPUTE_SESSION_OWNER</code> if it exists.  
+    &sysuserid</code> or <code>SYS_COMPUTE_SESSION_OWNER</code> if it exists.
     In a Stored Process session, <code>&sysuserid</code>
     resolves to a system account (default=sassrv) and instead there are several
     metadata username variables to choose from (_metauser, _metaperson
@@ -12,7 +12,7 @@
 
         %let user= %mf_getUser();
         %put &user;
-        
+
   @param type - do not use, may be deprecated in a future release
 
   @return SYSUSERID (if workspace server)
@@ -23,20 +23,22 @@
   @author Allan Bowe
 **/
 
-%macro mf_getuser(type=META
+%macro mf_getuser(
 )/*/STORE SOURCE*/;
-  %local user metavar;
-  %if &type=OS %then %let metavar=_secureusername;
-  %else %let metavar=_metaperson;
+  %local user;
 
-  %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %let user=&SYS_COMPUTE_SESSION_OWNER;
-  %else %if %symexist(&metavar) %then %do;
-    %if %length(&&&metavar)=0 %then %let user=&sysuserid;
+  %if %symexist(_sasjs_username) %then %let user=&_sasjs_username;
+  %else %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %do;
+    %let user=&SYS_COMPUTE_SESSION_OWNER;
+  %end;
+  %else %if %symexist(_metaperson) %then %do;
+    %if %length(&_metaperson)=0 %then %let user=&sysuserid;
     /* sometimes SAS will add @domain extension - remove for consistency */
-    %else %let user=%scan(&&&metavar,1,@);
+    /* but be sure to quote in case of usernames with commas */
+    %else %let user=%unquote(%scan(%quote(&_metaperson),1,@));
   %end;
   %else %let user=&sysuserid;
 
   %quote(&user)
 
-%mend;
+%mend mf_getuser;

base/mf_getvalue.sas

@@ -7,9 +7,12 @@
       %put %mf_getvalue(sashelp.class,name,filter=%quote(age=15));
       %put %mf_getvalue(sashelp.class,name);
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_getattrn.sas
 
+  <h4> Related Macros </h4>
+  @li mp_setkeyvalue.sas
+
   @param libds dataset to query
   @param variable the variable which contains the value to return.
   @param filter contents of where clause
@@ -20,7 +23,7 @@
 
 %macro mf_getvalue(libds,variable,filter=1
 )/*/STORE SOURCE*/;
- %if %mf_getattrn(&libds,NLOBS)>0 %then %do;
+  %if %mf_getattrn(&libds,NLOBS)>0 %then %do;
     %local dsid rc &variable;
     %let dsid=%sysfunc(open(&libds(where=(&filter))));
     %syscall set(dsid);
@@ -30,4 +33,4 @@
     %trim(&&&variable)
 
   %end;
-%mend;
+%mend mf_getvalue;

base/mf_getvarcount.sas

@@ -2,31 +2,48 @@
   @file
   @brief Returns number of variables in a dataset
   @details Useful to identify those renagade datasets that have no columns!
+  Can also be used to count for numeric, or character columns
 
-        %put Number of Variables=%mf_getvarcount(sashelp.class);
+      %put Number of Variables=%mf_getvarcount(sashelp.class);
+      %put Character Variables=%mf_getvarcount(sashelp.class,typefilter=C);
+      %put Numeric Variables = %mf_getvarcount(sashelp.class,typefilter=N);
 
   returns:
   > Number of Variables=4
 
-  @param libds Two part dataset (or view) reference.
+
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] typefilter= (A) Filter for certain types of column.  Valid values:
+    @li A Count All columns
+    @li C Count Character columns only
+    @li N Count Numeric columns only
 
   @version 9.2
   @author Allan Bowe
 
 **/
 
-%macro mf_getvarcount(libds
+%macro mf_getvarcount(libds,typefilter=A
 )/*/STORE SOURCE*/;
-  %local dsid nvars rc ;
+  %local dsid nvars rc outcnt x;
   %let dsid=%sysfunc(open(&libds));
   %let nvars=.;
+  %let outcnt=0;
+  %let typefilter=%upcase(&typefilter);
   %if &dsid %then %do;
     %let nvars=%sysfunc(attrn(&dsid,NVARS));
+    %if &typefilter=A %then %let outcnt=&nvars;
+    %else %if &nvars>0 %then %do x=1 %to &nvars;
+      /* increment based on variable type */
+      %if %sysfunc(vartype(&dsid,&x))=&typefilter %then %do;
+        %let outcnt=%eval(&outcnt+1);
+      %end;
+    %end;
     %let rc=%sysfunc(close(&dsid));
   %end;
   %else %do;
     %put unable to open &libds (rc=&dsid);
     %let rc=%sysfunc(close(&dsid));
   %end;
-  &nvars
-%mend;
+  &outcnt
+%mend mf_getvarcount;

base/mf_getvarformat.sas

@@ -5,9 +5,9 @@
   Usage:
 
       data test;
-         format str1 $1.  num1 datetime19.;
-         str2='hello mum!'; num2=666;
-         stop;
+        format str1 $1.  num1 datetime19.;
+        str2='hello mum!'; num2=666;
+        stop;
       run;
       %put %mf_getVarFormat(test,str1);
       %put %mf_getVarFormat(work.test,num1);
@@ -23,9 +23,9 @@
       8.
       NOTE: Variable renegade does not exist in test
 
-  @param libds Two part dataset (or view) reference.
-  @param var Variable name for which a format should be returned
-  @param force Set to 1 to supply a default if the variable has no format
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] var Variable name for which a format should be returned
+  @param [in] force=(0) Set to 1 to supply a default if the variable has no format
   @returns outputs format
 
   @author Allan Bowe
@@ -45,13 +45,14 @@
     /* Get variable format */
     %if(&vnum > 0) %then %let vformat=%sysfunc(varfmt(&dsid, &vnum));
     %else %do;
-       %put NOTE: Variable &var does not exist in &libds;
-       %let rc = %sysfunc(close(&dsid));
-       %return;
+      %put NOTE: Variable &var does not exist in &libds;
+      %let rc = %sysfunc(close(&dsid));
+      %return;
     %end;
   %end;
   %else %do;
-    %put dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
     %return;
   %end;
 
@@ -60,7 +61,7 @@
     %let vlen = %sysfunc(varlen(&dsid, &vnum));
     %let vtype = %sysfunc(vartype(&dsid, &vnum.));
     %if &vtype=C %then %let vformat=$&vlen..;
-    %else %let vformat=8.;
+    %else %let vformat=best.;
   %end;
 
 
@@ -68,4 +69,4 @@
   %let rc = %sysfunc(close(&dsid));
   /* Return variable format */
   &vformat
-%mend;
+%mend mf_getVarFormat;

base/mf_getvarlen.sas

@@ -5,8 +5,8 @@
   Usage:
 
       data test;
-         format str $1.  num datetime19.;
-         stop;
+        format str $1.  num datetime19.;
+        stop;
       run;
       %put %mf_getVarLen(test,str);
       %put %mf_getVarLen(work.test,num);
@@ -39,14 +39,18 @@
     /* Get variable format */
     %if(&vnum > 0) %then %let vlen = %sysfunc(varlen(&dsid, &vnum));
     %else %do;
-       %put NOTE: Variable &var does not exist in &libds;
-       %let vlen = %str( );
+      %put NOTE: Variable &var does not exist in &libds;
+      %let vlen = %str( );
     %end;
   %end;
-  %else %put dataset &libds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));
   /* Return variable format */
   &vlen
-%mend;
+%mend mf_getVarLen;

base/mf_getvarlist.sas

@@ -10,14 +10,21 @@
   returns:
   > List of Variables=Name Sex Age Height Weight
 
+  For a seperated list of column values:
+
         %put %mf_getvarlist(sashelp.class,dlm=%str(,),quote=double);
 
   returns:
   > "Name","Sex","Age","Height","Weight"
 
-  @param libds Two part dataset (or view) reference.
-  @param dlm= provide a delimiter (eg comma or space) to separate the vars
-  @param quote= use either DOUBLE or SINGLE to quote the results
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] dlm= ( ) Provide a delimiter (eg comma or space) to separate the
+    variables
+  @param [in] quote= (none) use either DOUBLE or SINGLE to quote the results
+  @param [in] typefilter= (A) Filter for certain types of column.  Valid values:
+    @li A Return All columns
+    @li C Return Character columns
+    @li N Return Numeric columns
 
   @version 9.2
   @author Allan Bowe
@@ -27,9 +34,10 @@
 %macro mf_getvarlist(libds
       ,dlm=%str( )
       ,quote=no
+      ,typefilter=A
 )/*/STORE SOURCE*/;
   /* declare local vars */
-  %local outvar dsid nvars x rc dlm q var;
+  %local outvar dsid nvars x rc dlm q var vtype;
 
   /* credit Rowland Hale  - byte34 is double quote, 39 is single quote */
   %if %upcase(&quote)=DOUBLE %then %let q=%qsysfunc(byte(34));
@@ -37,28 +45,30 @@
   /* open dataset in macro */
   %let dsid=%sysfunc(open(&libds));
 
-
   %if &dsid %then %do;
     %let nvars=%sysfunc(attrn(&dsid,NVARS));
     %if &nvars>0 %then %do;
-      /* add first dataset variable to global macro variable */
-      %let outvar=&q.%sysfunc(varname(&dsid,1))&q.;
-      /* add remaining variables with supplied delimeter */
+      /* add variables with supplied delimeter */
       %do x=1 %to &nvars;
-        %let var=&q.%sysfunc(varname(&dsid,&x))&q.;
-        %if &var=&q&q %then %do;
-          %put &sysmacroname: Empty column found in &libds!;
-          %let var=&q. &q.;
+        /* get variable type */
+        %let vtype=%sysfunc(vartype(&dsid,&x));
+        %if &vtype=&typefilter or &typefilter=A %then %do;
+          %let var=&q.%sysfunc(varname(&dsid,&x))&q.;
+          %if &var=&q&q %then %do;
+            %put &sysmacroname: Empty column found in &libds!;
+            %let var=&q. &q.;
+          %end;
+          %if %quote(&outvar)=%quote() %then %let outvar=&var;
+          %else %let outvar=&outvar.&dlm.&var.;
         %end;
-        %if &x=1 %then %let outvar=&var;
-        %else %let outvar=&outvar.&dlm.&var.;
       %end;
     %end;
     %let rc=%sysfunc(close(&dsid));
   %end;
   %else %do;
-    %put unable to open &libds (rc=&dsid);
+    %put &sysmacroname: Unable to open &libds (rc=&dsid);
+    %put &sysmacroname: SYSMSG= %sysfunc(sysmsg());
     %let rc=%sysfunc(close(&dsid));
   %end;
-  &outvar
-%mend;
+  %do;%unquote(&outvar)%end;
+%mend mf_getvarlist;

base/mf_getvarnum.sas

@@ -6,8 +6,8 @@
 Usage:
 
     data work.test;
-       format str $1.  num datetime19.;
-       stop;
+      format str $1.  num datetime19.;
+      stop;
     run;
     %put %mf_getVarNum(work.test,str);
     %put %mf_getVarNum(work.test,num);
@@ -39,11 +39,15 @@ returns:
     /* Get variable number */
     %let vnum = %sysfunc(varnum(&dsid, &var));
     %if(&vnum <= 0) %then %do;
-       %put NOTE: Variable &var does not exist in &libds;
-       %let vnum = %str( );
+      %put NOTE: Variable &var does not exist in &libds;
+      %let vnum = %str( );
     %end;
   %end;
-  %else %put dataset &ds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));
@@ -51,4 +55,4 @@ returns:
   /* Return variable number */
     &vnum.
 
-%mend;
+%mend mf_getVarNum;

base/mf_getvartype.sas

@@ -5,8 +5,8 @@
 Usage:
 
       data test;
-         length str $1.  num 8.;
-         stop;
+        length str $1.  num 8.;
+        stop;
       run;
       %put %mf_getvartype(test,str);
       %put %mf_getvartype(work.test,num);
@@ -35,14 +35,18 @@ Usage:
     /* Get variable type (C/N) */
     %if(&vnum. > 0) %then %let vtype = %sysfunc(vartype(&dsid, &vnum.));
     %else %do;
-       %put NOTE: Variable &var does not exist in &libds;
-       %let vtype = %str( );
+      %put NOTE: Variable &var does not exist in &libds;
+      %let vtype = %str( );
     %end;
   %end;
-  %else %put dataset &libds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));
   /* Return variable type */
   &vtype
-%mend;
+%mend mf_getvartype;

base/mf_getxengine.sas

@@ -0,0 +1,43 @@
+/**
+  @file
+  @brief Returns the engine type of a SAS fileref
+  @details Queries sashelp.vextfl to get the xengine value.
+  Usage:
+
+      filename feng temp;
+      %put %mf_getxengine(feng);
+
+  returns:
+  > TEMP
+
+  @param fref The fileref to check
+
+  @returns The XENGINE value in sashelp.vextfl or 0 if not found.
+
+  @version 9.2
+  @author Allan Bowe
+
+  <h4> Related Macros </h4>
+  @li mf_getengine.sas
+
+**/
+
+%macro mf_getxengine(fref
+)/*/STORE SOURCE*/;
+  %local dsid engnum rc engine;
+
+  %let dsid=%sysfunc(
+    open(sashelp.vextfl(where=(fileref="%upcase(&fref)")),i)
+  );
+  %if (&dsid ^= 0) %then %do;
+    %let engnum=%sysfunc(varnum(&dsid,XENGINE));
+    %let rc=%sysfunc(fetch(&dsid));
+    %let engine=%sysfunc(getvarc(&dsid,&engnum));
+    %* put &fref. ENGINE is &engine.;
+    %let rc= %sysfunc(close(&dsid));
+  %end;
+  %else %let engine=0;
+
+  &engine
+
+%mend mf_getxengine;

base/mf_increment.sas

@@ -0,0 +1,29 @@
+/**
+  @file
+  @brief Increments a macro variable
+  @details Useful outside of do-loops - will increment a macro variable every
+  time it is called.
+
+  Example:
+
+      %let cnt=1;
+      %put We have run %mf_increment(cnt) lines;
+      %put Now we have run %mf_increment(cnt) lines;
+      %put There are %mf_increment(cnt) lines in total;
+
+  @param [in] MACRO_NAME the name of the macro variable to increment
+  @param [in] ITER= The amount to add or subtract to the macro
+
+  <h4> Related Files </h4>
+  @li mf_increment.test.sas
+
+**/
+
+%macro mf_increment(macro_name,incr=1);
+
+  /* iterate the value */
+  %let &macro_name=%eval(&&&macro_name+&incr);
+  /* return the value */
+  &&&macro_name
+
+%mend mf_increment;

base/mf_isblank.sas

@@ -6,12 +6,13 @@
       %sysevalf(%superq(param)=,boolean)
 
   Usage:
-     
-     %put mf_isblank(&var);
 
-  inspiration:  https://support.sas.com/resources/papers/proceedings09/022-2009.pdf
+      %put mf_isblank(&var);
 
-  @param param VALUE to be checked 
+  inspiration:
+  https://support.sas.com/resources/papers/proceedings09/022-2009.pdf
+
+  @param param VALUE to be checked
 
   @return output returns 1 (if blank) else 0
 
@@ -23,4 +24,4 @@
 
   %sysevalf(%superq(param)=,boolean)
 
-%mend;
+%mend mf_isblank;

base/mf_isdir.sas

@@ -0,0 +1,34 @@
+/**
+  @file
+  @brief Checks whether a path is a valid directory
+  @details
+  Usage:
+
+      %let isdir=%mf_isdir(/tmp);
+
+  With thanks and full credit to Andrea Defronzo -
+  https://www.linkedin.com/in/andrea-defronzo-b1a47460/
+
+  @param path full path of the file/directory to be checked
+
+  @return output returns 1 if path is a directory, 0 if it is not
+
+  @version 9.2
+**/
+
+%macro mf_isdir(path
+)/*/STORE SOURCE*/;
+  %local rc did is_directory fref_t;
+
+  %let is_directory = 0;
+  %let rc = %sysfunc(filename(fref_t, %superq(path)));
+  %let did = %sysfunc(dopen(&fref_t.));
+  %if &did. ^= 0 %then %do;
+    %let is_directory = 1;
+    %let rc = %sysfunc(dclose(&did.));
+  %end;
+  %let rc = %sysfunc(filename(fref_t));
+
+  &is_directory
+
+%mend mf_isdir;

base/mf_isint.sas

@@ -0,0 +1,36 @@
+/**
+  @file
+  @brief Returns 1 if the variable contains only digits 0-9, else 0
+  @details Note that numerics containing any punctuation (including decimals
+    or exponents) will be flagged zero.
+
+  If you'd like support for this, then do raise an issue (or even better, a
+  pull request!)
+
+  Usage:
+
+      %put %mf_isint(1) returns 1;
+      %put %mf_isint(1.1) returns 0;
+      %put %mf_isint(%str(1,1)) returns 0;
+
+  @param [in] arg input value to check
+
+  @version 9.2
+**/
+
+%macro mf_isint(arg
+)/*/STORE SOURCE*/;
+
+  /* blank val is not an integer */
+  %if "&arg"="" %then %do;0%return;%end;
+
+  /* remove minus sign if exists */
+  %local val;
+  %if "%substr(%str(&arg),1,1)"="-" %then %let val=%substr(%str(&arg),2);
+  %else %let val=&arg;
+
+  /* check remaining chars */
+  %if %sysfunc(findc(%str(&val),,kd)) %then %do;0%end;
+  %else %do;1%end;
+
+%mend mf_isint;

base/mf_islibds.sas

@@ -0,0 +1,40 @@
+/**
+  @file
+  @brief Checks whether a string follows correct library.dataset format
+  @details Many macros in the core library accept a library.dataset parameter
+  referred to as 'libds'.  This macro validates the structure of that parameter,
+  eg:
+
+    @li 8 character libref?
+    @li 32 character dataset?
+    @li contains a period?
+
+  It does NOT check whether the dataset exists, or if the library is assigned.
+
+  Usage:
+
+      %put %mf_islibds(work.something)=1;
+      %put %mf_islibds(nolib)=0;
+      %put %mf_islibds(badlibref.ds)=0;
+      %put %mf_islibds(w.t.f)=0;
+
+  @param [in] libds The string to be checked
+
+  @return output Returns 1 if libds is valid, 0 if it is not
+
+  <h4> Related Macros </h4>
+  @li mf_islibds.test.sas
+  @li mp_validatecol.sas
+
+  @version 9.2
+**/
+
+%macro mf_islibds(libds
+)/*/STORE SOURCE*/;
+
+%local regex;
+%let regex=%sysfunc(prxparse(%str(/^[_a-z]\w{0,7}\.[_a-z]\w{0,31}$/i)));
+
+%sysfunc(prxmatch(&regex,&libds))
+
+%mend mf_islibds;

base/mf_loc.sas

@@ -26,4 +26,4 @@
   &root
 %end;
 
-%mend;
+%mend mf_loc;

base/mf_mkdir.sas

@@ -46,22 +46,22 @@ Usage:
     */
 
     %if (%length(&dir) gt %length(&child)) %then %do;
-       %let parent = %substr(&dir, 1, %length(&dir)-%length(&child));
-       %mf_mkdir(&parent)
+      %let parent = %substr(&dir, 1, %length(&dir)-%length(&child));
+      %mf_mkdir(&parent)
     %end;
 
     /*
-      Now create the directory.  Complain loudly of any errors.
+      Now create the directory.  Complain loudly of any errs.
     */
 
     %let dname = %sysfunc(dcreate(&child, &parent));
     %if (%bquote(&dname) eq ) %then %do;
-       %put %str(ERR)OR: could not create &parent + &child;
-       %abort cancel;
+      %put %str(ERR)OR: could not create &parent + &child;
+      %abort cancel;
     %end;
     %else %do;
-       %put Directory created:  &dir;
+      %put Directory created:  &dir;
     %end;
   %end;
   /* exit quietly if directory did exist.*/
-%mend;
+%mend mf_mkdir;

base/mf_mval.sas

@@ -1,8 +1,9 @@
 /**
   @file mf_mval.sas
   @brief Returns a macro variable value if the variable exists
-  @details Use this macro to avoid repetitive use of `%if %symexist(MACVAR) %then`
-  type logic.  
+  @details
+  Use this macro to avoid repetitive use of `%if %symexist(MACVAR) %then`
+  type logic.
   Usage:
 
       %if %mf_mval(maynotexist)=itdid %then %do;
@@ -15,4 +16,4 @@
   %if %symexist(&var) %then %do;
     %superq(&var)
   %end;
-%mend;
+%mend mf_mval;

base/mf_nobs.sas

@@ -6,13 +6,13 @@
 
         %put Number of observations=%mf_nobs(sashelp.class);
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_getattrn.sas
 
   @param libds library.dataset
 
   @return output returns result of the attrn value supplied, or log message
-    if error.
+    if err.
 
 
   @version 9.2
@@ -23,4 +23,4 @@
 %macro mf_nobs(libds
 )/*/STORE SOURCE*/;
   %mf_getattrn(&libds,NLOBS)
-%mend;
+%mend mf_nobs;

base/mf_trimstr.sas

@@ -0,0 +1,50 @@
+/**
+  @file mf_trimstr.sas
+  @brief Removes character(s) from the end, if they exist
+  @details If the designated characters exist at the end of the string, they
+  are removed
+
+        %put %mf_trimstr(/blah/,/); * /blah;
+        %put %mf_trimstr(/blah/,h); * /blah/;
+        %put %mf_trimstr(/blah/,h/);* /bla;
+
+  <h4> SAS Macros </h4>
+
+
+  @param basestr The string to be modified
+  @param trimstr The string to be removed from the end of `basestr`, if it
+    exists
+
+  @return output returns result with the value of `trimstr` removed from the end
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_trimstr(basestr,trimstr);
+%local baselen trimlen trimval;
+
+/* return if basestr is shorter than trimstr (or 0) */
+%let baselen=%length(%superq(basestr));
+%let trimlen=%length(%superq(trimstr));
+%if &baselen < &trimlen or &baselen=0 %then %return;
+
+/* obtain the characters from the end of basestr */
+%let trimval=%qsubstr(%superq(basestr)
+  ,%length(%superq(basestr))-&trimlen+1
+  ,&trimlen);
+
+/* compare and if matching, chop it off! */
+%if %superq(basestr)=%superq(trimstr) %then %do;
+  %return;
+%end;
+%else %if %superq(trimval)=%superq(trimstr) %then %do;
+  %qsubstr(%superq(basestr),1,%length(%superq(basestr))-&trimlen)
+%end;
+%else %do;
+  &basestr
+%end;
+
+%mend mf_trimstr;

base/mf_uid.sas

@@ -1,11 +1,11 @@
 /**
   @file
-  @brief Creates a Unique ID based on system time in a friendly format
+  @brief Creates a unique ID based on system time in friendly format
   @details format = YYYYMMDD_HHMMSSmmm_<sysjobid>_<3randomDigits>
 
         %put %mf_uid();
 
-  @version 9.2
+  @version 9.3
   @author Allan Bowe
 
 **/
@@ -14,8 +14,8 @@
 )/*/STORE SOURCE*/;
   %local today now;
   %let today=%sysfunc(today(),yymmddn8.);
-  %let now=%sysfunc(compress(%sysfunc(time(),time12.3),:.));
+  %let now=%sysfunc(compress(%sysfunc(time(),tod12.3),:.));
 
   &today._&now._&sysjobid._%sysevalf(%sysfunc(ranuni(0))*999,CEIL)
 
-%mend;
+%mend mf_uid;

base/mf_verifymacvars.sas

@@ -1,6 +1,6 @@
 /**
   @file
-  @brief Checks if a set of macro variables exist / contain values.
+  @brief Checks if a set of macro variables exist AND contain values.
   @details Writes ERROR to log if abortType is SOFT, else will call %mf_abort.
   Usage:
 
@@ -11,13 +11,14 @@
   Returns:
   > 1
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_abort.sas
 
-  @param verifyvars space separated list of macro variable names
-  @param makeupcase= set to YES to convert all variable VALUES to
+  @param [in] verifyvars Space separated list of macro variable names
+  @param [in] makeupcase= (NO) Set to YES to convert all variable VALUES to
     uppercase.
-  @param mAbort= Abort Type.  Default is SOFT (writes err to log).
+  @param [in] mAbort= (SOFT) Abort Type.  When SOFT, simply writes an err
+    message to the log.
     Set to any other value to call mf_abort (which can be configured to abort in
     various fashions according to context).
 
@@ -35,7 +36,7 @@
 
 
 %macro mf_verifymacvars(
-     verifyVars  /* list of macro variable NAMES */
+    verifyVars  /* list of macro variable NAMES */
     ,makeUpcase=NO  /* set to YES to make all the variable VALUES uppercase */
     ,mAbort=SOFT
 )/*/STORE SOURCE*/;
@@ -58,8 +59,14 @@
 
   %goto exit_success;
   %exit_err:
-    %if &mAbort=SOFT %then %put %str(ERR)OR: &abortmsg;
-    %else %mf_abort(mac=mf_verifymacvars,type=&mabort,msg=&abortmsg);
+    %put &abortmsg;
+    %mf_abort(iftrue=(&mabort ne SOFT),
+      mac=mf_verifymacvars,
+      msg=%str(&abortmsg)
+    )
+    0
+    %return;
   %exit_success:
+  1
 
-%mend;
+%mend mf_verifymacvars;

base/mf_wordsinstr1andstr2.sas

@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Returns words that are in both string 1 and string 2
+  @details  Compares two space separated strings and returns the words that are
+  in both.
+  Usage:
+
+      %put %mf_wordsInStr1andStr2(
+        Str1=blah sss blaaah brah bram boo
+        ,Str2=   blah blaaah brah ssss
+      );
+
+  returns:
+  > blah blaaah brah
+
+  @param str1= string containing words to extract
+  @param str2= used to compare with the extract string
+
+  @warning CASE SENSITIVE!
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_wordsInStr1andStr2(
+  Str1= /* string containing words to extract */
+  ,Str2= /* used to compare with the extract string */
+)/*/STORE SOURCE*/;
+
+%local count_base count_extr i i2 extr_word base_word match outvar;
+%if %length(&str1)=0 or %length(&str2)=0 %then %do;
+  %put base string (str1)= &str1;
+  %put compare string (str2) = &str2;
+  %return;
+%end;
+%let count_base=%sysfunc(countw(&Str2));
+%let count_extr=%sysfunc(countw(&Str1));
+
+%do i=1 %to &count_extr;
+  %let extr_word=%scan(&Str1,&i,%str( ));
+  %let match=0;
+  %do i2=1 %to &count_base;
+    %let base_word=%scan(&Str2,&i2,%str( ));
+    %if &extr_word=&base_word %then %let match=1;
+  %end;
+  %if &match=1 %then %let outvar=&outvar &extr_word;
+%end;
+
+  &outvar
+
+%mend mf_wordsInStr1andStr2;
+

base/mf_wordsinstr1butnotstr2.sas

@@ -3,20 +3,21 @@
   @brief Returns words that are in string 1 but not in string 2
   @details  Compares two space separated strings and returns the words that are
   in the first but not in the second.
+
+  Note - case sensitive!
+
   Usage:
 
       %let x= %mf_wordsInStr1ButNotStr2(
-         Str1=blah sss blaaah brah bram boo
+        Str1=blah sss blaaah brah bram boo
         ,Str2=   blah blaaah brah ssss
       );
 
   returns:
   > sss bram boo
 
-  @param str1= string containing words to extract
-  @param str2= used to compare with the extract string
-
-  @warning CASE SENSITIVE!
+  @param [in] str1= string containing words to extract
+  @param [in] str2= used to compare with the extract string
 
   @version 9.2
   @author Allan Bowe
@@ -24,13 +25,12 @@
 **/
 
 %macro mf_wordsInStr1ButNotStr2(
-    Str1= /* string containing words to extract */
-   ,Str2= /* used to compare with the extract string */
+  Str1= /* string containing words to extract */
+  ,Str2= /* used to compare with the extract string */
 )/*/STORE SOURCE*/;
 
 %local count_base count_extr i i2 extr_word base_word match outvar;
 %if %length(&str1)=0 or %length(&str2)=0 %then %do;
-  %put WARNING: empty string provided!;
   %put base string (str1)= &str1;
   %put compare string (str2) = &str2;
   %return;
@@ -50,5 +50,5 @@
 
   &outvar
 
-%mend;
+%mend mf_wordsInStr1ButNotStr2;
 

base/mf_writefile.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Creates a text file using pure macro
+  @details Creates a text file of up to 10 lines.  If further lines are
+    desired, feel free to [create an issue](
+    https://github.com/sasjs/core/issues/new), or make a pull request!
+
+    The use of PARMBUFF was considered for this macro, but it would have made
+    things problematic for writing lines containing commas.
+
+    Usage:
+
+        %mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=more content)
+        data _null_;
+          infile "&sasjswork/myfile.txt";
+          input;
+          list;
+        run;
+
+  @param [in] fpath Full path to file to be created or appended to
+  @param [in] mode= (O) Available options are A or O as follows:
+    @li A APPEND mode, writes new records after the current end of the file.
+    @li O OUTPUT mode, writes new records from the beginning of the file.
+  @param [in] l1= First line
+  @param [in] l2= Second line (etc through to l10)
+
+  <h4> Related Macros </h4>
+  @li mf_writefile.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mf_writefile(fpath,mode=O,l1=,l2=,l3=,l4=,l5=,l6=,l7=,l8=,l9=,l10=
+)/*/STORE SOURCE*/;
+%local fref rc fid i total_lines;
+
+/* find number of lines by reference to first non-blank param */
+%do i=10 %to 1 %by -1;
+  %if %str(&&l&i) ne %str() %then %goto continue;
+%end;
+%continue:
+%let total_lines=&i;
+
+%if %sysfunc(filename(fref,&fpath)) ne 0 %then %do;
+  %put &=fref &=fpath;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let fid=%sysfunc(fopen(&fref,&mode));
+
+%if &fid=0 %then %do;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%do i=1 %to &total_lines;
+  %let rc=%sysfunc(fput(&fid, &&l&i));
+  %let rc=%sysfunc(fwrite(&fid));
+%end;
+%let rc=%sysfunc(fclose(&fid));
+%let rc=%sysfunc(filename(&fref));
+
+%mend mf_writefile;
+/** @endcond */

base/mp_abort.sas

@@ -4,137 +4,298 @@
   @details Configures an abort mechanism according to site specific policies or
     the particulars of an environment.  For instance, can stream custom
     results back to the client in an STP Web App context, or completely stop
-    in the case of a batch run.
+    in the case of a batch run.  For STP sessions
 
-  @param mac= to contain the name of the calling macro
+  The method used varies according to the context.  Important points:
+
+  @li should not use endsas or abort cancel in 9.4m3 WIN environments as this
+    can cause hung multibridge sessions and result in a frozen STP server
+  @li The use of endsas in 9.4m6+ windows environments for POST requests to the
+    STP server can result in an empty response body
+  @li should not use endsas in viya 3.5 as this destroys the session and cannot
+    fetch results (although both mv_getjoblog.sas and the @sasjs/adapter will
+    recognise this and fetch the log of the parent session instead)
+  @li STP environments must finish cleanly to avoid the log being sent to
+    _webout.  To assist with this, we also run stpsrvset('program error', 0)
+    and set SYSCC=0.
+    Where possible, we take a unique "soft abort" approach - we open a macro
+    but don't close it!  This works everywhere EXCEPT inside a \%include inside
+    a macro.  For that, we recommend you use mp_include.sas to perform the
+    include, and then call \%mp_abort(mode=INCLUDE) from the source program (ie,
+    OUTSIDE of the top-parent macro).
+    The soft abort has become ineffective in 9.4m6 WINDOWS environments.  We are
+    currently investigating approaches to deal with this.
+
+
+  @param mac= (mp_abort.sas) To contain the name of the calling macro. Do not
+    use &sysmacroname as this will always resolve to MP_ABORT.
   @param msg= message to be returned
-  @param iftrue= supply a condition under which the macro should be executed.
+  @param iftrue= (1=1) Supply a condition for which the macro should be executed
+  @param errds= (work.mp_abort_errds) There is no clean way to end a process
+    within a %include called within a macro.  Furthermore, there is no way to
+    test if a macro is called within a %include.  To handle this particular
+    scenario, the %include should be switched for the mp_include.sas macro.
+    This provides an indicator that we are running a macro within a \%include
+    (`_SYSINCLUDEFILEDEVICE`) and allows us to provide a dataset with the abort
+    values (msg, mac).
+    We can then run an abort cancel FILE to stop the include running, and pass
+    the dataset back to the calling program to run a regular \%mp_abort().
+    The dataset will contain the following fields:
+    @li iftrue (1=1)
+    @li msg (the message)
+    @li mac (the mac param)
 
-  @version 9.4M3
+  @param mode= (REGULAR) If mode=INCLUDE then the &errds dataset is checked for
+    an abort status.
+    Valid values:
+    @li REGULAR (default)
+    @li INCLUDE
+
+  @version 9.4
   @author Allan Bowe
+
+  <h4> Related Macros </h4>
+  @li mp_include.sas
+
+  @cond
 **/
 
 %macro mp_abort(mac=mp_abort.sas, type=, msg=, iftrue=%str(1=1)
+  , errds=work.mp_abort_errds
+  , mode=REGULAR
 )/*/STORE SOURCE*/;
 
-  %if not(%eval(%unquote(&iftrue))) %then %return;
+%global sysprocessmode sysprocessname sasjs_stpsrv_header_loc sasjsprocessmode;
+%local fref fid i;
 
-  %put NOTE: ///  mp_abort macro executing //;
-  %if %length(&mac)>0 %then %put NOTE- called by &mac;
-  %put NOTE - &msg;
+%if not(%eval(%unquote(&iftrue))) %then %return;
 
-  /* Stored Process Server web app context */
-  %if %symexist(_metaperson) 
-  or (%symexist(SYSPROCESSNAME) and "&SYSPROCESSNAME"="Compute Server" )
+%put NOTE: ///  mp_abort macro executing //;
+%if %length(&mac)>0 %then %put NOTE- called by &mac;
+%put NOTE - &msg;
+
+%if %symexist(_SYSINCLUDEFILEDEVICE)
+/* abort cancel FILE does not restart outside the INCLUDE on Viya 3.5 */
+and %superq(SYSPROCESSNAME) ne %str(Compute Server)
+%then %do;
+  %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do;
+    data &errds;
+      iftrue='1=1';
+      length mac $100 msg $5000;
+      mac=symget('mac');
+      msg=symget('msg');
+    run;
+    data _null_;
+      abort cancel FILE;
+    run;
+    %return;
+  %end;
+%end;
+
+/* Web App Context */
+%if %symexist(_PROGRAM)
+  or %superq(SYSPROCESSNAME) = %str(Compute Server)
+  or &mode=INCLUDE
+%then %do;
+  options obs=max replace mprint;
+  %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5"
   %then %do;
-    options obs=max replace nosyntaxcheck mprint;
-    /* extract log errs / warns, if exist */
-    %local logloc logline;
-    %global logmsg; /* capture global messages */
-    %if %symexist(SYSPRINTTOLOG) %then %let logloc=&SYSPRINTTOLOG;
-    %else %let logloc=%qsysfunc(getoption(LOG));
-    proc printto log=log;run;
-    %if %length(&logloc)>0 %then %do;
-      %let logline=0;
+    options nosyntaxcheck;
+  %end;
+
+  %if &mode=INCLUDE %then %do;
+    %if %sysfunc(exist(&errds))=1 %then %do;
+      data _null_;
+        set &errds;
+        call symputx('iftrue',iftrue,'l');
+        call symputx('mac',mac,'l');
+        call symputx('msg',msg,'l');
+        putlog (_all_)(=);
+      run;
+      %if (&iftrue)=0 %then %return;
+    %end;
+    %else %do;
+      %put &sysmacroname: No include errors found;
+      %return;
+    %end;
+  %end;
+
+  /* extract log errs / warns, if exist */
+  %local logloc logline;
+  %global logmsg; /* capture global messages */
+  %if %symexist(SYSPRINTTOLOG) %then %let logloc=&SYSPRINTTOLOG;
+  %else %let logloc=%qsysfunc(getoption(LOG));
+  proc printto log=log;run;
+  %let logline=0;
+  %if %length(&logloc)>0 %then %do;
+    data _null_;
+      infile &logloc lrecl=5000;
+      input; putlog _infile_;
+      i=1;
+      retain logonce 0;
+      if (
+          _infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR"
+        ) and logonce=0 then
+      do;
+        call symputx('logline',_n_);
+        logonce+1;
+      end;
+    run;
+    /* capture log including lines BEFORE the err */
+    %if &logline>0 %then %do;
       data _null_;
         infile &logloc lrecl=5000;
-        input; putlog _infile_;
+        input;
         i=1;
-        retain logonce 0;
-        if (_infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR") and logonce=0 then do;
-          call symputx('logline',_n_);
-          logonce+1;
-        end;
-      run;
-      /* capture log including lines BEFORE the err */
-      %if &logline>0 %then %do;
-        data _null_;
-          infile &logloc lrecl=5000;
+        stoploop=0;
+        if _n_ ge &logline-15 and stoploop=0 then do until (i>22);
+          call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
           input;
-          i=1;
-          stoploop=0;
-          if _n_ ge &logline-5 and stoploop=0 then do until (i>12);
-            call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
-            input;
-            i+1;
-            stoploop=1;
-          end;
-          if stoploop=1 then stop;
-        run;
-      %end;
-    %end;
-
-    %if %symexist(SYS_JES_JOB_URI) %then %do;
-      /* refer web service output to file service in one hit */
-      filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json";
-    %end;
-
-    /* send response in SASjs JSON format */
-    data _null_;
-      file _webout mod lrecl=32000;
-      length msg $32767 debug $8;
-      sasdatetime=datetime();
-      msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
-      /* escape the quotes */
-      msg=tranwrd(msg,'"','\"');
-      /* ditch the CRLFs as chrome complains */
-      msg=compress(msg,,'kw');
-      /* quote without quoting the quotes (which are escaped instead) */
-      msg=cats('"',msg,'"');
-      if symexist('_debug') then debug=quote(trim(symget('_debug')));
-      else debug='""';
-      if debug ge '"131"' then put '>>weboutBEGIN<<';
-      put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
-      put ',"sasjsAbort" : [{';
-      put ' "MSG":' msg ;
-      put ' ,"MAC": "' "&mac" '"}]';
-      put ",""SYSUSERID"" : ""&sysuserid"" ";
-      put ',"_DEBUG":' debug ;
-      if symexist('_metauser') then do;
-        _METAUSER=quote(trim(symget('_METAUSER')));
-        put ",""_METAUSER"": " _METAUSER;
-        _METAPERSON=quote(trim(symget('_METAPERSON')));
-        put ',"_METAPERSON": ' _METAPERSON;
-      end;
-      if symexist('SYS_JES_JOB_URI') then do;
-        SYS_JES_JOB_URI=quote(trim(symget('SYS_JES_JOB_URI')));
-        put ',"SYS_JES_JOB_URI": ' SYS_JES_JOB_URI;
-      end;
-      _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
-      put ',"_PROGRAM" : ' _PROGRAM ;
-      put ",""SYSCC"" : ""&syscc"" ";
-      put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
-      put ",""SYSJOBID"" : ""&sysjobid"" ";
-      put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
-      put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
-      put "}" @;
-      if debug ge '"131"' then put '>>weboutEND<<';
-    run;
-
-    %let syscc=0;
-    %if %symexist(_metaport) %then %do;
-      data _null_;
-        if symexist('sysprocessmode')
-         then if symget("sysprocessmode")="SAS Stored Process Server"
-          then rc=stpsrvset('program error', 0);
+          i+1;
+          stoploop=1;
+        end;
+        if stoploop=1 then stop;
       run;
     %end;
-    /**
-     * endsas is reliable but kills some deployments.
-     * Abort variants are ungraceful (non zero return code)
-     * This approach lets SAS run silently until the end :-)
-     */
-    %put _all_;
-    filename skip temp;
+  %end;
+
+  %if %symexist(SYS_JES_JOB_URI) %then %do;
+    /* setup webout for Viya */
+    options nobomfile;
+    %if "X&SYS_JES_JOB_URI.X"="XX" %then %do;
+        filename _webout temp lrecl=999999 mod;
+    %end;
+    %else %do;
+      filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
+        name="_webout.json" lrecl=999999 mod;
+    %end;
+  %end;
+  %else %if %sysfunc(filename(fref,&sasjs_stpsrv_header_loc))=0 %then %do;
+    options nobomfile;
+    /* set up http header for SASjs Server */
+    %let fid=%sysfunc(fopen(&fref,A));
+    %if &fid=0 %then %do;
+      %put %str(ERR)OR: %sysfunc(sysmsg());
+      %return;
+    %end;
+    %let rc=%sysfunc(fput(&fid,%str(Content-Type: application/json)));
+    %let rc=%sysfunc(fwrite(&fid));
+    %let rc=%sysfunc(fclose(&fid));
+    %let rc=%sysfunc(filename(&fref));
+  %end;
+
+  /* send response in SASjs JSON format */
+  data _null_;
+    file _webout mod lrecl=32000 encoding='utf-8';
+    length msg syswarningtext syserrortext $32767 mode $10 ;
+    sasdatetime=datetime();
+    msg=symget('msg');
+  %if &logline>0 %then %do;
+    msg=cats(msg,'\n\nLog Extract:\n',symget('logmsg'));
+  %end;
+    /* escape the escapes */
+    msg=tranwrd(msg,'\','\\');
+    /* escape the quotes */
+    msg=tranwrd(msg,'"','\"');
+    /* ditch the CRLFs as chrome complains */
+    msg=compress(msg,,'kw');
+    /* quote without quoting the quotes (which are escaped instead) */
+    msg=cats('"',msg,'"');
+    if symexist('_debug') then debug=quote(trim(symget('_debug')));
+    else debug='""';
+    if symget('sasjsprocessmode')='Stored Program' then mode='SASJS';
+    if mode ne 'SASJS' then put '>>weboutBEGIN<<';
+    put '{"SYSDATE" : "' "&SYSDATE" '"';
+    put ',"SYSTIME" : "' "&SYSTIME" '"';
+    put ',"sasjsAbort" : [{';
+    put ' "MSG":' msg ;
+    put ' ,"MAC": "' "&mac" '"}]';
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
+    put ',"_DEBUG":' debug ;
+    if symexist('_metauser') then do;
+      _METAUSER=quote(trim(symget('_METAUSER')));
+      put ",""_METAUSER"": " _METAUSER;
+      _METAPERSON=quote(trim(symget('_METAPERSON')));
+      put ',"_METAPERSON": ' _METAPERSON;
+    end;
+    if symexist('SYS_JES_JOB_URI') then do;
+      SYS_JES_JOB_URI=quote(trim(symget('SYS_JES_JOB_URI')));
+      put ',"SYS_JES_JOB_URI": ' SYS_JES_JOB_URI;
+    end;
+    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
+    put ',"_PROGRAM" : ' _PROGRAM ;
+    put ",""SYSCC"" : ""&syscc"" ";
+    syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
+    put ",""SYSERRORTEXT"" : " syserrortext;
+    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSJOBID"" : ""&sysjobid"" ";
+    put ",""SYSSCPL"" : ""&sysscpl"" ";
+    put ",""SYSSITE"" : ""&syssite"" ";
+    sysvlong=quote(trim(symget('sysvlong')));
+    put ',"SYSVLONG" : ' sysvlong;
+    syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');
+    put ",""SYSWARNINGTEXT"" : " syswarningtext;
+    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
+    put "}" ;
+    if mode ne 'SASJS' then put '>>weboutEND<<';
+  run;
+
+  %put _all_;
+
+  %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
     data _null_;
-      file skip;
-      put '%macro skip(); %macro skippy();';
+      putlog 'stpsrvset program err and syscc';
+      rc=stpsrvset('program error', 0);
+      call symputx("syscc",0,"g");
+    run;
+    %if &sysscp=WIN
+    and 1=0 /* deprecating this logic until we figure out a consistent abort */
+    and "%substr(%str(&sysvlong         ),1,8)"="9.04.01M"
+    and "%substr(%str(&sysvlong         ),9,1)">"5" %then %do;
+      /* skip approach (below) does not work in windows m6+ envs */
+      endsas;
+    %end;
+    %else %do;
+      /**
+        * endsas kills 9.4m3 deployments by orphaning multibridges.
+        * Abort variants are ungraceful (non zero return code)
+        * This approach lets SAS run silently until the end :-)
+        * Caution - fails when called within a %include within a macro
+        * Use mp_include() to handle this.
+        */
+      filename skip temp;
+      data _null_;
+        file skip;
+        put '%macro skip();';
+        comment '%mend skip; -> fix lint ';
+        put '%macro skippy();';
+        comment '%mend skippy; -> fix lint ';
+      run;
+      %inc skip;
+    %end;
+  %end;
+  %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
+    /* endsas kills the session making it harder to fetch results */
+    data _null_;
+      syswarningtext=symget('syswarningtext');
+      syserrortext=symget('syserrortext');
+      abort_msg=symget('msg');
+      syscc=symget('syscc');
+      sysuserid=symget('sysuserid');
+      iftrue=symget('iftrue');
+      put (_all_)(/=);
+      call symputx('syscc',0);
+      abort cancel nolist;
     run;
-    %inc skip;
   %end;
   %else %do;
-    %put _all_;
     %abort cancel;
   %end;
-%mend;
+%end;
+%else %do;
+  %put _all_;
+  %abort cancel;
+%end;
+%mend mp_abort;
 
+/** @endcond */

base/mp_appendfile.sas

@@ -0,0 +1,57 @@
+/**
+  @file
+  @brief Append (concatenate) two or more files.
+  @details Will append one more more `appendrefs` (filerefs) to a `baseref`.
+  Uses a binary mechanism, so will work with any file type.  For that reason -
+  use with care!   And supply your own trailing carriage returns in each file..
+
+  Usage:
+
+        filename tmp1 temp;
+        filename tmp2 temp;
+        filename tmp3 temp;
+        data _null_; file tmp1; put 'base file';
+        data _null_; file tmp2; put 'append1';
+        data _null_; file tmp3; put 'append2';
+        run;
+        %mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3)
+
+
+  @param [in] baseref= Fileref of the base file (should exist)
+  @param [in] appendrefs= One or more filerefs to be appended to the base
+    fileref.  Space separated.
+
+  @version 9.2
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+
+
+**/
+
+%macro mp_appendfile(
+  baseref=0,
+  appendrefs=0
+)/*/STORE SOURCE*/;
+
+%mp_abort(iftrue= (&baseref=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Baseref NOT specified!)
+)
+%mp_abort(iftrue= (&appendrefs=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Appendrefs NOT specified!)
+)
+
+%local i;
+%do i=1 %to %sysfunc(countw(&appendrefs));
+  %mp_abort(iftrue= (&syscc>0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc)
+  )
+  %mp_binarycopy(inref=%scan(&appendrefs,&i), outref=&baseref, mode=APPEND)
+%end;
+
+%mend mp_appendfile;

base/mp_applyformats.sas

@@ -0,0 +1,181 @@
+/**
+  @file
+  @brief Apply a set of formats to a table
+  @details Applies a set of formats to the metadata of one or more SAS datasets.
+    Can be used to migrate formats from one table to another.  The input table
+    must contain the following columns:
+
+    @li lib - the libref of the table to be updated
+    @li ds - the dataset to be updated
+    @li var - the variable to be updated
+    @li fmt - the format to apply.  Missing or default ($CHAR, 8.) formats are
+      ignored.
+
+  The macro will abort in the following scenarios:
+
+    @li Libref not assigned
+    @li Dataset does not exist
+    @li Input table contains null or invalid values
+
+  Example usage:
+
+      data work.example;
+        set sashelp.prdsale;
+        format _all_ clear;
+      run;
+
+      %mp_getcols(sashelp.prdsale,outds=work.cols)
+
+      data work.cols2;
+        set work.cols;
+        lib='WORK';
+        ds='EXAMPLE';
+        var=name;
+        fmt=format;
+        keep lib ds var fmt;
+      run;
+
+      %mp_applyformats(work.cols2)
+
+  @param [in] inds The input dataset containing the formats to apply (and where
+    to apply them).  Example structure:
+  |LIB:$8.|DS:$32.|VAR:$32.|FMT:$49.|
+  |---|---|---|---|
+  |`WORK `|`EXAMPLE `|`ACTUAL `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`COUNTRY `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`DIVISION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`MONTH `|`MONNAME3. `|
+  |`WORK `|`EXAMPLE `|`PREDICT `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`PRODTYPE `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`PRODUCT `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`QUARTER `|`8. `|
+  |`WORK `|`EXAMPLE `|`REGION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`YEAR `|`8. `|
+
+  @param [out] errds= (0) Provide a libds reference here to export the
+    error messages to a table.  In this case, they will not be printed to the
+    log.
+
+  <h4> SAS Macros </h4>
+  @li mf_getengine.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_validatecol.sas
+
+
+  <h4> Related Macros </h4>
+  @li mp_getformats.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_applyformats(inds,errds=0
+)/*/STORE SOURCE*/;
+%local outds liblist i engine lib msg ;
+
+/**
+  * Validations
+  */
+proc sort data=&inds;
+  by lib ds var fmt;
+run;
+
+%if &errds=0 %then %let outds=%mf_getuniquename(prefix=mp_applyformats);
+%else %let outds=&errds;
+
+data &outds;
+  set &inds;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  length msg $128;
+  by lib ds var fmt;
+  if libref(lib) ne 0 then do;
+    msg=catx(' ','libref',lib,'is not assigned!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  if exist(cats(lib,'.',ds)) ne 1 then do;
+    msg=catx(' ','libds',lib,'.',ds,'does not exist!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  %mp_validatecol(fmt,FORMAT,is_fmt)
+  if is_fmt=0 then do;
+    msg=catx(' ','format',fmt,'on libds',lib,'.',ds,'.',var,'is not valid!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+
+  if first.ds then do;
+    retain dsid;
+    dsid=open(cats(lib,'.',ds));
+    if dsid=0 then do;
+      msg=catx(' ','libds',lib,'.',ds,' could not be opened!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+      return;
+    end;
+    if varnum(dsid,var)<1 then do;
+      msg=catx(' ','Variable',lib,'.',ds,'.',var,' was not found!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+    end;
+  end;
+  if last.ds then rc=close(dsid);
+run;
+
+proc sql noprint;
+select distinct lib into: liblist separated by ' ' from &inds;
+%put &=liblist;
+%if %length(&liblist)>0 %then %do i=1 %to %sysfunc(countw(&liblist));
+  %let lib=%scan(&liblist,1);
+  %let engine=%mf_getengine(&lib);
+  %if &engine ne V9 and &engine ne BASE %then %do;
+    %let msg=&lib has &engine engine - formats cannot be applied;
+    insert into &outds set lib="&lib",ds="_all_",var="_all", msg="&msg" ;
+    %if &errds=0 %then %put %str(ERR)OR: &msg;
+  %end;
+%end;
+quit;
+
+%if %mf_nobs(&outds)>0 %then %return;
+
+/**
+  * Validations complete - now apply the actual formats!
+  */
+%let fref=%mf_getuniquefileref();
+data _null_;
+  set &inds;
+  by lib ds var fmt;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  file &fref;
+  if first.lib then put 'proc datasets nolist lib=' lib ';';
+  if first.ds then put '  modify ' ds ';';
+  put '    format ' var fmt ';';
+  if last.ds then put '  run;';
+  if last.lib then put 'quit;';
+run;
+
+%inc &fref/source2;
+
+%if &errds=0 %then %do;
+  proc sql;
+  drop table &outds;
+%end;
+
+%mend mp_applyformats;

base/mp_assert.sas

@@ -0,0 +1,56 @@
+/**
+  @file
+  @brief Generic assertion
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      %mp_assert(iftrue=(1=1),
+        desc=Obviously true
+      )
+
+      %mp_assert(iftrue=(1=0),
+        desc=Will fail
+      )
+
+  @param [in] iftrue= (1=1) A condition where, if true, the test is a PASS.
+  Else, the test is a fail.
+
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Column &inds contained ALL columns|
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assert(iftrue=(1=1),
+  desc=0,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  data ;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_comments="&sysmacroname: Test result of "!!symget('iftrue');
+  %if %eval(%unquote(&iftrue)) %then %do;
+    test_result='PASS';
+  %end;
+  %else %do;
+    test_result='FAIL';
+  %end;
+  run;
+
+  %local ds ;
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+
+%mend mp_assert;

base/mp_assertcols.sas

@@ -0,0 +1,145 @@
+/**
+  @file
+  @brief Asserts the existence (or not) of columns
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      %mp_assertcols(sashelp.class,
+        cols=name age sex,
+        test=ALL,
+        desc=check all columns exist
+      )
+
+      %mp_assertcols(sashelp.class,
+        cols=a b c,
+        test=NONE
+      )
+
+      %mp_assertcols(sashelp.class,
+        cols=age depth,
+        test=ANY
+      )
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_existvarlist.sas
+  @li mf_getvarlist.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_abort.sas
+
+
+  @param [in] inds The input library.dataset to test for values
+  @param [in] cols= The list of columns to check for
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [in] test= (ALL) The test to apply.  Valid values are:
+    @li ALL - Test is a PASS if ALL columns exist in &inds
+    @li ANY - Test is a PASS if ANY of the columns exist in &inds
+    @li NONE - Test is a PASS if NONE of the columns exist in &inds
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Column &inds contained ALL columns|
+
+
+  <h4> Related Macros </h4>
+  @li mp_assertdsobs.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertcols(inds,
+  cols=0,
+  test=ALL,
+  desc=0,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  %mp_abort(iftrue= (&syscc ne 0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc - on macro entry)
+  )
+
+  %local lib ds ;
+  %let lib=%scan(&inds,1,%str(.));
+  %let ds=%scan(&inds,2,%str(.));
+  %let cols=%upcase(&cols);
+
+  %mp_abort(iftrue= (%mf_existds(&lib..&ds)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(&lib..&ds not found!)
+  )
+
+  %mp_abort(iftrue= (&cols=0)
+    ,mac=&sysmacroname
+    ,msg=%str(No cols provided)
+  )
+
+
+  %let test=%upcase(&test);
+
+  %if &test ne ANY and &test ne ALL and &test ne NONE %then %do;
+    %mp_abort(
+      mac=&sysmacroname,
+      msg=%str(Invalid test - &test)
+    )
+  %end;
+
+  /**
+    * now do the actual test!
+    */
+  %local result;
+  %if %mf_existVarList(&inds,&cols)=1 %then %let result=ALL;
+  %else %do;
+    %local targetcols compare;
+    %let targetcols=%upcase(%mf_getvarlist(&inds));
+    %let compare=%mf_wordsinstr1butnotstr2(
+        Str1=&cols,
+        Str2=&targetcols
+      );
+    %if %cmpres(&compare)=%cmpres(&cols) %then %let result=NONE;
+    %else %let result=SOME;
+  %end;
+
+  data;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    if test_description='0'
+    then test_description="Testing &inds for existence of &test of: &cols";
+
+    test_result='FAIL';
+    test_comments="&sysmacroname: &inds has &result columns ";
+  %if &test=ALL %then %do;
+    %if &result=ALL %then %do;
+      test_result='PASS';
+    %end;
+  %end;
+  %else %if &test=ANY %then %do;
+    %if &result=SOME %then %do;
+      test_result='PASS';
+    %end;
+  %end;
+  %else %if &test=NONE %then %do;
+    %if &result=NONE %then %do;
+      test_result='PASS';
+    %end;
+  %end;
+  %else %do;
+    test_comments="&sysmacroname: Unsatisfied test condition - &test";
+  %end;
+  run;
+
+  %local ds;
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+
+%mend mp_assertcols;

base/mp_assertcolvals.sas

@@ -0,0 +1,168 @@
+/**
+  @file
+  @brief Asserts the values in a column
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      data work.checkds;
+        do checkval='Jane','James','Jill';
+          output;
+        end;
+      run;
+      %mp_assertcolvals(sashelp.class.name,
+        checkvals=work.checkds.checkval,
+        desc=At least one value has a match,
+        test=ANYVAL
+      )
+
+      data work.check;
+        do val='M','F';
+          output;
+        end;
+      run;
+      %mp_assertcolvals(sashelp.class.sex,
+        checkvals=work.check.val,
+        desc=All values have a match,
+        test=ALLVALS
+      )
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+
+  @param [in] indscol The input library.dataset.column to test for values
+  @param [in] checkvals= A library.dataset.column value containing a UNIQUE
+    list of values to be compared against the source (indscol).
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [in] test= (ALLVALS) The test to apply.  Valid values are:
+    @li ALLVALS - Test is a PASS if ALL values have a match in checkvals
+    @li ANYVAL - Test is a PASS if at least 1 value has a match in checkvals
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Column &indscol contained ALL target vals|
+
+
+  <h4> Related Macros </h4>
+  @li mp_assertdsobs.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertcolvals(indscol,
+  checkvals=0,
+  test=ALLVALS,
+  desc=mp_assertcolvals - no desc provided,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  %mp_abort(iftrue= (&syscc ne 0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc - on macro entry)
+  )
+
+  %local lib ds col clib cds ccol nobs;
+  %let lib=%scan(&indscol,1,%str(.));
+  %let ds=%scan(&indscol,2,%str(.));
+  %let col=%scan(&indscol,3,%str(.));
+  %mp_abort(iftrue= (%mf_existds(&lib..&ds)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(&lib..&ds not found!)
+  )
+
+  %mp_abort(iftrue= (&checkvals=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Set CHECKVALS to a library.dataset.column containing check vals)
+  )
+  %let clib=%scan(&checkvals,1,%str(.));
+  %let cds=%scan(&checkvals,2,%str(.));
+  %let ccol=%scan(&checkvals,3,%str(.));
+  %mp_abort(iftrue= (%mf_existds(&clib..&cds)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(&clib..&cds not found!)
+  )
+  %let nobs=%mf_nobs(&clib..&cds);
+  %mp_abort(iftrue= (&nobs=0)
+    ,mac=&sysmacroname
+    ,msg=%str(&clib..&cds is empty!)
+  )
+
+  %let test=%upcase(&test);
+
+  %if &test ne ALLVALS and &test ne ANYVAL %then %do;
+    %mp_abort(
+      mac=&sysmacroname,
+      msg=%str(Invalid test - &test)
+    )
+  %end;
+
+  %local result orig;
+  %let result=-1;
+  %let orig=-1;
+  proc sql noprint;
+  select count(*) into: result
+    from &lib..&ds
+    where &col not in (
+      select &ccol from &clib..&cds
+    );
+  select count(*) into: orig from &lib..&ds;
+  quit;
+
+  %local notfound tmp1 tmp2;
+  %let tmp1=%mf_getuniquename();
+  %let tmp2=%mf_getuniquename();
+
+  /* this is a bit convoluted - but using sql outobs=10 throws warnings */
+  proc sql noprint;
+  create view &tmp1 as
+    select distinct &col
+    from &lib..&ds
+    where &col not in (
+      select &ccol from &clib..&cds
+    );
+  data &tmp2;
+    set &tmp1;
+    if _n_>10 then stop;
+  run;
+  proc sql;
+  select distinct &col  into: notfound separated by ' ' from &tmp2;
+
+
+  %mp_abort(iftrue= (&syscc ne 0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc after macro query)
+  )
+
+  data;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_result='FAIL';
+    test_comments="&sysmacroname: &lib..&ds..&col has &result values "
+      !!"not in &clib..&cds..&ccol.. First 10 vals:"!!symget('notfound');
+  %if &test=ANYVAL %then %do;
+    if &result < &orig then test_result='PASS';
+  %end;
+  %else %if &test=ALLVALS %then %do;
+    if &result=0 then test_result='PASS';
+  %end;
+  %else %do;
+    test_comments="&sysmacroname: Unsatisfied test condition - &test";
+  %end;
+  run;
+
+  %local ds;
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+
+%mend mp_assertcolvals;

base/mp_assertdsobs.sas

@@ -0,0 +1,121 @@
+/**
+  @file
+  @brief Asserts the number of observations in a dataset
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      %mp_assertdsobs(sashelp.class) %* tests if any observations are present;
+
+      %mp_assertdsobs(sashelp.class,test=ATLEAST 10) %* pass if >9 obs present;
+
+      %mp_assertdsobs(sashelp.class,test=ATMOST 20) %* pass if <21 obs present;
+
+
+  @param [in] inds input dataset to test for presence of observations
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [in] test= (HASOBS) The test to apply.  Valid values are:
+    @li HASOBS - Test is a PASS if the input dataset has any observations
+    @li EMPTY - Test is a PASS if input dataset is empty
+    @li EQUALS [integer] - Test passes if row count matches the provided integer
+    @li ATLEAST [integer] - Test passes if row count is more than or equal to
+      the provided integer
+    @li ATMOST [integer] - Test passes if row count is less than or equal to
+      the provided integer
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Dataset &inds has XX obs|
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+  <h4> Related Macros </h4>
+  @li mp_assertcolvals.sas
+  @li mp_assert.sas
+  @li mp_assertcols.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertdsobs(inds,
+  test=HASOBS,
+  desc=Testing observations,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  %local nobs ds;
+  %let nobs=%mf_nobs(&inds);
+  %let test=%upcase(&test);
+  %let ds=%mf_getuniquename(prefix=mp_assertdsobs);
+
+  %if %substr(&test.xxxxx,1,6)=EQUALS %then %do;
+    %let val=%scan(&test,2,%str( ));
+    %mp_abort(iftrue= (%DATATYP(&val)=CHAR)
+      ,mac=&sysmacroname
+      ,msg=%str(Invalid test - &test, expected EQUALS [integer])
+    )
+    %let test=EQUALS;
+  %end;
+  %else %if %substr(&test.xxxxxxx,1,7)=ATLEAST %then %do;
+    %let val=%scan(&test,2,%str( ));
+    %mp_abort(iftrue= (%DATATYP(&val)=CHAR)
+      ,mac=&sysmacroname
+      ,msg=%str(Invalid test - &test, expected ATLEAST [integer])
+    )
+    %let test=ATLEAST;
+  %end;
+  %else %if %substr(&test.xxxxxxx,1,7)=ATMOST %then %do;
+    %let val=%scan(&test,2,%str( ));
+    %mp_abort(iftrue= (%DATATYP(&val)=CHAR)
+      ,mac=&sysmacroname
+      ,msg=%str(Invalid test - &test, expected ATMOST [integer])
+    )
+    %let test=ATMOST;
+  %end;
+  %else %if &test ne HASOBS and &test ne EMPTY %then %do;
+    %mp_abort(
+      mac=&sysmacroname,
+      msg=%str(Invalid test - &test)
+    )
+  %end;
+
+  data &ds;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_result='FAIL';
+    test_comments="&sysmacroname: Dataset &inds has &nobs observations.";
+    test_comments=test_comments!!" Test was "!!symget('test');
+  %if &test=HASOBS %then %do;
+    if &nobs>0 then test_result='PASS';
+  %end;
+  %else %if &test=EMPTY %then %do;
+    if &nobs=0 then test_result='PASS';
+  %end;
+  %else %if &test=EQUALS %then %do;
+    if &nobs=&val then test_result='PASS';
+  %end;
+  %else %if &test=ATLEAST %then %do;
+    if &nobs ge &val then test_result='PASS';
+  %end;
+  %else %if &test=ATMOST %then %do;
+    if &nobs le &val then test_result='PASS';
+  %end;
+  %else %do;
+    test_comments="&sysmacroname: Unsatisfied test condition - &test";
+  %end;
+  run;
+
+  proc append base=&outds data=&ds;
+  run;
+
+  proc sql;
+  drop table &ds;
+
+%mend mp_assertdsobs;

base/mp_assertscope.sas

@@ -0,0 +1,147 @@
+/**
+  @file
+  @brief Used to capture scope leakage of macro variables
+  @details
+
+  A common 'difficult to detect' bug in macros is where a nested macro
+  over-writes variables in a higher level macro.
+
+  This assertion takes a snapshot of the macro variables before and after
+  a macro invocation.  Differences are captured in the `&outds` table. This
+  makes it easy to detect whether any macro variables were modified or
+  changed.
+
+  The following variables are NOT tested (as they are known, global variables
+  used in SASjs):
+
+  @li &sasjs_prefix._FUNCTIONS
+
+  Global variables are initialised in mp_init.sas - which will also trigger
+  "strict mode" in your SAS session.  Whilst this is a default in SASjs
+  produced apps, if you prefer not to use this mode, simply instantiate the
+  following variable to prevent the macro from running:  `SASJS_PREFIX`
+
+  Example usage:
+
+      %mp_assertscope(SNAPSHOT)
+
+      %let oops=I did it again;
+
+      %mp_assertscope(COMPARE,
+        desc=Checking macro variables against previous snapshot
+      )
+
+  This macro is designed to work alongside `sasjs test` - for more information
+  about this facility, visit [cli.sasjs.io/test](https://cli.sasjs.io/test).
+
+  @param [in] action (SNAPSHOT) The action to take.  Valid values:
+    @li SNAPSHOT - take a copy of the current macro variables
+    @li COMPARE - compare the current macro variables against previous values
+  @param [in] scope= (GLOBAL) The scope of the variables to be checked.  This
+    corresponds to the values in the SCOPE column in `sashelp.vmacro`.
+  @param [in] desc= (Testing scope leakage) The user provided test description
+  @param [in] ignorelist= Provide a list of macro variable names to ignore from
+    the comparison
+  @param [in,out] scopeds= (work.mp_assertscope) The dataset to contain the
+    scope snapshot
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|No out of scope variables created or modified|
+
+  <h4> SAS Macros </h4>
+  @li mf_getquotedstr.sas
+  @li mp_init.sas
+
+  <h4> Related Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertcols.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertscope(action,
+  desc=Testing Scope Leakage,
+  scope=GLOBAL,
+  scopeds=work.mp_assertscope,
+  ignorelist=,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+%local ds test_result test_comments del add mod ilist;
+%let ilist=%upcase(&sasjs_prefix._FUNCTIONS SYS_PROCHTTP_STATUS_CODE
+  SYS_PROCHTTP_STATUS_CODE SYS_PROCHTTP_STATUS_PHRASE &ignorelist);
+
+/**
+  * this sets up the global vars, it will also enter STRICT mode.  If this
+  * behaviour is not desired, simply initiate the following global macro
+  * variable to prevent the macro from running: SASJS_PREFIX
+  */
+%mp_init()
+
+/* get current variables */
+%if &action=SNAPSHOT %then %do;
+  proc sql;
+  create table &scopeds as
+    select name,offset,value
+    from dictionary.macros
+    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
+    order by name,offset;
+%end;
+%else %if &action=COMPARE %then %do;
+
+  proc sql;
+  create table _data_ as
+    select name,offset,value
+    from dictionary.macros
+    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
+    order by name,offset;
+
+  %let ds=&syslast;
+
+  proc compare
+    base=&scopeds(where=(upcase(name) not in (%mf_getquotedstr(&ilist))))
+    compare=&ds noprint;
+  run;
+
+  %if &sysinfo=0 %then %do;
+    %let test_result=PASS;
+    %let test_comments=&scope Variables Unmodified;
+  %end;
+  %else %do;
+    proc sql noprint undo_policy=none;
+    select distinct name into: del separated by ' '  from &scopeds
+      where name not in (select name from &ds);
+    select distinct name into: add separated by ' '  from &ds
+      where name not in (select name from &scopeds);
+    select distinct a.name into: mod separated by ' '
+      from &scopeds a
+      inner join &ds b
+      on a.name=b.name
+        and a.offset=b.offset
+      where a.value ne b.value;
+    %let test_result=FAIL;
+    %let test_comments=%str(Mod:(&mod) Add:(&add) Del:(&del));
+  %end;
+
+
+  data ;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_comments=symget('test_comments');
+    test_result=symget('test_result');
+  run;
+
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+%end;
+
+%mend mp_assertscope;

base/mp_base64copy.sas

@@ -0,0 +1,117 @@
+/**
+  @file
+  @brief Convert a file to/from base64 format
+  @details Creates a new version of a file either encoded or decoded using
+  Base64.  Inspired by this post by Michael Dixon:
+  https://support.selerity.com.au/hc/en-us/articles/223345708-Tip-SAS-and-Base64
+
+  Usage:
+
+        filename tmp temp;
+        data _null_;
+          file tmp;
+          put 'base ik ally';
+        run;
+        %mp_base64copy(inref=tmp, outref=myref, action=ENCODE)
+
+        data _null_;
+          infile myref;
+          input;
+          put _infile_;
+        run;
+
+        %mp_base64copy(inref=myref, outref=mynewref, action=DECODE)
+
+        data _null_;
+          infile mynewref;
+          input;
+          put _infile_;
+        run;
+
+  @param [in] inref= Fileref of the input file (should exist)
+  @param [out] outref= Output fileref. If it does not exist, it is created.
+  @param [in] action= (ENCODE) The action to take. Valid values:
+    @li ENCODE - Convert the file to base64 format
+    @li DECODE - Decode the file from base64 format
+
+  @version 9.2
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+
+
+**/
+
+%macro mp_base64copy(
+  inref=0,
+  outref=0,
+  action=ENCODE
+)/*/STORE SOURCE*/;
+
+%let inref=%upcase(&inref);
+%let outref=%upcase(&outref);
+%let action=%upcase(&action);
+%local infound outfound;
+%let infound=0;
+%let outfound=0;
+data _null_;
+  set sashelp.vextfl(where=(fileref="&inref" or fileref="&outref"));
+  if fileref="&inref" then call symputx('infound',1,'l');
+  if fileref="&outref" then call symputx('outfound',1,'l');
+run;
+
+%mp_abort(iftrue= (&infound=0)
+  ,mac=&sysmacroname
+  ,msg=%str(INREF &inref NOT FOUND!)
+)
+%mp_abort(iftrue= (&outref=0)
+  ,mac=&sysmacroname
+  ,msg=%str(OUTREF NOT PROVIDED!)
+)
+%mp_abort(iftrue= (&action ne ENCODE and &action ne DECODE)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid action! Should be ENCODE OR DECODE)
+)
+
+%if &outfound=0 %then %do;
+  filename &outref temp lrecl=2097088;
+%end;
+
+%if &action=ENCODE %then %do;
+  data _null_;
+    length b64 $ 76 line $ 57;
+    retain line "";
+    infile &inref recfm=F lrecl= 1 end=eof;
+    input @1 stream $char1.;
+    file &outref recfm=N;
+    substr(line,(_N_-(CEIL(_N_/57)-1)*57),1) = byte(rank(stream));
+    if mod(_N_,57)=0 or EOF then do;
+      if eof then b64=put(trim(line),$base64X76.);
+      else b64=put(line, $base64X76.);
+      put b64 + (-1) @;
+      line="";
+    end;
+  run;
+%end;
+%else %if &action=DECODE %then %do;
+  data _null_;
+    length filein 8 fileout 8;
+    filein = fopen("&inref",'I',4,'B');
+    fileout = fopen("&outref",'O',3,'B');
+    char= '20'x;
+    do while(fread(filein)=0);
+      length raw $4;
+      do i=1 to 4;
+        rc=fget(filein,char,1);
+        substr(raw,i,1)=char;
+      end;
+      rc = fput(fileout,input(raw,$base64X4.));
+      rc = fwrite(fileout);
+    end;
+    rc = fclose(filein);
+    rc = fclose(fileout);
+  run;
+%end;
+
+%mend mp_base64copy;

base/mp_binarycopy.sas

@@ -2,16 +2,37 @@
   @file
   @brief Copy any file using binary input / output streams
   @details Reads in a file byte by byte and writes it back out.  Is an
-    os-independent method to copy files.  In case of naming collision, the
-    default filerefs can be modified.
-    Based on http://stackoverflow.com/questions/13046116/using-sas-to-copy-a-text-file
+  os-independent method to copy files.  In case of naming collision, the
+  default filerefs can be modified.
+  Note that if you have a new enough version of SAS, and you don't need features
+  such as APPEND, you may be better of using the fcopy() function instead.
 
         %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
 
-  @param inloc full, quoted "path/and/filename.ext" of the object to be copied
-  @param outloc full, quoted "path/and/filename.ext" of object to be created
-  @param inref can override default input fileref to avoid naming clash
-  @param outref an override default output fileref to avoid naming clash
+  To append to a file, use the mode option, eg:
+
+      filename tmp1 temp;
+      filename tmp2 temp;
+      data _null_;
+        file tmp1;
+        put 'stacking';
+      run;
+
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+
+
+  @param [in] inloc quoted "path/and/filename.ext" of the file to be copied
+  @param [out] outloc quoted "path/and/filename.ext" of the file to be created
+  @param [in] inref (____in) If provided, this fileref will take precedence over
+    the `inloc` parameter
+  @param [out] outref (____in) If provided, this fileref will take precedence
+    over the `outloc` parameter.  It must already exist!
+  @param [in] mode (CREATE) Valid values:
+    @li CREATE - Create the file (even if it already exists)
+    @li APPEND - Append to the file (don't overwrite)
+  @param iftrue= (1=1) Supply a condition for which the macro should be executed
+
   @returns nothing
 
   @version 9.2
@@ -19,37 +40,40 @@
 **/
 
 %macro mp_binarycopy(
-     inloc=           /* full path and filename of the object to be copied */
+    inloc=           /* full path and filename of the object to be copied */
     ,outloc=          /* full path and filename of object to be created */
     ,inref=____in   /* override default to use own filerefs */
     ,outref=____out /* override default to use own filerefs */
+    ,mode=CREATE
+    ,iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
-   /* these IN and OUT filerefs can point to anything */
+  %local mod;
+
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
+  %if &mode=APPEND %then %let mod=mod;
+
+  /* these IN and OUT filerefs can point to anything */
   %if &inref = ____in %then %do;
     filename &inref &inloc lrecl=1048576 ;
   %end;
   %if &outref=____out %then %do;
-    filename &outref &outloc lrecl=1048576 ;
+    filename &outref &outloc lrecl=1048576 &mod;
   %end;
 
-   /* copy the file byte-for-byte  */
-   data _null_;
-     length filein 8 fileid 8;
-     filein = fopen("&inref",'I',1,'B');
-     fileid = fopen("&outref",'O',1,'B');
-     rec = '20'x;
-     do while(fread(filein)=0);
-        rc = fget(filein,rec,1);
-        rc = fput(fileid, rec);
-        rc =fwrite(fileid);
-     end;
-     rc = fclose(filein);
-     rc = fclose(fileid);
-   run;
+  /* copy the file byte-for-byte  */
+  data _null_;
+    infile &inref lrecl=1 recfm=n;
+    file &outref &mod recfm=n;
+    input sourcechar $char1. @@;
+    format sourcechar hex2.;
+    put sourcechar char1. @@;
+  run;
+
   %if &inref = ____in %then %do;
     filename &inref clear;
   %end;
   %if &outref=____out %then %do;
     filename &outref clear;
   %end;
-%mend;
+%mend mp_binarycopy;

base/mp_chop.sas

@@ -0,0 +1,194 @@
+/**
+  @file
+  @brief Splits a file of ANY SIZE by reference to a search string.
+  @details Provide a fileref and a search string to chop off part of a file.
+
+  Works by reading in the file byte by byte, then marking the beginning and end
+  of each matched string, before finally doing the chop.
+
+  Choose whether to keep the FIRST or the LAST section of the file.  Optionally,
+  use an OFFSET to fix the precise chop point.
+
+  Usage:
+
+      %let src="%sysfunc(pathname(work))/file.txt";
+      %let str=Chop here!;
+      %let out1="%sysfunc(pathname(work))/file1.txt";
+      %let out2="%sysfunc(pathname(work))/file2.txt";
+      %let out3="%sysfunc(pathname(work))/file3.txt";
+      %let out4="%sysfunc(pathname(work))/file4.txt";
+
+      data _null_;
+        file &src;
+        put "startsection&str.endsection";
+      run;
+
+      %mp_chop(&src, matchvar=str, keep=FIRST, outfile=&out1)
+      %mp_chop(&src, matchvar=str, keep=LAST, outfile=&out2)
+      %mp_chop(&src, matchvar=str, keep=FIRST, matchpoint=END, outfile=&out3)
+      %mp_chop(&src, matchvar=str, keep=LAST, matchpoint=END, outfile=&out4)
+
+      filename results (&out1 &out2 &out3 &out4);
+      data _null_;
+        infile results;
+        input;
+        list;
+      run;
+
+  Results:
+    @li `startsection`
+    @li `Chop here!endsection`
+    @li `startsectionChop here!`
+    @li `endsection`
+
+  For more examples, see mp_chop.test.sas
+
+  @param [in] infile The QUOTED path to the file on which to perform the chop
+  @param [in] matchvar= Macro variable NAME containing the string to split by
+  @param [in] matchpoint= (START) Valid values:
+    @li START - chop at the beginning of the string in `matchvar`.
+    @li END - chop at the end of the string in `matchvar`.
+  @param [in] offset= (0) An adjustment to the precise chop location, by
+    by reference to the `matchpoint`. Should be a positive or negative integer.
+  @param [in] keep= (FIRST) Valid values:
+    @li FIRST - keep the section of the file before the chop
+    @li LAST - keep the section of the file after the chop
+  @param [in] mdebug= (0) Set to 1 to provide macro debugging
+  @param outfile= (0) Optional QUOTED path to the adjusted output file (avoids
+    overwriting the first file).
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mp_abort.sas
+  @li mp_gsubfile.sas
+  @li mp_replace.sas
+  @li mp_chop.test.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mp_chop(infile,
+  matchvar=,
+  matchpoint=START,
+  keep=FIRST,
+  offset=0,
+  mdebug=0,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+%local fref0 dttm ds1 outref;
+%let fref0=%mf_getuniquefileref();
+%let ds1=%mf_getuniquename(prefix=allchars);
+%let ds2=%mf_getuniquename(prefix=startmark);
+
+%if &outfile=0 %then %let outfile=&infile;
+
+%mp_abort(iftrue= (%length(%superq(&matchvar))=0)
+  ,mac=mp_chop.sas
+  ,msg=%str(&matchvar is an empty variable)
+)
+
+/* START */
+%let dttm=%sysfunc(datetime());
+
+filename &fref0 &infile lrecl=1 recfm=n;
+
+/* create dataset with one char per row */
+data &ds1;
+  infile &fref0;
+  input sourcechar $char1. @@;
+  format sourcechar hex2.;
+run;
+
+/* get start & stop position of first matchvar string (one row, two vars) */
+data &ds2;
+  /* set find string to length in bytes to cover trailing spaces */
+  length string $ %length(%superq(&matchvar));
+  string =symget("&matchvar");
+  drop string;
+
+  firstchar=char(string,1);
+  findlen=lengthm(string); /* <- for trailing bytes */
+
+  do _N_=1 to nobs;
+    set &ds1 nobs=nobs point=_N_;
+    if sourcechar=firstchar then do;
+      pos=1;
+      s=0;
+      do point=_N_ to min(_N_ + findlen -1,nobs);
+        set &ds1 point=point;
+        if sourcechar=char(string, pos) then s + 1;
+        else goto _leave_;
+        pos+1;
+      end;
+      _leave_:
+      if s=findlen then do;
+        START =_N_;
+        _N_ =_N_+ s - 1;
+        STOP =_N_;
+        output;
+        /* matched! */
+        stop;
+      end;
+    end;
+  end;
+  stop;
+  keep START STOP;
+run;
+
+%local split;
+%let split=0;
+data _null_;
+  set &ds2;
+  if "&matchpoint"='START' then do;
+    if "&keep"='FIRST' then mp=start;
+    else if "&keep"='LAST' then mp=start-1;
+  end;
+  else if "&matchpoint"='END' then do;
+    if "&keep"='FIRST' then mp=stop+1;
+    else if "&keep"='LAST' then mp=stop;
+  end;
+  split=mp+&offset;
+  call symputx('split',split,'l');
+%if &mdebug=1 %then %do;
+  put (_all_)(=);
+  %put &=offset;
+%end;
+run;
+%if &split=0 %then %do;
+  %put &sysmacroname: No match found in &infile for string %superq(&matchvar);
+  %return;
+%end;
+
+data _null_;
+  file &outfile recfm=n;
+  set &ds1;
+%if &keep=FIRST %then %do;
+  if _n_ ge &split then stop;
+%end;
+%else %do;
+  if _n_ gt &split;
+%end;
+  put sourcechar char1.;
+run;
+
+%if &mdebug=0 %then %do;
+  filename &fref0 clear;
+%end;
+%else %do;
+  data _null_;
+    infile &outfile lrecl=32767;
+    input;
+    list;
+    if _n_>50 then stop;
+  run;
+%end;
+/* END */
+%put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) seconds to run;
+
+%mend mp_chop;

base/mp_cleancsv.sas

@@ -1,27 +1,31 @@
 /**
-  @file mp_cleancsv.sas
+  @file
   @brief Fixes embedded cr / lf / crlf in CSV
   @details CSVs will sometimes contain lf or crlf within quotes (eg when
-    saved by excel).  When the termstr is ALSO lf or crlf that can be tricky
-    to process using SAS defaults.
-    This macro converts any csv to follow the convention of a windows excel file,
-    applying CRLF line endings and converting embedded cr and crlf to lf.
+  saved by excel).  When the termstr is ALSO lf or crlf that can be tricky
+  to process using SAS defaults.
+  This macro converts any csv to follow the convention of a windows excel file,
+  applying CRLF line endings and converting embedded cr and crlf to lf.
+
+  Usage:
 
-  usage:
       fileref mycsv "/path/your/csv";
       %mp_cleancsv(in=mycsv,out=/path/new.csv)
 
-  @param in= provide path or fileref to input csv
-  @param out= output path or fileref to output csv
-  @param qchar= quote char - hex code 22 is the double quote.
+  @param in= (NOTPROVIDED) Provide path or fileref to input csv. If a period is
+    found, it is assumed to be a file.
+  @param out= (NOTPROVIDED) Output path or fileref to output csv.  If a period
+    is found, it is assumed to be a file.
+  @param qchar= ('22'x) Quote char - hex code 22 is the double quote.
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
 %macro mp_cleancsv(in=NOTPROVIDED,out=NOTPROVIDED,qchar='22'x);
 %if "&in"="NOTPROVIDED" or "&out"="NOTPROVIDED" %then %do;
-  %put %str(ERR)OR: Please provide valid input (&in) and output (&out) locations;
+  %put %str(ERR)OR: Please provide valid input (&in) & output (&out) locations;
   %return;
 %end;
 
@@ -30,9 +34,9 @@
 %if %index(&out,.) %then %let out="&out";
 
 /**
- * convert all cr and crlf within quotes to lf
- * convert all other cr or lf to crlf
- */
+  * convert all cr and crlf within quotes to lf
+  * convert all other cr or lf to crlf
+  */
   data _null_;
     infile &in recfm=n ;
     file &out recfm=n;
@@ -54,9 +58,14 @@
     else do;
       /* outside a quote, change cr and lf to crlf */
       if inchar='0D'x then do;
+        crblank:
         put '0D0A'x;
         input inchar $char1.;
-        if inchar ne '0A'x then do;
+        if inchar='0D'x then do;
+          /* multiple CR indicates CR formatted file with blank lines */
+          goto crblank;
+        end;
+        else if inchar ne '0A'x then do;
           put inchar $char1.;
           if inchar=qchar then isq = mod(isq+1,2);
         end;
@@ -65,4 +74,5 @@
       else put inchar $char1.;
     end;
   run;
-%mend;
+%mend mp_cleancsv;
+/** @endcond */

base/mp_cntlout.sas

@@ -0,0 +1,85 @@
+/**
+  @file mp_cntlout.sas
+  @brief Creates a cntlout dataset in a consistent format
+  @details The dataset produced by proc format in the cntlout option will vary
+  according to its contents.
+
+  When dealing with formats from an ETL perspective (eg in [Data Controller for
+  SAS](https://datacontroller.io)), it is important that the output dataset
+  has a consistent model (and compariable values).
+
+  This macro makes use of mddl_sas_cntlout.sas to provide the consistent model,
+  and will left-align the start and end values when dealing with numeric ranges
+  to enable consistency when checking for differences.
+
+  usage:
+
+      %mp_cntlout(libcat=yourlib.cat,cntlout=work.formatexport)
+
+  @param [in] libcat The library.catalog reference
+  @param [in] fmtlist= (0) provide a space separated list of specific formats to
+    extract
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [out] cntlout= (work.fmtextract) Libds reference for the output dataset
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mf_getvarformat.sas
+  @li mp_getformats.sas
+  @li mp_loadformat.sas
+  @li mp_ds2fmtds.sas
+
+  @version 9.2
+  @author Allan Bowe
+  @cond
+**/
+
+%macro mp_cntlout(
+  iftrue=(1=1)
+  ,libcat=
+  ,cntlout=work.fmtextract
+  ,fmtlist=0
+)/*/STORE SOURCE*/;
+%local ddlds cntlds i;
+
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+%let ddlds=%mf_getuniquename();
+%let cntlds=%mf_getuniquename();
+
+%mddl_sas_cntlout(libds=&ddlds)
+
+%if %index(&libcat,-)>0 and %scan(&libcat,2,-)=FC %then %do;
+  %let libcat=%scan(&libcat,1,-);
+%end;
+
+proc format lib=&libcat cntlout=&cntlds;
+%if "&fmtlist" ne "0" %then %do;
+  select
+  %do i=1 %to %sysfunc(countw(&fmtlist));
+    %scan(&fmtlist,&i,%str( ))
+  %end;
+  ;
+%end;
+run;
+
+data &cntlout;
+  if 0 then set &ddlds;
+  set &cntlds;
+  if type="N" then do;
+    start=cats(start);
+    end=cats(end);
+  end;
+run;
+proc sort;
+  by fmtname start;
+run;
+
+proc sql;
+drop table &ddlds,&cntlds;
+
+%mend mp_cntlout;
+/** @endcond */

base/mp_copyfolder.sas

@@ -0,0 +1,84 @@
+/**
+  @file
+  @brief A macro to recursively copy a directory
+  @details Performs a recursive directory listing then works from top to bottom
+    copying files and creating subdirectories.
+
+  Usage:
+
+      %let rootdir=%sysfunc(pathname(work))/demo;
+      %let copydir=%sysfunc(pathname(work))/demo_copy;
+      %mf_mkdir(&rootdir)
+      %mf_mkdir(&rootdir/subdir)
+      %mf_mkdir(&rootdir/subdir/subsubdir)
+      data "&rootdir/subdir/example.sas7bdat";
+      run;
+
+      %mp_copyfolder(&rootdir,&copydir)
+
+  @param [in] source Unquoted path to the folder to copy from.
+  @param [out] target Unquoted path to the folder to copy to.
+  @param [in] copymax=(MAX) Set to a positive integer to indicate the level of
+    subdirectory copy recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_isdir.sas
+  @li mf_mkdir.sas
+  @li mp_abort.sas
+  @li mp_dirlist.sas
+
+  <h4> Related Macros </h4>
+  @li mp_copyfolder.test.sas
+
+**/
+
+%macro mp_copyfolder(source,target,copymax=MAX);
+
+  %mp_abort(iftrue=(%mf_isdir(&source)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Source dir does not exist (&source))
+  )
+
+  %mf_mkdir(&target)
+
+  %mp_abort(iftrue=(%mf_isdir(&target)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Target dir could not be created (&target))
+  )
+
+  /* prep temp table */
+  %local tempds;
+  %let tempds=%mf_getuniquename();
+
+  /* recursive directory listing */
+  %mp_dirlist(path=&source,outds=work.&tempds,maxdepth=&copymax)
+
+  /* create folders and copy content */
+  data _null_;
+    length msg $200;
+    call missing(msg);
+    set work.&tempds;
+    if _n_ = 1 then dpos+sum(length(directory),2);
+    filepath2="&target/"!!substr(filepath,dpos);
+    if file_or_folder='folder' then call execute('%mf_mkdir('!!filepath2!!')');
+    else do;
+      length fref1 fref2 $8;
+      rc1=filename(fref1,filepath,'disk','recfm=n');
+      rc2=filename(fref2,filepath2,'disk','recfm=n');
+      if fcopy(fref1,fref2) ne 0 then do;
+        msg=sysmsg();
+        putlog "%str(ERR)OR: Unable to copy " filepath " to " filepath2;
+        putlog msg=;
+      end;
+    end;
+    rc=filename(fref1);
+    rc=filename(fref2);
+  run;
+
+  /* tidy up */
+  proc sql;
+  drop table work.&tempds;
+
+%mend mp_copyfolder;

base/mp_coretable.sas

@@ -0,0 +1,73 @@
+/**
+  @file
+  @brief Create the permanent Core tables
+  @details Several macros in the [core](https://github.com/sasjs/core) library
+    make use of permanent tables.  To avoid duplication in definitions, this
+    macro provides a central location for managing the corresponding DDL.
+
+  Note - this macro is likely to be deprecated in future in favour of a
+  dedicated "datamodel" folder (prefix mddl)
+
+  Any corresponding data would go in a seperate repo, to avoid this one
+  ballooning in size!
+
+  Example usage:
+
+      %mp_coretable(LOCKTABLE,libds=work.locktable)
+
+  @param [in] table_ref The type of table to create.  Example values:
+    @li DIFFTABLE
+    @li FILTER_DETAIL
+    @li FILTER_SUMMARY
+    @li LOCKANYTABLE
+    @li MAXKEYTABLE
+  @param [in] libds= (0) The library.dataset reference used to create the table.
+    If not provided, then the DDL is simply printed to the log.
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mddl_dc_filterdetail.sas
+  @li mddl_dc_filtersummary.sas
+  @li mddl_dc_locktable.sas
+  @li mddl_dc_maxkeytable.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filterstore.sas
+  @li mp_lockanytable.sas
+  @li mp_retainedkey.sas
+  @li mp_storediffs.sas
+  @li mp_stackdiffs.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_coretable(table_ref,libds=0
+)/*/STORE SOURCE*/;
+%local outds ;
+%let outds=%sysfunc(ifc(&libds=0,%mf_getuniquename(),&libds));
+proc sql;
+%if &table_ref=DIFFTABLE %then %do;
+  %mddl_dc_difftable(libds=&outds)
+%end;
+%else %if &table_ref=LOCKTABLE %then %do;
+  %mddl_dc_locktable(libds=&outds)
+%end;
+%else %if &table_ref=FILTER_SUMMARY %then %do;
+  %mddl_dc_filtersummary(libds=&outds)
+%end;
+%else %if &table_ref=FILTER_DETAIL %then %do;
+  %mddl_dc_filterdetail(libds=&outds)
+%end;
+%else %if &table_ref=MAXKEYTABLE %then %do;
+  %mddl_dc_maxkeytable(libds=&outds)
+%end;
+
+%if &libds=0 %then %do;
+  proc sql;
+  describe table &syslast;
+  drop table &syslast;
+%end;
+%mend mp_coretable;

base/mp_createconstraints.sas

@@ -13,16 +13,19 @@
           constraint unq unique(tx_from, dd_type),
           constraint nnn not null(DD_SHORTDESC)
         );
-      
+
       %mp_getconstraints(lib=work,ds=example,outds=work.constraints)
       %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
       %mp_createconstraints(inds=work.constraints,outds=created,execute=YES)
 
-  @param inds= The input table containing the constraint info
-  @param outds= a table containing the create statements (create_statement column)
-  @param execute= `YES|NO` - default is NO. To actually create, use YES.
+  @param inds= (work.mp_getconstraints) The input table containing the
+    constraint info
+  @param outds= (work.mp_createconstraints) A table containing the create
+    statements (create_statement column)
+  @param execute= (NO) To actually create, use YES.
 
-  <h4> Dependencies </h4>
+  <h4> Related Files </h4>
+  @li mp_getconstraints.sas
 
   @version 9.2
   @author Allan Bowe
@@ -30,7 +33,7 @@
 **/
 
 %macro mp_createconstraints(inds=mp_getconstraints
-  ,outds=mp_createconstraints
+  ,outds=work.mp_createconstraints
   ,execute=NO
 )/*/STORE SOURCE*/;
 
@@ -48,7 +51,7 @@ data &outds;
     else type=constraint_type;
     create_statement=catx(" ","alter table",libref,".",table_name
       ,"add constraint",constraint_name,type,"(");
-    if last.constraint_name then 
+    if last.constraint_name then
       create_statement=cats(create_statement,column_name,");");
     else create_statement=cats(create_statement,column_name,",");
     if "&execute"="YES" then call execute(create_statement);
@@ -64,4 +67,4 @@ data &outds;
   output;
 run;
 
-%mend;
+%mend mp_createconstraints;

base/mp_createwebservice.sas

@@ -1,47 +1,13 @@
 /**
   @file mp_createwebservice.sas
-  @brief Create a web service in SAS 9 or Viya
-  @details Creates a SASJS ready Stored Process in SAS 9 or Job Execution
-  Service in SAS Viya
-
-Usage:
-
-    %* compile macros ;
-    filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
-    %inc mc;
-
-    %* write some code;
-    filename ft15f001 temp;
-    parmcards4;
-        %* fetch any data from frontend ;
-        %webout(FETCH) 
-        data example1 example2;
-          set sashelp.class;
-        run;
-        %* send data back;
-        %webout(OPEN)
-        %webout(ARR,example1) * Array format, fast, suitable for large tables ;
-        %webout(OBJ,example2) * Object format, easier to work with ;
-        %webout(CLOSE)
-    ;;;;
-    %mp_createwebservice(path=/Public/app/common,name=appInit,code=ft15f001,replace=YES)
-
-  <h4> Dependencies </h4>
-  @li mf_getplatform.sas
-  @li mm_createwebservice.sas
-  @li mv_createwebservice.sas
-
-  @param path= The full folder path where the service will be created
-  @param name= Service name.  Avoid spaces.
-  @param desc= The description of the service (optional)
-  @param precode= Space separated list of filerefs, pointing to the code that
-    needs to be attached to the beginning of the service (optional)
-  @param code= Space seperated fileref(s) of the actual code to be added
-  @param replace= select YES to replace any existing service in that location
+  @brief Create a web service in SAS 9, Viya or SASjs Server
+  @details This is actually a wrapper for mx_createwebservice.sas, remaining
+  for legacy purposes.  For new apps, use mx_createwebservice.sas.
 
 
-  @version 9.2
-  @author Allan Bowe
+  <h4> SAS Macros </h4>
+  @li mx_createwebservice.sas
+
 
 **/
 
@@ -51,33 +17,16 @@ Usage:
     ,code=ft15f001
     ,desc=This service was created by the mp_createwebservice macro
     ,replace=YES
+    ,mdebug=0
 )/*/STORE SOURCE*/;
 
-%if &syscc ge 4 %then %do;
-  %put syscc=&syscc - &sysmacroname will not execute in this state;
-  %return;
-%end;
-
-%local platform; %let platform=%mf_getplatform();
-%if &platform=SASVIYA %then %do;
-  %if "&path"="HOME" %then %let path=/Users/&sysuserid/My Folder;
-  %mv_createwebservice(path=&path
+  %mx_createwebservice(path=&path
     ,name=&name
-    ,code=&code
     ,precode=&precode
+    ,code=&code
     ,desc=&desc
     ,replace=&replace
+    ,mdebug=&mdebug
   )
-%end;
-%else %do;
-  %if "&path"="HOME" %then %let path=/User Folders/&sysuserid/My Folder;
-  %mm_createwebservice(path=&path
-    ,name=&name
-    ,code=&code
-    ,precode=&precode
-    ,desc=&desc
-    ,replace=&replace
-  )
-%end;
 
-%mend;
+%mend mp_createwebservice;

base/mp_csv2ds.sas

@@ -0,0 +1,140 @@
+/**
+  @file mp_csv2ds.sas
+  @brief Efficient import of arbitrary CSV using a dataset as template
+  @details Used to import relevant columns from a large CSV using
+  a dataset to provide the types and lengths.  Assumes that a header
+  row is provided, and datarows start on line 2.  Extra columns in
+  both the CSV and base dataset are ignored.
+
+  Usage:
+
+      filename mycsv temp;
+      data _null_;
+        file mycsv;
+        put 'name,age,nickname';
+        put 'John,48,Jonny';
+        put 'Jennifer,23,Jen';
+      run;
+
+      %mp_csv2ds(inref=mycsv,outds=myds,baseds=sashelp.class)
+
+
+  @param inref= fileref to the CSV
+  @param outds= output ds (lib.ds format)
+  @param view= Set to YES or NO to determine whether the output should be
+    a view or not.  Default is NO (not a view).
+  @param baseds= Template dataset on which to create the input statement.
+    Is used to determine types, lengths, and any informats.
+
+  @version 9.2
+  @author Allan Bowe
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_existds.sas
+
+**/
+
+%macro mp_csv2ds(inref=0,outds=0,baseds=0,view=NO);
+
+%mp_abort(iftrue=( &inref=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the INREF variable must be provided)
+)
+%mp_abort(iftrue=( %superq(outds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the OUTDS variable must be provided)
+)
+%mp_abort(iftrue=( &baseds=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS variable must be provided)
+)
+%mp_abort(iftrue=( %mf_existds(&baseds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS dataset (&baseds) needs to be assigned, and to exist)
+)
+
+/* count rows */
+%local hasheader; %let hasheader=0;
+data _null_;
+  if _N_ > 1 then do;
+    call symputx('hasheader',1,'l');
+    stop;
+  end;
+  infile &inref;
+  input;
+run;
+%mp_abort(iftrue=( &hasheader=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(No header row in &inref)
+)
+
+/* get the variables in the CSV */
+data _data_;
+  infile &inref;
+  input;
+  length name $32;
+  do i=1 to countc(_infile_,',')+1;
+    name=upcase(scan(_infile_,i,','));
+    output;
+  end;
+  stop;
+run;
+%local csv_vars;%let csv_vars=&syslast;
+
+/* get the variables in the dataset */
+proc contents noprint data=&baseds
+  out=_data_ (keep=name type length format: informat);
+run;
+%local base_vars; %let base_vars=&syslast;
+
+proc sql undo_policy=none;
+create table &csv_vars as
+  select a.*
+    ,b.type
+    ,b.length
+    ,b.format
+    ,b.formatd
+    ,b.formatl
+    ,b.informat
+  from &csv_vars a
+  left join &base_vars b
+  on a.name=upcase(b.name)
+  order by i;
+
+/* prepare the input statement */
+%local instat dropvars;
+data _null_;
+  set &syslast end=last;
+  length in dropvars $32767;
+  retain in dropvars;
+  if missing(type) then do;
+    informat='$1.';
+    dropvars=catx(' ',dropvars,name);
+  end;
+  else if missing(informat) then do;
+    if type=1 then informat='best.';
+    else informat=cats('$',length,'.');
+  end;
+  else informat=cats(informat,'.');
+  in=catx(' ',in,name,':',informat);
+  if last then do;
+    call symputx('instat',in,'l');
+    call symputx('dropvars',dropvars,'l');
+  end;
+run;
+
+/* import the CSV */
+data &outds
+  %if %upcase(&view)=YES %then %do;
+    /view=&outds
+  %end;
+  ;
+  infile &inref dsd firstobs=2;
+  input &instat;
+  %if %length(&dropvars)>0 %then %do;
+    drop &dropvars;
+  %end;
+run;
+
+%mend mp_csv2ds;

base/mp_deleteconstraints.sas

@@ -13,7 +13,7 @@
           constraint unq unique(tx_from, dd_type),
           constraint nnn not null(DD_SHORTDESC)
         );
-      
+
       %mp_getconstraints(lib=work,ds=example,outds=work.constraints)
       %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
 
@@ -49,4 +49,4 @@ data &outds;
   end;
 run;
 
-%mend;
+%mend mp_deleteconstraints;

base/mp_deletefolder.sas

@@ -0,0 +1,70 @@
+/**
+  @file
+  @brief A macro to delete a directory
+  @details Will delete all folder content (including subfolder content) and
+    finally, the folder itself.
+
+  Usage:
+
+      %let rootdir=%sysfunc(pathname(work))/demo;
+      %mf_mkdir(&rootdir)
+      %mf_mkdir(&rootdir/subdir)
+      %mf_mkdir(&rootdir/subdir/subsubdir)
+      data "&rootdir/subdir/example.sas7bdat";
+      run;
+
+      %mp_deletefolder(&rootdir)
+
+  @param path Unquoted path to the folder to delete.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_isdir.sas
+  @li mp_dirlist.sas
+
+  <h4> Related Macros </h4>
+  @li mp_deletefolder.test.sas
+
+**/
+
+%macro mp_deletefolder(folder);
+  /* proceed if valid directory */
+  %if %mf_isdir(&folder)=1 %then %do;
+
+    /* prep temp table */
+    %local tempds;
+    %let tempds=%mf_getuniquename();
+
+    /* recursive directory listing */
+    %mp_dirlist(path=&folder,outds=work.&tempds, maxdepth=MAX)
+
+    /* sort descending level so can delete folder contents before folders */
+    proc sort data=work.&tempds;
+      by descending level;
+    run;
+
+    /* ensure top level folder is removed at the end */
+    proc sql;
+    insert into work.&tempds set filepath="&folder";
+
+    /* delete everything */
+    data _null_;
+      set work.&tempds end=last;
+      length fref $8;
+      fref='';
+      rc=filename(fref,filepath);
+      rc=fdelete(fref);
+      if rc then do;
+        msg=sysmsg();
+        put "&sysmacroname:" / rc= / msg= / filepath=;
+      end;
+      rc=filename(fref);
+    run;
+
+    /* tidy up */
+    proc sql;
+    drop table work.&tempds;
+
+  %end;
+  %else %put &sysmacroname: &folder: is not a valid / accessible folder. ;
+%mend mp_deletefolder;

base/mp_dictionary.sas

@@ -0,0 +1,52 @@
+/**
+  @file mp_dictionary.sas
+  @brief Creates a portal (libref) into the SQL Dictionary Views
+  @details Provide a libref and the macro will create a series of views against
+  each view in the special PROC SQL dictionary libref.
+
+  This is useful if you would like to visualise (navigate) the views in a SAS
+  client such as Base SAS, Enterprise Guide, or Studio (or [Data Controller](
+  https://datacontroller.io)).
+
+  It works by extracting the dictionary.dictionaries view into
+  YOURLIB.dictionaries, then uses that to create a YOURLIB.{viewName} for every
+  other dictionary.view, eg:
+
+      proc sql;
+      create view YOURLIB.columns as select * from dictionary.columns;
+
+  Usage:
+
+      libname demo "/lib/directory";
+      %mp_dictionary(lib=demo)
+
+  Or, to just create them in WORK:
+
+      %mp_dictionary()
+
+  If you'd just like to browse the dictionary data model, you can also check
+  out [this article](https://rawsas.com/dictionary-of-dictionaries/).
+
+  ![](https://user-images.githubusercontent.com/4420615/188278365-2987db97-0594-4a39-ac81-dbacdef5cdc8.png)
+
+  @param lib= (WORK) The libref in which to create the views
+
+  <h4> Related Files </h4>
+  @li mp_dictionary.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_dictionary(lib=WORK)/*/STORE SOURCE*/;
+  %local list i mem;
+  proc sql noprint;
+  create view &lib..dictionaries as select * from dictionary.dictionaries;
+  select distinct memname into: list separated by ' '  from &lib..dictionaries;
+  %do i=1 %to %sysfunc(countw(&list,%str( )));
+    %let mem=%scan(&list,&i,%str( ));
+    create view &lib..&mem as select * from dictionary.&mem;
+  %end;
+  quit;
+%mend mp_dictionary;

base/mp_dirlist.sas

@@ -1,69 +1,118 @@
 /**
   @file
   @brief Returns all files and subdirectories within a specified parent
-  @details When used with getattrs=NO, is not OS specific (uses dopen / dread). 
-
-  If getattrs=YES then the doptname / foptname functions are used to scan all
-  properties - any characters that are not valid in a SAS name (v7) are simply 
-  stripped, and the table is transposed so theat each property is a column
-  and there is one file per row.  An attempt is made to get all properties 
-  whether a file or folder, but some files/folders cannot be accessed, and so
-  not all properties can / will be populated.
+  @details When used with getattrs=NO, is not OS specific (uses dopen / dread).
 
   Credit for the rename approach:
-  https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003 
+  https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003
 
+  Usage:
 
-  usage:
-
-      %mp_dirlist(path=/some/location,outds=myTable)
+      %mp_dirlist(path=/some/location, outds=myTable, maxdepth=MAX)
 
       %mp_dirlist(outds=cwdfileprops, getattrs=YES)
 
-  @warning In a Unix environment, the existence of a named pipe will cause this 
+      %mp_dirlist(fref=MYFREF)
+
+  @warning In a Unix environment, the existence of a named pipe will cause this
   macro to hang.  Therefore this tool should be used with caution in a SAS 9 web
   application, as it can use up all available multibridge sessions if requests
   are resubmitted.
-  If anyone finds a way to positively identify a named pipe using SAS (without 
+  If anyone finds a way to positively identify a named pipe using SAS (without
   X CMD) do please raise an issue!
 
 
-  @param path= for which to return contents
-  @param outds= the output dataset to create
-  @param getattrs= YES/NO (default=NO).  Uses doptname and foptname to return 
-  all attributes for each file / folder.  
+  @param [in] path= (%sysfunc(pathname(work))) Path for which to return contents
+  @param [in] fref= (0) Provide a DISK engine fileref as an alternative to PATH
+  @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
+    subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
+  @param [in] showparent= (NO) By default, the initial parent directory is not
+    part of the results.  Set to YES to include it.  For this record only,
+    directory=filepath.
+  @param [out] outds= (work.mp_dirlist) The output dataset to create
+  @param [out] getattrs= (NO)  If getattrs=YES then the doptname / foptname
+    functions are used to scan all properties - any characters that are not
+    valid in a SAS name (v7) are simply stripped, and the table is transposed
+    so theat each property is a column and there is one file per row.  An
+    attempt is made to get all properties whether a file or folder, but some
+    files/folders cannot be accessed, and so not all properties can / will be
+    populated.
 
 
   @returns outds contains the following variables:
-   - file_or_folder (file / folder)
-   - filepath (path/to/file.name)
-   - filename (just the file name)
-   - ext (.extension)
-   - msg (system message if any issues)
-   - OS SPECIFIC variables, if <code>getattrs=</code> is used.
+    - directory (containing folder)
+    - file_or_folder (file / folder)
+    - filepath (path/to/file.name)
+    - filename (just the file name)
+    - ext (.extension)
+    - msg (system message if any issues)
+    - level (depth of folder)
+    - OS SPECIFIC variables, if <code>getattrs=</code> is used.
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_getvarlist.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_dropmembers.sas
+
+  <h4> Related Macros </h4>
+  @li mp_dirlist.test.sas
 
   @version 9.2
-  @author Allan Bowe
 **/
 
 %macro mp_dirlist(path=%sysfunc(pathname(work))
+    , fref=0
     , outds=work.mp_dirlist
     , getattrs=NO
+    , showparent=NO
+    , maxdepth=0
+    , level=0 /* The level of recursion to perform.  For internal use only. */
 )/*/STORE SOURCE*/;
 %let getattrs=%upcase(&getattrs)XX;
 
-data &outds (compress=no keep=file_or_folder filepath filename ext msg);
-  length filepath $500 fref fref2 $8 file_or_folder $6 filename $80 ext $20 msg $200;
-  rc = filename(fref, "&path");
+/* temp table */
+%local out_ds;
+data;run;
+%let out_ds=%str(&syslast);
+
+/* drop main (top) table if it exists */
+%if &level=0 %then %do;
+  %mp_dropmembers(%scan(&outds,-1,.), libref=WORK)
+%end;
+
+data &out_ds(compress=no
+    keep=file_or_folder filepath filename ext msg directory level
+  );
+  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
+    ext $20 msg $200 foption $16;
+  if _n_=1 then call missing(of _all_);
+  retain level &level;
+  %if &fref=0 %then %do;
+    rc = filename(fref, "&path");
+  %end;
+  %else %do;
+    fref="&fref";
+    rc=0;
+  %end;
   if rc = 0 then do;
-     did = dopen(fref);
-     if did=0 then do;
-        putlog "NOTE: This directory is empty - &path";
-        msg=sysmsg();
-        put _all_;
-        stop;
-     end;
-     rc = filename(fref);
+    did = dopen(fref);
+    if did=0 then do;
+      putlog "NOTE: This directory is empty, or does not exist - &path";
+      msg=sysmsg();
+      put msg;
+      put _all_;
+      stop;
+    end;
+    /* attribute is OS-dependent - could be "Directory" or "Directory Name" */
+    numopts=doptnum(did);
+    do i=1 to numopts;
+      foption=doptname(did,i);
+      if foption=:'Directory' then i=numopts;
+    end;
+    directory=dinfo(did,foption);
+    rc = filename(fref);
   end;
   else do;
     msg=sysmsg();
@@ -73,7 +122,8 @@ data &outds (compress=no keep=file_or_folder filepath filename ext msg);
   dnum = dnum(did);
   do i = 1 to dnum;
     filename = dread(did, i);
-    rc = filename(fref2, "&path/"!!filename);
+    filepath=cats(directory,'/',filename);
+    rc = filename(fref2,filepath);
     midd=dopen(fref2);
     dmsg=sysmsg();
     if did > 0 then file_or_folder='folder';
@@ -82,12 +132,12 @@ data &outds (compress=no keep=file_or_folder filepath filename ext msg);
     fmsg=sysmsg();
     if midf > 0 then file_or_folder='file';
     rc=fclose(midf);
-    
-    if index(fmsg,'File is in use') or index(dmsg,'is not a directory') 
+
+    if index(fmsg,'File is in use') or index(dmsg,'is not a directory')
       then file_or_folder='file';
-    else if index(fmsg, 'Insufficient authorization') then file_or_folder='file';
+    else if index(fmsg,'Insufficient authorization') then file_or_folder='file';
     else if file_or_folder='' then file_or_folder='locked';
-      
+
     if file_or_folder='file' then do;
       ext = prxchange('s/.*\.{1,1}(.*)/$1/', 1, filename);
       if filename = ext then ext = ' ';
@@ -96,17 +146,26 @@ data &outds (compress=no keep=file_or_folder filepath filename ext msg);
       ext='';
       file_or_folder='folder';
     end;
-    filepath="&path/"!!filename;
     output;
   end;
   rc = dclose(did);
+  %if &showparent=YES and &level=0 %then %do;
+    filepath=directory;
+    file_or_folder='folder';
+    ext='';
+    filename=scan(directory,-1,'/\');
+    msg='';
+    level=&level;
+    output;
+  %end;
   stop;
 run;
 
 %if %substr(&getattrs,1,1)=Y %then %do;
-  data &outds;
-    set &outds;
+  data &out_ds;
+    set &out_ds;
     length infoname infoval $60 fref $8;
+    if _n_=1 then call missing(fref);
     rc=filename(fref,filepath);
     drop rc infoname fid i close fref;
     if file_or_folder='file' then do;
@@ -120,7 +179,7 @@ run;
       else do i=1 to foptnum(fid);
         infoname=foptname(fid,i);
         infoval=finfo(fid,infoname);
-        sasname=compress(infoname, '_', 'adik');	
+        sasname=compress(infoname, '_', 'adik');
         if anydigit(sasname)=1 then sasname=substr(sasname,anyalpha(sasname));
         if upcase(sasname) ne 'FILENAME' then output;
       end;
@@ -137,7 +196,7 @@ run;
       else do i=1 to doptnum(fid);
         infoname=doptname(fid,i);
         infoval=dinfo(fid,infoname);
-        sasname=compress(infoname, '_', 'adik');	
+        sasname=compress(infoname, '_', 'adik');
         if anydigit(sasname)=1 then sasname=substr(sasname,anyalpha(sasname));
         if upcase(sasname) ne 'FILENAME' then output;
       end;
@@ -146,10 +205,59 @@ run;
   run;
   proc sort;
     by filepath sasname;
-  proc transpose data=&outds out=&outds(drop=_:);
+  proc transpose data=&out_ds out=&out_ds(drop=_:);
     id sasname;
     var infoval;
     by filepath file_or_folder filename ext ;
   run;
 %end;
-%mend;
+
+data &out_ds;
+  set &out_ds(where=(filepath ne ''));
+run;
+
+/**
+  * The above transpose can mean that some updates create additional columns.
+  * This necessitates the occasional use of datastep over proc append.
+  */
+%if %mf_existds(&outds) %then %do;
+  %local basevars appvars newvars;
+  %let basevars=%mf_getvarlist(&outds);
+  %let appvars=%mf_getvarlist(&out_ds);
+  %let newvars=%length(%mf_wordsinstr1butnotstr2(Str1=&appvars,Str2=&basevars));
+  %if &newvars>0 %then %do;
+    data &outds;
+      set &outds &out_ds;
+    run;
+  %end;
+  %else %do;
+    proc append base=&outds data=&out_ds force nowarn;
+    run;
+  %end;
+%end;
+%else %do;
+  proc append base=&outds data=&out_ds;
+  run;
+%end;
+
+/* recursive call */
+%if &maxdepth>&level or &maxdepth=MAX %then %do;
+  data _null_;
+    set &out_ds;
+    where file_or_folder='folder';
+  %if &showparent=YES and &level=0 %then %do;
+    if filepath ne directory;
+  %end;
+    length code $10000;
+    code=cats('%nrstr(%mp_dirlist(path=',filepath,",outds=&outds"
+      ,",getattrs=&getattrs,level=%eval(&level+1),maxdepth=&maxdepth))");
+    put code=;
+    call execute(code);
+  run;
+%end;
+
+/* tidy up */
+proc sql;
+drop table &out_ds;
+
+%mend mp_dirlist;

base/mp_distinctfmtvalues.sas

@@ -19,7 +19,7 @@
 **/
 
 %macro mp_distinctfmtvalues(
-     libds=
+    libds=
     ,var=
     ,outvar=formatted_value
     ,outds=work.mp_distinctfmtvalues
@@ -34,7 +34,7 @@
   create table &outds as
     select distinct
     %if &vtype=C & %trim(&fmt)=%str() %then %do;
-       &var
+      &var
     %end;
     %else %if &vtype=C %then %do;
       put(&var,&fmt)
@@ -45,6 +45,6 @@
     %else %do;
       put(&var,&fmt)
     %end;
-       as &outvar length=&varlen
+        as &outvar length=&varlen
     from &libds;
-%mend;
+%mend mp_distinctfmtvalues;

base/mp_dropmembers.sas

@@ -1,20 +1,25 @@
 /**
   @file
   @brief Drops tables / views (if they exist) without warnings in the log
-  @details
+  @details Useful for dropping tables when you're not sure they exist, or if
+  you are not sure whether they are a dataset or view.  Also efficient for
+  dropping multiple tables / views.
+
   Example usage:
 
       proc sql;
       create table data1 as select * from sashelp.class;
       create view view2 as select * from sashelp.class;
-      %mp_dropmembers(list=data1 view2)
+      %mp_dropmembers(data1 view2, libref=WORK)
 
-  <h4> Dependencies </h4>
+
+  <h4> SAS Macros </h4>
   @li mf_isblank.sas
 
 
-  @param list space separated list of datasets / views
-  @param libref= can only drop from a single library at a time
+  @param [in] list space separated list of datasets / views, WITHOUT libref
+  @param [in] libref= (WORK) Note - you can only drop from one library at a time
+  @param [in] iftrue= (1=1) Conditionally drop tables, eg if &debug=N
 
   @version 9.2
   @author Allan Bowe
@@ -22,10 +27,13 @@
 **/
 
 %macro mp_dropmembers(
-     list /* space separated list of datasets / views */
+    list /* space separated list of datasets / views */
     ,libref=WORK  /* can only drop from a single library at a time */
+    ,iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
   %if %mf_isblank(&list) %then %do;
     %put NOTE: nothing to drop!;
     %return;
@@ -35,4 +43,4 @@
     delete &list;
     delete &list /mtype=view;
   run;
-%mend;
+%mend mp_dropmembers;

base/mp_ds2cards.sas

@@ -2,67 +2,95 @@
   @file
   @brief Create a CARDS file from a SAS dataset.
   @details Uses dataset attributes to convert all data into datalines.
-    Running the generated file will rebuild the original dataset.
-  usage:
+    Running the generated file will rebuild the original dataset.  Includes
+    support for large decimals, binary data, PROCESSED_DTTM columns, and
+    alternative encoding.  If the input dataset is empty, the cards file will
+    still be created.
 
-      %mp_ds2cards(base_ds=sashelp.class
+    Additional support to generate a random sample and max rows.
+
+  Usage:
+
+      %mp_ds2cards(sashelp.class
+        , tgt_ds=work.class
         , cards_file= "C:\temp\class.sas"
-        , maxobs=5)
+        , showlog=NO
+        , maxobs=5
+      )
 
-    stuff to add
-     - labelling the dataset
-     - explicity setting a unix LF
-     - constraints / indexes etc
+  TODO:
+    - labelling the dataset
+    - explicity setting a unix LF
+    - constraints / indexes etc
 
-  @param base_ds= Should be two level - eg work.blah.  This is the table that
-                   is converted to a cards file.
-  @param tgt_ds= Table that the generated cards file would create. Optional -
-                  if omitted, will be same as BASE_DS.
-  @param cards_file= Location in which to write the (.sas) cards file
-  @param maxobs= to limit output to the first <code>maxobs</code> observations
-  @param showlog= whether to show generated cards file in the SAS log (YES/NO)
-  @param outencoding= provide encoding value for file statement (eg utf-8)
+  @param [in] base_ds Should be two level - eg work.blah.  This is the table
+                  that is converted to a cards file.
+  @param [in] tgt_ds= Table that the generated cards file would create.
+    Optional - if omitted, will be same as BASE_DS.
+  @param [out] cards_file= ("%sysfunc(pathname(work))/cardgen.sas") Location in
+    which to write the (.sas) cards file
+  @param [in] maxobs= (max) To limit output to the first <code>maxobs</code>
+    observations, enter an integer here.
+  @param [in] random_sample= (NO) Set to YES to generate a random sample of
+    data.  Can be quite slow.
+  @param [in] showlog= (YES) Whether to show generated cards file in the SAS
+    log.  Valid values:
+    @li YES
+    @li NO
+  @param [in] outencoding= Provide encoding value for file statement (eg utf-8)
+  @param [in] append= (NO) If NO then will rebuild the cards file if it already
+    exists, otherwise will append to it.  Used by the mp_lib2cards.sas macro.
 
+  <h4> Related Macros </h4>
+  @li mp_lib2cards.sas
+  @li mp_ds2inserts.sas
+  @li mp_mdtablewrite.sas
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
-%macro mp_ds2cards(base_ds=, tgt_ds=
+%macro mp_ds2cards(base_ds, tgt_ds=
     ,cards_file="%sysfunc(pathname(work))/cardgen.sas"
     ,maxobs=max
     ,random_sample=NO
     ,showlog=YES
     ,outencoding=
+    ,append=NO
 )/*/STORE SOURCE*/;
 %local i setds nvars;
 
 %if not %sysfunc(exist(&base_ds)) %then %do;
-   %put WARNING:  &base_ds does not exist;
-   %return;
+  %put %str(WARN)ING:  &base_ds does not exist;
+  %return;
 %end;
 
 %if %index(&base_ds,.)=0 %then %let base_ds=WORK.&base_ds;
 %if (&tgt_ds = ) %then %let tgt_ds=&base_ds;
 %if %index(&tgt_ds,.)=0 %then %let tgt_ds=WORK.%scan(&base_ds,2,.);
 %if ("&outencoding" ne "") %then %let outencoding=encoding="&outencoding";
+%if ("&append" = "" or "&append" = "NO") %then %let append=;
+%else %let append=mod;
 
 /* get varcount */
 %let nvars=0;
 proc sql noprint;
 select count(*) into: nvars from dictionary.columns
-  where libname="%scan(%upcase(&base_ds),1)"
-    and memname="%scan(%upcase(&base_ds),2)";
+  where upcase(libname)="%scan(%upcase(&base_ds),1)"
+    and upcase(memname)="%scan(%upcase(&base_ds),2)";
 %if &nvars=0 %then %do;
-  %put WARNING:  Dataset &base_ds has no variables!  It will not be converted.;
+  %put %str(WARN)ING: Dataset &base_ds has no variables, will not be converted.;
   %return;
 %end;
 
 /* get indexes */
-proc sort data=sashelp.vindex
-    (where=(upcase(libname)="%scan(%upcase(&base_ds),1)"
-       and upcase(memname)="%scan(%upcase(&base_ds),2)"))
-	out=_data_;
+proc sort
+  data=sashelp.vindex(
+    where=(upcase(libname)="%scan(%upcase(&base_ds),1)"
+      and upcase(memname)="%scan(%upcase(&base_ds),2)")
+    )
+  out=_data_;
   by indxname indxpos;
 run;
 
@@ -77,7 +105,7 @@ data _null_;
     idxcnt+1;
     nom='';
     uni='';
-  	vars=name;
+    vars=name;
   end;
   else vars=catx(' ',vars,name);
   if last.indxname then do;
@@ -103,18 +131,18 @@ proc sql
   order by ranuni(42)
 %end;
   ;
-
-
+reset outobs=max;
 create table datalines1 as
-   select name,type,length,varnum,format,label from dictionary.columns
-   where libname="%upcase(%scan(&base_ds,1))"
-    and memname="%upcase(%scan(&base_ds,2))";
+  select name,type,length,varnum,format,label from dictionary.columns
+  where upcase(libname)="%upcase(%scan(&base_ds,1))"
+    and upcase(memname)="%upcase(%scan(&base_ds,2))";
 
 /**
   Due to long decimals cannot use best. format
   So - use bestd. format and then use character functions to strip trailing
-    zeros, if NOT an integer!!
-  resolved code = ifc(int(VARIABLE)=VARIABLE
+    zeros, if NOT an integer or missing!!  Cannot use int() as it upsets
+    note2err when there are missings.
+  resolved code = ifc( mod(coalesce(VARIABLE,0),1)=0
     ,put(VARIABLE,best32.)
     ,substrn(put(VARIABLE,bestd32.),1
     ,findc(put(VARIABLE,bestd32.),'0','TBK')));
@@ -122,23 +150,34 @@ create table datalines1 as
 
 data datalines_2;
   format dataline $32000.;
- set datalines1 (where=(upcase(name) not in
+  set datalines1 (where=(upcase(name) not in
     ('PROCESSED_DTTM','VALID_FROM_DTTM','VALID_TO_DTTM')));
   if type='num' then dataline=
-    cats('ifc(int(',name,')=',name,'
+    cats('ifc(mod(coalesce(',name,',0),1)=0
       ,put(',name,',best32.-l)
       ,substrn(put(',name,',bestd32.-l),1
       ,findc(put(',name,',bestd32.-l),"0","TBK")))');
-  else dataline=name;
+  /**
+    * binary data must be converted, to store in text format.  It is identified
+    * by the presence of the $HEX keyword in the format.
+    */
+  else if upcase(format)=:'$HEX' then
+    dataline=cats('put(trim(',name,'),',format,')');
+  /**
+    * There is no easy way to store line breaks in a cards file.
+    * To discuss this, use: https://github.com/sasjs/core/issues/80
+    * Removing all nonprintables with kw (keep writeable)
+    */
+  else dataline=cats('compress(',name,', ,"kw")');
 run;
 
 proc sql noprint;
 select dataline into: datalines separated by ',' from datalines_2;
 
 %local
-   process_dttm_flg
-   valid_from_dttm_flg
-   valid_to_dttm_flg
+  process_dttm_flg
+  valid_from_dttm_flg
+  valid_to_dttm_flg
 ;
 %let process_dttm_flg = N;
 %let valid_from_dttm_flg = N;
@@ -154,7 +193,8 @@ data _null_;
 
 
 /* Build input statement */
-  if type='char' then type3=':$char.';
+  if upcase(format)=:'$HEX' then type3=':'!!format;
+  else if type='char' then type3=':$char.';
   str2=put(name,$33.)||type3;
 
 
@@ -173,14 +213,16 @@ data _null_;
 run;
 
 data _null_;
-  file &cards_file. &outencoding lrecl=32767 termstr=nl;
+  file &cards_file. &outencoding lrecl=32767 termstr=nl &append;
   length __attrib $32767;
   if _n_=1 then do;
-    put '/*******************************************************************';
-    put " Datalines for %upcase(%scan(&base_ds,2)) dataset ";
-    put " Generated by %nrstr(%%)mp_ds2cards()";
-    put " Available on github.com/sasjs/core";
-    put '********************************************************************/';
+    put '/**';
+    put '  @file';
+    put "  @brief Datalines for %upcase(%scan(&base_ds,2)) dataset";
+    put "  @details Generated by %nrstr(%%)mp_ds2cards()";
+    put "    Source: https://github.com/sasjs/core";
+    put '  @cond ';
+    put '**/';
     put "data &tgt_ds &indexes;";
     put "attrib ";
     %do i = 1 %to &nvars;
@@ -204,14 +246,15 @@ data _null_;
       put 'run;';
     end;
     else do;
-      put "infile cards dsd delimiter=',';";
+      put "infile cards dsd;";
       put "input ";
       %do i = 1 %to &nvars.;
         %if(%length(&&input_stmt_&i..)) %then
-           put "   &&input_stmt_&i..";
+          put "  &&input_stmt_&i..";
         ;
       %end;
       put ";";
+      put 'missing a b c d e f g h i j k l m n o p q r s t u v w x y z _;';
       put "datalines4;";
     end;
   end;
@@ -224,6 +267,7 @@ data _null_;
   if __lastobs then do;
     put ';;;;';
     put 'run;';
+    put '/** @endcond **/';
     stop;
   end;
 run;
@@ -243,4 +287,5 @@ quit;
 %put NOTE-;%put NOTE-;
 %put NOTE- %sysfunc(dequote(&cards_file.));
 %put NOTE-;%put NOTE-;
-%mend;
+%mend mp_ds2cards;
+/** @endcond **/