Merge pull request #409 from sasjs/build/disable-npm-scripts

Disable npm scripts

.all-contributorsrc

@@ -135,8 +135,36 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "henrik-forsell",
+      "name": "Henrik Forsell",
+      "avatar_url": "https://avatars.githubusercontent.com/u/109935936?v=4",
+      "profile": "https://github.com/henrik-forsell",
+      "contributions": [
+        "doc"
+      ]
+    },
+    {
+      "login": "rudvfaden",
+      "name": "Rud Faden",
+      "avatar_url": "https://avatars.githubusercontent.com/u/2445577?v=4",
+      "profile": "http://rudvfaden.github.io/",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "andyjessen",
+      "name": "andyjessen",
+      "avatar_url": "https://avatars.githubusercontent.com/u/62343929?v=4",
+      "profile": "https://github.com/andyjessen",
+      "contributions": [
+        "doc"
+      ]
     }
   ],
   "contributorsPerLine": 7,
-  "skipCi": true
+  "skipCi": true,
+  "commitType": "docs"
 }

.git-hooks/pre-commit

@@ -2,7 +2,7 @@
 
 # Ensure lint is passing
 LINT=`sasjs lint`
-if [[ "$LINT" != "✔ All matched files use @sasjs/lint code style!" ]]; then
+if [[ "$LINT" != *"All matched files use @sasjs/lint code style!" ]]; then
   echo "$LINT"
   echo "To commit in spite of these warnings, use the -n parameter."
   exit 1

.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

.github/CONTRIBUTING.md

@@ -27,5 +27,6 @@ To contribute:
 1.  Create your feature branch (`git checkout -b myfeature`)
 2.  Make your change
 3.  Update the `all.sas` file (`python3 build.py`)
-4.  Push and make a PR
+4.  Commit using a [Conventional Commit](https://www.conventionalcommits.org)
+5.  Push and make a PR
 

.github/FUNDING.yml

@@ -1,3 +1,3 @@
 # These are supported funding model platforms
 
-github: [sasjs]
+custom: https://getalby.com/p/sasjs

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,3 +15,4 @@ What code changes have been made to achieve the intent.
 - [ ] Code is formatted correctly (`sasjs lint`).
 - [ ] Any new functionality has been unit tested.
 - [ ] All unit tests are passing (`sasjs test`).
+- [ ] The PR desc or underlying commits follow the [Conventional Commit](https://www.conventionalcommits.org) standard

.github/vpn/config.ovpn

@@ -1,30 +1,27 @@
-cipher AES-256-CBC
-setenv FORWARD_COMPATIBLE 1
+# Client
 client
-server-poll-timeout 4
-nobind
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 443 tcp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
+tls-client
 dev tun
-dev-type tun
-ns-cert-type server
-setenv opt tls-version-min 1.0 or-highest
-reneg-sec 604800
-sndbuf 0
-rcvbuf 0
-# NOTE: LZO commands are pushed by the Access Server at connect time.
-# NOTE: The below line doesn't disable LZO.
-comp-lzo no
-verb 3
-setenv PUSH_PEER_INFO
+# this will connect with whatever proto DNS tells us (https://community.openvpn.net/openvpn/ticket/934)
+proto udp
+remote vpn.4gl.io 7194
+resolv-retry infinite
+# this will fallback from udp6 to udp4 as well
+connect-timeout 5
+data-ciphers AES-256-CBC:AES-256-GCM
+auth SHA256
+script-security 2
+keepalive 10 120
+remote-cert-tls server
 
+# Keys
 ca ca.crt
 cert user.crt
 key user.key
 tls-auth tls.key 1
+
+# Security
+nobind
+persist-key
+persist-tun
+verb 3

.github/workflows/main.yml

@@ -13,9 +13,54 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-      - name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v2
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          npm ci
+
+      - name: Check code style (aborts if errors found)
+        run: npx @sasjs/cli lint
+
+      - name: Write VPN Files
+        run: |
+          echo "$CA_CRT" > .github/vpn/ca.crt
+          echo "$USER_CRT" > .github/vpn/user.crt
+          echo "$USER_KEY" > .github/vpn/user.key
+          echo "$TLS_KEY" > .github/vpn/tls.key
+        shell: bash
         env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+          CA_CRT: ${{ secrets.CA_CRT}}
+          USER_CRT: ${{ secrets.USER_CRT }}
+          USER_KEY: ${{ secrets.USER_KEY }}
+          TLS_KEY: ${{ secrets.TLS_KEY }}
+
+      - name: Install Open VPN
+        run: |
+          #sudo apt install apt-transport-https
+          #sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
+          #sudo apt-key add openvpn-repo-pkg-key.pub
+          #sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
+          #sudo apt update
+          #sudo apt install openvpn3=17~betaUb22042+jammy
+      - name: Start Open VPN 3
+        run: |
+          # openvpn3 session-start --config .github/vpn/config.ovpn
+
+      - name: Add credentials
+        run: |
+          echo "CLIENT=${{secrets.SAS9_4GL_IO_CLIENT}}"> .env.server
+          echo "ACCESS_TOKEN=${{secrets.SAS9_4GL_IO_ACCESS_TOKEN}}" >> .env.server
+          echo "REFRESH_TOKEN=${{secrets.SAS9_4GL_IO_REFRESH_TOKEN}}" >> .env.server
+
+      - name: Semantic Release
+        uses: cycjimmy/semantic-release-action@v4
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      - name: SAS Packages Release
+        run: |
+          npx @sasjs/cli compile job -s sasjs/utils/create_sas_package.sas -o sasjsbuild -t server
+          # need long duration token per https://github.com/sasjs/server/issues/307
+          # npx @sasjs/cli run sasjsbuild/jobs/utils/create_sas_package.sas -t server

.github/workflows/notmain.yml

@@ -0,0 +1,32 @@
+# This workflow will run tests using node and then publish a package to GitHub Packages when a release is created
+# For more information see: https://help.github.com/actions/language-and-framework-guides/publishing-nodejs-packages
+
+name: SASjs Core - Update all.sas
+
+on:
+  push:
+    branches-ignore:
+      - main
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          npm ci
+          npm i -g @sasjs/cli@latest
+
+      - name: Ensure all.sas is always up to date
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          python3 build.py
+          git add all.sas
+          git commit -m "chore: updating all.sas" --allow-empty
+          git push
+
+

.github/workflows/run-tests.yml

@@ -8,11 +8,11 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     strategy:
       matrix:
-        node-version: [lts/fermium]
+        node-version: [lts/hydrogen]
 
     steps:
       - uses: actions/checkout@v2
@@ -34,51 +34,53 @@ jobs:
           USER_KEY: ${{ secrets.USER_KEY }}
           TLS_KEY: ${{ secrets.TLS_KEY }}
 
+      - name: Chmod VPN files
+        run: |
+          chmod 600 .github/vpn/ca.crt .github/vpn/user.crt .github/vpn/user.key .github/vpn/tls.key
+
       - name: Install Open VPN
         run: |
           sudo apt install apt-transport-https
           sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
           sudo apt-key add openvpn-repo-pkg-key.pub
-          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-focal.list
+          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
           sudo apt update
-          sudo apt install openvpn3
+          sudo apt install openvpn3=17~betaUb22042+jammy
 
       - name: Start Open VPN 3
         run: openvpn3 session-start --config .github/vpn/config.ovpn
 
+      - name: Fetch SASJS server
+        run: curl ${{ secrets.SASJS_SERVER_URL }}/SASjsApi/info
+
       - name: Install Doxygen
         run: sudo apt-get install doxygen
 
       - name: Install dependencies
         run: npm ci
 
-      - name: Check code style
-        run: npm run lint
+      - name: Check code style (aborts if errors found)
+        run: npx @sasjs/cli lint
 
       - name: Add client
-        run: echo "CLIENT=${{secrets.CLIENT}}"> .env.viya
-
-      - name: Add secret
-        run: echo "SECRET=${{secrets.SECRET}}" >> .env.viya
+        run: echo "CLIENT=${{secrets.SAS9_4GL_IO_CLIENT}}"> .env.server
 
       - name: Add access token
-        run: echo "ACCESS_TOKEN=${{secrets.ACCESS_TOKEN}}" >> .env.viya
+        run: echo "ACCESS_TOKEN=${{secrets.SAS9_4GL_IO_ACCESS_TOKEN}}" >> .env.server
 
       - name: Add refresh token
-        run: echo "REFRESH_TOKEN=${{secrets.REFRESH_TOKEN}}" >> .env.viya
+        run: echo "REFRESH_TOKEN=${{secrets.SAS9_4GL_IO_REFRESH_TOKEN}}" >> .env.server
 
-      - name: Build Project
-        run: npm run build
+      - name: Build & Deploy Project to SAS server
+        run: npx @sasjs/cli cbd -t server
 
-      - name: Run SASjs  tests
-        run: npm run test
+      - name: Run all tests
+        run: npx @sasjs/cli test -t server
         env:
           CI: true
           CLIENT: ${{secrets.CLIENT}}
           SECRET: ${{secrets.SECRET}}
           SAS_USERNAME: ${{secrets.SAS_USERNAME}}
           SAS_PASSWORD: ${{secrets.SAS_PASSWORD}}
-          SERVER_URL: ${{secrets.SERVER_URL}}
-          SERVER_TYPE: ${{secrets.SERVER_TYPE}}
           ACCESS_TOKEN: ${{secrets.ACCESS_TOKEN}}
           REFRESH_TOKEN: ${{secrets.REFRESH_TOKEN}}

.gitignore

@@ -10,4 +10,7 @@ sasjsresults/
 mc_*
 
 # ignore .env files as they can contain sasjs access tokens
-*.env*
+*.env*
+
+~
+

.npmrc

@@ -0,0 +1 @@
+ignore-scripts=true

.sasjslint

@@ -1,13 +1,15 @@
 {
-    "noTrailingSpaces": true,
-    "noEncodedPasswords": true,
-    "hasDoxygenHeader": true,
-    "hasMacroNameInMend": true,
-    "hasMacroParentheses": true,
-    "noNestedMacros": false,
-    "noSpacesInFileNames": true,
-    "maxLineLength": 300,
-    "lowerCaseFileNames": true,
-    "noTabIndentation": true,
-    "indentationMultiple": 2
-}
+  "noTrailingSpaces": true,
+  "noEncodedPasswords": true,
+  "hasDoxygenHeader": true,
+  "hasMacroNameInMend": true,
+  "hasMacroParentheses": true,
+  "lineEndings": "lf",
+  "noGremlins": true,
+  "noNestedMacros": false,
+  "noSpacesInFileNames": true,
+  "maxLineLength": 300,
+  "lowerCaseFileNames": true,
+  "noTabs": true,
+  "indentationMultiple": 2
+}

.vscode/settings.json

@@ -6,5 +6,7 @@
   "editor.rulers": [
     80
   ],
-  "files.trimTrailingWhitespace": true
+  "files.trimTrailingWhitespace": true,
+  "sasjs-for-vscode.target": "docsonly",
+  "sasjs-for-vscode.isLocal": true
 }

README.md

@@ -1,21 +1,12 @@
 # Macro Core
-[![npm package][npm-image]][npm-url]
-[![Github Workflow][githubworkflow-image]][githubworkflow-url]
+
+[![npm package](https://img.shields.io/npm/v/@sasjs/core.svg)](http://npmjs.org/package/@sasjs/core)
+[![Github Workflow](https://github.com/sasjs/core/actions/workflows/main.yml/badge.svg)](https://github.com/sasjs/core/blob/main/.github/workflows/main.yml)
 ![npm](https://img.shields.io/npm/dt/@sasjs/core)
-![Snyk Vulnerabilities for npm package](https://img.shields.io/snyk/vulnerabilities/npm/@sasjs/core)
-[![License](https://img.shields.io/apm/l/atomic-design-ui.svg)](/LICENSE)
 ![GitHub top language](https://img.shields.io/github/languages/top/sasjs/core)
 [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sasjs/core)](https://github.com/sasjs/core/issues?q=is%3Aissue+is%3Aclosed)
 [![GitHub issues](https://img.shields.io/github/issues-raw/sasjs/core)](https://github.com/sasjs/core/issues)
 ![total lines](https://tokei.rs/b1/github/sasjs/core)
-[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-908a85?logo=gitpod)](https://gitpod.io/#https://github.com/sasjs/core)
-
-
-[npm-image]:https://img.shields.io/npm/v/@sasjs/core.svg
-[npm-url]:http://npmjs.org/package/@sasjs/core
-[githubworkflow-image]:https://github.com/sasjs/core/actions/workflows/main.yml/badge.svg
-[githubworkflow-url]:https://github.com/sasjs/core/blob/main/.github/workflows/main.yml
-[dependency-url]:https://github.com/sasjs/core/blob/main/package.json
 
 
 Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/.github/CONTRIBUTING.md) are welcome.
@@ -36,21 +27,21 @@ Documentation: https://core.sasjs.io
 - OS independent
 - Works on all SAS Platforms
 - No X command
-- Prefixes: _mf_, _mp_
+- Prefixes: `mf_`, `mp_`
 
 ### DDL folder (All Platforms)
 
 - OS independent
 - Works on all SAS Platforms
 - No X command
-- Prefixes: _mddl_(lib)_ -> where lib can be "SAS" (in relation to a SAS component) or "DC" (in relation to a Data Controller component)
+- Prefixes: `mddl_(lib)_` -> where lib can be "SAS" (in relation to a SAS component) or "DC" (in relation to a Data Controller component)
 
 This library will not be used for storing data entries (such as formats or datalines).  Where this becomes necessary in the future, a new repo will be created, in order to keep the NPM bundle size down (for the benefit of those looking to embed purely macros in their applications).
 
 ### FCMP folder (All Platforms)
 
 - Function and macro names are identical, except for special cases
-- Prefixes: _mcf_
+- Prefixes: `mcf_`
 
 The fcmp macros are used to generate fcmp functions, and can be used with or without the `proc fcmp` wrapper.
 
@@ -72,7 +63,7 @@ endsubmit;
 run;
 ```
 
-- Prefixes: _ml_
+- Prefixes: `ml_`
 
 ### META folder (SAS9 only)
 
@@ -81,14 +72,14 @@ Macros used in SAS EBI, which connect to the metadata server.
 - OS independent
 - Metadata aware
 - No X command
-- Prefixes: _mm_
+- Prefixes: `mm_`
 
 ### METAX folder (SAS9 only)
 
 - OS specific
 - Metadata aware
 - X command enabled
-- Prefixes: _mmw_,_mmu_,_mmx_
+- Prefixes: `mmx_`
 
 ### SERVER folder (@sasjs/server only)
 These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
@@ -96,7 +87,7 @@ These macros are used for building applications using [@sasjs/server](https://se
 - OS independent
 - @sasjs/server aware
 - No X command
-- Prefixes: _ms_
+- Prefixes: `ms_`
 
 ### VIYA folder (Viya only)
 
@@ -104,7 +95,7 @@ Macros used for interfacing with SAS Viya.
 
 - OS independent
 - No X command
-- Prefixes: _mv_, _mvf_
+- Prefixes: `mv_`, `mvf_`
 
 ### XPLATFORM folder (Viya, Meta, and Server)
 
@@ -112,7 +103,7 @@ Sometimes it is helpful to use a macro that can be used interchangeably regardle
 
 - OS independent
 - No X command
-- Prefixes: _mx_
+- Prefixes: `mx_`
 
 ## Installation
 
@@ -149,16 +140,17 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 - macro names must be lowercase
 - one macro per file
 - prefixes:
-  - _mcf_ for macro compiled functions (proc fcmp)
-  - _mddl_ for macros containing DDL (Data Definition Language)
-  - _mf_ for macro functions (can be used in open code).
-  - _ml_ for macros that are used to compile LUA modules
-  - _mm_ for metadata macros (interface with the metadata server).
-  - _mmx_ for macros that use metadata and are XCMD enabled (working on both windows and unix)
-  - _mp_ for macro procedures (which generate sas code)
-  - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
-  - _mv_ for macro procedures that will only work in Viya
-  - _mx_ for macros that work on Viya, SAS 9 EBI and SASjs Server
+  - _mcf__: macro compiled functions (proc fcmp)
+  - _mddl__: macros containing DDL (Data Definition Language)
+  - _mf__: macro functions (can be used in open code).
+  - _mfv__: macro functions that work only in Viya
+  - _ml__: macros that are used to compile LUA modules
+  - _mm__: metadata macros (interface with the metadata server).
+  - _mmx__: macros that use metadata and are XCMD enabled (working on both windows and unix)
+  - _mp__: macro procedures (which generate sas code)
+  - _ms__: macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
+  - _mv__: macro procedures that will only work in Viya
+  - _mx__: macros that work on Viya, SAS 9 EBI and SASjs Server
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
@@ -214,18 +206,20 @@ When contributing to this library, it is therefore important to ensure that all
 
 ## General Notes
 
-- All macros should be compatible with SAS versions from support level B and above (so currently 9.2 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully (eg `%if %sysevalf(&sysver<9.3) %then %return`).
+- All macros should be compatible with SAS versions from support level B and above (so currently 9.3 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully.
+- It's [best to avoid](https://git.datacontroller.io/dc/dc/issues/50) special / non-ASCII characters for compatibility with the widest variety of SAS installations.
 
 ## Breaking Changes
 
 We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major/breaking release (v5) becomes necessary:
 
-* mf_getuniquelibref.sas to have the deprecated maxtried parameter removed (no longer needed)
-* mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
-* `insert_cmplib` option of mcf_xxx macros will be deprecated (the option is now checked automatically with value inserted only if needed)
+* mcf_xxx macros to have `insert_cmplib` option deprecated (the option is now checked automatically with value inserted only if needed)
 * mcf_xxx macros to have `wrap=` option defaulted to YES for convenience.  Set this option explicitly to avoid issues.
-* mp_getddl.sas to be renamed to mp_ds2ddl.sas (consistent with other ds2xxx macros).  A wrapper macro is already in place, and you are able to use this immediately.  The default for SHOWLOG will also be YES instead of NO.
+* mf_getuniquelibref.sas to have the deprecated maxtries parameter removed (no longer needed)
+* mp_abort.sas will have the redundant type= parameter removed.
 * mp_coretable.sas will be replaced by the standalone macros in the `ddl` folder (which are already available)
+* mp_getddl.sas to be renamed to mp_ds2ddl.sas (consistent with other ds2xxx macros).  A wrapper macro is already in place, and you are able to use this immediately.  The default for SHOWLOG will also be YES instead of NO.
+* mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
 
 ## Star Gazing
 
@@ -238,15 +232,19 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 The following repositories are also worth checking out:
 
 * [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
+* [Criptic/sas_snippets](https://github.com/Criptic/sas_snippets)
 * [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)
 * [KatjaGlassConsulting/SMILE-SmartSASMacros](https://github.com/KatjaGlassConsulting/SMILE-SmartSASMacros)
+* [paul-canals/toolbox](https://github.com/paul-canals/toolbox)
 * [rogerjdeangelis](https://github.com/rogerjdeangelis)
+* [SASJedi/sas-macros](https://github.com/SASJedi/sas-macros)
 * [scottbass/sas](https://github.com/scottbass/SAS)
+* [xieliaing/SAS](https://github.com/xieliaing/SAS)
 * [yabwon/sas_packages](https://github.com/yabwon/SAS_PACKAGES)
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-12-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-15-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -254,22 +252,29 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
 <!-- prettier-ignore-start -->
 <!-- markdownlint-disable -->
 <table>
-  <tr>
-    <td align="center"><a href="https://github.com/allanbowe"><img src="https://avatars.githubusercontent.com/u/4420615?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Allan Bowe</b></sub></a><br /><a href="#business-allanbowe" title="Business development">💼</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Code">💻</a> <a href="#content-allanbowe" title="Content">🖋</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Documentation">📖</a> <a href="#infra-allanbowe" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#maintenance-allanbowe" title="Maintenance">🚧</a> <a href="#mentoring-allanbowe" title="Mentoring">🧑‍🏫</a> <a href="#question-allanbowe" title="Answering Questions">💬</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3Aallanbowe" title="Reviewed Pull Requests">👀</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Tests">⚠️</a></td>
-    <td align="center"><a href="https://github.com/rafgag"><img src="https://avatars.githubusercontent.com/u/69139928?v=4?s=100" width="100px;" alt=""/><br /><sub><b>rafgag</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rafgag" title="Code">💻</a></td>
-    <td align="center"><a href="https://github.com/tmoody"><img src="https://avatars.githubusercontent.com/u/79837106?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Trevor Moody</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=tmoody" title="Code">💻</a></td>
-    <td align="center"><a href="https://krishna-acondy.io/"><img src="https://avatars.githubusercontent.com/u/2980428?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Krishna Acondy</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=krishna-acondy" title="Code">💻</a> <a href="#infra-krishna-acondy" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#blog-krishna-acondy" title="Blogposts">📝</a> <a href="#content-krishna-acondy" title="Content">🖋</a> <a href="#ideas-krishna-acondy" title="Ideas, Planning, & Feedback">🤔</a> <a href="#video-krishna-acondy" title="Videos">📹</a></td>
-    <td align="center"><a href="https://github.com/saadjutt01"><img src="https://avatars.githubusercontent.com/u/8914650?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Muhammad Saad </b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=saadjutt01" title="Code">💻</a> <a href="#ideas-saadjutt01" title="Ideas, Planning, & Feedback">🤔</a></td>
-    <td align="center"><a href="https://www.erudicat.com/"><img src="https://avatars.githubusercontent.com/u/25773492?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Yury Shkoda</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=YuryShkoda" title="Code">💻</a> <a href="#infra-YuryShkoda" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#video-YuryShkoda" title="Videos">📹</a></td>
-    <td align="center"><a href="https://github.com/medjedovicm"><img src="https://avatars.githubusercontent.com/u/18329105?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Mihajlo Medjedovic</b></sub></a><br /><a href="#infra-medjedovicm" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
-  </tr>
-  <tr>
-    <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
-    <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
-    <td align="center"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
-    <td align="center"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
-    <td align="center"><a href="https://bandism.net/"><img src="https://avatars.githubusercontent.com/u/22633385?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Ikko Ashimine</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=eltociear" title="Code">💻</a></td>
-  </tr>
+  <tbody>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/allanbowe"><img src="https://avatars.githubusercontent.com/u/4420615?v=4?s=100" width="100px;" alt="Allan Bowe"/><br /><sub><b>Allan Bowe</b></sub></a><br /><a href="#business-allanbowe" title="Business development">💼</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Code">💻</a> <a href="#content-allanbowe" title="Content">🖋</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Documentation">📖</a> <a href="#infra-allanbowe" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#maintenance-allanbowe" title="Maintenance">🚧</a> <a href="#mentoring-allanbowe" title="Mentoring">🧑‍🏫</a> <a href="#question-allanbowe" title="Answering Questions">💬</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3Aallanbowe" title="Reviewed Pull Requests">👀</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Tests">⚠️</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/rafgag"><img src="https://avatars.githubusercontent.com/u/69139928?v=4?s=100" width="100px;" alt="rafgag"/><br /><sub><b>rafgag</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rafgag" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/tmoody"><img src="https://avatars.githubusercontent.com/u/79837106?v=4?s=100" width="100px;" alt="Trevor Moody"/><br /><sub><b>Trevor Moody</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=tmoody" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://krishna-acondy.io/"><img src="https://avatars.githubusercontent.com/u/2980428?v=4?s=100" width="100px;" alt="Krishna Acondy"/><br /><sub><b>Krishna Acondy</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=krishna-acondy" title="Code">💻</a> <a href="#infra-krishna-acondy" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#blog-krishna-acondy" title="Blogposts">📝</a> <a href="#content-krishna-acondy" title="Content">🖋</a> <a href="#ideas-krishna-acondy" title="Ideas, Planning, & Feedback">🤔</a> <a href="#video-krishna-acondy" title="Videos">📹</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/saadjutt01"><img src="https://avatars.githubusercontent.com/u/8914650?v=4?s=100" width="100px;" alt="Muhammad Saad "/><br /><sub><b>Muhammad Saad </b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=saadjutt01" title="Code">💻</a> <a href="#ideas-saadjutt01" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://www.erudicat.com/"><img src="https://avatars.githubusercontent.com/u/25773492?v=4?s=100" width="100px;" alt="Yury Shkoda"/><br /><sub><b>Yury Shkoda</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=YuryShkoda" title="Code">💻</a> <a href="#infra-YuryShkoda" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#video-YuryShkoda" title="Videos">📹</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/medjedovicm"><img src="https://avatars.githubusercontent.com/u/18329105?v=4?s=100" width="100px;" alt="Mihajlo Medjedovic"/><br /><sub><b>Mihajlo Medjedovic</b></sub></a><br /><a href="#infra-medjedovicm" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt="kkchandok"/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt="Vladislav Parhomchik"/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt="Vignesh T."/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt="Bart Jablonski"/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://bandism.net/"><img src="https://avatars.githubusercontent.com/u/22633385?v=4?s=100" width="100px;" alt="Ikko Ashimine"/><br /><sub><b>Ikko Ashimine</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=eltociear" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/henrik-forsell"><img src="https://avatars.githubusercontent.com/u/109935936?v=4?s=100" width="100px;" alt="Henrik Forsell"/><br /><sub><b>Henrik Forsell</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=henrik-forsell" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://rudvfaden.github.io/"><img src="https://avatars.githubusercontent.com/u/2445577?v=4?s=100" width="100px;" alt="Rud Faden"/><br /><sub><b>Rud Faden</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rudvfaden" title="Code">💻</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/andyjessen"><img src="https://avatars.githubusercontent.com/u/62343929?v=4?s=100" width="100px;" alt="andyjessen"/><br /><sub><b>andyjessen</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=andyjessen" title="Documentation">📖</a></td>
+    </tr>
+  </tbody>
 </table>
 
 <!-- markdownlint-restore -->

base/mf_abort.sas

@@ -3,6 +3,11 @@
   @brief Abort, ungracefully
   @details Will abort with a straightforward %abort if the condition is true.
 
+  @param [in] mac= (mf_abort.sas) Name of calling macro (is printed to the log)
+  @param [in] msg= ( ) Additional string to print to the log
+  @param [in] iftrue= (%str(1=1)) Conditional logic under which to perform the
+    abort
+
   <h4> Related Macros </h4>
   @li mp_abort.sas
 
@@ -12,7 +17,7 @@
 **/
 
 %macro mf_abort(mac=mf_abort.sas, msg=, iftrue=%str(1=1)
-)/*/STORE SOURCE*/;
+)/des='ungraceful abort' /*STORE SOURCE*/;
 
   %if not(%eval(%unquote(&iftrue))) %then %return;
 
@@ -24,4 +29,4 @@
 
 %mend mf_abort;
 
-/** @endcond */
+/** @endcond */

base/mf_deletefile.sas

@@ -10,7 +10,7 @@
       %mf_deletefile(&sasjswork/myfile.txt)
 
 
-  @param filepath Full path to the target file
+  @param [in] file Full path to the target file
 
   @returns The return code from the fdelete() invocation
 

base/mf_existds.sas

@@ -10,7 +10,7 @@
     expected results (depending on whether you 'expect' the result to be
     case insensitive in this context!)
 
-  @param libds library.dataset
+  @param [in] libds library.dataset
   @return output returns 1 or 0
 
   <h4> Related Macros </h4>

base/mf_existfileref.sas

@@ -4,7 +4,7 @@
   @details You can probably do without this macro as it is just a one liner.
   Mainly it is here as a convenient way to remember the syntax!
 
-  @param fref the fileref to detect
+  @param [in] fref the fileref to detect
 
   @return output Returns 1 if found and 0 if not found.  Note - it is possible
   that the fileref is found, but the file does not (yet) exist. If you need
@@ -30,4 +30,4 @@
     0
   %end;
 
-%mend mf_existfileref;
+%mend mf_existfileref;

base/mf_existfunction.sas

@@ -16,7 +16,7 @@
 https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionexists-macro
   ).
 
-  @param [in] name (positional) - function name
+  @param [in] name function name
 
   @author Allan Bowe
 **/

base/mf_existvar.sas

@@ -25,13 +25,17 @@
   %local dsid rc;
   %let dsid=%sysfunc(open(&libds,is));
 
-  %if &dsid=0 or %length(&var)=0 %then %do;
+  %if &dsid=0 %then %do;
     %put %sysfunc(sysmsg());
-      0
+    0
+  %end;
+  %else %if %length(&var)=0 %then %do;
+    0
+    %let rc=%sysfunc(close(&dsid));
   %end;
   %else %do;
-      %sysfunc(varnum(&dsid,&var))
-      %let rc=%sysfunc(close(&dsid));
+    %sysfunc(varnum(&dsid,&var))
+    %let rc=%sysfunc(close(&dsid));
   %end;
 
 %mend mf_existvar;

base/mf_existvarlist.sas

@@ -6,8 +6,8 @@
 
       %put %mf_existVarList(sashelp.class, age sex name dummyvar);
 
-  @param libds 2 part dataset or view reference
-  @param varlist space separated variable names
+  @param [in] libds 2 part dataset or view reference
+  @param [in] varlist space separated variable names
 
   @version 9.2
   @author Allan Bowe

base/mf_getapploc.sas

@@ -31,6 +31,7 @@
       %put %mf_getapploc(/some/location/jobs/extract/somejob/);
       %put %mf_getapploc(/some/location/tests/jobs/somejob/);
 
+  @param [in] pgm The _program value from which to extract the appLoc
 
   @author Allan Bowe
 **/

base/mf_getattrc.sas

@@ -6,8 +6,8 @@
       %put Dataset label = %mf_getattrc(sashelp.class,LABEL);
       %put Member Type = %mf_getattrc(sashelp.class,MTYPE);
 
-  @param libds library.dataset
-  @param attr full list in [documentation](
+  @param [in] libds library.dataset
+  @param [in] attr full list in [documentation](
     https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000147794.htm)
   @return output returns result of the attrc value supplied, or -1 and log
     message if err.

base/mf_getattrn.sas

@@ -6,8 +6,8 @@
       %put Number of observations=%mf_getattrn(sashelp.class,NLOBS);
       %put Number of variables = %mf_getattrn(sashelp.class,NVARS);
 
-  @param libds library.dataset
-  @param attr Common values are NLOBS and NVARS, full list in [documentation](
+  @param [in] libds library.dataset
+  @param [in] attr Common values are NLOBS and NVARS, full list in [documentation](
   http://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000212040.htm)
   @return output returns result of the attrn value supplied, or -1 and log
     message if err.

base/mf_getfmtlist.sas

@@ -10,10 +10,9 @@
 
   returns:
 
-  > DOLLAR $CHAR W MONNAME
-  > $CHAR BEST DOLLAR
-  > BEST Z $CHAR COMMA PERCENTN
-
+      DOLLAR $CHAR W MONNAME
+      $CHAR BEST DOLLAR
+      BEST Z $CHAR COMMA PERCENTN
 
   @param [in] libds Two part library.dataset reference.
 

base/mf_getgitbranch.sas

@@ -0,0 +1,37 @@
+/**
+  @file
+  @brief Retrieves the current branch from a local GIT repo
+  @details In a local git repository, the current branch is always available in
+  the `.git/HEAD` file in a format like this:  `ref: refs/heads/master`
+
+  This macro simply reads the file and returns the last word (eg `master`).
+
+  Example usage:
+
+      %let gitdir=%sysfunc(pathname(work))/core;
+      %let repo=https://github.com/sasjs/core;
+      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&gitdir));
+
+      %put The current branch is %mf_getgitbranch(&gitdir);
+
+  @param [in] gitdir The directory containing the GIT repository
+
+  <h4> SAS Macros </h4>
+  @li mf_readfile.sas
+
+  <h4> Related Macros </h4>
+  @li mp_gitadd.sas
+  @li mp_gitlog.sas
+  @li mp_gitreleaseinfo.sas
+  @li mp_gitstatus.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_getgitbranch(gitdir
+)/*/STORE SOURCE*/;
+
+  %scan(%mf_readfile(&gitdir/.git/HEAD),-1)
+
+%mend mf_getgitbranch;

base/mf_getkeyvalue.sas

@@ -7,8 +7,9 @@
       %put %mf_getkeyvalue(someindex)
 
 
-  @param key Provide a key on which to perform the lookup
-  @param libds= define the target table which holds the parameters
+  @param [in] key Provide a key on which to perform the lookup
+  @param [in] libds= (work.mp_setkeyvalue) The library.dataset which holds the
+    parameters
 
   @version 9.2
   @author Allan Bowe

base/mf_getplatform.sas

@@ -12,7 +12,7 @@
   @li SASJS
   @li BASESAS
 
-  @param switch the param for which to return a platform specific variable
+  @param [in] switch the param for which to return a platform specific variable
 
   <h4> SAS Macros </h4>
   @li mf_mval.sas

base/mf_getschema.sas

@@ -8,7 +8,7 @@
   returns:
   > dbo
 
-  @param libref Library reference (also accepts a 2 level libds ref).
+  @param [in] libref Library reference (also accepts a 2 level libds ref).
 
   @return output returns the library schema for the FIRST library encountered
 

base/mf_getuniquelibref.sas

@@ -19,14 +19,14 @@
   @param [in] prefix= (mclib) first part of the returned libref. As librefs can
     be as long as 8 characters, a maximum length of 7 characters is premitted
     for this prefix.
-  @param [in] maxtries= Deprecated parameter. Remains here to ensure a
+  @param [in] maxtries= (1000) Deprecated parameter. Remains here to ensure a
     non-breaking change.  Will be removed in v5.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-%macro mf_getuniquelibref(prefix=mclib,maxtries=1000);
+%macro mf_getuniquelibref(prefix=mc,maxtries=1000);
   %local x;
 
   %if ( %length(&prefix) gt 7 ) %then %do;

base/mf_getuniquename.sas

@@ -10,7 +10,7 @@
 
 > MCc59c750610321d4c8bf75faadbcd22
 
-  @param prefix= set a prefix for the new name
+  @param [in] prefix= (MC) Sets a prefix for the new name
 
   @version 9.3
   @author Allan Bowe

base/mf_getuser.sas

@@ -13,8 +13,6 @@
         %let user= %mf_getUser();
         %put &user;
 
-  @param type - do not use, may be deprecated in a future release
-
   @return SYSUSERID (if workspace server)
   @return _METAPERSON (if stored process server)
   @return SYS_COMPUTE_SESSION_OWNER (if Viya compute session)

base/mf_getvalue.sas

@@ -13,9 +13,9 @@
   <h4> Related Macros </h4>
   @li mp_setkeyvalue.sas
 
-  @param libds dataset to query
-  @param variable the variable which contains the value to return.
-  @param filter contents of where clause
+  @param [in] libds dataset to query
+  @param [in] variable the variable which contains the value to return.
+  @param [in] filter= (1) contents of where clause
 
   @version 9.2
   @author Allan Bowe

base/mf_getvarformat.sas

@@ -25,7 +25,8 @@
 
   @param [in] libds Two part dataset (or view) reference.
   @param [in] var Variable name for which a format should be returned
-  @param [in] force=(0) Set to 1 to supply a default if the variable has no format
+  @param [in] force= (0) Set to 1 to supply a default if the variable has no
+    format
   @returns outputs format
 
   @author Allan Bowe

base/mf_getvarlen.sas

@@ -18,8 +18,8 @@
       8
       NOTE: Variable renegade does not exist in test
 
-  @param libds Two part dataset (or view) reference.
-  @param var Variable name for which a length should be returned
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] var Variable name for which a length should be returned
   @returns outputs length
 
   @author Allan Bowe

base/mf_getvarnum.sas

@@ -21,8 +21,8 @@ returns:
 
   > NOTE: Variable renegade does not exist in test
 
-  @param libds Two part dataset (or view) reference.
-  @param var Variable name for which a position should be returned
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] var Variable name for which a position should be returned
 
   @author Allan Bowe
   @version 9.2

base/mf_getvartype.sas

@@ -13,8 +13,8 @@ Usage:
 
 
 
-  @param libds Two part dataset (or view) reference.
-  @param var the variable name to be checked
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] var the variable name to be checked
   @return output returns C or N depending on variable type.  If variable
     does not exist then a blank is returned and a note is written to the log.
 

base/mf_getxengine.sas

@@ -10,7 +10,7 @@
   returns:
   > TEMP
 
-  @param fref The fileref to check
+  @param [in] fref The fileref to check
 
   @returns The XENGINE value in sashelp.vextfl or 0 if not found.
 

base/mf_increment.sas

@@ -0,0 +1,29 @@
+/**
+  @file
+  @brief Increments a macro variable
+  @details Useful outside of do-loops - will increment a macro variable every
+  time it is called.
+
+  Example:
+
+      %let cnt=1;
+      %put We have run %mf_increment(cnt) lines;
+      %put Now we have run %mf_increment(cnt) lines;
+      %put There are %mf_increment(cnt) lines in total;
+
+  @param [in] macro_name The name of the macro variable to increment
+  @param [in] incr= (1) The amount to add or subtract to the macro
+
+  <h4> Related Files </h4>
+  @li mf_increment.test.sas
+
+**/
+
+%macro mf_increment(macro_name,incr=1);
+
+  /* iterate the value */
+  %let &macro_name=%eval(&&&macro_name+&incr);
+  /* return the value */
+  &&&macro_name
+
+%mend mf_increment;

base/mf_isblank.sas

@@ -7,12 +7,12 @@
 
   Usage:
 
-      %put mf_isblank(&var);
+      %put %mf_isblank(&var);
 
   inspiration:
   https://support.sas.com/resources/papers/proceedings09/022-2009.pdf
 
-  @param param VALUE to be checked
+  @param [in] Param VALUE to be checked
 
   @return output returns 1 (if blank) else 0
 

base/mf_isdir.sas

@@ -9,7 +9,7 @@
   With thanks and full credit to Andrea Defronzo -
   https://www.linkedin.com/in/andrea-defronzo-b1a47460/
 
-  @param path full path of the file/directory to be checked
+  @param [in] path Full path of the file/directory to be checked
 
   @return output returns 1 if path is a directory, 0 if it is not
 

base/mf_loc.sas

@@ -6,6 +6,10 @@
 
       %put %mf_loc(POF); %*location of PlatformObjectFramework tools;
 
+  @param [in] loc The item to locate, eg:
+    @li PLAATFORMOBJECTFRAMEWORK (or POF)
+    @li VIYACONFG
+
   @version 9.2
   @author Allan Bowe
 **/
@@ -15,7 +19,8 @@
 %local root;
 
 %if &loc=POF or &loc=PLATFORMOBJECTFRAMEWORK %then %do;
-  %let root=%substr(%sysget(SASROOT),1,%index(%sysget(SASROOT),SASFoundation)-2);
+  %let root=%sysget(SASROOT);
+  %let root=%substr(&root,1,%index(&root,SASFoundation)-2);
   %let root=&root/SASPlatformObjectFramework/&sysver;
   %put Batch tools located at: &root;
   &root

base/mf_mimetype.sas

@@ -0,0 +1,436 @@
+/**
+  @file
+  @brief Returns a mime type from a file extension
+  @details Provide a file extension and get a mime type.
+    The mappings were derived from this source:
+    https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
+
+
+
+  @param [in] ext The file extension (unquoted)
+  @return output The mime type
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_mimetype(
+    ext
+)/*/STORE SOURCE*/ /minoperator mindelimiter=' ';
+
+%let ext=%lowcase(&ext);
+
+%if &ext in (sas txt text conf def list log)
+%then %do;%str(text/plain)%end;
+%else %if &ext=xlsx %then %do;
+%str(application/vnd.openxmlformats-officedocument.spreadsheetml.sheet)%end;
+%else %if &ext in (xls xlm xla xlc xlt xlw)
+%then %do;%str(application/vnd.ms-excel)%end;
+%else %if &ext=xlsm
+%then %do;%str(application/vnd.ms-excel.sheet.macroenabled.12)%end;
+%else %if &ext=xlsb
+%then %do;%str(application/vnd.ms-excel.sheet.binary.macroenabled.12)%end;
+%else %if &ext in (css csv html n3 sgml vcard)
+%then %do;%str(text/&ext)%end;
+%else %if &ext in (avif bmp cgm gif ief jxl ktx png sgi tiff webp)
+%then %do;%str(image/&ext)%end;
+%else %if &ext in (exi gxf ipfix json mbox mp21 mxf oda oxps pdf rtf sdp wasm
+  xml yang zip)
+%then %do;%str(application/&ext)%end;
+%else %if &ext in (jpeg jpg jpe) %then %do;%str(image/jpeg)%end;
+%else %if &ext in (mp4 mp4v mpg4) %then %do;%str(video/mp4)%end;
+%else %if &ext in (otf ttf woff woff2) %then %do;%str(font/&ext)%end;
+%else %if &ext in (mpeg mpg mpe m1v m2v)
+%then %do;%str(video/mpeg)%end;
+%else %if &ext in (h261 h263 h264 jpm mj2 webm)
+%then %do;%str(video/&ext)%end;
+%else %if &ext in (f4v fli flv m4v mng smv)
+%then %do;%str(video/x-&ext)%end;
+%else %if &ext in (3ds cmx pcx rgb tga)
+%then %do;%str(image/x-&ext)%end;
+%else %if &ext in (asm nfo opml sfv)
+%then %do;%str(text/x-&ext)%end;
+%else %if &ext in (aac caf flac wav)
+%then %do;%str(audio/x-&ext)%end;
+%else %if &ext in (ts m2t m2ts mts)
+%then %do;%str(video/mp2t)%end;
+%else %if &ext in (pfa pfb pfm afm)
+%then %do;%str(application/x-font-type1)%end;
+%else %if &ext in (oga ogg spx opus)
+%then %do;%str(audio/ogg)%end;
+%else %if &ext in (mid midi kar rmi)
+%then %do;%str(audio/midi)%end;
+%else %if &ext in (onetoc onetoc2 onetmp onepkg)
+%then %do;%str(application/onenote)%end;
+%else %if &ext in (mxml xhvml xvml xvm)
+%then %do;%str(application/xv+xml)%end;
+%else %if &ext in (f for f77 f90)
+%then %do;%str(text/x-fortran)%end;
+%else %if &ext in (wmf wmz emf emz)
+%then %do;%str(application/x-msmetafile)%end;
+%else %if &ext in (exe dll com bat msi)
+%then %do;%str(application/x-msdownload)%end;
+%else %if &ext in (bin dms lrf mar so dist distz pkg bpk dump elc deploy)
+%then %do;%str(application/octet-stream)%end;
+%else %if &ext in (atom atomcat atomsvc ccxml davmount emma gml gpx inkml mads
+  mathml metalink mets mods omdoc pls rdf rsd rss sbml shf smil sru ssdl ssml
+  tei wsdl wspolicy xaml xenc xhtml xop xslt xspf yin)
+%then %do;%str(application/&ext+xml)%end;
+%else %if &ext in (dir dcr dxr cst cct cxt w3d fgd swa)
+%then %do;%str(application/x-director)%end;
+%else %if &ext in (z1 z2 z3 z4 z5 z6 z7 z8)
+%then %do;%str(application/x-zmachine)%end;
+%else %if &ext in (c cc cxx cpp h hh dic)
+%then %do;%str(text/x-c)%end;
+%else %if &ext in (mpga mp2 mp2a mp3 m2a m3a)
+%then %do;%str(audio/mpeg)%end;
+%else %if &ext in (t tr roff man me ms)
+%then %do;%str(text/troff)%end;
+%else %if &ext in (cbr cba cbt cbz cb7)
+%then %do;%str(application/x-cbr)%end;
+%else %if &ext in (fh fhc fh4 fh5 fh7)
+%then %do;%str(image/x-freehand)%end;
+%else %if &ext in (aab x32 u32 vox)
+%then %do;%str(application/x-authorware-bin)%end;
+%else %if &ext in (uvi uvvi uvg uvvg)
+%then %do;%str(image/vnd.dece.graphic)%end;
+%else %if &ext in (cdx cif cmdf cml csml xyz)
+%then %do;%str(chemical/x-&ext)%end;
+%else %if &ext in (aif aiff aifc) %then %do;%str(audio/x-aiff)%end;
+%else %if &ext in (ma nb mb) %then %do;%str(application/mathematica)%end;
+%else %if &ext in (mvb m13 m14) %then %do;%str(application/x-msmediaview)%end;
+%else %if &ext in (msh mesh silo) %then %do;%str(model/mesh)%end;
+%else %if &ext in (uri uris urls) %then %do;%str(text/uri-list)%end;
+%else %if &ext in (mkv mk3d mks) %then %do;%str(video/x-matroska)%end;
+%else %if &ext=ez %then %do;%str(application/andrew-inset)%end;
+%else %if &ext=aw %then %do;%str(application/applixware)%end;
+%else %if &ext=cdmia %then %do;%str(application/cdmi-capability)%end;
+%else %if &ext=cdmic %then %do;%str(application/cdmi-container)%end;
+%else %if &ext=cdmid %then %do;%str(application/cdmi-domain)%end;
+%else %if &ext=cdmio %then %do;%str(application/cdmi-object)%end;
+%else %if &ext=cdmiq %then %do;%str(application/cdmi-queue)%end;
+%else %if &ext=cu %then %do;%str(application/cu-seeme)%end;
+%else %if &ext=dssc %then %do;%str(application/dssc+der)%end;
+%else %if &ext=xdssc %then %do;%str(application/dssc+xml)%end;
+%else %if &ext=ecma %then %do;%str(application/ecmascript)%end;
+%else %if &ext=epub %then %do;%str(application/epub+zip)%end;
+%else %if &ext=pfr %then %do;%str(application/font-tdpfr)%end;
+%else %if &ext=stk %then %do;%str(application/hyperstudio)%end;
+%else %if &ext=ink %then %do;%str(application/inkml+xml)%end;
+%else %if &ext=jar %then %do;%str(application/java-archive)%end;
+%else %if &ext=ser %then %do;%str(application/java-serialized-object)%end;
+%else %if &ext=class %then %do;%str(application/java-vm)%end;
+%else %if &ext=jsonml %then %do;%str(application/jsonml+json)%end;
+%else %if &ext=lostxml %then %do;%str(application/lost+xml)%end;
+%else %if &ext=hqx %then %do;%str(application/mac-binhex40)%end;
+%else %if &ext=cpt %then %do;%str(application/mac-compactpro)%end;
+%else %if &ext=mrc %then %do;%str(application/marc)%end;
+%else %if &ext=mrcx %then %do;%str(application/marcxml+xml)%end;
+%else %if &ext=mscml %then %do;%str(application/mediaservercontrol+xml)%end;
+%else %if &ext=meta4 %then %do;%str(application/metalink4+xml)%end;
+%else %if &ext=m21 %then %do;%str(application/mp21)%end;
+%else %if &ext=mp4s %then %do;%str(application/mp4)%end;
+%else %if &ext=doc %then %do;%str(application/msword)%end;
+%else %if &ext=dot %then %do;%str(application/msword)%end;
+%else %if &ext=opf %then %do;%str(application/oebps-package+xml)%end;
+%else %if &ext=ogx %then %do;%str(application/ogg)%end;
+%else %if &ext=xer %then %do;%str(application/patch-ops-error+xml)%end;
+%else %if &ext=pgp %then %do;%str(application/pgp-encrypted)%end;
+%else %if &ext=asc %then %do;%str(application/pgp-signature)%end;
+%else %if &ext=sig %then %do;%str(application/pgp-signature)%end;
+%else %if &ext=prf %then %do;%str(application/pics-rules)%end;
+%else %if &ext=p10 %then %do;%str(application/pkcs10)%end;
+%else %if &ext=p7m %then %do;%str(application/pkcs7-mime)%end;
+%else %if &ext=p7c %then %do;%str(application/pkcs7-mime)%end;
+%else %if &ext=p7s %then %do;%str(application/pkcs7-signature)%end;
+%else %if &ext=p8 %then %do;%str(application/pkcs8)%end;
+%else %if &ext=ac %then %do;%str(application/pkix-attr-cert)%end;
+%else %if &ext=cer %then %do;%str(application/pkix-cert)%end;
+%else %if &ext=crl %then %do;%str(application/pkix-crl)%end;
+%else %if &ext=pkipath %then %do;%str(application/pkix-pkipath)%end;
+%else %if &ext=pki %then %do;%str(application/pkixcmp)%end;
+%else %if &ext=cww %then %do;%str(application/prs.cww)%end;
+%else %if &ext=pskcxml %then %do;%str(application/pskc+xml)%end;
+%else %if &ext=rif %then %do;%str(application/reginfo+xml)%end;
+%else %if &ext=rnc %then %do;%str(application/relax-ng-compact-syntax)%end;
+%else %if &ext=rld %then %do;%str(application/resource-lists-diff+xml)%end;
+%else %if &ext=rl %then %do;%str(application/resource-lists+xml)%end;
+%else %if &ext=gbr %then %do;%str(application/rpki-ghostbusters)%end;
+%else %if &ext=mft %then %do;%str(application/rpki-manifest)%end;
+%else %if &ext=roa %then %do;%str(application/rpki-roa)%end;
+%else %if &ext=scq %then %do;%str(application/scvp-cv-request)%end;
+%else %if &ext=scs %then %do;%str(application/scvp-cv-response)%end;
+%else %if &ext=spq %then %do;%str(application/scvp-vp-request)%end;
+%else %if &ext=spp %then %do;%str(application/scvp-vp-response)%end;
+%else %if &ext=setpay %then %do;%str(application/set-payment-initiation)%end;
+%else %if &ext=setreg %then %do;%str(application/set-registration-initiation)%end;
+%else %if &ext=smi %then %do;%str(application/smil+xml)%end;
+%else %if &ext=rq %then %do;%str(application/sparql-query)%end;
+%else %if &ext=srx %then %do;%str(application/sparql-results+xml)%end;
+%else %if &ext=gram %then %do;%str(application/srgs)%end;
+%else %if &ext=grxml %then %do;%str(application/srgs+xml)%end;
+%else %if &ext=teicorpus %then %do;%str(application/tei+xml)%end;
+%else %if &ext=tfi %then %do;%str(application/thraud+xml)%end;
+%else %if &ext=tsd %then %do;%str(application/timestamped-data)%end;
+%else %if &ext=vxml %then %do;%str(application/voicexml+xml)%end;
+%else %if &ext=wgt %then %do;%str(application/widget)%end;
+%else %if &ext=hlp %then %do;%str(application/winhlp)%end;
+%else %if &ext=7z %then %do;%str(application/x-7z-compressed)%end;
+%else %if &ext=abw %then %do;%str(application/x-abiword)%end;
+%else %if &ext=ace %then %do;%str(application/x-ace-compressed)%end;
+%else %if &ext=dmg %then %do;%str(application/x-apple-diskimage)%end;
+%else %if &ext=aam %then %do;%str(application/x-authorware-map)%end;
+%else %if &ext=aas %then %do;%str(application/x-authorware-seg)%end;
+%else %if &ext=bcpio %then %do;%str(application/x-bcpio)%end;
+%else %if &ext=torrent %then %do;%str(application/x-bittorrent)%end;
+%else %if &ext=blb %then %do;%str(application/x-blorb)%end;
+%else %if &ext=blorb %then %do;%str(application/x-blorb)%end;
+%else %if &ext=bz %then %do;%str(application/x-bzip)%end;
+%else %if &ext=bz2 %then %do;%str(application/x-bzip2)%end;
+%else %if &ext=boz %then %do;%str(application/x-bzip2)%end;
+%else %if &ext=vcd %then %do;%str(application/x-cdlink)%end;
+%else %if &ext=cfs %then %do;%str(application/x-cfs-compressed)%end;
+%else %if &ext=chat %then %do;%str(application/x-chat)%end;
+%else %if &ext=pgn %then %do;%str(application/x-chess-pgn)%end;
+%else %if &ext=nsc %then %do;%str(application/x-conference)%end;
+%else %if &ext=cpio %then %do;%str(application/x-cpio)%end;
+%else %if &ext=csh %then %do;%str(application/x-csh)%end;
+%else %if &ext=deb %then %do;%str(application/x-debian-package)%end;
+%else %if &ext=udeb %then %do;%str(application/x-debian-package)%end;
+%else %if &ext=dgc %then %do;%str(application/x-dgc-compressed)%end;
+%else %if &ext=wad %then %do;%str(application/x-doom)%end;
+%else %if &ext=ncx %then %do;%str(application/x-dtbncx+xml)%end;
+%else %if &ext=dtb %then %do;%str(application/x-dtbook+xml)%end;
+%else %if &ext=res %then %do;%str(application/x-dtbresource+xml)%end;
+%else %if &ext=dvi %then %do;%str(application/x-dvi)%end;
+%else %if &ext=evy %then %do;%str(application/x-envoy)%end;
+%else %if &ext=eva %then %do;%str(application/x-eva)%end;
+%else %if &ext=bdf %then %do;%str(application/x-font-bdf)%end;
+%else %if &ext=gsf %then %do;%str(application/x-font-ghostscript)%end;
+%else %if &ext=psf %then %do;%str(application/x-font-linux-psf)%end;
+%else %if &ext=pcf %then %do;%str(application/x-font-pcf)%end;
+%else %if &ext=snf %then %do;%str(application/x-font-snf)%end;
+%else %if &ext=arc %then %do;%str(application/x-freearc)%end;
+%else %if &ext=spl %then %do;%str(application/x-futuresplash)%end;
+%else %if &ext=gca %then %do;%str(application/x-gca-compressed)%end;
+%else %if &ext=ulx %then %do;%str(application/x-glulx)%end;
+%else %if &ext=gnumeric %then %do;%str(application/x-gnumeric)%end;
+%else %if &ext=gramps %then %do;%str(application/x-gramps-xml)%end;
+%else %if &ext=gtar %then %do;%str(application/x-gtar)%end;
+%else %if &ext=hdf %then %do;%str(application/x-hdf)%end;
+%else %if &ext=install %then %do;%str(application/x-install-instructions)%end;
+%else %if &ext=iso %then %do;%str(application/x-iso9660-image)%end;
+%else %if &ext=jnlp %then %do;%str(application/x-java-jnlp-file)%end;
+%else %if &ext=latex %then %do;%str(application/x-latex)%end;
+%else %if &ext=lzh %then %do;%str(application/x-lzh-compressed)%end;
+%else %if &ext=lha %then %do;%str(application/x-lzh-compressed)%end;
+%else %if &ext=mie %then %do;%str(application/x-mie)%end;
+%else %if &ext=prc %then %do;%str(application/x-mobipocket-ebook)%end;
+%else %if &ext=mobi %then %do;%str(application/x-mobipocket-ebook)%end;
+%else %if &ext=application %then %do;%str(application/x-ms-application)%end;
+%else %if &ext=lnk %then %do;%str(application/x-ms-shortcut)%end;
+%else %if &ext=wmd %then %do;%str(application/x-ms-wmd)%end;
+%else %if &ext=wmz %then %do;%str(application/x-ms-wmz)%end;
+%else %if &ext=xbap %then %do;%str(application/x-ms-xbap)%end;
+%else %if &ext=mdb %then %do;%str(application/x-msaccess)%end;
+%else %if &ext=obd %then %do;%str(application/x-msbinder)%end;
+%else %if &ext=crd %then %do;%str(application/x-mscardfile)%end;
+%else %if &ext=clp %then %do;%str(application/x-msclip)%end;
+%else %if &ext=mny %then %do;%str(application/x-msmoney)%end;
+%else %if &ext=pub %then %do;%str(application/x-mspublisher)%end;
+%else %if &ext=scd %then %do;%str(application/x-msschedule)%end;
+%else %if &ext=trm %then %do;%str(application/x-msterminal)%end;
+%else %if &ext=wri %then %do;%str(application/x-mswrite)%end;
+%else %if &ext=nc %then %do;%str(application/x-netcdf)%end;
+%else %if &ext=cdf %then %do;%str(application/x-netcdf)%end;
+%else %if &ext=nzb %then %do;%str(application/x-nzb)%end;
+%else %if &ext=p12 %then %do;%str(application/x-pkcs12)%end;
+%else %if &ext=pfx %then %do;%str(application/x-pkcs12)%end;
+%else %if &ext=p7b %then %do;%str(application/x-pkcs7-certificates)%end;
+%else %if &ext=spc %then %do;%str(application/x-pkcs7-certificates)%end;
+%else %if &ext=p7r %then %do;%str(application/x-pkcs7-certreqresp)%end;
+%else %if &ext=rar %then %do;%str(application/x-rar-compressed)%end;
+%else %if &ext=ris %then %do;%str(application/x-research-info-systems)%end;
+%else %if &ext=sh %then %do;%str(application/x-sh)%end;
+%else %if &ext=shar %then %do;%str(application/x-shar)%end;
+%else %if &ext=swf %then %do;%str(application/x-shockwave-flash)%end;
+%else %if &ext=xap %then %do;%str(application/x-silverlight-app)%end;
+%else %if &ext=sql %then %do;%str(application/x-sql)%end;
+%else %if &ext=sit %then %do;%str(application/x-stuffit)%end;
+%else %if &ext=sitx %then %do;%str(application/x-stuffitx)%end;
+%else %if &ext=srt %then %do;%str(application/x-subrip)%end;
+%else %if &ext=sv4cpio %then %do;%str(application/x-sv4cpio)%end;
+%else %if &ext=sv4crc %then %do;%str(application/x-sv4crc)%end;
+%else %if &ext=t3 %then %do;%str(application/x-t3vm-image)%end;
+%else %if &ext=gam %then %do;%str(application/x-tads)%end;
+%else %if &ext=tar %then %do;%str(application/x-tar)%end;
+%else %if &ext=tcl %then %do;%str(application/x-tcl)%end;
+%else %if &ext=tex %then %do;%str(application/x-tex)%end;
+%else %if &ext=tfm %then %do;%str(application/x-tex-tfm)%end;
+%else %if &ext=texinfo %then %do;%str(application/x-texinfo)%end;
+%else %if &ext=texi %then %do;%str(application/x-texinfo)%end;
+%else %if &ext=obj %then %do;%str(application/x-tgif)%end;
+%else %if &ext=ustar %then %do;%str(application/x-ustar)%end;
+%else %if &ext=src %then %do;%str(application/x-wais-source)%end;
+%else %if &ext=der %then %do;%str(application/x-x509-ca-cert)%end;
+%else %if &ext=crt %then %do;%str(application/x-x509-ca-cert)%end;
+%else %if &ext=fig %then %do;%str(application/x-xfig)%end;
+%else %if &ext=xlf %then %do;%str(application/x-xliff+xml)%end;
+%else %if &ext=xpi %then %do;%str(application/x-xpinstall)%end;
+%else %if &ext=xz %then %do;%str(application/x-xz)%end;
+%else %if &ext=xdf %then %do;%str(application/xcap-diff+xml)%end;
+%else %if &ext=xht %then %do;%str(application/xhtml+xml)%end;
+%else %if &ext=xsl %then %do;%str(application/xml)%end;
+%else %if &ext=dtd %then %do;%str(application/xml-dtd)%end;
+%else %if &ext=xpl %then %do;%str(application/xproc+xml)%end;
+%else %if &ext=adp %then %do;%str(audio/adpcm)%end;
+%else %if &ext=au %then %do;%str(audio/basic)%end;
+%else %if &ext=snd %then %do;%str(audio/basic)%end;
+%else %if &ext=m4a %then %do;%str(audio/mp4)%end;
+%else %if &ext=mp4a %then %do;%str(audio/mp4)%end;
+%else %if &ext=s3m %then %do;%str(audio/s3m)%end;
+%else %if &ext=sil %then %do;%str(audio/silk)%end;
+%else %if &ext=uva %then %do;%str(audio/vnd.dece.audio)%end;
+%else %if &ext=uvva %then %do;%str(audio/vnd.dece.audio)%end;
+%else %if &ext=eol %then %do;%str(audio/vnd.digital-winds)%end;
+%else %if &ext=dra %then %do;%str(audio/vnd.dra)%end;
+%else %if &ext=dts %then %do;%str(audio/vnd.dts)%end;
+%else %if &ext=dtshd %then %do;%str(audio/vnd.dts.hd)%end;
+%else %if &ext=lvp %then %do;%str(audio/vnd.lucent.voice)%end;
+%else %if &ext=pya %then %do;%str(audio/vnd.ms-playready.media.pya)%end;
+%else %if &ext=ecelp4800 %then %do;%str(audio/vnd.nuera.ecelp4800)%end;
+%else %if &ext=ecelp7470 %then %do;%str(audio/vnd.nuera.ecelp7470)%end;
+%else %if &ext=ecelp9600 %then %do;%str(audio/vnd.nuera.ecelp9600)%end;
+%else %if &ext=rip %then %do;%str(audio/vnd.rip)%end;
+%else %if &ext=weba %then %do;%str(audio/webm)%end;
+%else %if &ext=mka %then %do;%str(audio/x-matroska)%end;
+%else %if &ext=m3u %then %do;%str(audio/x-mpegurl)%end;
+%else %if &ext=wax %then %do;%str(audio/x-ms-wax)%end;
+%else %if &ext=wma %then %do;%str(audio/x-ms-wma)%end;
+%else %if &ext=ra %then %do;%str(audio/x-pn-realaudio)%end;
+%else %if &ext=ram %then %do;%str(audio/x-pn-realaudio)%end;
+%else %if &ext=rmp %then %do;%str(audio/x-pn-realaudio-plugin)%end;
+%else %if &ext=xm %then %do;%str(audio/xm)%end;
+%else %if &ext=ttc %then %do;%str(font/collection)%end;
+%else %if &ext=g3 %then %do;%str(image/g3fax)%end;
+%else %if &ext=btif %then %do;%str(image/prs.btif)%end;
+%else %if &ext=svg %then %do;%str(image/svg+xml)%end;
+%else %if &ext=svgz %then %do;%str(image/svg+xml)%end;
+%else %if &ext=tif %then %do;%str(image/tiff)%end;
+%else %if &ext=psd %then %do;%str(image/vnd.adobe.photoshop)%end;
+%else %if &ext=djv %then %do;%str(image/vnd.djvu)%end;
+%else %if &ext=djvu %then %do;%str(image/vnd.djvu)%end;
+%else %if &ext=sub %then %do;%str(image/vnd.dvb.subtitle)%end;
+%else %if &ext=dwg %then %do;%str(image/vnd.dwg)%end;
+%else %if &ext=dxf %then %do;%str(image/vnd.dxf)%end;
+%else %if &ext=fbs %then %do;%str(image/vnd.fastbidsheet)%end;
+%else %if &ext=fpx %then %do;%str(image/vnd.fpx)%end;
+%else %if &ext=fst %then %do;%str(image/vnd.fst)%end;
+%else %if &ext=mmr %then %do;%str(image/vnd.fujixerox.edmics-mmr)%end;
+%else %if &ext=rlc %then %do;%str(image/vnd.fujixerox.edmics-rlc)%end;
+%else %if &ext=mdi %then %do;%str(image/vnd.ms-modi)%end;
+%else %if &ext=wdp %then %do;%str(image/vnd.ms-photo)%end;
+%else %if &ext=npx %then %do;%str(image/vnd.net-fpx)%end;
+%else %if &ext=wbmp %then %do;%str(image/vnd.wap.wbmp)%end;
+%else %if &ext=xif %then %do;%str(image/vnd.xiff)%end;
+%else %if &ext=ras %then %do;%str(image/x-cmu-raster)%end;
+%else %if &ext=ico %then %do;%str(image/x-icon)%end;
+%else %if &ext=sid %then %do;%str(image/x-mrsid-image)%end;
+%else %if &ext=pct %then %do;%str(image/x-pict)%end;
+%else %if &ext=pic %then %do;%str(image/x-pict)%end;
+%else %if &ext=pnm %then %do;%str(image/x-portable-anymap)%end;
+%else %if &ext=pbm %then %do;%str(image/x-portable-bitmap)%end;
+%else %if &ext=pgm %then %do;%str(image/x-portable-graymap)%end;
+%else %if &ext=ppm %then %do;%str(image/x-portable-pixmap)%end;
+%else %if &ext=xbm %then %do;%str(image/x-xbitmap)%end;
+%else %if &ext=xpm %then %do;%str(image/x-xpixmap)%end;
+%else %if &ext=xwd %then %do;%str(image/x-xwindowdump)%end;
+%else %if &ext=eml %then %do;%str(message/rfc822)%end;
+%else %if &ext=mime %then %do;%str(message/rfc822)%end;
+%else %if &ext=iges %then %do;%str(model/iges)%end;
+%else %if &ext=igs %then %do;%str(model/iges)%end;
+%else %if &ext=dae %then %do;%str(model/vnd.collada+xml)%end;
+%else %if &ext=dwf %then %do;%str(model/vnd.dwf)%end;
+%else %if &ext=gdl %then %do;%str(model/vnd.gdl)%end;
+%else %if &ext=gtw %then %do;%str(model/vnd.gtw)%end;
+%else %if &ext=vtu %then %do;%str(model/vnd.vtu)%end;
+%else %if &ext=vrml %then %do;%str(model/vrml)%end;
+%else %if &ext=wrl %then %do;%str(model/vrml)%end;
+%else %if &ext=x3db %then %do;%str(model/x3d+binary)%end;
+%else %if &ext=x3dbz %then %do;%str(model/x3d+binary)%end;
+%else %if &ext=x3dv %then %do;%str(model/x3d+vrml)%end;
+%else %if &ext=x3dvz %then %do;%str(model/x3d+vrml)%end;
+%else %if &ext=x3d %then %do;%str(model/x3d+xml)%end;
+%else %if &ext=x3dz %then %do;%str(model/x3d+xml)%end;
+%else %if &ext=appcache %then %do;%str(text/cache-manifest)%end;
+%else %if &ext=ics %then %do;%str(text/calendar)%end;
+%else %if &ext=ifb %then %do;%str(text/calendar)%end;
+%else %if &ext=htm %then %do;%str(text/html)%end;
+%else %if &ext=js %then %do;%str(text/javascript)%end;
+%else %if &ext=mjs %then %do;%str(text/javascript)%end;
+%else %if &ext=dsc %then %do;%str(text/prs.lines.tag)%end;
+%else %if &ext=rtx %then %do;%str(text/richtext)%end;
+%else %if &ext=sgm %then %do;%str(text/sgml)%end;
+%else %if &ext=tsv %then %do;%str(text/tab-separated-values)%end;
+%else %if &ext=ttl %then %do;%str(text/turtle)%end;
+%else %if &ext=curl %then %do;%str(text/vnd.curl)%end;
+%else %if &ext=dcurl %then %do;%str(text/vnd.curl.dcurl)%end;
+%else %if &ext=mcurl %then %do;%str(text/vnd.curl.mcurl)%end;
+%else %if &ext=scurl %then %do;%str(text/vnd.curl.scurl)%end;
+%else %if &ext=sub %then %do;%str(text/vnd.dvb.subtitle)%end;
+%else %if &ext=fly %then %do;%str(text/vnd.fly)%end;
+%else %if &ext=flx %then %do;%str(text/vnd.fmi.flexstor)%end;
+%else %if &ext=gv %then %do;%str(text/vnd.graphviz)%end;
+%else %if &ext=3dml %then %do;%str(text/vnd.in3d.3dml)%end;
+%else %if &ext=spot %then %do;%str(text/vnd.in3d.spot)%end;
+%else %if &ext=jad %then %do;%str(text/vnd.sun.j2me.app-descriptor)%end;
+%else %if &ext=wml %then %do;%str(text/vnd.wap.wml)%end;
+%else %if &ext=wmls %then %do;%str(text/vnd.wap.wmlscript)%end;
+%else %if &ext=s %then %do;%str(text/x-asm)%end;
+%else %if &ext=java %then %do;%str(text/x-java-source)%end;
+%else %if &ext=p %then %do;%str(text/x-pascal)%end;
+%else %if &ext=pas %then %do;%str(text/x-pascal)%end;
+%else %if &ext=etx %then %do;%str(text/x-setext)%end;
+%else %if &ext=uu %then %do;%str(text/x-uuencode)%end;
+%else %if &ext=vcs %then %do;%str(text/x-vcalendar)%end;
+%else %if &ext=vcf %then %do;%str(text/x-vcard)%end;
+%else %if &ext=3gp %then %do;%str(video/3gpp)%end;
+%else %if &ext=3g2 %then %do;%str(video/3gpp2)%end;
+%else %if &ext=jpgv %then %do;%str(video/jpeg)%end;
+%else %if &ext=jpgm %then %do;%str(video/jpm)%end;
+%else %if &ext=mjp2 %then %do;%str(video/mj2)%end;
+%else %if &ext=ogv %then %do;%str(video/ogg)%end;
+%else %if &ext=mov %then %do;%str(video/quicktime)%end;
+%else %if &ext=qt %then %do;%str(video/quicktime)%end;
+%else %if &ext=uvh %then %do;%str(video/vnd.dece.hd)%end;
+%else %if &ext=uvvh %then %do;%str(video/vnd.dece.hd)%end;
+%else %if &ext=uvm %then %do;%str(video/vnd.dece.mobile)%end;
+%else %if &ext=uvvm %then %do;%str(video/vnd.dece.mobile)%end;
+%else %if &ext=uvp %then %do;%str(video/vnd.dece.pd)%end;
+%else %if &ext=uvvp %then %do;%str(video/vnd.dece.pd)%end;
+%else %if &ext=uvs %then %do;%str(video/vnd.dece.sd)%end;
+%else %if &ext=uvvs %then %do;%str(video/vnd.dece.sd)%end;
+%else %if &ext=uvv %then %do;%str(video/vnd.dece.video)%end;
+%else %if &ext=uvvv %then %do;%str(video/vnd.dece.video)%end;
+%else %if &ext=dvb %then %do;%str(video/vnd.dvb.file)%end;
+%else %if &ext=fvt %then %do;%str(video/vnd.fvt)%end;
+%else %if &ext=m4u %then %do;%str(video/vnd.mpegurl)%end;
+%else %if &ext=mxu %then %do;%str(video/vnd.mpegurl)%end;
+%else %if &ext=pyv %then %do;%str(video/vnd.ms-playready.media.pyv)%end;
+%else %if &ext=uvu %then %do;%str(video/vnd.uvvu.mp4)%end;
+%else %if &ext=uvvu %then %do;%str(video/vnd.uvvu.mp4)%end;
+%else %if &ext=viv %then %do;%str(video/vnd.vivo)%end;
+%else %if &ext=asf %then %do;%str(video/x-ms-asf)%end;
+%else %if &ext=asx %then %do;%str(video/x-ms-asf)%end;
+%else %if &ext=vob %then %do;%str(video/x-ms-vob)%end;
+%else %if &ext=wm %then %do;%str(video/x-ms-wm)%end;
+%else %if &ext=wmv %then %do;%str(video/x-ms-wmv)%end;
+%else %if &ext=wmx %then %do;%str(video/x-ms-wmx)%end;
+%else %if &ext=wvx %then %do;%str(video/x-ms-wvx)%end;
+%else %if &ext=avi %then %do;%str(video/x-msvideo)%end;
+%else %if &ext=movie %then %do;%str(video/x-sgi-movie)%end;
+%else %if &ext=ice %then %do;%str(x-conference/x-cooltalk)%end;
+%else %if "&ext"="in" %then %do;%str(text/plain)%end;
+%else %do;%str(application/octet-stream)%end;
+
+%mend mf_mimetype;

base/mf_mkdir.sas

@@ -7,7 +7,7 @@ Usage:
     %mf_mkdir(/some/path/name)
 
 
-  @param dir relative or absolute pathname.  Unquoted.
+  @param [in] dir Relative or absolute pathname.  Unquoted.
   @version 9.2
 
 **/

base/mf_mval.sas

@@ -8,6 +8,8 @@
 
       %if %mf_mval(maynotexist)=itdid %then %do;
 
+  @param [in] var The macro variable NAME to return the (possible) value for
+
   @version 9.2
   @author Allan Bowe
 **/

base/mf_nobs.sas

@@ -9,7 +9,7 @@
   <h4> SAS Macros </h4>
   @li mf_getattrn.sas
 
-  @param libds library.dataset
+  @param [in] libds library.dataset
 
   @return output returns result of the attrn value supplied, or log message
     if err.

base/mf_readfile.sas

@@ -0,0 +1,63 @@
+/**
+  @file
+  @brief Reads the first line of a file using pure macro
+  @details Reads the first line of a file and returns it.  Future versions may
+  read each line into a macro variable array.
+
+  Generally, reading data into macro variables is not great as certain
+  nonprintable characters (such as CR, LF) may be dropped in the conversion.
+
+    Usage:
+
+        %mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=more content)
+
+        %put %mf_readfile(&sasjswork/myfile.txt);
+
+
+  @param [in] fpath Full path to file to be read
+
+  <h4> Related Macros </h4>
+  @li mf_deletefile.sas
+  @li mf_writefile.sas
+  @li mf_readfile.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mf_readfile(fpath
+)/*/STORE SOURCE*/;
+%local fref rc fid fcontent;
+
+/* check file exists */
+%if %sysfunc(filename(fref,&fpath)) ne 0 %then %do;
+  %put &=fref &=fpath;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let fid=%sysfunc(fopen(&fref,I));
+
+%if &fid=0 %then %do;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%if %sysfunc(fread(&fid)) = 0 %then %do;
+  %let rc=%sysfunc(fget(&fid,fcontent,65534));
+  &fcontent
+%end;
+
+/*
+%do %while(%sysfunc(fread(&fid)) = 0);
+  %let rc=%sysfunc(fget(&fid,fcontent,65534));
+  &fcontent
+%end;
+*/
+
+%let rc=%sysfunc(fclose(&fid));
+%let rc=%sysfunc(filename(&fref));
+
+%mend mf_readfile;
+/** @endcond */

base/mf_trimstr.sas

@@ -11,8 +11,8 @@
   <h4> SAS Macros </h4>
 
 
-  @param basestr The string to be modified
-  @param trimstr The string to be removed from the end of `basestr`, if it
+  @param [in] basestr The string to be modified
+  @param [in] trimstr The string to be removed from the end of `basestr`, if it
     exists
 
   @return output returns result with the value of `trimstr` removed from the end

base/mf_wordsinstr1andstr2.sas

@@ -13,8 +13,8 @@
   returns:
   > blah blaaah brah
 
-  @param str1= string containing words to extract
-  @param str2= used to compare with the extract string
+  @param [in] str1= () string containing words to extract
+  @param [in] str2= () used to compare with the extract string
 
   @warning CASE SENSITIVE!
 

base/mf_wordsinstr1butnotstr2.sas

@@ -16,8 +16,8 @@
   returns:
   > sss bram boo
 
-  @param [in] str1= string containing words to extract
-  @param [in] str2= used to compare with the extract string
+  @param [in] str1= () String containing words to extract
+  @param [in] str2= () Used to compare with the extract string
 
   @version 9.2
   @author Allan Bowe

base/mf_writefile.sas

@@ -21,8 +21,8 @@
   @param [in] mode= (O) Available options are A or O as follows:
     @li A APPEND mode, writes new records after the current end of the file.
     @li O OUTPUT mode, writes new records from the beginning of the file.
-  @param [in] l1= First line
-  @param [in] l2= Second line (etc through to l10)
+  @param [in] l1= () First line
+  @param [in] l2= () Second line (etc through to l10)
 
   <h4> Related Macros </h4>
   @li mf_writefile.test.sas

base/mp_abort.sas

@@ -27,14 +27,15 @@
     currently investigating approaches to deal with this.
 
 
-  @param mac= (mp_abort.sas) To contain the name of the calling macro. Do not
-    use &sysmacroname as this will always resolve to MP_ABORT.
-  @param msg= message to be returned
-  @param iftrue= (1=1) Supply a condition for which the macro should be executed
-  @param errds= (work.mp_abort_errds) There is no clean way to end a process
-    within a %include called within a macro.  Furthermore, there is no way to
-    test if a macro is called within a %include.  To handle this particular
-    scenario, the %include should be switched for the mp_include.sas macro.
+  @param [in] mac= (mp_abort.sas) To contain the name of the calling macro. Do
+    not use &sysmacroname as this will always resolve to MP_ABORT.
+  @param [out] msg= message to be returned
+  @param [in] iftrue= (1=1) Condition under which the macro should be executed
+  @param [in] errds= (work.mp_abort_errds) There is no clean way to end a
+    process within a %include called within a macro.  Furthermore, there is no
+    way to test if a macro is called within a %include.  To handle this
+    particular scenario, the %include should be switched for the mp_include.sas
+    macro.
     This provides an indicator that we are running a macro within a \%include
     (`_SYSINCLUDEFILEDEVICE`) and allows us to provide a dataset with the abort
     values (msg, mac).
@@ -45,8 +46,8 @@
     @li msg (the message)
     @li mac (the mac param)
 
-  @param mode= (REGULAR) If mode=INCLUDE then the &errds dataset is checked for
-    an abort status.
+  @param [in] mode= (REGULAR) If mode=INCLUDE then the &errds dataset is checked
+    for an abort status.
     Valid values:
     @li REGULAR (default)
     @li INCLUDE
@@ -225,15 +226,51 @@ and %superq(SYSPROCESSNAME) ne %str(Compute Server)
     _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
-    syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
-    put ",""SYSERRORTEXT"" : " syserrortext;
+    syserrortext=cats(symget('syserrortext'));
+    if findc(syserrortext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syserrortext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syserrortext)
+      )))))))))))))!!'"';
+    end;
+    else syserrortext=cats('"',syserrortext,'"');
+    put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');
+    syswarningtext=cats(symget('syswarningtext'));
+    if findc(syswarningtext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syswarningtext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syswarningtext)
+      )))))))))))))!!'"';
+    end;
+    else syswarningtext=cats('"',syswarningtext,'"');
     put ",""SYSWARNINGTEXT"" : " syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     put "}" ;

base/mp_aligndecimal.sas

@@ -0,0 +1,95 @@
+/**
+  @file
+  @brief Apply leading blanks to align numbers vertically in a char variable
+  @details This is particularly useful when storing numbers (as character) that
+  need to be sorted.
+
+  It works by splitting the number left and right of the decimal place, and
+  aligning it accordingly.  A temporary variable is created as part of this
+  process (which is automatically dropped)
+
+  The macro can be used only in data step, eg as follows:
+
+      data _null_;
+        length myvar $50;
+        do i=1 to 1000 by 50;
+          if mod(i,2)=0 then j=ranuni(0)*i*100;
+          else j=i*100;
+
+          %mp_aligndecimal(myvar,width=7)
+
+          leading_spaces=length(myvar)-length(cats(myvar));
+          putlog +leading_spaces myvar;
+        end;
+      run;
+
+  The generated code will look something like this:
+
+      length aligndp4e49996 $7;
+      if index(myvar,'.') then do;
+        aligndp4e49996=cats(scan(myvar,1,'.'));
+        aligndp4e49996=right(aligndp4e49996);
+        myvar=aligndp4e49996!!'.'!!cats(scan(myvar,2,'.'));
+      end;
+      else do;
+        aligndp4e49996=myvar;
+        aligndp4e49996=right(aligndp4e49996);
+        myvar=aligndp4e49996;
+      end;
+      drop aligndp4e49996;
+
+  Results (myvar variable):
+
+                0.7683559324
+              122.8232796
+            99419.50552
+            42938.5143414
+              763.3799189
+            15170.606073
+            15083.285773
+            85443.198707
+          2022999.2251
+            12038.658867
+          1350582.6734
+            52777.258221
+            11723.347628
+            33101.268376
+          6181622.8603
+          7390614.0669
+            73384.537893
+          1788362.1016
+          2774586.2219
+          7998580.8415
+
+
+  @param [in] var The (data step, character) variable to modify
+  @param [in] width= (8) The number of characters BEFORE the decimal point
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+
+  <h4> Related Programs </h4>
+  @li mp_aligndecimal.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_aligndecimal(var,width=8);
+
+  %local tmpvar;
+  %let tmpvar=%mf_getuniquename(prefix=aligndp);
+  length &tmpvar $&width;
+  if index(&var,'.') then do;
+    &tmpvar=cats(scan(&var,1,'.'));
+    &tmpvar=right(&tmpvar);
+    &var=&tmpvar!!'.'!!cats(scan(&var,2,'.'));
+  end;
+  else do;
+    &tmpvar=cats(&var);
+    &tmpvar=right(&tmpvar);
+    &var=&tmpvar;
+  end;
+  drop &tmpvar;
+
+%mend mp_aligndecimal;

base/mp_appendfile.sas

@@ -17,8 +17,8 @@
         %mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3)
 
 
-  @param [in] baseref= Fileref of the base file (should exist)
-  @param [in] appendrefs= One or more filerefs to be appended to the base
+  @param [in] baseref= (0) Fileref of the base file (should exist)
+  @param [in] appendrefs= (0) One or more filerefs to be appended to the base
     fileref.  Space separated.
 
   @version 9.2

base/mp_applyformats.sas

@@ -94,7 +94,7 @@ data &outds;
   if libref(lib) ne 0 then do;
     msg=catx(' ','libref',lib,'is not assigned!');
     %if &errds=0 %then %do;
-      putlog "%str(ERR)OR: " msg;
+      putlog 'ERR' +(-1) "OR: " msg;
     %end;
     output;
     return;
@@ -102,7 +102,7 @@ data &outds;
   if exist(cats(lib,'.',ds)) ne 1 then do;
     msg=catx(' ','libds',lib,'.',ds,'does not exist!');
     %if &errds=0 %then %do;
-      putlog "%str(ERR)OR: " msg;
+      putlog 'ERR' +(-1) "OR: " msg;
     %end;
     output;
     return;
@@ -111,7 +111,7 @@ data &outds;
   if is_fmt=0 then do;
     msg=catx(' ','format',fmt,'on libds',lib,'.',ds,'.',var,'is not valid!');
     %if &errds=0 %then %do;
-      putlog "%str(ERR)OR: " msg;
+      putlog 'ERR' +(-1) "OR: " msg;
     %end;
     output;
     return;
@@ -123,7 +123,7 @@ data &outds;
     if dsid=0 then do;
       msg=catx(' ','libds',lib,'.',ds,' could not be opened!');
       %if &errds=0 %then %do;
-        putlog "%str(ERR)OR: " msg;
+        putlog 'ERR' +(-1) "OR: " msg;
       %end;
       output;
       return;
@@ -131,7 +131,7 @@ data &outds;
     if varnum(dsid,var)<1 then do;
       msg=catx(' ','Variable',lib,'.',ds,'.',var,' was not found!');
       %if &errds=0 %then %do;
-        putlog "%str(ERR)OR: " msg;
+        putlog 'ERR' +(-1) "OR: " msg;
       %end;
       output;
     end;

base/mp_assert.sas

@@ -22,7 +22,7 @@
   results.  If it does not exist, it will be created, with the following format:
   |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
   |---|---|---|
-  |User Provided description|PASS|Column &inds contained ALL columns|
+  |User Provided description|PASS|Dataset &inds contained ALL columns|
 
   @version 9.2
   @author Allan Bowe

base/mp_assertcols.sas

@@ -31,8 +31,8 @@
 
 
   @param [in] inds The input library.dataset to test for values
-  @param [in] cols= The list of columns to check for
-  @param [in] desc= (Testing observations) The user provided test description
+  @param [in] cols= (0) The list of columns to check for
+  @param [in] desc= (0) The user provided test description
   @param [in] test= (ALL) The test to apply.  Valid values are:
     @li ALL - Test is a PASS if ALL columns exist in &inds
     @li ANY - Test is a PASS if ANY of the columns exist in &inds
@@ -41,7 +41,7 @@
   results.  If it does not exist, it will be created, with the following format:
   |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
   |---|---|---|
-  |User Provided description|PASS|Column &inds contained ALL columns|
+  |User Provided description|PASS|Dataset &inds contained ALL columns|
 
 
   <h4> Related Macros </h4>

base/mp_assertcolvals.sas

@@ -36,12 +36,13 @@
 
 
   @param [in] indscol The input library.dataset.column to test for values
-  @param [in] checkvals= A library.dataset.column value containing a UNIQUE
+  @param [in] checkvals= (0) A library.dataset.column value containing a UNIQUE
     list of values to be compared against the source (indscol).
   @param [in] desc= (Testing observations) The user provided test description
   @param [in] test= (ALLVALS) The test to apply.  Valid values are:
     @li ALLVALS - Test is a PASS if ALL values have a match in checkvals
     @li ANYVAL - Test is a PASS if at least 1 value has a match in checkvals
+    @li NOVAL - Test is a PASS if there are NO matches in checkvals
   @param [out] outds= (work.test_results) The output dataset to contain the
   results.  If it does not exist, it will be created, with the following format:
   |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
@@ -97,7 +98,7 @@
 
   %let test=%upcase(&test);
 
-  %if &test ne ALLVALS and &test ne ANYVAL %then %do;
+  %if &test ne ALLVALS and &test ne ANYVAL and &test ne NOVAL %then %do;
     %mp_abort(
       mac=&sysmacroname,
       msg=%str(Invalid test - &test)
@@ -108,12 +109,12 @@
   %let result=-1;
   %let orig=-1;
   proc sql noprint;
-  select count(*) into: result
+  select count(*) into: result trimmed
     from &lib..&ds
     where &col not in (
       select &ccol from &clib..&cds
     );
-  select count(*) into: orig from &lib..&ds;
+  select count(*) into: orig trimmed from &lib..&ds;
   quit;
 
   %local notfound tmp1 tmp2;
@@ -145,7 +146,7 @@
     length test_description $256 test_result $4 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
-    test_comments="&sysmacroname: &lib..&ds..&col has &result values "
+    test_comments="&sysmacroname: &lib..&ds..&col has &result/&orig values "
       !!"not in &clib..&cds..&ccol.. First 10 vals:"!!symget('notfound');
   %if &test=ANYVAL %then %do;
     if &result < &orig then test_result='PASS';
@@ -153,6 +154,9 @@
   %else %if &test=ALLVALS %then %do;
     if &result=0 then test_result='PASS';
   %end;
+  %else %if &test=NOVAL %then %do;
+    if &result=&orig then test_result='PASS';
+  %end;
   %else %do;
     test_comments="&sysmacroname: Unsatisfied test condition - &test";
   %end;

base/mp_assertscope.sas

@@ -73,10 +73,6 @@
   ignorelist=,
   outds=work.test_results
 )/*/STORE SOURCE*/;
-%local ds test_result test_comments del add mod ilist;
-%let ilist=%upcase(&sasjs_prefix._FUNCTIONS SYS_PROCHTTP_STATUS_CODE
-  SYS_PROCHTTP_STATUS_CODE SYS_PROCHTTP_STATUS_PHRASE &ignorelist);
-
 /**
   * this sets up the global vars, it will also enter STRICT mode.  If this
   * behaviour is not desired, simply initiate the following global macro
@@ -84,6 +80,10 @@
   */
 %mp_init()
 
+%local ds test_result test_comments del add mod ilist;
+%let ilist=%upcase(&sasjs_prefix._FUNCTIONS SYS_PROCHTTP_STATUS_CODE
+  SYS_PROCHTTP_STATUS_CODE SYS_PROCHTTP_STATUS_PHRASE &ignorelist);
+
 /* get current variables */
 %if &action=SNAPSHOT %then %do;
   proc sql;

base/mp_base64copy.sas

@@ -28,8 +28,8 @@
           put _infile_;
         run;
 
-  @param [in] inref= Fileref of the input file (should exist)
-  @param [out] outref= Output fileref. If it does not exist, it is created.
+  @param [in] inref= (0) Fileref of the input file (should exist)
+  @param [out] outref= (0) Output fileref. If it does not exist, it is created.
   @param [in] action= (ENCODE) The action to take. Valid values:
     @li ENCODE - Convert the file to base64 format
     @li DECODE - Decode the file from base64 format

base/mp_binarycopy.sas

@@ -22,15 +22,17 @@
       %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
 
 
-  @param [in] inloc quoted "path/and/filename.ext" of the file to be copied
-  @param [out] outloc quoted "path/and/filename.ext" of the file to be created
-  @param [in] inref (____in) If provided, this fileref will take precedence over
+  @param [in] inloc= () quoted "path/and/filename.ext" of the file to be copied
+  @param [out] outloc= () quoted "path/and/filename.ext" of the file to create
+  @param [in] inref= (____in) If provided, this fileref takes precedence over
     the `inloc` parameter
-  @param [out] outref (____in) If provided, this fileref will take precedence
+  @param [out] outref= (____in) If provided, this fileref takes precedence
     over the `outloc` parameter.  It must already exist!
-  @param [in] mode (CREATE) Valid values:
+  @param [in] mode= (CREATE) Valid values:
     @li CREATE - Create the file (even if it already exists)
     @li APPEND - Append to the file (don't overwrite)
+  @param [in] iftrue= (1=1)
+    Supply a condition for which the macro should be executed
 
   @returns nothing
 
@@ -44,8 +46,12 @@
     ,inref=____in   /* override default to use own filerefs */
     ,outref=____out /* override default to use own filerefs */
     ,mode=CREATE
+    ,iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
   %local mod;
+
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
   %if &mode=APPEND %then %let mod=mod;
 
   /* these IN and OUT filerefs can point to anything */

base/mp_chop.sas

@@ -44,7 +44,7 @@
   For more examples, see mp_chop.test.sas
 
   @param [in] infile The QUOTED path to the file on which to perform the chop
-  @param [in] matchvar= Macro variable NAME containing the string to split by
+  @param [in] matchvar= () Macro variable NAME containing the string to split by
   @param [in] matchpoint= (START) Valid values:
     @li START - chop at the beginning of the string in `matchvar`.
     @li END - chop at the end of the string in `matchvar`.
@@ -54,7 +54,8 @@
     @li FIRST - keep the section of the file before the chop
     @li LAST - keep the section of the file after the chop
   @param [in] mdebug= (0) Set to 1 to provide macro debugging
-  @param outfile= (0) Optional QUOTED path to the adjusted output file (avoids
+  @param [out] outfile= (0)
+    Optional QUOTED path to the adjusted output file (avoids
     overwriting the first file).
 
   <h4> SAS Macros </h4>
@@ -185,7 +186,7 @@ run;
     infile &outfile lrecl=32767;
     input;
     list;
-    if _n_>50 then stop;
+    if _n_>200 then stop;
   run;
 %end;
 /* END */

base/mp_cleancsv.sas

@@ -1,5 +1,5 @@
 /**
-  @file mp_cleancsv.sas
+  @file
   @brief Fixes embedded cr / lf / crlf in CSV
   @details CSVs will sometimes contain lf or crlf within quotes (eg when
   saved by excel).  When the termstr is ALSO lf or crlf that can be tricky
@@ -7,14 +7,17 @@
   This macro converts any csv to follow the convention of a windows excel file,
   applying CRLF line endings and converting embedded cr and crlf to lf.
 
-  usage:
+  Usage:
 
       fileref mycsv "/path/your/csv";
       %mp_cleancsv(in=mycsv,out=/path/new.csv)
 
-  @param in= provide path or fileref to input csv
-  @param out= output path or fileref to output csv
-  @param qchar= quote char - hex code 22 is the double quote.
+  @param [in] in= (NOTPROVIDED)
+    Provide path or fileref to input csv. If a period is
+    found, it is assumed to be a file.
+  @param [in] out= (NOTPROVIDED) Output path or fileref to output csv.
+    If a period is found, it is assumed to be a file.
+  @param [in] qchar= ('22'x) Quote char - hex code 22 is the double quote.
 
   @version 9.2
   @author Allan Bowe
@@ -56,9 +59,14 @@
     else do;
       /* outside a quote, change cr and lf to crlf */
       if inchar='0D'x then do;
+        crblank:
         put '0D0A'x;
         input inchar $char1.;
-        if inchar ne '0A'x then do;
+        if inchar='0D'x then do;
+          /* multiple CR indicates CR formatted file with blank lines */
+          goto crblank;
+        end;
+        else if inchar ne '0A'x then do;
           put inchar $char1.;
           if inchar=qchar then isq = mod(isq+1,2);
         end;

base/mp_cntlout.sas

@@ -25,6 +25,7 @@
   <h4> SAS Macros </h4>
   @li mddl_sas_cntlout.sas
   @li mf_getuniquename.sas
+  @li mp_aligndecimal.sas
 
   <h4> Related Macros </h4>
   @li mf_getvarformat.sas
@@ -57,25 +58,33 @@
 %end;
 
 proc format lib=&libcat cntlout=&cntlds;
-%if "&fmtlist" ne "0" %then %do;
+%if "&fmtlist" ne "0" and "&fmtlist" ne "" %then %do;
   select
-  %do i=1 %to %sysfunc(countw(&fmtlist));
+  %do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
     %scan(&fmtlist,&i,%str( ))
   %end;
   ;
 %end;
 run;
 
-data &cntlout;
+data &cntlout/nonote2err;
   if 0 then set &ddlds;
   set &cntlds;
-  if type="N" then do;
-    start=cats(start);
-    end=cats(end);
+  by type fmtname notsorted;
+
+  /* align the numeric values to avoid overlapping ranges */
+  if type in ("I","N") then do;
+    %mp_aligndecimal(start,width=16)
+    %mp_aligndecimal(end,width=16)
   end;
+
+  /* create row marker. Data cannot be sorted without it! */
+  if first.fmtname then fmtrow=1;
+  else fmtrow+1;
+
 run;
 proc sort;
-  by fmtname start;
+  by type fmtname fmtrow;
 run;
 
 proc sql;

base/mp_copyfolder.sas

@@ -18,7 +18,7 @@
 
   @param [in] source Unquoted path to the folder to copy from.
   @param [out] target Unquoted path to the folder to copy to.
-  @param [in] copymax=(MAX) Set to a positive integer to indicate the level of
+  @param [in] copymax= (MAX) Set to a positive integer to indicate the level of
     subdirectory copy recursion - eg 3, to go `./3/levels/deep`.  For unlimited
     recursion, set to MAX.
 
@@ -69,7 +69,7 @@
       rc2=filename(fref2,filepath2,'disk','recfm=n');
       if fcopy(fref1,fref2) ne 0 then do;
         msg=sysmsg();
-        putlog "%str(ERR)OR: Unable to copy " filepath " to " filepath2;
+        putlog 'ERR' +(-1) "OR: Unable to copy " filepath " to " filepath2;
         putlog msg=;
       end;
     end;

base/mp_createconstraints.sas

@@ -18,11 +18,14 @@
       %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
       %mp_createconstraints(inds=work.constraints,outds=created,execute=YES)
 
-  @param inds= The input table containing the constraint info
-  @param outds= a table containing the create statements (create_statement column)
-  @param execute= `YES|NO` - default is NO. To actually create, use YES.
+  @param [in] inds= (work.mp_getconstraints) The input table containing the
+    constraint info
+  @param [out] outds= (work.mp_createconstraints) A table containing the create
+    statements (create_statement column)
+  @param [in] execute= (NO) To actually create, use YES.
 
-  <h4> SAS Macros </h4>
+  <h4> Related Files </h4>
+  @li mp_getconstraints.sas
 
   @version 9.2
   @author Allan Bowe
@@ -30,7 +33,7 @@
 **/
 
 %macro mp_createconstraints(inds=mp_getconstraints
-  ,outds=mp_createconstraints
+  ,outds=work.mp_createconstraints
   ,execute=NO
 )/*/STORE SOURCE*/;
 
@@ -64,4 +67,4 @@ data &outds;
   output;
 run;
 
-%mend mp_createconstraints;
+%mend mp_createconstraints;

base/mp_createwebservice.sas

@@ -1,6 +1,6 @@
 /**
   @file mp_createwebservice.sas
-  @brief Create a web service in SAS 9, Viya or SASjs Server
+  @brief Create a web service in SAS 9, Viya or SASjs Server (legacy)
   @details This is actually a wrapper for mx_createwebservice.sas, remaining
   for legacy purposes.  For new apps, use mx_createwebservice.sas.
 

base/mp_csv2ds.sas

@@ -19,11 +19,12 @@
       %mp_csv2ds(inref=mycsv,outds=myds,baseds=sashelp.class)
 
 
-  @param inref= fileref to the CSV
-  @param outds= output ds (lib.ds format)
-  @param view= Set to YES or NO to determine whether the output should be
-    a view or not.  Default is NO (not a view).
-  @param baseds= Template dataset on which to create the input statement.
+  @param [in] inref= (0) Fileref to the CSV
+  @param [out] outds= (0) Output ds (lib.ds format)
+  @param [in] view= (NO) Set to YES or NO to determine whether the output
+    should be a view or not.  Default is NO (not a view).
+  @param [in] baseds= (0)
+    Template dataset on which to create the input statement.
     Is used to determine types, lengths, and any informats.
 
   @version 9.2

base/mp_deleteconstraints.sas

@@ -17,9 +17,11 @@
       %mp_getconstraints(lib=work,ds=example,outds=work.constraints)
       %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
 
-  @param inds= The input table containing the constraint info
-  @param outds= a table containing the drop statements (drop_statement column)
-  @param execute= `YES|NO` - default is NO. To actually drop, use YES.
+  @param [in] inds= (mp_getconstraints)
+    The input table containing constraint info
+  @param [out] outds= (mp_deleteconstraints)
+    Table containing the drop statements (drop_statement column)
+  @param [in] execute= (NO) `YES|NO` - default is NO. To actually drop, use YES.
 
 
   @version 9.2

base/mp_deletefolder.sas

@@ -15,7 +15,7 @@
 
       %mp_deletefolder(&rootdir)
 
-  @param path Unquoted path to the folder to delete.
+  @param [in] folder Unquoted path to the folder to delete.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas

base/mp_dictionary.sas

@@ -0,0 +1,52 @@
+/**
+  @file mp_dictionary.sas
+  @brief Creates a portal (libref) into the SQL Dictionary Views
+  @details Provide a libref and the macro will create a series of views against
+  each view in the special PROC SQL dictionary libref.
+
+  This is useful if you would like to visualise (navigate) the views in a SAS
+  client such as Base SAS, Enterprise Guide, or Studio (or [Data Controller](
+  https://datacontroller.io)).
+
+  It works by extracting the dictionary.dictionaries view into
+  YOURLIB.dictionaries, then uses that to create a YOURLIB.{viewName} for every
+  other dictionary.view, eg:
+
+      proc sql;
+      create view YOURLIB.columns as select * from dictionary.columns;
+
+  Usage:
+
+      libname demo "/lib/directory";
+      %mp_dictionary(lib=demo)
+
+  Or, to just create them in WORK:
+
+      %mp_dictionary()
+
+  If you'd just like to browse the dictionary data model, you can also check
+  out [this article](https://rawsas.com/dictionary-of-dictionaries/).
+
+  ![](https://user-images.githubusercontent.com/4420615/188278365-2987db97-0594-4a39-ac81-dbacdef5cdc8.png)
+
+  @param [in] lib= (WORK) The libref in which to create the views
+
+  <h4> Related Files </h4>
+  @li mp_dictionary.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_dictionary(lib=WORK)/*/STORE SOURCE*/;
+  %local list i mem;
+  proc sql noprint;
+  create view &lib..dictionaries as select * from dictionary.dictionaries;
+  select distinct memname into: list separated by ' '  from &lib..dictionaries;
+  %do i=1 %to %sysfunc(countw(&list,%str( )));
+    %let mem=%scan(&list,&i,%str( ));
+    create view &lib..&mem as select * from dictionary.&mem;
+  %end;
+  quit;
+%mend mp_dictionary;

base/mp_dirlist.sas

@@ -27,6 +27,9 @@
   @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
     subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
     recursion, set to MAX.
+  @param [in] showparent= (NO) By default, the initial parent directory is not
+    part of the results.  Set to YES to include it.  For this record only,
+    directory=filepath.
   @param [out] outds= (work.mp_dirlist) The output dataset to create
   @param [out] getattrs= (NO)  If getattrs=YES then the doptname / foptname
     functions are used to scan all properties - any characters that are not
@@ -63,6 +66,7 @@
     , fref=0
     , outds=work.mp_dirlist
     , getattrs=NO
+    , showparent=NO
     , maxdepth=0
     , level=0 /* The level of recursion to perform.  For internal use only. */
 )/*/STORE SOURCE*/;
@@ -81,7 +85,7 @@ data;run;
 data &out_ds(compress=no
     keep=file_or_folder filepath filename ext msg directory level
   );
-  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
+  length directory filepath $2000 fref fref2 $8 file_or_folder $6 filename $255
     ext $20 msg $200 foption $16;
   if _n_=1 then call missing(of _all_);
   retain level &level;
@@ -97,8 +101,7 @@ data &out_ds(compress=no
     if did=0 then do;
       putlog "NOTE: This directory is empty, or does not exist - &path";
       msg=sysmsg();
-      put msg;
-      put _all_;
+      put (_all_)(=);
       stop;
     end;
     /* attribute is OS-dependent - could be "Directory" or "Directory Name" */
@@ -145,6 +148,15 @@ data &out_ds(compress=no
     output;
   end;
   rc = dclose(did);
+  %if &showparent=YES and &level=0 %then %do;
+    filepath=directory;
+    file_or_folder='folder';
+    ext='';
+    filename=scan(directory,-1,'/\');
+    msg='';
+    level=&level;
+    output;
+  %end;
   stop;
 run;
 
@@ -232,6 +244,9 @@ run;
   data _null_;
     set &out_ds;
     where file_or_folder='folder';
+  %if &showparent=YES and &level=0 %then %do;
+    if filepath ne directory;
+  %end;
     length code $10000;
     code=cats('%nrstr(%mp_dirlist(path=',filepath,",outds=&outds"
       ,",getattrs=&getattrs,level=%eval(&level+1),maxdepth=&maxdepth))");

base/mp_distinctfmtvalues.sas

@@ -7,11 +7,11 @@
 
         %mp_distinctfmtvalues(libds=sashelp.class,var=age,outvar=age,outds=test)
 
-  @param libds input dataset
-  @param var variable to get distinct values for
-  @param outvar variable to create.  Default:  `formatted_value`
-  @param outds dataset to create.  Default:  work.mp_distinctfmtvalues
-  @param varlen length of variable to create (default 200)
+  @param [in] libds= () input dataset
+  @param [in] var= (0) variable to get distinct values for
+  @param [out] outvar= (formatteed_value) variable to create.
+  @param [out] outds= (work.mp_distinctfmtvalues) dataset to create.
+  @param [in] varlen= (2000) length of variable to create
 
   @version 9.2
   @author Allan Bowe

base/mp_ds2csv.sas

@@ -118,13 +118,21 @@ data _null_;
     header = cats(coalescec(varlabel(dsid,i),varnm),dlm);
   %end;
   %else %if &headerformat=SASJS %then %do;
-    if vartype(dsid,i)='C' then header=cats(varnm,':$char',varlen(dsid,i),'.');
+    vlen=varlen(dsid,i);
+    if vartype(dsid,i)='C' then header=cats(varnm,':$char',vlen,'.');
     else do;
       vfmt=coalescec(varfmt(dsid,i),'0');
       fmttype=mcf_getfmttype(vfmt);
       if fmttype='DATE' then header=cats(varnm,':date9.');
       else if fmttype='DATETIME' then header=cats(varnm,':E8601DT26.6');
       else if fmttype='TIME' then header=cats(varnm,':TIME12.');
+      /**
+        * there is not much point importing a short length numeric like this,
+        * eg with best4., as the resulting variable will still be stored as
+        * length 8.  We need a length or format statement to ensure variable
+        * is creatd with the smaller length...
+        **/
+      else if vlen<8 then header=cats(varnm,':best',vlen,'.');
       else header=cats(varnm,':best.');
     end;
   %end;
@@ -151,6 +159,7 @@ data _null_;
   set &ds end=last;
 %do i=1 %to &vcnt;
   %let var=%scan(&varlist,&i);
+  %local vlen&i;
   %if %mf_getvartype(&ds,&var)=C %then %do;
     %let dsv1=%mf_getuniquename(prefix=csvcol1_);
     %let dsv2=%mf_getuniquename(prefix=csvcol2_);

base/mp_ds2ddl.sas

@@ -1,8 +1,20 @@
 /**
   @file
-  @brief A wrapper for mp_getddl.sas
-  @details In the next release, this will be the main version.
+  @brief Fetches DDL for a specific table
+  @details Uses mp_getddl under the hood
 
+  @param [in] libds library.dataset to create ddl for
+  @param [in] fref= (getddl) the fileref to which to _append_ the DDL.  If it
+    does not exist, it will be created.
+  @param [in] flavour= (SAS) The type of DDL to create. Options:
+    @li SAS
+    @li TSQL
+
+  @param [in]showlog= (NO) Set to YES to show the DDL in the log
+  @param [in] schema= () Choose a preferred schema name (default is to use
+    actual schema, else libref)
+  @param [in] applydttm= (NO) For non SAS DDL, choose if columns are created with
+    native datetime2 format or regular decimal type
 
   <h4> SAS Macros </h4>
   @li mp_getddl.sas

base/mp_ds2fmtds.sas

@@ -17,7 +17,7 @@
   <h4> SAS Macros </h4>
   @li mf_existds.sas
 
-  <h4> Related Macros <h4>
+  <h4> Related Macros </h4>
   @li mp_jsonout.sas
 
   @version 9.2

base/mp_ds2md.sas

@@ -92,12 +92,13 @@ data _null_;
 run;
 
 %if %upcase(&showlog)=YES %then %do;
-  options ps=max;
+  options ps=max lrecl=max;
   data _null_;
     infile &outref;
+    if _n_=1 then putlog "# &libds" /;
     input;
     putlog _infile_;
   run;
 %end;
 
-%mend mp_ds2md;
+%mend mp_ds2md;

base/mp_dsmeta.sas

@@ -0,0 +1,110 @@
+/**
+  @file
+  @brief Export dataset metadata to a single output table
+  @details Exports the dataset attributes and enginehost information, then
+  converts the datasets into a single output table in the following format:
+
+|ODS_TABLE:$10.|NAME:$100.|VALUE:$1000.|
+|---|---|---|
+|`ATTRIBUTES `|`Data Set Name `|`SASHELP.CLASS `|
+|`ATTRIBUTES `|`Observations `|`19 `|
+|`ATTRIBUTES `|`Member Type `|`DATA `|
+|`ATTRIBUTES `|`Variables `|`5 `|
+|`ATTRIBUTES `|`Engine `|`V9 `|
+|`ATTRIBUTES `|`Indexes `|`0 `|
+|`ATTRIBUTES `|`Created `|`06/08/2020 00:59:14 `|
+|`ATTRIBUTES `|`Observation Length `|`40 `|
+|`ATTRIBUTES `|`Last Modified `|`06/08/2020 00:59:14 `|
+|`ATTRIBUTES `|`Deleted Observations `|`0 `|
+|`ATTRIBUTES `|`Protection `|`. `|
+|`ATTRIBUTES `|`Compressed `|`NO `|
+|`ATTRIBUTES `|`Data Set Type `|`. `|
+|`ATTRIBUTES `|`Sorted `|`NO `|
+|`ATTRIBUTES `|`Label `|`Student Data `|
+|`ATTRIBUTES `|`Data Representation `|`SOLARIS_X86_64, LINUX_X86_64, ALPHA_TRU64, LINUX_IA64 `|
+|`ATTRIBUTES `|`Encoding `|`us-ascii  ASCII (ANSI) `|
+|`ENGINEHOST `|`Data Set Page Size `|`65536 `|
+|`ENGINEHOST `|`Number of Data Set Pages `|`1 `|
+|`ENGINEHOST `|`First Data Page `|`1 `|
+|`ENGINEHOST `|`Max Obs per Page `|`1632 `|
+|`ENGINEHOST `|`Obs in First Data Page `|`19 `|
+|`ENGINEHOST `|`Number of Data Set Repairs `|`0 `|
+|`ENGINEHOST `|`Filename `|`/opt/sas/sas9/SASHome/SASFoundation/9.4/sashelp/class.sas7bdat `|
+|`ENGINEHOST `|`Release Created `|`9.0401M7 `|
+|`ENGINEHOST `|`Host Created `|`Linux `|
+|`ENGINEHOST `|`Inode Number `|`28314616 `|
+|`ENGINEHOST `|`Access Permission `|`rw-r--r-- `|
+|`ENGINEHOST `|`Owner Name `|`sas `|
+|`ENGINEHOST `|`File Size `|`128KB `|
+|`ENGINEHOST `|`File Size (bytes) `|`131072 `|
+
+  Example usage:
+
+      %mp_dsmeta(sashelp.class,outds=work.mymeta)
+      proc print data=work.mymeta;
+      run;
+
+  For more details on creating datasets from PROC CONTENTS check out this
+  excellent [paper](
+  https://support.sas.com/resources/papers/proceedings14/1549-2014.pdf) by
+  [Louise Hadden](https://www.linkedin.com/in/louisehadden/).
+
+  @param [in] libds The library.dataset to export the metadata for
+  @param [out] outds= (work.dsmeta) The output table to contain the metadata
+
+  <h4> Related Files </h4>
+  @li mp_dsmeta.test.sas
+  @li mp_getcols.sas
+  @li mp_getdbml.sas
+  @li mp_getddl.sas
+  @li mp_getformats.sas
+  @li mp_getpk.sas
+  @li mp_guesspk.sas
+
+**/
+
+%macro mp_dsmeta(libds,outds=work.dsmeta);
+
+%local ds1 ds2;
+data;run; %let ds1=&syslast;
+data;run; %let ds2=&syslast;
+
+/* setup the ODS capture */
+ods output attributes=&ds1 enginehost=&ds2;
+
+/* export the metadata */
+proc contents data=&libds;
+run;
+
+/* load it into a single table */
+data &outds (keep=ods_table name value);
+  length ods_table $10 name label2 label1 label $100
+    value cvalue cvalue1 cvalue2 $1000
+    nvalue nvalue1 nvalue2 8;
+  if _n_=1 then call missing (of _all_);
+  * putlog (_all_)(=);
+  set &ds1 (in=atrs) &ds2 (in=eng);
+  if atrs then do;
+    ods_table='ATTRIBUTES';
+    name=coalescec(label1,label);
+    value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.));
+    output;
+    if label2 ne '' then do;
+    name=label2;
+    value=coalescec(cvalue2,put(nvalue2,best.));
+    output;
+    end;
+  end;
+  else if eng then do;
+    ods_table='ENGINEHOST';
+    name=coalescec(label1,label);
+    value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.));
+    output;
+  end;
+run;
+
+proc sql;
+drop table &ds1, &ds2;
+
+%mend mp_dsmeta;
+

base/mp_filtercheck.sas

@@ -37,10 +37,12 @@
   @param [in] targetds= The target dataset against which to verify VARIABLE_NM.
     This must be available (ie, the library must be assigned).
   @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
-  @param [out] outds= The output table, which is a copy of the &inds. table
-    plus a REASON_CD column, containing only bad records.  If bad records found,
-    the SYSCC value will be set to 1008 (general data problem).  Downstream
-    processes should check this table (and return code) before continuing.
+  @param [out] outds= (work.badrecords) The output table, which is a copy of the
+    &inds. table plus a REASON_CD column, containing only bad records.
+    If bad records are found, the SYSCC value will be set to 1008
+    (a general data problem).
+    Downstream processes should check this table (and return code) before
+    continuing.
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas
@@ -84,15 +86,15 @@
 /**
   * Sanitise the values based on valid value lists, then strip out
   * quotes, commas, periods and spaces.
-  * Only numeric values should remain
   */
 %local reason_cd nobs;
 %let nobs=0;
 data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
-  set &inds;
-  length reason_cd $4032 vtype $1 vnum dsid 8;
+  set &inds end=last;
+  length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000;
+  drop tmp;
 
   /* quick check to ensure column exists */
   if upcase(VARIABLE_NM) not in
@@ -107,7 +109,8 @@ data &outds;
   end;
 
   /* need to open the dataset to get the column type */
-  dsid=open("&targetds","i");
+  retain dsid;
+  if _n_=1 then dsid=open("&targetds","i");
   if dsid>0 then do;
     vnum=varnum(dsid,VARIABLE_NM);
     if vnum<1 then do;
@@ -117,11 +120,19 @@ data &outds;
       call symputx('reason_cd',reason_cd,'l');
       call symputx('nobs',_n_,'l');
       output;
-      return;
+      goto endstep;
     end;
     /* now we can get the type */
     else vtype=vartype(dsid,vnum);
   end;
+  else do;
+      REASON_CD=cats("Could not open &targetds");
+      putlog REASON_CD= dsid=;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
+      output;
+      stop;
+  end;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
@@ -156,30 +167,95 @@ data &outds;
   end;
 
   /* special missing logic */
-  if vtype='N'
-    and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE')
-    and cats(upcase(raw_value)) in (
+  if vtype='N' & OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE') then do;
+    if cats(upcase(raw_value)) in (
       '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N'
       '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._'
     )
-  then do;
-    /* valid numeric - exit data step loop */
-    return;
+    then do;
+      /* valid numeric - exit data step loop */
+      return;
+    end;
+    else if subpad(upcase(raw_value),1,1) in (
+      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+    )
+    then do;
+      /* check if the raw_value contains a valid variable NAME */
+      vnum=varnum(dsid,subpad(raw_value,1,32));
+      if vnum>0 then do;
+        /* now we can get the type */
+        vtype2=vartype(dsid,vnum);
+        /* check type matches */
+        if vtype2=vtype then do;
+          /* valid target var - exit loop */
+          return;
+        end;
+        else do;
+          REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
+          putlog REASON_CD= dsid=;
+          call symputx('reason_cd',reason_cd,'l');
+          call symputx('nobs',_n_,'l');
+          output;
+          goto endstep;
+        end;
+      end;
+    end;
   end;
 
   /* special logic */
-  if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ','');
-  else if OPERATOR_NM in ('IN','NOT IN') then do;
-    if substr(raw_value,1,1) ne '('
-    or substr(cats(reverse(raw_value)),1,1) ne ')'
-    then do;
-      REASON_CD='Missing start/end bracket in RAW_VALUE';
-      putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
-      call symputx('reason_cd',reason_cd,'l');
-      call symputx('nobs',_n_,'l');
-      output;
+  if OPERATOR_NM in ('IN','NOT IN','BETWEEN') then do;
+    if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ',',');
+    else do;
+      if substr(raw_value,1,1) ne '('
+      or substr(cats(reverse(raw_value)),1,1) ne ')'
+      then do;
+        REASON_CD='Missing start/end bracket in RAW_VALUE';
+        putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+        call symputx('reason_cd',reason_cd,'l');
+        call symputx('nobs',_n_,'l');
+        output;
+      end;
+      else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
+    end;
+    /* we now have a comma seperated list of values */
+    if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
+      tmp=scan(raw_value1,i,',');
+      if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
+        if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in (
+          'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+          'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+        )
+        then do;
+          /* check if the raw_value contains a valid variable NAME */
+          /* is not valid syntax for IN or NOT IN */
+          vnum=varnum(dsid,subpad(tmp,1,32));
+          if vnum>0 then do;
+            /* now we can get the type */
+            vtype2=vartype(dsid,vnum);
+            /* check type matches */
+            if vtype2=vtype then do;
+              /* valid target var - exit loop */
+              return;
+            end;
+            else do;
+              REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
+              putlog REASON_CD= dsid=;
+              call symputx('reason_cd',reason_cd,'l');
+              call symputx('nobs',_n_,'l');
+              output;
+              goto endstep;
+            end;
+          end;
+        end;
+        REASON_CD='Non Numeric value provided';
+        putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+        call symputx('reason_cd',reason_cd,'l');
+        call symputx('nobs',_n_,'l');
+        output;
+      end;
+      return;
     end;
-    else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
   end;
   else raw_value1=raw_value;
 
@@ -196,14 +272,42 @@ data &outds;
 
   /* output records that contain values other than digits and spaces */
   if notdigit(compress(raw_value3,' '))>0 then do;
+    if vtype='C' and subpad(upcase(raw_value),1,1) in (
+      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+    )
+    then do;
+      /* check if the raw_value contains a valid variable NAME */
+      vnum=varnum(dsid,subpad(raw_value,1,32));
+      if vnum>0 then do;
+        /* now we can get the type */
+        vtype2=vartype(dsid,vnum);
+        /* check type matches */
+        if vtype2=vtype then do;
+          /* valid target var - exit loop */
+          return;
+        end;
+        else do;
+          REASON_CD=cats("Compared Char Type (",vtype2,") is not (",vtype,")");
+          putlog REASON_CD= dsid=;
+          call symputx('reason_cd',reason_cd,'l');
+          call symputx('nobs',_n_,'l');
+          output;
+          goto endstep;
+        end;
+      end;
+    end;
+
     putlog raw_value3= $hex32.;
     REASON_CD=cats('Invalid RAW_VALUE:',raw_value);
-    putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
+    putlog (_all_)(=);
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');
     output;
   end;
 
+  endstep:
+  if last then rc=close(dsid);
 run;
 
 

base/mp_filtergenerate.sas

@@ -51,8 +51,8 @@
   > )
 
   @param [in] inds The input table with query values
-  @param [out] outref= The output fileref to contain the filter clause.  Will
-    be created (or replaced).
+  @param [out] outref= (filter) The output fileref to contain the filter clause.
+    Will be created (or replaced).
 
   <h4> Related Macros </h4>
   @li mp_filtercheck.sas

base/mp_filterstore.sas

@@ -44,9 +44,10 @@
     mp_coretable.sas as follows:  `mp_coretable(LOCKTABLE)`.
   @param [in] maxkeytable= (0) Optional permanent reference table used for
     retained key tracking.  Described in mp_retainedkey.sas.
-  @param [in] mdebug= set to 1 to enable DEBUG messages
-  @param [out] outresult= The result table with the FILTER_RK
-  @param [out] outquery= The original query, taken as extract after table load
+  @param [in] mdebug= (1) set to 1 to enable DEBUG messages
+  @param [out] outresult= (work.result) The result table with the FILTER_RK
+  @param [out] outquery= (work.query) The original query, taken as extract
+    after table load
 
 
   <h4> SAS Macros </h4>

base/mp_getcols.sas

@@ -12,14 +12,15 @@
 
       %mp_getcols(sashelp.airline,outds=work.myds)
 
-  @param ds The dataset from which to obtain column metadata
-  @param outds= (work.cols) The output dataset to create. Sample data:
+  @param [in] ds The dataset from which to obtain column metadata
+  @param [out] outds= (work.cols) The output dataset to create. Sample data:
 |NAME:$32.|LENGTH:best.|VARNUM:best.|LABEL:$256.|FMTNAME:$32.|FORMAT:$49.|TYPE:$1.|DDTYPE:$9.|
 |---|---|---|---|---|---|---|---|
 |`AIR `|`8 `|`2 `|`international airline travel (thousands) `|` `|`8. `|`N `|`NUMERIC `|
 |`DATE `|`8 `|`1 `|`DATE `|`MONYY `|`MONYY. `|`N `|`DATE `|
 |`REGION `|`3 `|`3 `|`REGION `|` `|`$3. `|`C `|`CHARACTER `|
 
+
   <h4> Related Macros </h4>
   @li mf_getvarlist.sas
   @li mm_getcols.sas
@@ -52,10 +53,11 @@ data &outds(keep=name type length varnum format label ddtype fmtname);
     else if formatd=0 then format=cats(fmtname,formatl,'.');
     else format=cats(fmtname,formatl,'.',formatd);
     type='N';
-    if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
+    if format=:'DATETIME' or format=:'E8601DT' or format=:'NLDATM'
+    then ddtype='DATETIME';
     else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
       or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
-      or format=:'MONYY'
+      or format=:'MONYY' or format=:'NLDATE'
     then ddtype='DATE';
     else if format=:'TIME' then ddtype='TIME';
     else ddtype='NUMERIC';

base/mp_getconstraints.sas

@@ -94,8 +94,11 @@ create table &outds as
 /**
   * We cannot apply this clause to the underlying dictionary table.  See:
   * https://communities.sas.com/t5/SAS-Programming/Unexpected-Where-Clause-behaviour-in-dictionary-TABLE/m-p/771554#M244867
+  * cannot use`where calculated libref="&lib"` either as it will STILL execute
+  * all the underlying constraint queries, causing exception errors in some
+  * cases: https://github.com/sasjs/core/issues/283
   */
-  where calculated libref="&lib"
+  where a.TABLE_CATALOG="&lib"
   %if "&ds" ne "" %then %do;
     and upcase(a.TABLE_NAME)="&ds"
     and upcase(b.TABLE_NAME)="&ds"

base/mp_getdbml.sas

@@ -23,10 +23,10 @@
   @li mf_getquotedstr.sas
   @li mp_getconstraints.sas
 
-  @param liblist= Space seperated list of librefs to take as
-    input (Default=SASHELP)
-  @param outref= Fileref to contain the DBML (Default=getdbml)
-  @param showlog= set to YES to show the DBML in the log (Default is NO)
+  @param [in] liblist= (SASHELP) Space seperated list of librefs to take as
+    input
+  @param [out] outref= (getdbml) Fileref to contain the DBML
+  @param [in] showlog= (NO) set to YES to show the DBML in the log
 
   @version 9.3
   @author Allan Bowe

base/mp_getddl.sas

@@ -22,16 +22,21 @@
   @li mf_getvarcount.sas
   @li mp_getconstraints.sas
 
-  @param lib libref of the library to create DDL for.  Should be assigned.
-  @param ds dataset to create ddl for (optional)
-  @param fref= the fileref to which to _append_ the DDL.  If it does not exist,
-    it will be created.
-  @param flavour= The type of DDL to create (default=SAS). Supported=TSQL
-  @param showlog= Set to YES to show the DDL in the log
-  @param schema= Choose a preferred schema name (default is to use actual schema
-    ,else libref)
-  @param applydttm= for non SAS DDL, choose if columns are created with native
-    datetime2 format or regular decimal type
+  @param [in] libref Libref of the library to create DDL for. Should already
+    be assigned.
+  @param [in] ds dataset to create ddl for (optional)
+  @param [in] fref= (getddl) the fileref to which to _append_ the DDL.  If it
+    does not exist, it will be created.
+  @param [in] flavour= (SAS) The type of DDL to create. Options:
+    @li SAS
+    @li TSQL
+
+  @param [in]showlog= (NO) Set to YES to show the DDL in the log
+  @param [in] schema= () Choose a preferred schema name (default is to use
+    actual schema, else libref)
+  @param [in] applydttm= (NO) For non SAS DDL, choose if columns are created
+    with native datetime2 format or regular decimal type
+
   @version 9.3
   @author Allan Bowe
 **/
@@ -130,13 +135,13 @@ run;
 
 %local x curds;
 %if &flavour=SAS %then %do;
-  data _null_;
-    file &fref mod;
-    put "/* SAS Flavour DDL for %upcase(&libref).&curds */";
-    put "proc sql;";
-  run;
   %do x=1 %to %sysfunc(countw(&dsnlist));
     %let curds=%scan(&dsnlist,&x);
+    data _null_;
+      file &fref mod;
+      put "/* SAS Flavour DDL for %upcase(&libref).&curds */";
+      put "proc sql;";
+    run;
     data _null_;
       file &fref mod;
       length lab $1024 typ $20;

base/mp_getformats.sas

@@ -7,10 +7,15 @@
   Formats are taken from the library / dataset reference and / or a static
   format list.
 
+  Note - the source for this information is the dictionary.formats table. This
+  cannot show formats that are not already declared in the FMTSEARCH path.
+
   Example usage:
 
       %mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable)
 
+      %mp_getformats(fmtlist=FORMAT1 $FORMAT2 @INFMT3,outsummary=work.table2)
+
   @param [in] lib= (0) The libref for which to return formats.
   @todo Enable exporting of formats for an entire library
   @param [in] ds= (0) The dataset from which to obtain format definitions
@@ -49,7 +54,9 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 
 
   <h4> Related Macros </h4>
+  @li mf_getfmtlist.sas
   @li mp_applyformats.sas
+  @li mp_cntlout.sas
   @li mp_getformats.test.sas
 
   @version 9.2
@@ -66,7 +73,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 
 %local i fmt allfmts tempds fmtcnt;
 
-%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( )));
+%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
   /* ensure format list contains format _name_ only */
   %let fmt=%scan(&fmtlist,&i,%str( ));
   %let fmt=%mf_getfmtname(&fmt);
@@ -90,8 +97,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 proc sql;
 create table &outsummary as
   select * from dictionary.formats
-  where fmtname in (%mf_getquotedstr(&allfmts,quote=D))
-    and fmttype='F';
+  where fmtname in (%mf_getquotedstr(&allfmts,quote=D));
 
 %if "&outdetail" ne "0" %then %do;
   /* ensure base table always exists */
@@ -115,6 +121,10 @@ create table &outsummary as
     data &tempds;
       if 0 then set &outdetail;
       set &tempds;
+      /* set fmtrow (position of record within the format) */
+      by type fmtname notsorted;
+      if first.fmtname then fmtrow=1;
+      else fmtrow+1;
     run;
     proc append base=&outdetail data=&tempds ;
     run;

base/mp_gitadd.sas

@@ -0,0 +1,45 @@
+/**
+  @file
+  @brief Stages files in a GIT repo
+  @details Uses the output dataset from mp_gitstatus.sas to determine the files
+  that should be staged.
+
+  If `STAGED ne "TRUE"` then the file is staged.
+
+  Usage:
+
+      %let dir=%sysfunc(pathname(work))/core;
+      %let repo=https://github.com/sasjs/core;
+      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir));
+      %mf_writefile(&dir/somefile.txt,l1=some content)
+      %mf_deletefile(&dir/package.json)
+      %mp_gitstatus(&dir,outds=work.gitstatus)
+
+      %mp_gitadd(&dir,inds=work.gitstatus)
+
+  @param [in] gitdir The directory containing the GIT repository
+  @param [in] inds= (work.mp_gitadd) The input dataset with the list of files
+  to stage.  Will accept the output from mp_gitstatus(), else just use a table
+  with the following columns:
+    @li path $1024 - relative path to the file in the repo
+    @li staged $32 - whether the file is staged (TRUE or FALSE)
+    @li status $64 - either new, deleted, or modified
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> Related Files </h4>
+  @li mp_gitadd.test.sas
+  @li mp_gitstatus.sas
+
+**/
+
+%macro mp_gitadd(gitdir,inds=work.mp_gitadd,mdebug=0);
+
+data _null_;
+  set &inds;
+  if STAGED ne "TRUE";
+  rc=git_index_add("&gitdir",cats(path),status);
+  if rc ne 0 or &mdebug=1 then put rc=;
+run;
+
+%mend mp_gitadd;

base/mp_gitlog.sas

@@ -0,0 +1,104 @@
+/**
+  @file
+  @brief Creates a dataset with the commit history of a local repository
+  @details Returns the commit history from a local repository. The name of the
+  branch is also returned.
+
+  More details here:
+https://documentation.sas.com/doc/ko/pgmsascdc/v_033/lefunctionsref/n1qo5miyvry1nen111js203hlwrh.htm
+
+  Usage:
+
+      %let gitdir=%sysfunc(pathname(work))/core;
+      %let repo=https://github.com/sasjs/core;
+      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir));
+
+      %mp_gitlog(&gitdir,outds=work.mp_gitlog)
+
+  @param [in] gitdir The directory containing the GIT repository
+  @param [in] filter= (BRANCHONLY) To return only the commits for the current
+  branch, use BRANCHONLY (the default).  Anything else will return the entire
+  commit history.
+  @param [out] outds= (work.mp_gitlog) The output dataset to create.
+  All vars are $128 except `message` which is $4000.
+    @li author returns the author who submitted the commit.
+    @li children_ids returns a list of the children commit IDs
+    @li committer returns the name of the committer.
+    @li committer_email returns the email of the committer.
+    @li email returns the email of the commit author.
+    @li id returns the commit ID of the commit object.
+    @li in_current_branch returns "TRUE" or "FALSE" to indicate if the commit is
+      in the current branch.
+    @li message returns the commit message.
+    @li parent_ids returns a list of the parent commit IDs.
+    @li stash returns "TRUE" or "FALSE" to indicate if the commit is a stash
+      commit.
+    @li time returns the time of the commit as numeric string
+    @li commit_time_num time of the commit as numeric SAS datetime
+    @li commit_time_str the commit_time_num variable cast as string
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] nobs= (0) Set to an integer greater than 0 to restrict the number
+  of rows returned
+
+  <h4> SAS Macros </h4>
+  @li mf_getgitbranch.sas
+
+  <h4> Related Files </h4>
+  @li mp_gitadd.sas
+  @li mp_gitreleaseinfo.sas
+  @li mp_gitstatus.sas
+
+**/
+
+%macro mp_gitlog(gitdir,outds=work.mp_gitlog,mdebug=0,filter=BRANCHONLY,nobs=0);
+
+%local varlist i var;
+%let varlist=author children_ids committer committer_email email id
+  in_current_branch parent_ids stash time ;
+
+data &outds;
+  LENGTH gitdir branch $ 1024 message $4000 &varlist $128 commit_time_num 8.
+    commit_time_str $32;
+  call missing (of _all_);
+  branch="%mf_getgitbranch(&gitdir)";
+  gitdir=symget('gitdir');
+  rc=git_status_free(trim(gitdir));
+  if rc=-1 then do;
+    put "The libgit2 library is unavailable and no Git operations can be used.";
+    put "See: https://stackoverflow.com/questions/74082874";
+    stop;
+  end;
+  else if rc=-2 then do;
+    put "The libgit2 library is available, but the status function failed.";
+    put "See the log for details.";
+    stop;
+  end;
+  entries=git_commit_log(trim(gitdir));
+  do n=1 to entries;
+
+  %do i=1 %to %sysfunc(countw(&varlist message));
+    %let var=%scan(&varlist message,&i,%str( ));
+    rc=git_commit_get(n,trim(gitdir),"&var",&var);
+  %end;
+    /* convert unix time to SAS time - https://4gl.uk/corelink0 */
+    /* Number of seconds between 01JAN1960 and 01JAN1970: 315619200 */
+    format commit_time_num datetime19.;
+    commit_time_num=sum(input(cats(time),best.),315619200);
+    commit_time_str=put(commit_time_num,datetime19.);
+  %if &mdebug=1 %then %do;
+    putlog (_all_)(=);
+  %end;
+    if "&filter"="BRANCHONLY" then do;
+      if cats(in_current_branch)='TRUE' then output;
+    end;
+    else output;
+  %if &nobs>0 %then %do;
+    if n ge &nobs then stop;
+  %end;
+  end;
+  rc=git_commit_free(trim(gitdir));
+  keep gitdir branch &varlist message time commit_time_num commit_time_str;
+run;
+
+%mend mp_gitlog;

base/mp_gitreleaseinfo.sas

@@ -0,0 +1,74 @@
+/**
+  @file
+  @brief Pulls latest release info from a GIT repository
+  @details Useful for grabbing the latest version number or other attributes
+  from a GIT server.  Supported providers are GitLab and GitHub. Pull requests
+  are welcome if you'd like to see additional providers!
+
+  Note that each provider provides slightly different JSON output.  Therefore
+  the macro simply extracts the JSON and assigns the libname (using the JSON
+  engine).
+
+  Example usage (eg, to grab latest release version from github):
+
+      %mp_gitreleaseinfo(GITHUB,sasjs/core,outlib=mylibref)
+
+      data _null_;
+        set mylibref.root;
+        putlog TAG_NAME=;
+      run;
+
+  @param [in] provider The GIT provider for the release info.  Accepted values:
+    @li GITLAB
+    @li GITHUB - Tables include root, assets, author, alldata
+  @param [in] project The link to the repository.  This has different formats
+    depending on the vendor:
+    @li GITHUB - org/repo, eg sasjs/core
+    @li GITLAB - project, eg 1343223
+  @param [in] server= (0) If your repo is self-hosted, then provide the domain
+    here.  Otherwise it will default to the provider domain (eg gitlab.com).
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outlib= (GITREL) The JSON-engine libref to be created, which will
+    point at the returned JSON
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+
+  <h4> Related Files </h4>
+  @li mp_gitreleaseinfo.test.sas
+
+**/
+
+%macro mp_gitreleaseinfo(provider,project,server=0,outlib=GITREL,mdebug=0);
+%local url fref;
+
+%let provider=%upcase(&provider);
+
+%if &provider=GITHUB %then %do;
+  %if "&server"="0" %then %let server=https://api.github.com;
+  %let url=&server/repos/&project/releases/latest;
+%end;
+%else %if &provider=GITLAB %then %do;
+  %if "&server"="0" %then %let server=https://gitlab.com;
+  %let url=&server/api/v4/projects/&project/releases;
+%end;
+
+%let fref=%mf_getuniquefileref();
+
+proc http method='GET' out=&fref url="&url";
+%if &mdebug=1 %then %do;
+  debug level = 3;
+%end;
+run;
+
+libname &outlib JSON fileref=&fref;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fref;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_gitreleaseinfo;

base/mp_gitstatus.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Creates a dataset with the output from `GIT_STATUS()`
+  @details Uses `git_status()` to fetch the number of changed files, then
+  iterates with `git_status_get()`, inserting all attributes into an output
+  dataset.
+
+  Usage:
+
+      %let dir=%sysfunc(pathname(work))/core;
+      %let repo=https://github.com/sasjs/core;
+      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir));
+      %mf_writefile(&dir/somefile.txt,l1=some content)
+      %mf_deletefile(&dir/package.json)
+
+      %mp_gitstatus(&dir,outds=work.gitstatus)
+
+  More info on these functions is in this [helpful paper]
+(https://www.sas.com/content/dam/SAS/support/en/sas-global-forum-proceedings/2019/3057-2019.pdf)
+  by Danny Zimmerman.
+
+  @param [in] gitdir The directory containing the GIT repository
+  @param [out] outds= (work.git_status) The output dataset to create.  Vars:
+    @li gitdir $1024 - directory of repo
+    @li path $1024 - relative path to the file in the repo
+    @li staged $32 - whether the file is staged (TRUE or FALSE)
+    @li status $64 - either new, deleted, or modified
+    @li cnt - number of files
+    @li n - the "nth" file in the list from git_status()
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> Related Files </h4>
+  @li mp_gitstatus.test.sas
+  @li mp_gitadd.sas
+
+**/
+
+%macro mp_gitstatus(gitdir,outds=work.mp_gitstatus,mdebug=0);
+
+data &outds;
+  LENGTH gitdir path $ 1024 STATUS $ 64 STAGED $ 32;
+  call missing (of _all_);
+  gitdir=symget('gitdir');
+  cnt=git_status(trim(gitdir));
+  if cnt=-1 then do;
+    put "The libgit2 library is unavailable and no Git operations can be used.";
+    put "See: https://stackoverflow.com/questions/74082874";
+  end;
+  else if cnt=-2 then do;
+    put "The libgit2 library is available, but the status function failed.";
+    put "See the log for details.";
+  end;
+  else do n=1 to cnt;
+    rc=GIT_STATUS_GET(n,gitdir,'PATH',path);
+    rc=GIT_STATUS_GET(n,gitdir,'STAGED',staged);
+    rc=GIT_STATUS_GET(n,gitdir,'STATUS',status);
+    output;
+  %if &mdebug=1 %then %do;
+    putlog (_all_)(=);
+  %end;
+  end;
+  rc=git_status_free(trim(gitdir));
+  drop rc cnt;
+run;
+
+%mend mp_gitstatus;

base/mp_gsubfile.sas

@@ -22,15 +22,15 @@
         list;
       run;
 
-  @param file= (0) The file to perform the substitution on
-  @param patternvar= A macro variable containing the Lua
+  @param [in] file= (0) The file to perform the substitution on
+  @param [in] patternvar= A macro variable containing the Lua
     [pattern](https://www.lua.org/pil/20.2.html) to search for.  Due to the use
     of special (magic) characters in Lua patterns, it is safer to pass the NAME
     of the macro variable containing the string, rather than the value itself.
-  @param replacevar= The name of the macro variable containing the replacement
-    _string_.
-  @param outfile= (0) The file to write the output to. If zero, then the file
-    is overwritten in-place.
+  @param [in] replacevar= ()
+    The name of the macro variable containing the replacement _string_.
+  @param [out] outfile= (0) The file to write the output to.
+    If zero, then the file is overwritten in-place.
 
   <h4> SAS Macros </h4>
   @li ml_gsubfile.sas

base/mp_guesspk.sas

@@ -18,7 +18,7 @@
       %mp_guesspk(sashelp.class,outds=classpks)
 
   @param [in] baseds The dataset to analyse
-  @param [out] outds= The output dataset to contain the possible PKs
+  @param [out] outds= (mp_guesspk) Output dataset to contain the possible PKs
   @param [in] max_guesses= (3) The total number of possible primary keys to
     generate. A table may have multiple (unlikely) PKs, so no need to list them
     all.

base/mp_hashdataset.sas

@@ -11,7 +11,7 @@
         put hashkey=;
       run;
 
-  ![sas md5 hash dataset log results](https://i.imgur.com/MqF98vk.png)
+  ![sas md5 hash dataset log results](https://i.4gl.io/1/KorUKoyE05.png/raw)
 
   <h4> SAS Macros </h4>
   @li mf_getattrn.sas
@@ -21,11 +21,12 @@
 
   <h4> Related Files </h4>
   @li mp_hashdataset.test.sas
+  @li mp_hashdirectory.sas
 
   @param [in] libds dataset to hash
-  @param [in] salt= Provide a salt (could be, for instance, the dataset name)
-  @param [in] iftrue= A condition under which the macro should be executed.
-  @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
+  @param [in] salt= () Provide a salt (could be, for instance, the dataset name)
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [out] outds= (work._data_) The output dataset to create. This
   will contain one column (hashkey) with one observation (a $hex32.
   representation of the input hash)
   |hashkey:$32.|

base/mp_hashdirectory.sas

@@ -0,0 +1,164 @@
+/**
+  @file
+  @brief Returns a unique hash for each file in a directory
+  @details Hashes each file in each directory, and then hashes the hashes to
+  create a hash for each directory also.
+
+  This makes use of the new `hashing_file()` and `hashing` functions, available
+  since 9.4m6. Interestingly, those functions can be used in pure macro, eg:
+
+      %put %sysfunc(hashing_file(md5,/path/to/file.blob,0));
+
+  Actual usage:
+
+      %let fpath=/some/directory;
+
+      %mp_hashdirectory(&fpath,outds=myhash,maxdepth=2)
+
+      data _null_;
+        set work.myhash;
+        put (_all_)(=);
+      run;
+
+  Whilst files are hashed in their entirety, the logic for creating a folder
+  hash is as follows:
+
+  @li Sort the files by filename (case sensitive, uppercase then lower)
+  @li Take the first 100 hashes, concatenate and hash
+  @li Concatenate this hash with another 100 hashes and hash again
+  @li Continue until the end of the folder.  This is the folder hash
+  @li If a folder contains other folders, start from the bottom of the tree -
+    the folder hashes cascade upwards so you know immediately if there is a
+    change in a sub/sub directory
+  @li If a subfolder has no content (empty) then it is ignored. No hash created.
+  @li If the file is empty, it is also ignored / no hash created.
+  @li If the target directory (&inloc) is empty, &outds will also be empty
+
+  <h4> SAS Macros </h4>
+  @li mp_dirlist.sas
+
+  <h4> Related Files </h4>
+  @li mp_hashdataset.sas
+  @li mp_hashdirectory.test.sas
+  @li mp_md5.sas
+
+  @param [in] inloc Full filepath of the file to be hashed (unquoted)
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
+    subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
+  @param [in] method= (MD5) the hashing method to use.  Available options:
+    @li MD5
+    @li SH1
+    @li SHA256
+    @li SHA384
+    @li SHA512
+    @li CRC32
+  @param [out] outds= (work.mp_hashdirectory) The output dataset.  Contains:
+    @li directory - the parent folder
+    @li file_hash - the hash output
+    @li hash_duration - how long the hash took (first hash always takes longer)
+    @li file_path - /full/path/to/each/file.ext
+    @li file_or_folder - contains either "file" or "folder"
+    @li level - the depth of the directory (top level is 0)
+
+  @version 9.4m6
+  @author Allan Bowe
+**/
+
+%macro mp_hashdirectory(inloc,
+  outds=work.mp_hashdirectory,
+  method=MD5,
+  maxdepth=0,
+  iftrue=%str(1=1)
+)/*/STORE SOURCE*/;
+
+%local curlevel tempds maxlevel;
+
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+/* get the directory listing */
+%mp_dirlist(path=&inloc, outds=&outds, maxdepth=&maxdepth, showparent=YES)
+
+/* create the hashes */
+data &outds;
+  set &outds (rename=(filepath=file_path));
+  length FILE_HASH $32 HASH_DURATION 8;
+  keep directory file_hash hash_duration file_path file_or_folder level;
+
+  ts=datetime();
+  if file_or_folder='file' then do;
+    /* if file is empty, hashing_file will break - so ignore / delete */
+    length fname val $8;
+    drop fname val fid is_empty;
+    rc=filename(fname,file_path);
+    fid=fopen(fname);
+    if fid > 0 then do;
+      rc=fread(fid);
+      is_empty=fget(fid,val);
+    end;
+    rc=fclose(fid);
+    rc=filename(fname);
+    if is_empty ne 0 then delete;
+    else file_hash=hashing_file("&method",cats(file_path),0);
+  end;
+  hash_duration=datetime()-ts;
+run;
+
+proc sort data=&outds ;
+  by descending level directory file_path;
+run;
+
+%let maxlevel=0;
+data _null_;
+  set &outds;
+  call symputx('maxlevel',level,'l');
+  stop;
+run;
+
+/* now hash the hashes to populate folder hashes, starting from the bottom */
+%do curlevel=&maxlevel %to 0 %by -1;
+  data work._data_ (keep=directory file_hash);
+    set &outds;
+    where level=&curlevel;
+    by descending level directory file_path;
+    length str $32767 tmp_hash $32;
+    retain str tmp_hash ;
+    /* reset vars when starting a new directory */
+    if first.directory then do;
+      str='';
+      tmp_hash='';
+      i=0;
+    end;
+    /* hash each chunk of 100 file paths */
+    i+1;
+    str=cats(str,file_hash);
+    if mod(i,100)=0 or last.directory then do;
+      tmp_hash=hashing("&method",cats(tmp_hash,str));
+      str='';
+    end;
+    /* output the hash at directory level */
+    if last.directory then do;
+      file_hash=tmp_hash;
+      output;
+    end;
+    if last.level then stop;
+  run;
+  %let tempds=&syslast;
+  /* join the hash back into the main table */
+  proc sql undo_policy=none;
+  create table &outds as
+    select a.directory
+      ,coalesce(b.file_hash,a.file_hash) as file_hash
+      ,a.hash_duration
+      ,a.file_path
+      ,a.file_or_folder
+      ,a.level
+    from &outds a
+    left join &tempds b
+    on a.file_path=b.directory
+    order by level desc, directory, file_path;
+  drop table &tempds;
+%end;
+
+%mend mp_hashdirectory;

base/mp_init.sas

@@ -70,6 +70,7 @@ options
 %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
   noautocorrect           /* disallow misspelled procedure names            */
   dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
+                          /* turn off with dsoptions=nonote2err             */
 %end;
 ;
 

base/mp_jsonout.sas

@@ -62,7 +62,7 @@
   @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
     that should be converted to JSON
 
-  <h4> Related Macros <h4>
+  <h4> Related Files </h4>
   @li mp_ds2fmtds.sas
 
   @version 9.2
@@ -70,14 +70,14 @@
   @source https://github.com/sasjs/core
 
 **/
-
 %macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y
   ,engine=DATASTEP
   ,missing=NULL
   ,showmeta=N
   ,maxobs=MAX
 )/*/STORE SOURCE*/;
-%local tempds colinfo fmtds i numcols stmt_obs;
+%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval
+  tmpds1 tmpds2 tmpds3 tmpds4;
 %let numcols=0;
 %if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;);
 
@@ -115,7 +115,7 @@
     by varnum;
   run;
   /* move meta to mac vars */
-  data _null_;
+  data &colinfo;
     if _n_=1 then call symputx('numcols',nobs,'l');
     set &colinfo end=last nobs=nobs;
     name=upcase(name);
@@ -126,7 +126,6 @@
       if format='' then fmt=cats('$',length,'.');
       else if formatl=0 then fmt=cats(format,'.');
       else fmt=cats(format,formatl,'.');
-      newlen=max(formatl,length);
     end;
     else do;
       typelong='num';
@@ -134,23 +133,26 @@
       else if formatl=0 then fmt=cats(format,'.');
       else if formatd=0 then fmt=cats(format,formatl,'.');
       else fmt=cats(format,formatl,'.',formatd);
-      /* needs to be wide, for datetimes etc */
-      newlen=max(length,formatl,24);
     end;
     /* 32 char unique name */
     newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
 
     call symputx(cats('name',_n_),name,'l');
     call symputx(cats('newname',_n_),newname,'l');
-    call symputx(cats('len',_n_),newlen,'l');
     call symputx(cats('length',_n_),length,'l');
     call symputx(cats('fmt',_n_),fmt,'l');
     call symputx(cats('type',_n_),type,'l');
     call symputx(cats('typelong',_n_),typelong,'l');
     call symputx(cats('label',_n_),coalescec(label,name),'l');
+    /* overwritten when fmt=Y and a custom format exists in catalog */
+    if typelong='num' then call symputx(cats('fmtlen',_n_),200,'l');
+    else call symputx(cats('fmtlen',_n_),min(32767,ceil((length+10)*1.5)),'l');
   run;
 
   %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+  proc sql;
+  select count(*) into: lastobs from &ds;
+  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs));
 
   %if &engine=PROCJSON %then %do;
     %if &missing=STRING %then %do;
@@ -187,26 +189,99 @@
     %end;
 
     %if &fmt=Y %then %do;
-      data _data_;
+      /**
+        * Extract format definitions
+        * First, by getting library locations from dictionary.formats
+        * Then, by exporting the width using proc format
+        * Cannot use maxw from sashelp.vformat as not always populated
+        * Cannot use fmtinfo() as not supported in all flavours
+        */
+      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      proc sql noprint;
+      create table &tmpds1 as
+          select cats(libname,'.',memname) as FMTCAT,
+          FMTNAME
+        from dictionary.formats
+        where fmttype='F' and libname is not null
+          and fmtname in (select format from &colinfo where format is not null)
+        order by 1;
+      create table &tmpds2(
+          FMTNAME char(32),
+          LENGTH num
+      );
+      %local catlist cat fmtlist i;
+      select distinct fmtcat into: catlist separated by ' ' from &tmpds1;
+      %do i=1 %to %sysfunc(countw(&catlist,%str( )));
+        %let cat=%scan(&catlist,&i,%str( ));
+        proc sql;
+        select distinct fmtname into: fmtlist separated by ' '
+          from &tmpds1 where fmtcat="&cat";
+        proc format lib=&cat cntlout=&tmpds3(keep=fmtname length);
+          select &fmtlist;
+        run;
+        proc sql;
+        insert into &tmpds2 select distinct fmtname,length from &tmpds3;
+      %end;
+
+      proc sql;
+      create table &tmpds4 as
+        select a.*, b.length as MAXW
+        from &colinfo a
+        left join &tmpds2 b
+        on cats(a.format)=cats(upcase(b.fmtname))
+        order by a.varnum;
+      data _null_;
+        set &tmpds4;
+        if not missing(maxw);
+        call symputx(
+          cats('fmtlen',_n_),
+          /* vars need extra padding due to JSON escaping of special chars */
+          min(32767,ceil((max(length,maxw)+10)*1.5))
+          ,'l'
+        );
+      run;
+
+      /* configure varlenchk - as we are explicitly shortening the variables */
+      %let optval=%sysfunc(getoption(varlenchk));
+      options varlenchk=NOWARN;
+      data _data_(compress=char);
+        /* shorten the new vars */
+        length
+      %do i=1 %to &numcols;
+          &&name&i $&&fmtlen&i
+      %end;
+          ;
         /* rename on entry */
         set &ds(rename=(
       %do i=1 %to &numcols;
-        &&name&i=&&newname&i
+          &&name&i=&&newname&i
       %end;
         ));
+      &stmt_obs;
+
+      drop
+      %do i=1 %to &numcols;
+        &&newname&i
+      %end;
+        ;
       %do i=1 %to &numcols;
-        length &&name&i $&&len&i;
         %if &&typelong&i=num %then %do;
-          &&name&i=left(put(&&newname&i,&&fmt&i));
+          &&name&i=cats(put(&&newname&i,&&fmt&i));
         %end;
         %else %do;
           &&name&i=put(&&newname&i,&&fmt&i);
         %end;
-        drop &&newname&i;
       %end;
-        if _error_ then call symputx('syscc',1012);
+        if _error_ then do;
+          call symputx('syscc',1012);
+          stop;
+        end;
       run;
       %let fmtds=&syslast;
+      options varlenchk=&optval;
     %end;
 
     proc format; /* credit yabwon for special null removal */
@@ -225,8 +300,8 @@
       attrib _all_ label='';
       %do i=1 %to &numcols;
         %if &&typelong&i=char or &fmt=Y %then %do;
-          length &&name&i $32767;
-          format &&name&i $32767.;
+          length &&name&i $&&fmtlen&i...;
+          format &&name&i $&&fmtlen&i...;
         %end;
       %end;
       %if &fmt=Y %then %do;
@@ -239,7 +314,7 @@
       format _numeric_ bart.;
     %do i=1 %to &numcols;
       %if &&typelong&i=char or &fmt=Y %then %do;
-        if findc(&&name&i,'"\'!!'0A0D09000E0F01021011'x) then do;
+        if findc(&&name&i,'"\'!!'0A0D09000E0F010210111A'x) then do;
           &&name&i='"'!!trim(
             prxchange('s/"/\\"/',-1,        /* double quote */
             prxchange('s/\x0A/\n/',-1,      /* new line */
@@ -252,8 +327,9 @@
             prxchange('s/\x02/\\u0002/',-1, /* STX */
             prxchange('s/\x10/\\u0010/',-1, /* DLE */
             prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+            prxchange('s/\x1A/\\u001A/',-1, /* SUB */
             prxchange('s/\\/\\\\/',-1,&&name&i)
-          ))))))))))))!!'"';
+          )))))))))))))!!'"';
         end;
         else &&name&i=quote(cats(&&name&i));
       %end;

base/mp_lib2cards.sas

@@ -24,19 +24,19 @@
   @li mf_trimstr.sas
   @li mp_ds2cards.sas
 
-  @param [in] lib= Library in which to convert all datasets
-  @param [out] outloc= Location in which to store output.  Defaults to WORK
-    library. No quotes.
-  @param [out] outfile= Optional output file NAME - if provided, then will create
-  a single output file instead of one file per input table.
-  @param [in] maxobs= limit output to the first <code>maxobs</code> observations
+  @param [in] lib= () Library in which to convert all datasets
+  @param [out] outloc= (%sysfunc(pathname(work))) Location in which to store
+    output. No quotes.
+  @param [out] outfile= (0) Optional output file NAME - if provided, then
+    will create a single output file instead of one file per input table.
+  @param [in] maxobs= (max) limit output to the first <code>maxobs</code> rows
 
   @version 9.2
   @author Allan Bowe
 **/
 
 %macro mp_lib2cards(lib=
-    ,outloc=%sysfunc(pathname(work)) /* without trailing slash */
+    ,outloc=%sysfunc(pathname(work))
     ,maxobs=max
     ,random_sample=NO
     ,outfile=0

base/mp_loadformat.sas

@@ -9,6 +9,9 @@
   format, to prevent loss of data - UNLESS the input dataset contains a marker
   column, specifying that a particular row needs to be deleted (`delete_col=`).
 
+  Positions of formats are made using the FMTROW variable - this must be present
+  and unique (on TYPE / FMTNAME / FMTROW).
+
   This macro can also be used to identify which records would be (or were)
   considered new, modified or deleted (`loadtarget=`) by creating the following
   tables:
@@ -17,7 +20,7 @@
   @li work.outds_del
   @li work.outds_mod
 
-  For example usage, see mp_loadformat.test.sas
+  For example usage, see test (under Related Macros)
 
   @param [in] libcat The format catalog to be loaded
   @param [in] libds The staging table to load
@@ -34,18 +37,22 @@
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
-  @li mddl_sas_cntlout.sas
+  @li mf_existds.sas
+  @li mf_existvar.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
+  @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
+  @li mp_md5.sas
   @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
   @li mddl_dc_difftable.sas
   @li mddl_dc_locktable.sas
-  @li mp_loadformat.test.sas
+  @li mp_loadformat.test.1.sas
+  @li mp_loadformat.test.2.sas
   @li mp_lockanytable.sas
   @li mp_stackdiffs.sas
 
@@ -67,7 +74,7 @@
 );
 /* set up local macro variables and temporary tables (with a prefix) */
 %local err msg prefix dslist i var fmtlist ibufsize;
-%let dslist=base_fmts template inlibds ds1 stagedata storediffs;
+%let dslist=base_fmts template inlibds ds1 stagedata storediffs del1 del2;
 %if &outds_add=0 %then %let dslist=&dslist outds_add;
 %if &outds_del=0 %then %let dslist=&dslist outds_del;
 %if &outds_mod=0 %then %let dslist=&dslist outds_mod;
@@ -78,17 +85,20 @@
   %let &var=%upcase(&prefix._&var);
 %end;
 
-/*
-format values can be up to 32767 wide.  SQL joins on such a wide column can
-cause buffer issues.  Update ibufsize and reset at the end.
-*/
-%let ibufsize=%sysfunc(getoption(ibufsize));
-options ibufsize=32767 ;
-
 /* in DC, format catalogs maybe specified in the libds with a -FC extension */
 %let libcat=%scan(&libcat,1,-);
 
 /* perform input validations */
+%mp_abort(
+  iftrue=(%mf_existds(&libds)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(&libds could not be found)
+)
+%mp_abort(
+  iftrue=(%mf_existvar(&libds,FMTROW)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(FMTROW not found in &libds)
+)
 %let err=0;
 %let msg=0;
 data _null_;
@@ -109,13 +119,6 @@ data _null_;
       stop;
     end;
   end;
-  else if name='LIBDS' then do;
-    if exist(value) le 0 then do;
-      call symputx('msg',"Unable to open staging table: "!!value);
-      call symputx('err',1);
-      stop;
-    end;
-  end;
   else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
   and missing(value) then do;
     call symputx('msg',"missing value in var: "!!name);
@@ -123,6 +126,14 @@ data _null_;
     stop;
   end;
 run;
+data _null_;
+  set &libds;
+  if missing(fmtrow) then do;
+    call symputx('msg',"missing fmtrow in format: "!!FMTNAME);
+    call symputx('err',1);
+    stop;
+  end;
+run;
 
 %mp_abort(
   iftrue=(&err ne 0)
@@ -130,33 +141,69 @@ run;
   ,msg=%str(&msg)
 )
 
+%local cnt;
+proc sql noprint;
+select count(distinct catx('|',type,fmtname,fmtrow)) into: cnt from &libds;
+%mp_abort(
+  iftrue=(&cnt ne %mf_nobs(&libds))
+  ,mac=&sysmacroname
+  ,msg=%str(Non-unique primary key on &libds)
+)
+
 /**
   * First, extract only relevant formats from the catalog
   */
 proc sql noprint;
-select distinct upcase(fmtname) into: fmtlist separated by ' ' from &libds;
+select distinct
+  case
+    when type='N' then upcase(fmtname)
+    when type='C' then cats('$',upcase(fmtname))
+    when type='I' then cats('@',upcase(fmtname))
+    when type='J' then cats('@$',upcase(fmtname))
+    else "&sysmacroname:UNHANDLED"
+  end
+  into: fmtlist separated by ' '
+  from &libds;
 
 %mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
 
+/* get a hash of the row */
+%local cvars nvars;
+%let cvars=TYPE FMTNAME START END LABEL PREFIX FILL SEXCL EEXCL HLO DECSEP
+  DIG3SEP DATATYPE LANGUAGE;
+%let nvars=FMTROW MIN MAX DEFAULT LENGTH FUZZ MULT NOEDIT;
+data &base_fmts/note2err;
+  set &base_fmts;
+  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
+run;
 
 /**
   * Ensure input table and base_formats have consistent lengths and types
   */
-%mddl_sas_cntlout(libds=&template)
-data &inlibds;
-  length &delete_col $3;
-  if 0 then set &template;
+data &inlibds/nonote2err;
+  length &delete_col $3 FMTROW 8 start end label $32767;
+  if 0 then set &base_fmts;
   set &libds;
+  by type fmtname notsorted;
   if &delete_col='' then &delete_col='No';
   fmtname=upcase(fmtname);
+  type=upcase(type);
   if missing(type) then do;
-    if substr(fmtname,1,1)='$' then type='C';
-    else type='N';
+    if substr(fmtname,1,1)='@' then do;
+      if substr(fmtname,2,1)='$' then type='J';
+      else type='I';
+    end;
+    else do;
+      if substr(fmtname,1,1)='$' then type='C';
+      else type='N';
+    end;
   end;
-  if type='N' then do;
-    start=cats(start);
-    end=cats(end);
+  if type in ('N','I') then do;
+    %mp_aligndecimal(start,width=16)
+    %mp_aligndecimal(end,width=16)
   end;
+
+  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
 run;
 
 /**
@@ -167,23 +214,10 @@ create table &outds_add(drop=&delete_col) as
   select a.*
   from &inlibds a
   left join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where b.fmtname is null
     and upcase(a.&delete_col) ne "YES"
-  order by fmtname, start;;
-
-/**
-  * Identify deleted records
-  */
-create table &outds_del(drop=&delete_col) as
-  select a.*
-  from &inlibds a
-  inner join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
-  where upcase(a.&delete_col)="YES"
-  order by fmtname, start;
+  order by type, fmtname, fmtrow;
 
 /**
   * Identify modified records
@@ -192,12 +226,40 @@ create table &outds_mod (drop=&delete_col) as
   select a.*
   from &inlibds a
   inner join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where upcase(a.&delete_col) ne "YES"
-  order by fmtname, start;
+    and a.fmthash ne b.fmthash
+  order by type, fmtname, fmtrow;
+
+/**
+  * Identify deleted records
+  */
+create table &outds_del(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
+  where upcase(a.&delete_col)="YES"
+  order by type, fmtname, fmtrow;
+
+/**
+  * Identify fully deleted formats (where every record is removed)
+  * These require to be explicitly deleted in proc format
+  * del1 - identify _partial_ deletes
+  * del2 - exclude these, and also formats that come with _additions_
+  */
+create table &del1 as
+  select a.*
+  from &base_fmts a
+  left join &outds_del b
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
+  where b.fmtrow is null;
+
+create table &del2 as
+  select * from &outds_del
+  where cats(type,fmtname) not in (select cats(type,fmtname) from &outds_add)
+    and cats(type,fmtname) not in (select cats(type,fmtname) from &del1);
 
-options ibufsize=&ibufsize;
 
 %mp_abort(
   iftrue=(&syscc ne 0)
@@ -206,19 +268,21 @@ options ibufsize=&ibufsize;
 )
 
 %if &loadtarget=YES %then %do;
+  /* new records plus base records that are not deleted or modified */
   data &ds1;
     merge &base_fmts(in=base)
       &outds_mod(in=mod)
       &outds_add(in=add)
       &outds_del(in=del);
     if not del and not mod;
-    by fmtname start;
+    by type fmtname fmtrow;
   run;
+  /* add back the modified records */
   data &stagedata;
     set &ds1 &outds_mod;
   run;
   proc sort;
-    by fmtname start;
+    by type fmtname fmtrow;
   run;
 %end;
 /* mp abort needs to run outside of conditional blocks */
@@ -228,7 +292,7 @@ options ibufsize=&ibufsize;
   ,msg=%str(SYSCC=&syscc prior to actual load)
 )
 %if &loadtarget=YES %then %do;
-  %if %mf_nobs(&stagedata)=0 %then %do;
+  %if %mf_nobs(&stagedata)=0 and %mf_nobs(&del2)=0 %then %do;
     %put There are no changes to load in &libcat!;
     %return;
   %end;
@@ -244,6 +308,22 @@ options ibufsize=&ibufsize;
   /* do the actual load */
   proc format lib=&libcat cntlin=&stagedata;
   run;
+  /* apply any full deletes */
+  %if %mf_nobs(&del2)>0 %then %do;
+    %local delfmtlist;
+    proc sql noprint;
+    select distinct case when type='N' then cats(fmtname,'.FORMAT')
+        when type='C' then cats(fmtname,'.FORMATC')
+        when type='J' then cats(fmtname,'.INFMTC')
+        when type='I' then cats(fmtname,'.INFMT')
+        else cats(fmtname,'.BADENTRY!!!') end
+      into: delfmtlist
+      separated by ' '
+      from &del2;
+    proc catalog catalog=&libcat;
+      delete &delfmtlist;
+    quit;
+  %end;
   %if &locklibds ne 0 %then %do;
     /* unlock the table */
     %mp_lockanytable(UNLOCK
@@ -266,7 +346,7 @@ options ibufsize=&ibufsize;
 
     %mp_storediffs(&libcat-FC
       ,&base_fmts
-      ,FMTNAME START
+      ,TYPE FMTNAME FMTROW
       ,delds=&outds_del
       ,modds=&outds_mod
       ,appds=&outds_add