Merge pull request #342 from sasjs/issue341

Full Format Deletion

.all-contributorsrc

@@ -135,6 +135,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "henrik-forsell",
+      "name": "Henrik Forsell",
+      "avatar_url": "https://avatars.githubusercontent.com/u/109935936?v=4",
+      "profile": "https://github.com/henrik-forsell",
+      "contributions": [
+        "doc"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.git-hooks/pre-commit

@@ -2,7 +2,7 @@
 
 # Ensure lint is passing
 LINT=`sasjs lint`
-if [[ "$LINT" != "✔ All matched files use @sasjs/lint code style!" ]]; then
+if [[ "$LINT" != *"✔ All matched files use @sasjs/lint code style!" ]]; then
   echo "$LINT"
   echo "To commit in spite of these warnings, use the -n parameter."
   exit 1

.github/FUNDING.yml

@@ -1,3 +1,3 @@
 # These are supported funding model platforms
 
-github: [sasjs]
+custom: https://getalby.com/p/sasjs

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,3 +15,4 @@ What code changes have been made to achieve the intent.
 - [ ] Code is formatted correctly (`sasjs lint`).
 - [ ] Any new functionality has been unit tested.
 - [ ] All unit tests are passing (`sasjs test`).
+- [ ] `all.sas` has been regenerated (`python3 build.py`)

.github/vpn/config.ovpn

@@ -1,30 +1,25 @@
-cipher AES-256-CBC
-setenv FORWARD_COMPATIBLE 1
+# Client
 client
-server-poll-timeout 4
-nobind
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 443 tcp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
+tls-client
 dev tun
-dev-type tun
-ns-cert-type server
-setenv opt tls-version-min 1.0 or-highest
-reneg-sec 604800
-sndbuf 0
-rcvbuf 0
-# NOTE: LZO commands are pushed by the Access Server at connect time.
-# NOTE: The below line doesn't disable LZO.
-comp-lzo no
-verb 3
-setenv PUSH_PEER_INFO
+# this will connect with whatever proto DNS tells us (https://community.openvpn.net/openvpn/ticket/934)
+proto tcp
+remote vpn.4gl.io 7494
+resolv-retry infinite
+cipher AES-256-CBC
+auth SHA256
+script-security 2
+keepalive 10 120
+remote-cert-tls server
 
+# Keys
 ca ca.crt
 cert user.crt
 key user.key
 tls-auth tls.key 1
+
+# Security
+nobind
+persist-key
+persist-tun
+verb 3

.github/workflows/main.yml

@@ -15,7 +15,14 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v2
+        uses: cycjimmy/semantic-release-action@v3
         env:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+      - name: SAS Packages Release
+        run: |
+          npx @sasjs/cli compile job -s sasjs/utils/create_sas_package.sas -o sasjsbuild
+          # this part depends on https://github.com/sasjs/server/issues/307
+          # sasjs run sasjsbuild/jobs/utils/create_sas_package.sas -t sas9
+
+

.github/workflows/run-tests.yml

@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [lts/fermium]
+        node-version: [lts/hydrogen]
 
     steps:
       - uses: actions/checkout@v2
@@ -39,46 +39,39 @@ jobs:
           sudo apt install apt-transport-https
           sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
           sudo apt-key add openvpn-repo-pkg-key.pub
-          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-focal.list
+          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
           sudo apt update
-          sudo apt install openvpn3
-
+          sudo apt install openvpn3=17~betaUb22042+jammy
       - name: Start Open VPN 3
         run: openvpn3 session-start --config .github/vpn/config.ovpn
-
       - name: Install Doxygen
         run: sudo apt-get install doxygen
 
       - name: Install dependencies
         run: npm ci
 
-      - name: Check code style
-        run: npm run lint
+      - name: Check code style (aborts if errors found)
+        run: npx @sasjs/cli lint
 
       - name: Add client
-        run: echo "CLIENT=${{secrets.CLIENT}}"> .env.viya
-
-      - name: Add secret
-        run: echo "SECRET=${{secrets.SECRET}}" >> .env.viya
+        run: echo "CLIENT=${{secrets.SAS9_4GL_IO_CLIENT}}"> .env.server
 
       - name: Add access token
-        run: echo "ACCESS_TOKEN=${{secrets.ACCESS_TOKEN}}" >> .env.viya
+        run: echo "ACCESS_TOKEN=${{secrets.SAS9_4GL_IO_ACCESS_TOKEN}}" >> .env.server
 
       - name: Add refresh token
-        run: echo "REFRESH_TOKEN=${{secrets.REFRESH_TOKEN}}" >> .env.viya
+        run: echo "REFRESH_TOKEN=${{secrets.SAS9_4GL_IO_REFRESH_TOKEN}}" >> .env.server
 
-      - name: Build Project
-        run: npm run build
+      - name: Build & Deploy Project to SAS server
+        run: npx @sasjs/cli cbd -t server
 
-      - name: Run SASjs  tests
-        run: npm run test
+      - name: Run all tests
+        run: npx @sasjs/cli test -t server
         env:
           CI: true
           CLIENT: ${{secrets.CLIENT}}
           SECRET: ${{secrets.SECRET}}
           SAS_USERNAME: ${{secrets.SAS_USERNAME}}
           SAS_PASSWORD: ${{secrets.SAS_PASSWORD}}
-          SERVER_URL: ${{secrets.SERVER_URL}}
-          SERVER_TYPE: ${{secrets.SERVER_TYPE}}
           ACCESS_TOKEN: ${{secrets.ACCESS_TOKEN}}
           REFRESH_TOKEN: ${{secrets.REFRESH_TOKEN}}

.gitignore

@@ -10,4 +10,6 @@ sasjsresults/
 mc_*
 
 # ignore .env files as they can contain sasjs access tokens
-*.env*
+*.env*
+
+~

.sasjslint

@@ -1,13 +1,15 @@
 {
-    "noTrailingSpaces": true,
-    "noEncodedPasswords": true,
-    "hasDoxygenHeader": true,
-    "hasMacroNameInMend": true,
-    "hasMacroParentheses": true,
-    "noNestedMacros": false,
-    "noSpacesInFileNames": true,
-    "maxLineLength": 300,
-    "lowerCaseFileNames": true,
-    "noTabIndentation": true,
-    "indentationMultiple": 2
-}
+  "noTrailingSpaces": true,
+  "noEncodedPasswords": true,
+  "hasDoxygenHeader": true,
+  "hasMacroNameInMend": true,
+  "hasMacroParentheses": true,
+  "lineEndings": "lf",
+  "noGremlins": true,
+  "noNestedMacros": false,
+  "noSpacesInFileNames": true,
+  "maxLineLength": 300,
+  "lowerCaseFileNames": true,
+  "noTabs": true,
+  "indentationMultiple": 2
+}

.vscode/settings.json

@@ -6,5 +6,7 @@
   "editor.rulers": [
     80
   ],
-  "files.trimTrailingWhitespace": true
+  "files.trimTrailingWhitespace": true,
+  "sasjs-for-vscode.target": "docsonly",
+  "sasjs-for-vscode.isLocal": true
 }

README.md

@@ -2,8 +2,6 @@
 [![npm package][npm-image]][npm-url]
 [![Github Workflow][githubworkflow-image]][githubworkflow-url]
 ![npm](https://img.shields.io/npm/dt/@sasjs/core)
-![Snyk Vulnerabilities for npm package](https://img.shields.io/snyk/vulnerabilities/npm/@sasjs/core)
-[![License](https://img.shields.io/apm/l/atomic-design-ui.svg)](/LICENSE)
 ![GitHub top language](https://img.shields.io/github/languages/top/sasjs/core)
 [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sasjs/core)](https://github.com/sasjs/core/issues?q=is%3Aissue+is%3Aclosed)
 [![GitHub issues](https://img.shields.io/github/issues-raw/sasjs/core)](https://github.com/sasjs/core/issues)
@@ -36,21 +34,21 @@ Documentation: https://core.sasjs.io
 - OS independent
 - Works on all SAS Platforms
 - No X command
-- Prefixes: _mf_, _mp_
+- Prefixes: `mf_`, `mp_`
 
 ### DDL folder (All Platforms)
 
 - OS independent
 - Works on all SAS Platforms
 - No X command
-- Prefixes: _mddl_(lib)_ -> where lib can be "SAS" (in relation to a SAS component) or "DC" (in relation to a Data Controller component)
+- Prefixes: `mddl_(lib)_` -> where lib can be "SAS" (in relation to a SAS component) or "DC" (in relation to a Data Controller component)
 
 This library will not be used for storing data entries (such as formats or datalines).  Where this becomes necessary in the future, a new repo will be created, in order to keep the NPM bundle size down (for the benefit of those looking to embed purely macros in their applications).
 
 ### FCMP folder (All Platforms)
 
 - Function and macro names are identical, except for special cases
-- Prefixes: _mcf_
+- Prefixes: `mcf_`
 
 The fcmp macros are used to generate fcmp functions, and can be used with or without the `proc fcmp` wrapper.
 
@@ -72,7 +70,7 @@ endsubmit;
 run;
 ```
 
-- Prefixes: _ml_
+- Prefixes: `ml_`
 
 ### META folder (SAS9 only)
 
@@ -81,14 +79,14 @@ Macros used in SAS EBI, which connect to the metadata server.
 - OS independent
 - Metadata aware
 - No X command
-- Prefixes: _mm_
+- Prefixes: `mm_`
 
 ### METAX folder (SAS9 only)
 
 - OS specific
 - Metadata aware
 - X command enabled
-- Prefixes: _mmw_,_mmu_,_mmx_
+- Prefixes: `mmx_`
 
 ### SERVER folder (@sasjs/server only)
 These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
@@ -96,7 +94,7 @@ These macros are used for building applications using [@sasjs/server](https://se
 - OS independent
 - @sasjs/server aware
 - No X command
-- Prefixes: _ms_
+- Prefixes: `ms_`
 
 ### VIYA folder (Viya only)
 
@@ -104,7 +102,7 @@ Macros used for interfacing with SAS Viya.
 
 - OS independent
 - No X command
-- Prefixes: _mv_, _mvf_
+- Prefixes: `mv_`, `mvf_`
 
 ### XPLATFORM folder (Viya, Meta, and Server)
 
@@ -112,7 +110,7 @@ Sometimes it is helpful to use a macro that can be used interchangeably regardle
 
 - OS independent
 - No X command
-- Prefixes: _mx_
+- Prefixes: `mx_`
 
 ## Installation
 
@@ -237,6 +235,8 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 The following repositories are also worth checking out:
 
+* [xieliaing/SAS](https://github.com/xieliaing/SAS)
+* [SASJedi/sas-macros](https://github.com/SASJedi/sas-macros)
 * [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
 * [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)
 * [KatjaGlassConsulting/SMILE-SmartSASMacros](https://github.com/KatjaGlassConsulting/SMILE-SmartSASMacros)
@@ -246,7 +246,7 @@ The following repositories are also worth checking out:
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-12-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-13-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -254,22 +254,25 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
 <!-- prettier-ignore-start -->
 <!-- markdownlint-disable -->
 <table>
-  <tr>
-    <td align="center"><a href="https://github.com/allanbowe"><img src="https://avatars.githubusercontent.com/u/4420615?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Allan Bowe</b></sub></a><br /><a href="#business-allanbowe" title="Business development">💼</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Code">💻</a> <a href="#content-allanbowe" title="Content">🖋</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Documentation">📖</a> <a href="#infra-allanbowe" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#maintenance-allanbowe" title="Maintenance">🚧</a> <a href="#mentoring-allanbowe" title="Mentoring">🧑‍🏫</a> <a href="#question-allanbowe" title="Answering Questions">💬</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3Aallanbowe" title="Reviewed Pull Requests">👀</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Tests">⚠️</a></td>
-    <td align="center"><a href="https://github.com/rafgag"><img src="https://avatars.githubusercontent.com/u/69139928?v=4?s=100" width="100px;" alt=""/><br /><sub><b>rafgag</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rafgag" title="Code">💻</a></td>
-    <td align="center"><a href="https://github.com/tmoody"><img src="https://avatars.githubusercontent.com/u/79837106?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Trevor Moody</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=tmoody" title="Code">💻</a></td>
-    <td align="center"><a href="https://krishna-acondy.io/"><img src="https://avatars.githubusercontent.com/u/2980428?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Krishna Acondy</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=krishna-acondy" title="Code">💻</a> <a href="#infra-krishna-acondy" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#blog-krishna-acondy" title="Blogposts">📝</a> <a href="#content-krishna-acondy" title="Content">🖋</a> <a href="#ideas-krishna-acondy" title="Ideas, Planning, & Feedback">🤔</a> <a href="#video-krishna-acondy" title="Videos">📹</a></td>
-    <td align="center"><a href="https://github.com/saadjutt01"><img src="https://avatars.githubusercontent.com/u/8914650?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Muhammad Saad </b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=saadjutt01" title="Code">💻</a> <a href="#ideas-saadjutt01" title="Ideas, Planning, & Feedback">🤔</a></td>
-    <td align="center"><a href="https://www.erudicat.com/"><img src="https://avatars.githubusercontent.com/u/25773492?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Yury Shkoda</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=YuryShkoda" title="Code">💻</a> <a href="#infra-YuryShkoda" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#video-YuryShkoda" title="Videos">📹</a></td>
-    <td align="center"><a href="https://github.com/medjedovicm"><img src="https://avatars.githubusercontent.com/u/18329105?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Mihajlo Medjedovic</b></sub></a><br /><a href="#infra-medjedovicm" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
-  </tr>
-  <tr>
-    <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
-    <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
-    <td align="center"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
-    <td align="center"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
-    <td align="center"><a href="https://bandism.net/"><img src="https://avatars.githubusercontent.com/u/22633385?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Ikko Ashimine</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=eltociear" title="Code">💻</a></td>
-  </tr>
+  <tbody>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/allanbowe"><img src="https://avatars.githubusercontent.com/u/4420615?v=4?s=100" width="100px;" alt="Allan Bowe"/><br /><sub><b>Allan Bowe</b></sub></a><br /><a href="#business-allanbowe" title="Business development">💼</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Code">💻</a> <a href="#content-allanbowe" title="Content">🖋</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Documentation">📖</a> <a href="#infra-allanbowe" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#maintenance-allanbowe" title="Maintenance">🚧</a> <a href="#mentoring-allanbowe" title="Mentoring">🧑‍🏫</a> <a href="#question-allanbowe" title="Answering Questions">💬</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3Aallanbowe" title="Reviewed Pull Requests">👀</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Tests">⚠️</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/rafgag"><img src="https://avatars.githubusercontent.com/u/69139928?v=4?s=100" width="100px;" alt="rafgag"/><br /><sub><b>rafgag</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rafgag" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/tmoody"><img src="https://avatars.githubusercontent.com/u/79837106?v=4?s=100" width="100px;" alt="Trevor Moody"/><br /><sub><b>Trevor Moody</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=tmoody" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://krishna-acondy.io/"><img src="https://avatars.githubusercontent.com/u/2980428?v=4?s=100" width="100px;" alt="Krishna Acondy"/><br /><sub><b>Krishna Acondy</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=krishna-acondy" title="Code">💻</a> <a href="#infra-krishna-acondy" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#blog-krishna-acondy" title="Blogposts">📝</a> <a href="#content-krishna-acondy" title="Content">🖋</a> <a href="#ideas-krishna-acondy" title="Ideas, Planning, & Feedback">🤔</a> <a href="#video-krishna-acondy" title="Videos">📹</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/saadjutt01"><img src="https://avatars.githubusercontent.com/u/8914650?v=4?s=100" width="100px;" alt="Muhammad Saad "/><br /><sub><b>Muhammad Saad </b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=saadjutt01" title="Code">💻</a> <a href="#ideas-saadjutt01" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://www.erudicat.com/"><img src="https://avatars.githubusercontent.com/u/25773492?v=4?s=100" width="100px;" alt="Yury Shkoda"/><br /><sub><b>Yury Shkoda</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=YuryShkoda" title="Code">💻</a> <a href="#infra-YuryShkoda" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#video-YuryShkoda" title="Videos">📹</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/medjedovicm"><img src="https://avatars.githubusercontent.com/u/18329105?v=4?s=100" width="100px;" alt="Mihajlo Medjedovic"/><br /><sub><b>Mihajlo Medjedovic</b></sub></a><br /><a href="#infra-medjedovicm" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt="kkchandok"/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt="Vladislav Parhomchik"/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt="Vignesh T."/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt="Bart Jablonski"/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://bandism.net/"><img src="https://avatars.githubusercontent.com/u/22633385?v=4?s=100" width="100px;" alt="Ikko Ashimine"/><br /><sub><b>Ikko Ashimine</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=eltociear" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/henrik-forsell"><img src="https://avatars.githubusercontent.com/u/109935936?v=4?s=100" width="100px;" alt="Henrik Forsell"/><br /><sub><b>Henrik Forsell</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=henrik-forsell" title="Documentation">📖</a></td>
+    </tr>
+  </tbody>
 </table>
 
 <!-- markdownlint-restore -->

base/mf_abort.sas

@@ -12,7 +12,7 @@
 **/
 
 %macro mf_abort(mac=mf_abort.sas, msg=, iftrue=%str(1=1)
-)/*/STORE SOURCE*/;
+)/des='ungraceful abort' /*STORE SOURCE*/;
 
   %if not(%eval(%unquote(&iftrue))) %then %return;
 
@@ -24,4 +24,4 @@
 
 %mend mf_abort;
 
-/** @endcond */
+/** @endcond */

base/mf_existfeature.sas

@@ -40,6 +40,11 @@
     %else %if "&SYSVLONG" < "9.04.01M3" %then 0;
     %else 1;
   %end;
+  %else %if &feature=DBMS_MEMTYPE %then %do;
+    /* does dbms_memtype exist in dictionary.tables? */
+    %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 0;
+    %else 1;
+  %end;
   %else %if &feature=EXPORTXLS %then %do;
     /* is it possible to PROC EXPORT an excel file? */
     %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 1;

base/mf_existfileref.sas

@@ -30,4 +30,4 @@
     0
   %end;
 
-%mend mf_existfileref;
+%mend mf_existfileref;

base/mf_existvar.sas

@@ -25,13 +25,17 @@
   %local dsid rc;
   %let dsid=%sysfunc(open(&libds,is));
 
-  %if &dsid=0 or %length(&var)=0 %then %do;
+  %if &dsid=0 %then %do;
     %put %sysfunc(sysmsg());
-      0
+    0
+  %end;
+  %else %if %length(&var)=0 %then %do;
+    0
+    %let rc=%sysfunc(close(&dsid));
   %end;
   %else %do;
-      %sysfunc(varnum(&dsid,&var))
-      %let rc=%sysfunc(close(&dsid));
+    %sysfunc(varnum(&dsid,&var))
+    %let rc=%sysfunc(close(&dsid));
   %end;
 
 %mend mf_existvar;

base/mf_getfmtlist.sas

@@ -10,10 +10,9 @@
 
   returns:
 
-  > DOLLAR $CHAR W MONNAME
-  > $CHAR BEST DOLLAR
-  > BEST Z $CHAR COMMA PERCENTN
-
+      DOLLAR $CHAR W MONNAME
+      $CHAR BEST DOLLAR
+      BEST Z $CHAR COMMA PERCENTN
 
   @param [in] libds Two part library.dataset reference.
 

base/mf_getgitbranch.sas

@@ -0,0 +1,37 @@
+/**
+  @file
+  @brief Retrieves the current branch from a local GIT repo
+  @details In a local git repository, the current branch is always available in
+  the `.git/HEAD` file in a format like this:  `ref: refs/heads/master`
+
+  This macro simply reads the file and returns the last word (eg `master`).
+
+  Example usage:
+
+      %let gitdir=%sysfunc(pathname(work))/core;
+      %let repo=https://github.com/sasjs/core;
+      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&gitdir));
+
+      %put The current branch is %mf_getgitbranch(&gitdir);
+
+  @param [in] gitdir The directory containing the GIT repository
+
+  <h4> SAS Macros </h4>
+  @li mf_readfile.sas
+
+  <h4> Related Macros </h4>
+  @li mp_gitadd.sas
+  @li mp_gitlog.sas
+  @li mp_gitreleaseinfo.sas
+  @li mp_gitstatus.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_getgitbranch(gitdir
+)/*/STORE SOURCE*/;
+
+  %scan(%mf_readfile(&gitdir/.git/HEAD),-1)
+
+%mend mf_getgitbranch;

base/mf_increment.sas

@@ -0,0 +1,29 @@
+/**
+  @file
+  @brief Increments a macro variable
+  @details Useful outside of do-loops - will increment a macro variable every
+  time it is called.
+
+  Example:
+
+      %let cnt=1;
+      %put We have run %mf_increment(cnt) lines;
+      %put Now we have run %mf_increment(cnt) lines;
+      %put There are %mf_increment(cnt) lines in total;
+
+  @param [in] MACRO_NAME the name of the macro variable to increment
+  @param [in] ITER= The amount to add or subtract to the macro
+
+  <h4> Related Files </h4>
+  @li mf_increment.test.sas
+
+**/
+
+%macro mf_increment(macro_name,incr=1);
+
+  /* iterate the value */
+  %let &macro_name=%eval(&&&macro_name+&incr);
+  /* return the value */
+  &&&macro_name
+
+%mend mf_increment;

base/mf_readfile.sas

@@ -0,0 +1,63 @@
+/**
+  @file
+  @brief Reads the first line of a file using pure macro
+  @details Reads the first line of a file and returns it.  Future versions may
+  read each line into a macro variable array.
+
+  Generally, reading data into macro variables is not great as certain
+  nonprintable characters (such as CR, LF) may be dropped in the conversion.
+
+    Usage:
+
+        %mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=more content)
+
+        %put %mf_readfile(&sasjswork/myfile.txt);
+
+
+  @param [in] fpath Full path to file to be read
+
+  <h4> Related Macros </h4>
+  @li mf_deletefile.sas
+  @li mf_writefile.sas
+  @li mf_readfile.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mf_readfile(fpath
+)/*/STORE SOURCE*/;
+%local fref rc fid fcontent;
+
+/* check file exists */
+%if %sysfunc(filename(fref,&fpath)) ne 0 %then %do;
+  %put &=fref &=fpath;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let fid=%sysfunc(fopen(&fref,I));
+
+%if &fid=0 %then %do;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%if %sysfunc(fread(&fid)) = 0 %then %do;
+  %let rc=%sysfunc(fget(&fid,fcontent,65534));
+  &fcontent
+%end;
+
+/*
+%do %while(%sysfunc(fread(&fid)) = 0);
+  %let rc=%sysfunc(fget(&fid,fcontent,65534));
+  &fcontent
+%end;
+*/
+
+%let rc=%sysfunc(fclose(&fid));
+%let rc=%sysfunc(filename(&fref));
+
+%mend mf_readfile;
+/** @endcond */

base/mp_abort.sas

@@ -225,15 +225,51 @@ and %superq(SYSPROCESSNAME) ne %str(Compute Server)
     _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
-    syserrortext=quote(trim(symget('syserrortext')));
-    put ",""SYSERRORTEXT"" : " syserrortext;
+    syserrortext=cats(symget('syserrortext'));
+    if findc(syserrortext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syserrortext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syserrortext)
+      )))))))))))))!!'"';
+    end;
+    else syserrortext=cats('"',syserrortext,'"');
+    put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    syswarningtext=quote(trim(symget('syswarningtext')));
+    syswarningtext=cats(symget('syswarningtext'));
+    if findc(syswarningtext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syswarningtext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syswarningtext)
+      )))))))))))))!!'"';
+    end;
+    else syswarningtext=cats('"',syswarningtext,'"');
     put ",""SYSWARNINGTEXT"" : " syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     put "}" ;

base/mp_aligndecimal.sas

@@ -0,0 +1,95 @@
+/**
+  @file
+  @brief Apply leading blanks to align numbers vertically in a char variable
+  @details This is particularly useful when storing numbers (as character) that
+  need to be sorted.
+
+  It works by splitting the number left and right of the decimal place, and
+  aligning it accordingly.  A temporary variable is created as part of this
+  process (which is automatically dropped)
+
+  The macro can be used only in data step, eg as follows:
+
+      data _null_;
+        length myvar $50;
+        do i=1 to 1000 by 50;
+          if mod(i,2)=0 then j=ranuni(0)*i*100;
+          else j=i*100;
+
+          %mp_aligndecimal(myvar,width=7)
+
+          leading_spaces=length(myvar)-length(cats(myvar));
+          putlog +leading_spaces myvar;
+        end;
+      run;
+
+  The generated code will look something like this:
+
+      length aligndp4e49996 $7;
+      if index(myvar,'.') then do;
+        aligndp4e49996=cats(scan(myvar,1,'.'));
+        aligndp4e49996=right(aligndp4e49996);
+        myvar=aligndp4e49996!!'.'!!cats(scan(myvar,2,'.'));
+      end;
+      else do;
+        aligndp4e49996=myvar;
+        aligndp4e49996=right(aligndp4e49996);
+        myvar=aligndp4e49996;
+      end;
+      drop aligndp4e49996;
+
+  Results (myvar variable):
+
+                0.7683559324
+              122.8232796
+            99419.50552
+            42938.5143414
+              763.3799189
+            15170.606073
+            15083.285773
+            85443.198707
+          2022999.2251
+            12038.658867
+          1350582.6734
+            52777.258221
+            11723.347628
+            33101.268376
+          6181622.8603
+          7390614.0669
+            73384.537893
+          1788362.1016
+          2774586.2219
+          7998580.8415
+
+
+  @param var The (data step, character) variable to modify
+  @param width= (8) The number of characters BEFORE the decimal point
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+
+  <h4> Related Programs </h4>
+  @li mp_aligndecimal.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_aligndecimal(var,width=8);
+
+  %local tmpvar;
+  %let tmpvar=%mf_getuniquename(prefix=aligndp);
+  length &tmpvar $&width;
+  if index(&var,'.') then do;
+    &tmpvar=cats(scan(&var,1,'.'));
+    &tmpvar=right(&tmpvar);
+    &var=&tmpvar!!'.'!!cats(scan(&var,2,'.'));
+  end;
+  else do;
+    &tmpvar=cats(&var);
+    &tmpvar=right(&tmpvar);
+    &var=&tmpvar;
+  end;
+  drop &tmpvar;
+
+%mend mp_aligndecimal;

base/mp_assert.sas

@@ -22,7 +22,7 @@
   results.  If it does not exist, it will be created, with the following format:
   |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
   |---|---|---|
-  |User Provided description|PASS|Column &inds contained ALL columns|
+  |User Provided description|PASS|Dataset &inds contained ALL columns|
 
   @version 9.2
   @author Allan Bowe

base/mp_assertcols.sas

@@ -41,7 +41,7 @@
   results.  If it does not exist, it will be created, with the following format:
   |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
   |---|---|---|
-  |User Provided description|PASS|Column &inds contained ALL columns|
+  |User Provided description|PASS|Dataset &inds contained ALL columns|
 
 
   <h4> Related Macros </h4>

base/mp_assertcolvals.sas

@@ -42,6 +42,7 @@
   @param [in] test= (ALLVALS) The test to apply.  Valid values are:
     @li ALLVALS - Test is a PASS if ALL values have a match in checkvals
     @li ANYVAL - Test is a PASS if at least 1 value has a match in checkvals
+    @li NOVAL - Test is a PASS if there are NO matches in checkvals
   @param [out] outds= (work.test_results) The output dataset to contain the
   results.  If it does not exist, it will be created, with the following format:
   |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
@@ -97,7 +98,7 @@
 
   %let test=%upcase(&test);
 
-  %if &test ne ALLVALS and &test ne ANYVAL %then %do;
+  %if &test ne ALLVALS and &test ne ANYVAL and &test ne NOVAL %then %do;
     %mp_abort(
       mac=&sysmacroname,
       msg=%str(Invalid test - &test)
@@ -108,12 +109,12 @@
   %let result=-1;
   %let orig=-1;
   proc sql noprint;
-  select count(*) into: result
+  select count(*) into: result trimmed
     from &lib..&ds
     where &col not in (
       select &ccol from &clib..&cds
     );
-  select count(*) into: orig from &lib..&ds;
+  select count(*) into: orig trimmed from &lib..&ds;
   quit;
 
   %local notfound tmp1 tmp2;
@@ -145,7 +146,7 @@
     length test_description $256 test_result $4 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
-    test_comments="&sysmacroname: &lib..&ds..&col has &result values "
+    test_comments="&sysmacroname: &lib..&ds..&col has &result/&orig values "
       !!"not in &clib..&cds..&ccol.. First 10 vals:"!!symget('notfound');
   %if &test=ANYVAL %then %do;
     if &result < &orig then test_result='PASS';
@@ -153,6 +154,9 @@
   %else %if &test=ALLVALS %then %do;
     if &result=0 then test_result='PASS';
   %end;
+  %else %if &test=NOVAL %then %do;
+    if &result=&orig then test_result='PASS';
+  %end;
   %else %do;
     test_comments="&sysmacroname: Unsatisfied test condition - &test";
   %end;

base/mp_binarycopy.sas

@@ -31,6 +31,7 @@
   @param [in] mode (CREATE) Valid values:
     @li CREATE - Create the file (even if it already exists)
     @li APPEND - Append to the file (don't overwrite)
+  @param iftrue= (1=1) Supply a condition for which the macro should be executed
 
   @returns nothing
 
@@ -44,8 +45,12 @@
     ,inref=____in   /* override default to use own filerefs */
     ,outref=____out /* override default to use own filerefs */
     ,mode=CREATE
+    ,iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
   %local mod;
+
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
   %if &mode=APPEND %then %let mod=mod;
 
   /* these IN and OUT filerefs can point to anything */

base/mp_cleancsv.sas

@@ -1,5 +1,5 @@
 /**
-  @file mp_cleancsv.sas
+  @file
   @brief Fixes embedded cr / lf / crlf in CSV
   @details CSVs will sometimes contain lf or crlf within quotes (eg when
   saved by excel).  When the termstr is ALSO lf or crlf that can be tricky
@@ -7,14 +7,16 @@
   This macro converts any csv to follow the convention of a windows excel file,
   applying CRLF line endings and converting embedded cr and crlf to lf.
 
-  usage:
+  Usage:
 
       fileref mycsv "/path/your/csv";
       %mp_cleancsv(in=mycsv,out=/path/new.csv)
 
-  @param in= provide path or fileref to input csv
-  @param out= output path or fileref to output csv
-  @param qchar= quote char - hex code 22 is the double quote.
+  @param in= (NOTPROVIDED) Provide path or fileref to input csv. If a period is
+    found, it is assumed to be a file.
+  @param out= (NOTPROVIDED) Output path or fileref to output csv.  If a period
+    is found, it is assumed to be a file.
+  @param qchar= ('22'x) Quote char - hex code 22 is the double quote.
 
   @version 9.2
   @author Allan Bowe
@@ -56,9 +58,14 @@
     else do;
       /* outside a quote, change cr and lf to crlf */
       if inchar='0D'x then do;
+        crblank:
         put '0D0A'x;
         input inchar $char1.;
-        if inchar ne '0A'x then do;
+        if inchar='0D'x then do;
+          /* multiple CR indicates CR formatted file with blank lines */
+          goto crblank;
+        end;
+        else if inchar ne '0A'x then do;
           put inchar $char1.;
           if inchar=qchar then isq = mod(isq+1,2);
         end;

base/mp_cntlout.sas

@@ -25,6 +25,7 @@
   <h4> SAS Macros </h4>
   @li mddl_sas_cntlout.sas
   @li mf_getuniquename.sas
+  @li mp_aligndecimal.sas
 
   <h4> Related Macros </h4>
   @li mf_getvarformat.sas
@@ -57,25 +58,33 @@
 %end;
 
 proc format lib=&libcat cntlout=&cntlds;
-%if "&fmtlist" ne "0" %then %do;
+%if "&fmtlist" ne "0" and "&fmtlist" ne "" %then %do;
   select
-  %do i=1 %to %sysfunc(countw(&fmtlist));
+  %do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
     %scan(&fmtlist,&i,%str( ))
   %end;
   ;
 %end;
 run;
 
-data &cntlout;
+data &cntlout/nonote2err;
   if 0 then set &ddlds;
   set &cntlds;
-  if type="N" then do;
-    start=cats(start);
-    end=cats(end);
+  by type fmtname notsorted;
+
+  /* align the numeric values to avoid overlapping ranges */
+  if type in ("I","N") then do;
+    %mp_aligndecimal(start,width=16)
+    %mp_aligndecimal(end,width=16)
   end;
+
+  /* create row marker. Data cannot be sorted without it! */
+  if first.fmtname then fmtrow=0;
+  fmtrow+1;
+
 run;
 proc sort;
-  by fmtname start;
+  by type fmtname fmtrow;
 run;
 
 proc sql;

base/mp_createconstraints.sas

@@ -18,11 +18,14 @@
       %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
       %mp_createconstraints(inds=work.constraints,outds=created,execute=YES)
 
-  @param inds= The input table containing the constraint info
-  @param outds= a table containing the create statements (create_statement column)
-  @param execute= `YES|NO` - default is NO. To actually create, use YES.
+  @param inds= (work.mp_getconstraints) The input table containing the
+    constraint info
+  @param outds= (work.mp_createconstraints) A table containing the create
+    statements (create_statement column)
+  @param execute= (NO) To actually create, use YES.
 
-  <h4> SAS Macros </h4>
+  <h4> Related Files </h4>
+  @li mp_getconstraints.sas
 
   @version 9.2
   @author Allan Bowe
@@ -30,7 +33,7 @@
 **/
 
 %macro mp_createconstraints(inds=mp_getconstraints
-  ,outds=mp_createconstraints
+  ,outds=work.mp_createconstraints
   ,execute=NO
 )/*/STORE SOURCE*/;
 
@@ -64,4 +67,4 @@ data &outds;
   output;
 run;
 
-%mend mp_createconstraints;
+%mend mp_createconstraints;

base/mp_dictionary.sas

@@ -0,0 +1,52 @@
+/**
+  @file mp_dictionary.sas
+  @brief Creates a portal (libref) into the SQL Dictionary Views
+  @details Provide a libref and the macro will create a series of views against
+  each view in the special PROC SQL dictionary libref.
+
+  This is useful if you would like to visualise (navigate) the views in a SAS
+  client such as Base SAS, Enterprise Guide, or Studio (or [Data Controller](
+  https://datacontroller.io)).
+
+  It works by extracting the dictionary.dictionaries view into
+  YOURLIB.dictionaries, then uses that to create a YOURLIB.{viewName} for every
+  other dictionary.view, eg:
+
+      proc sql;
+      create view YOURLIB.columns as select * from dictionary.columns;
+
+  Usage:
+
+      libname demo "/lib/directory";
+      %mp_dictionary(lib=demo)
+
+  Or, to just create them in WORK:
+
+      %mp_dictionary()
+
+  If you'd just like to browse the dictionary data model, you can also check
+  out [this article](https://rawsas.com/dictionary-of-dictionaries/).
+
+  ![](https://user-images.githubusercontent.com/4420615/188278365-2987db97-0594-4a39-ac81-dbacdef5cdc8.png)
+
+  @param lib= (WORK) The libref in which to create the views
+
+  <h4> Related Files </h4>
+  @li mp_dictionary.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_dictionary(lib=WORK)/*/STORE SOURCE*/;
+  %local list i mem;
+  proc sql noprint;
+  create view &lib..dictionaries as select * from dictionary.dictionaries;
+  select distinct memname into: list separated by ' '  from &lib..dictionaries;
+  %do i=1 %to %sysfunc(countw(&list,%str( )));
+    %let mem=%scan(&list,&i,%str( ));
+    create view &lib..&mem as select * from dictionary.&mem;
+  %end;
+  quit;
+%mend mp_dictionary;

base/mp_dirlist.sas

@@ -27,6 +27,9 @@
   @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
     subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
     recursion, set to MAX.
+  @param [in] showparent= (NO) By default, the initial parent directory is not
+    part of the results.  Set to YES to include it.  For this record only,
+    directory=filepath.
   @param [out] outds= (work.mp_dirlist) The output dataset to create
   @param [out] getattrs= (NO)  If getattrs=YES then the doptname / foptname
     functions are used to scan all properties - any characters that are not
@@ -63,6 +66,7 @@
     , fref=0
     , outds=work.mp_dirlist
     , getattrs=NO
+    , showparent=NO
     , maxdepth=0
     , level=0 /* The level of recursion to perform.  For internal use only. */
 )/*/STORE SOURCE*/;
@@ -97,8 +101,7 @@ data &out_ds(compress=no
     if did=0 then do;
       putlog "NOTE: This directory is empty, or does not exist - &path";
       msg=sysmsg();
-      put msg;
-      put _all_;
+      put (_all_)(=);
       stop;
     end;
     /* attribute is OS-dependent - could be "Directory" or "Directory Name" */
@@ -145,6 +148,15 @@ data &out_ds(compress=no
     output;
   end;
   rc = dclose(did);
+  %if &showparent=YES and &level=0 %then %do;
+    filepath=directory;
+    file_or_folder='folder';
+    ext='';
+    filename=scan(directory,-1,'/\');
+    msg='';
+    level=&level;
+    output;
+  %end;
   stop;
 run;
 
@@ -232,6 +244,9 @@ run;
   data _null_;
     set &out_ds;
     where file_or_folder='folder';
+  %if &showparent=YES and &level=0 %then %do;
+    if filepath ne directory;
+  %end;
     length code $10000;
     code=cats('%nrstr(%mp_dirlist(path=',filepath,",outds=&outds"
       ,",getattrs=&getattrs,level=%eval(&level+1),maxdepth=&maxdepth))");

base/mp_ds2fmtds.sas

@@ -17,7 +17,7 @@
   <h4> SAS Macros </h4>
   @li mf_existds.sas
 
-  <h4> Related Macros <h4>
+  <h4> Related Macros </h4>
   @li mp_jsonout.sas
 
   @version 9.2

base/mp_ds2md.sas

@@ -92,12 +92,13 @@ data _null_;
 run;
 
 %if %upcase(&showlog)=YES %then %do;
-  options ps=max;
+  options ps=max lrecl=max;
   data _null_;
     infile &outref;
+    if _n_=1 then putlog "# &libds" /;
     input;
     putlog _infile_;
   run;
 %end;
 
-%mend mp_ds2md;
+%mend mp_ds2md;

base/mp_dsmeta.sas

@@ -0,0 +1,110 @@
+/**
+  @file
+  @brief Export dataset metadata to a single output table
+  @details Exports the dataset attributes and enginehost information, then
+  converts the datasets into a single output table in the following format:
+
+|ODS_TABLE:$10.|NAME:$100.|VALUE:$1000.|
+|---|---|---|
+|`ATTRIBUTES `|`Data Set Name `|`SASHELP.CLASS `|
+|`ATTRIBUTES `|`Observations `|`19 `|
+|`ATTRIBUTES `|`Member Type `|`DATA `|
+|`ATTRIBUTES `|`Variables `|`5 `|
+|`ATTRIBUTES `|`Engine `|`V9 `|
+|`ATTRIBUTES `|`Indexes `|`0 `|
+|`ATTRIBUTES `|`Created `|`06/08/2020 00:59:14 `|
+|`ATTRIBUTES `|`Observation Length `|`40 `|
+|`ATTRIBUTES `|`Last Modified `|`06/08/2020 00:59:14 `|
+|`ATTRIBUTES `|`Deleted Observations `|`0 `|
+|`ATTRIBUTES `|`Protection `|`. `|
+|`ATTRIBUTES `|`Compressed `|`NO `|
+|`ATTRIBUTES `|`Data Set Type `|`. `|
+|`ATTRIBUTES `|`Sorted `|`NO `|
+|`ATTRIBUTES `|`Label `|`Student Data `|
+|`ATTRIBUTES `|`Data Representation `|`SOLARIS_X86_64, LINUX_X86_64, ALPHA_TRU64, LINUX_IA64 `|
+|`ATTRIBUTES `|`Encoding `|`us-ascii  ASCII (ANSI) `|
+|`ENGINEHOST `|`Data Set Page Size `|`65536 `|
+|`ENGINEHOST `|`Number of Data Set Pages `|`1 `|
+|`ENGINEHOST `|`First Data Page `|`1 `|
+|`ENGINEHOST `|`Max Obs per Page `|`1632 `|
+|`ENGINEHOST `|`Obs in First Data Page `|`19 `|
+|`ENGINEHOST `|`Number of Data Set Repairs `|`0 `|
+|`ENGINEHOST `|`Filename `|`/opt/sas/sas9/SASHome/SASFoundation/9.4/sashelp/class.sas7bdat `|
+|`ENGINEHOST `|`Release Created `|`9.0401M7 `|
+|`ENGINEHOST `|`Host Created `|`Linux `|
+|`ENGINEHOST `|`Inode Number `|`28314616 `|
+|`ENGINEHOST `|`Access Permission `|`rw-r--r-- `|
+|`ENGINEHOST `|`Owner Name `|`sas `|
+|`ENGINEHOST `|`File Size `|`128KB `|
+|`ENGINEHOST `|`File Size (bytes) `|`131072 `|
+
+  Example usage:
+
+      %mp_dsmeta(sashelp.class,outds=work.mymeta)
+      proc print data=work.mymeta;
+      run;
+
+  For more details on creating datasets from PROC CONTENTS check out this
+  excellent [paper](
+  https://support.sas.com/resources/papers/proceedings14/1549-2014.pdf) by
+  [Louise Hadden](https://www.linkedin.com/in/louisehadden/).
+
+  @param libds The library.dataset to export the metadata for
+  @param outds= (work.dsmeta) The output table to contain the metadata
+
+  <h4> Related Files </h4>
+  @li mp_dsmeta.test.sas
+  @li mp_getcols.sas
+  @li mp_getdbml.sas
+  @li mp_getddl.sas
+  @li mp_getformats.sas
+  @li mp_getpk.sas
+  @li mp_guesspk.sas
+
+**/
+
+%macro mp_dsmeta(libds,outds=work.dsmeta);
+
+%local ds1 ds2;
+data;run; %let ds1=&syslast;
+data;run; %let ds2=&syslast;
+
+/* setup the ODS capture */
+ods output attributes=&ds1 enginehost=&ds2;
+
+/* export the metadata */
+proc contents data=&libds;
+run;
+
+/* load it into a single table */
+data &outds (keep=ods_table name value);
+  length ods_table $10 name label2 label1 label $100
+    value cvalue cvalue1 cvalue2 $1000
+    nvalue nvalue1 nvalue2 8;
+  if _n_=1 then call missing (of _all_);
+  * putlog (_all_)(=);
+  set &ds1 (in=atrs) &ds2 (in=eng);
+  if atrs then do;
+    ods_table='ATTRIBUTES';
+    name=coalescec(label1,label);
+    value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.));
+    output;
+    if label2 ne '' then do;
+    name=label2;
+    value=coalescec(cvalue2,put(nvalue2,best.));
+    output;
+    end;
+  end;
+  else if eng then do;
+    ods_table='ENGINEHOST';
+    name=coalescec(label1,label);
+    value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.));
+    output;
+  end;
+run;
+
+proc sql;
+drop table &ds1, &ds2;
+
+%mend mp_dsmeta;
+

base/mp_filtercheck.sas

@@ -92,7 +92,8 @@ data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
-  length reason_cd $4032 vtype $1 vnum dsid 8;
+  length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000;
+  drop tmp;
 
   /* quick check to ensure column exists */
   if upcase(VARIABLE_NM) not in
@@ -168,18 +169,32 @@ data &outds;
   end;
 
   /* special logic */
-  if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ','');
-  else if OPERATOR_NM in ('IN','NOT IN') then do;
-    if substr(raw_value,1,1) ne '('
-    or substr(cats(reverse(raw_value)),1,1) ne ')'
-    then do;
-      REASON_CD='Missing start/end bracket in RAW_VALUE';
-      putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
-      call symputx('reason_cd',reason_cd,'l');
-      call symputx('nobs',_n_,'l');
-      output;
+  if OPERATOR_NM in ('IN','NOT IN','BETWEEN') then do;
+    if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ',',');
+    else do;
+      if substr(raw_value,1,1) ne '('
+      or substr(cats(reverse(raw_value)),1,1) ne ')'
+      then do;
+        REASON_CD='Missing start/end bracket in RAW_VALUE';
+        putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+        call symputx('reason_cd',reason_cd,'l');
+        call symputx('nobs',_n_,'l');
+        output;
+      end;
+      else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
+    end;
+    /* we now have a comma seperated list of values */
+    if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
+      tmp=scan(raw_value1,i,',');
+      if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
+        REASON_CD='Non Numeric value provided';
+        putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+        call symputx('reason_cd',reason_cd,'l');
+        call symputx('nobs',_n_,'l');
+        output;
+      end;
+      return;
     end;
-    else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
   end;
   else raw_value1=raw_value;
 

base/mp_getconstraints.sas

@@ -94,8 +94,11 @@ create table &outds as
 /**
   * We cannot apply this clause to the underlying dictionary table.  See:
   * https://communities.sas.com/t5/SAS-Programming/Unexpected-Where-Clause-behaviour-in-dictionary-TABLE/m-p/771554#M244867
+  * cannot use`where calculated libref="&lib"` either as it will STILL execute
+  * all the underlying constraint queries, causing exception errors in some
+  * cases: https://github.com/sasjs/core/issues/283
   */
-  where calculated libref="&lib"
+  where a.TABLE_CATALOG="&lib"
   %if "&ds" ne "" %then %do;
     and upcase(a.TABLE_NAME)="&ds"
     and upcase(b.TABLE_NAME)="&ds"

base/mp_getddl.sas

@@ -130,13 +130,13 @@ run;
 
 %local x curds;
 %if &flavour=SAS %then %do;
-  data _null_;
-    file &fref mod;
-    put "/* SAS Flavour DDL for %upcase(&libref).&curds */";
-    put "proc sql;";
-  run;
   %do x=1 %to %sysfunc(countw(&dsnlist));
     %let curds=%scan(&dsnlist,&x);
+    data _null_;
+      file &fref mod;
+      put "/* SAS Flavour DDL for %upcase(&libref).&curds */";
+      put "proc sql;";
+    run;
     data _null_;
       file &fref mod;
       length lab $1024 typ $20;

base/mp_getformats.sas

@@ -7,10 +7,15 @@
   Formats are taken from the library / dataset reference and / or a static
   format list.
 
+  Note - the source for this information is the dictionary.formats table. This
+  cannot show formats that are not already declared in the FMTSEARCH path.
+
   Example usage:
 
       %mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable)
 
+      %mp_getformats(fmtlist=FORMAT1 $FORMAT2 @INFMT3,outsummary=work.table2)
+
   @param [in] lib= (0) The libref for which to return formats.
   @todo Enable exporting of formats for an entire library
   @param [in] ds= (0) The dataset from which to obtain format definitions
@@ -49,7 +54,9 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 
 
   <h4> Related Macros </h4>
+  @li mf_getfmtlist.sas
   @li mp_applyformats.sas
+  @li mp_cntlout.sas
   @li mp_getformats.test.sas
 
   @version 9.2
@@ -66,7 +73,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 
 %local i fmt allfmts tempds fmtcnt;
 
-%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( )));
+%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
   /* ensure format list contains format _name_ only */
   %let fmt=%scan(&fmtlist,&i,%str( ));
   %let fmt=%mf_getfmtname(&fmt);
@@ -90,8 +97,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 proc sql;
 create table &outsummary as
   select * from dictionary.formats
-  where fmtname in (%mf_getquotedstr(&allfmts,quote=D))
-    and fmttype='F';
+  where fmtname in (%mf_getquotedstr(&allfmts,quote=D));
 
 %if "&outdetail" ne "0" %then %do;
   /* ensure base table always exists */
@@ -115,6 +121,10 @@ create table &outsummary as
     data &tempds;
       if 0 then set &outdetail;
       set &tempds;
+      /* set fmtrow (position of record within the format) */
+      by type fmtname notsorted;
+      if first.fmtname then fmtrow=1;
+      else fmtrow+1;
     run;
     proc append base=&outdetail data=&tempds ;
     run;

base/mp_getpk.sas

@@ -34,6 +34,7 @@
   @param [out] outds= (work.mp_getpk) The name of the output table to create.
 
   <h4> SAS Macros </h4>
+  @li mf_existfeature.sas
   @li mf_getengine.sas
   @li mf_getschema.sas
   @li mp_dropmembers.sas
@@ -229,7 +230,12 @@ create table work.&tabs1 as select
   libname as libref
   ,upcase(memname) as dsn
   ,memtype
+%if %mf_existfeature(DBMS_MEMTYPE)=1 %then %do;
   ,dbms_memtype
+%end;
+%else %do;
+  ,'n/a' as dbms_memtype format=$32.
+%end;
   ,typemem
   ,memlabel
   ,nvar

base/mp_gitadd.sas

@@ -0,0 +1,45 @@
+/**
+  @file
+  @brief Stages files in a GIT repo
+  @details Uses the output dataset from mp_gitstatus.sas to determine the files
+  that should be staged.
+
+  If `STAGED ne "TRUE"` then the file is staged.
+
+  Usage:
+
+      %let dir=%sysfunc(pathname(work))/core;
+      %let repo=https://github.com/sasjs/core;
+      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir));
+      %mf_writefile(&dir/somefile.txt,l1=some content)
+      %mf_deletefile(&dir/package.json)
+      %mp_gitstatus(&dir,outds=work.gitstatus)
+
+      %mp_gitadd(&dir,inds=work.gitstatus)
+
+  @param [in] gitdir The directory containing the GIT repository
+  @param [in] inds= (work.mp_gitadd) The input dataset with the list of files
+  to stage.  Will accept the output from mp_gitstatus(), else just use a table
+  with the following columns:
+    @li path $1024 - relative path to the file in the repo
+    @li staged $32 - whether the file is staged (TRUE or FALSE)
+    @li status $64 - either new, deleted, or modified
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> Related Files </h4>
+  @li mp_gitadd.test.sas
+  @li mp_gitstatus.sas
+
+**/
+
+%macro mp_gitadd(gitdir,inds=work.mp_gitadd,mdebug=0);
+
+data _null_;
+  set &inds;
+  if STAGED ne "TRUE";
+  rc=git_index_add("&gitdir",cats(path),status);
+  if rc ne 0 or &mdebug=1 then put rc=;
+run;
+
+%mend mp_gitadd;

base/mp_gitlog.sas

@@ -0,0 +1,104 @@
+/**
+  @file
+  @brief Creates a dataset with the commit history of a local repository
+  @details Returns the commit history from a local repository. The name of the
+  branch is also returned.
+
+  More details here:
+https://documentation.sas.com/doc/ko/pgmsascdc/v_033/lefunctionsref/n1qo5miyvry1nen111js203hlwrh.htm
+
+  Usage:
+
+      %let gitdir=%sysfunc(pathname(work))/core;
+      %let repo=https://github.com/sasjs/core;
+      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir));
+
+      %mp_gitlog(&gitdir,outds=work.mp_gitlog)
+
+  @param [in] gitdir The directory containing the GIT repository
+  @param [in] filter= (BRANCHONLY) To return only the commits for the current
+  branch, use BRANCHONLY (the default).  Anything else will return the entire
+  commit history.
+  @param [out] outds= (work.mp_gitlog) The output dataset to create.
+  All vars are $128 except `message` which is $4000.
+    @li author returns the author who submitted the commit.
+    @li children_ids returns a list of the children commit IDs
+    @li committer returns the name of the committer.
+    @li committer_email returns the email of the committer.
+    @li email returns the email of the commit author.
+    @li id returns the commit ID of the commit object.
+    @li in_current_branch returns "TRUE" or "FALSE" to indicate if the commit is
+      in the current branch.
+    @li message returns the commit message.
+    @li parent_ids returns a list of the parent commit IDs.
+    @li stash returns "TRUE" or "FALSE" to indicate if the commit is a stash
+      commit.
+    @li time returns the time of the commit as numeric string
+    @li commit_time_num time of the commit as numeric SAS datetime
+    @li commit_time_str the commit_time_num variable cast as string
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] nobs= (0) Set to an integer greater than 0 to restrict the number
+  of rows returned
+
+  <h4> SAS Macros </h4>
+  @li mf_getgitbranch.sas
+
+  <h4> Related Files </h4>
+  @li mp_gitadd.sas
+  @li mp_gitreleaseinfo.sas
+  @li mp_gitstatus.sas
+
+**/
+
+%macro mp_gitlog(gitdir,outds=work.mp_gitlog,mdebug=0,filter=BRANCHONLY,nobs=0);
+
+%local varlist i var;
+%let varlist=author children_ids committer committer_email email id
+  in_current_branch parent_ids stash time ;
+
+data &outds;
+  LENGTH gitdir branch $ 1024 message $4000 &varlist $128 commit_time_num 8.
+    commit_time_str $32;
+  call missing (of _all_);
+  branch="%mf_getgitbranch(&gitdir)";
+  gitdir=symget('gitdir');
+  rc=git_status_free(trim(gitdir));
+  if rc=-1 then do;
+    put "The libgit2 library is unavailable and no Git operations can be used.";
+    put "See: https://stackoverflow.com/questions/74082874";
+    stop;
+  end;
+  else if rc=-2 then do;
+    put "The libgit2 library is available, but the status function failed.";
+    put "See the log for details.";
+    stop;
+  end;
+  entries=git_commit_log(trim(gitdir));
+  do n=1 to entries;
+
+  %do i=1 %to %sysfunc(countw(&varlist message));
+    %let var=%scan(&varlist message,&i,%str( ));
+    rc=git_commit_get(n,trim(gitdir),"&var",&var);
+  %end;
+    /* convert unix time to SAS time - https://4gl.uk/corelink0 */
+    /* Number of seconds between 01JAN1960 and 01JAN1970: 315619200 */
+    format commit_time_num datetime19.;
+    commit_time_num=sum(input(cats(time),best.),315619200);
+    commit_time_str=put(commit_time_num,datetime19.);
+  %if &mdebug=1 %then %do;
+    putlog (_all_)(=);
+  %end;
+    if "&filter"="BRANCHONLY" then do;
+      if cats(in_current_branch)='TRUE' then output;
+    end;
+    else output;
+  %if &nobs>0 %then %do;
+    if n ge &nobs then stop;
+  %end;
+  end;
+  rc=git_commit_free(trim(gitdir));
+  keep gitdir branch &varlist message time commit_time_num commit_time_str;
+run;
+
+%mend mp_gitlog;

base/mp_gitreleaseinfo.sas

@@ -0,0 +1,74 @@
+/**
+  @file
+  @brief Pulls latest release info from a GIT repository
+  @details Useful for grabbing the latest version number or other attributes
+  from a GIT server.  Supported providers are GitLab and GitHub. Pull requests
+  are welcome if you'd like to see additional providers!
+
+  Note that each provider provides slightly different JSON output.  Therefore
+  the macro simply extracts the JSON and assigns the libname (using the JSON
+  engine).
+
+  Example usage (eg, to grab latest release version from github):
+
+      %mp_gitreleaseinfo(GITHUB,sasjs/core,outlib=mylibref)
+
+      data _null_;
+        set mylibref.root;
+        putlog TAG_NAME=;
+      run;
+
+  @param [in] provider The GIT provider for the release info.  Accepted values:
+    @li GITLAB
+    @li GITHUB - Tables include root, assets, author, alldata
+  @param [in] project The link to the repository.  This has different formats
+    depending on the vendor:
+    @li GITHUB - org/repo, eg sasjs/core
+    @li GITLAB - project, eg 1343223
+  @param [in] server= (0) If your repo is self-hosted, then provide the domain
+    here.  Otherwise it will default to the provider domain (eg gitlab.com).
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outlib= (GITREL) The JSON-engine libref to be created, which will
+    point at the returned JSON
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+
+  <h4> Related Files </h4>
+  @li mp_gitreleaseinfo.test.sas
+
+**/
+
+%macro mp_gitreleaseinfo(provider,project,server=0,outlib=GITREL,mdebug=0);
+%local url fref;
+
+%let provider=%upcase(&provider);
+
+%if &provider=GITHUB %then %do;
+  %if "&server"="0" %then %let server=https://api.github.com;
+  %let url=&server/repos/&project/releases/latest;
+%end;
+%else %if &provider=GITLAB %then %do;
+  %if "&server"="0" %then %let server=https://gitlab.com;
+  %let url=&server/api/v4/projects/&project/releases;
+%end;
+
+%let fref=%mf_getuniquefileref();
+
+proc http method='GET' out=&fref url="&url";
+%if &mdebug=1 %then %do;
+  debug level = 3;
+%end;
+run;
+
+libname &outlib JSON fileref=&fref;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fref;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_gitreleaseinfo;

base/mp_gitstatus.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Creates a dataset with the output from `GIT_STATUS()`
+  @details Uses `git_status()` to fetch the number of changed files, then
+  iterates with `git_status_get()`, inserting all attributes into an output
+  dataset.
+
+  Usage:
+
+      %let dir=%sysfunc(pathname(work))/core;
+      %let repo=https://github.com/sasjs/core;
+      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir));
+      %mf_writefile(&dir/somefile.txt,l1=some content)
+      %mf_deletefile(&dir/package.json)
+
+      %mp_gitstatus(&dir,outds=work.gitstatus)
+
+  More info on these functions is in this [helpful paper]
+(https://www.sas.com/content/dam/SAS/support/en/sas-global-forum-proceedings/2019/3057-2019.pdf)
+  by Danny Zimmerman.
+
+  @param [in] gitdir The directory containing the GIT repository
+  @param [out] outds= (work.git_status) The output dataset to create.  Vars:
+    @li gitdir $1024 - directory of repo
+    @li path $1024 - relative path to the file in the repo
+    @li staged $32 - whether the file is staged (TRUE or FALSE)
+    @li status $64 - either new, deleted, or modified
+    @li cnt - number of files
+    @li n - the "nth" file in the list from git_status()
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> Related Files </h4>
+  @li mp_gitstatus.test.sas
+  @li mp_gitadd.sas
+
+**/
+
+%macro mp_gitstatus(gitdir,outds=work.mp_gitstatus,mdebug=0);
+
+data &outds;
+  LENGTH gitdir path $ 1024 STATUS $ 64 STAGED $ 32;
+  call missing (of _all_);
+  gitdir=symget('gitdir');
+  cnt=git_status(trim(gitdir));
+  if cnt=-1 then do;
+    put "The libgit2 library is unavailable and no Git operations can be used.";
+    put "See: https://stackoverflow.com/questions/74082874";
+  end;
+  else if cnt=-2 then do;
+    put "The libgit2 library is available, but the status function failed.";
+    put "See the log for details.";
+  end;
+  else do n=1 to cnt;
+    rc=GIT_STATUS_GET(n,gitdir,'PATH',path);
+    rc=GIT_STATUS_GET(n,gitdir,'STAGED',staged);
+    rc=GIT_STATUS_GET(n,gitdir,'STATUS',status);
+    output;
+  %if &mdebug=1 %then %do;
+    putlog (_all_)(=);
+  %end;
+  end;
+  rc=git_status_free(trim(gitdir));
+  drop rc cnt;
+run;
+
+%mend mp_gitstatus;

base/mp_hashdataset.sas

@@ -11,7 +11,7 @@
         put hashkey=;
       run;
 
-  ![sas md5 hash dataset log results](https://i.imgur.com/MqF98vk.png)
+  ![sas md5 hash dataset log results](https://i.4gl.io/1/KorUKoyE05.png/raw)
 
   <h4> SAS Macros </h4>
   @li mf_getattrn.sas
@@ -21,11 +21,12 @@
 
   <h4> Related Files </h4>
   @li mp_hashdataset.test.sas
+  @li mp_hashdirectory.sas
 
   @param [in] libds dataset to hash
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
-  @param [in] iftrue= A condition under which the macro should be executed.
-  @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [out] outds= (work._data_) The output dataset to create. This
   will contain one column (hashkey) with one observation (a $hex32.
   representation of the input hash)
   |hashkey:$32.|

base/mp_hashdirectory.sas

@@ -0,0 +1,164 @@
+/**
+  @file
+  @brief Returns a unique hash for each file in a directory
+  @details Hashes each file in each directory, and then hashes the hashes to
+  create a hash for each directory also.
+
+  This makes use of the new `hashing_file()` and `hashing` functions, available
+  since 9.4m6. Interestingly, those functions can be used in pure macro, eg:
+
+      %put %sysfunc(hashing_file(md5,/path/to/file.blob,0));
+
+  Actual usage:
+
+      %let fpath=/some/directory;
+
+      %mp_hashdirectory(&fpath,outds=myhash,maxdepth=2)
+
+      data _null_;
+        set work.myhash;
+        put (_all_)(=);
+      run;
+
+  Whilst files are hashed in their entirety, the logic for creating a folder
+  hash is as follows:
+
+  @li Sort the files by filename (case sensitive, uppercase then lower)
+  @li Take the first 100 hashes, concatenate and hash
+  @li Concatenate this hash with another 100 hashes and hash again
+  @li Continue until the end of the folder.  This is the folder hash
+  @li If a folder contains other folders, start from the bottom of the tree -
+    the folder hashes cascade upwards so you know immediately if there is a
+    change in a sub/sub directory
+  @li If a subfolder has no content (empty) then it is ignored. No hash created.
+  @li If the file is empty, it is also ignored / no hash created.
+  @li If the target directory (&inloc) is empty, &outds will also be empty
+
+  <h4> SAS Macros </h4>
+  @li mp_dirlist.sas
+
+  <h4> Related Files </h4>
+  @li mp_hashdataset.sas
+  @li mp_hashdirectory.test.sas
+  @li mp_md5.sas
+
+  @param [in] inloc Full filepath of the file to be hashed (unquoted)
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
+    subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
+  @param [in] method= (MD5) the hashing method to use.  Available options:
+    @li MD5
+    @li SH1
+    @li SHA256
+    @li SHA384
+    @li SHA512
+    @li CRC32
+  @param [out] outds= (work.mp_hashdirectory) The output dataset.  Contains:
+    @li directory - the parent folder
+    @li file_hash - the hash output
+    @li hash_duration - how long the hash took (first hash always takes longer)
+    @li file_path - /full/path/to/each/file.ext
+    @li file_or_folder - contains either "file" or "folder"
+    @li level - the depth of the directory (top level is 0)
+
+  @version 9.4m6
+  @author Allan Bowe
+**/
+
+%macro mp_hashdirectory(inloc,
+  outds=work.mp_hashdirectory,
+  method=MD5,
+  maxdepth=0,
+  iftrue=%str(1=1)
+)/*/STORE SOURCE*/;
+
+%local curlevel tempds maxlevel;
+
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+/* get the directory listing */
+%mp_dirlist(path=&inloc, outds=&outds, maxdepth=&maxdepth, showparent=YES)
+
+/* create the hashes */
+data &outds;
+  set &outds (rename=(filepath=file_path));
+  length FILE_HASH $32 HASH_DURATION 8;
+  keep directory file_hash hash_duration file_path file_or_folder level;
+
+  ts=datetime();
+  if file_or_folder='file' then do;
+    /* if file is empty, hashing_file will break - so ignore / delete */
+    length fname val $8;
+    drop fname val fid is_empty;
+    rc=filename(fname,file_path);
+    fid=fopen(fname);
+    if fid > 0 then do;
+      rc=fread(fid);
+      is_empty=fget(fid,val);
+    end;
+    rc=fclose(fid);
+    rc=filename(fname);
+    if is_empty ne 0 then delete;
+    else file_hash=hashing_file("&method",cats(file_path),0);
+  end;
+  hash_duration=datetime()-ts;
+run;
+
+proc sort data=&outds ;
+  by descending level directory file_path;
+run;
+
+%let maxlevel=0;
+data _null_;
+  set &outds;
+  call symputx('maxlevel',level,'l');
+  stop;
+run;
+
+/* now hash the hashes to populate folder hashes, starting from the bottom */
+%do curlevel=&maxlevel %to 0 %by -1;
+  data work._data_ (keep=directory file_hash);
+    set &outds;
+    where level=&curlevel;
+    by descending level directory file_path;
+    length str $32767 tmp_hash $32;
+    retain str tmp_hash ;
+    /* reset vars when starting a new directory */
+    if first.directory then do;
+      str='';
+      tmp_hash='';
+      i=0;
+    end;
+    /* hash each chunk of 100 file paths */
+    i+1;
+    str=cats(str,file_hash);
+    if mod(i,100)=0 or last.directory then do;
+      tmp_hash=hashing("&method",cats(tmp_hash,str));
+      str='';
+    end;
+    /* output the hash at directory level */
+    if last.directory then do;
+      file_hash=tmp_hash;
+      output;
+    end;
+    if last.level then stop;
+  run;
+  %let tempds=&syslast;
+  /* join the hash back into the main table */
+  proc sql undo_policy=none;
+  create table &outds as
+    select a.directory
+      ,coalesce(b.file_hash,a.file_hash) as file_hash
+      ,a.hash_duration
+      ,a.file_path
+      ,a.file_or_folder
+      ,a.level
+    from &outds a
+    left join &tempds b
+    on a.file_path=b.directory
+    order by level desc, directory, file_path;
+  drop table &tempds;
+%end;
+
+%mend mp_hashdirectory;

base/mp_jsonout.sas

@@ -28,7 +28,7 @@
 
         %mp_jsonout(OPEN,jref=tmp)
         %mp_jsonout(OBJ,class,jref=tmp)
-        %mp_jsonout(OBJ,class,dslabel=class2,jref=tmp,showmeta=YES)
+        %mp_jsonout(OBJ,class,dslabel=class2,jref=tmp,showmeta=Y)
         %mp_jsonout(CLOSE,jref=tmp)
 
         data _null_;
@@ -55,12 +55,14 @@
     @li DATASTEP (more reliable when data has non standard characters)
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
-  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+  @param [in] showmeta= (N) Set to Y to output metadata alongside each table,
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to JSON
 
-  <h4> Related Macros <h4>
+  <h4> Related Files </h4>
   @li mp_ds2fmtds.sas
 
   @version 9.2
@@ -68,14 +70,16 @@
   @source https://github.com/sasjs/core
 
 **/
-
 %macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y
   ,engine=DATASTEP
   ,missing=NULL
-  ,showmeta=NO
+  ,showmeta=N
+  ,maxobs=MAX
 )/*/STORE SOURCE*/;
-%local tempds colinfo fmtds i numcols;
+%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval
+  tmpds1 tmpds2 tmpds3 tmpds4;
 %let numcols=0;
+%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;);
 
 %if &action=OPEN %then %do;
   options nobomfile;
@@ -111,7 +115,7 @@
     by varnum;
   run;
   /* move meta to mac vars */
-  data _null_;
+  data &colinfo;
     if _n_=1 then call symputx('numcols',nobs,'l');
     set &colinfo end=last nobs=nobs;
     name=upcase(name);
@@ -122,7 +126,6 @@
       if format='' then fmt=cats('$',length,'.');
       else if formatl=0 then fmt=cats(format,'.');
       else fmt=cats(format,formatl,'.');
-      newlen=max(formatl,length);
     end;
     else do;
       typelong='num';
@@ -130,23 +133,26 @@
       else if formatl=0 then fmt=cats(format,'.');
       else if formatd=0 then fmt=cats(format,formatl,'.');
       else fmt=cats(format,formatl,'.',formatd);
-      /* needs to be wide, for datetimes etc */
-      newlen=max(length,formatl,24);
     end;
     /* 32 char unique name */
     newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
 
     call symputx(cats('name',_n_),name,'l');
     call symputx(cats('newname',_n_),newname,'l');
-    call symputx(cats('len',_n_),newlen,'l');
     call symputx(cats('length',_n_),length,'l');
     call symputx(cats('fmt',_n_),fmt,'l');
     call symputx(cats('type',_n_),type,'l');
     call symputx(cats('typelong',_n_),typelong,'l');
     call symputx(cats('label',_n_),coalescec(label,name),'l');
+    /* overwritten when fmt=Y and a custom format exists in catalog */
+    if typelong='num' then call symputx(cats('fmtlen',_n_),200,'l');
+    else call symputx(cats('fmtlen',_n_),min(32767,ceil((length+10)*1.5)),'l');
   run;
 
   %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+  proc sql;
+  select count(*) into: lastobs from &ds;
+  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs));
 
   %if &engine=PROCJSON %then %do;
     %if &missing=STRING %then %do;
@@ -154,7 +160,9 @@
       %put &sysmacroname: Switching to DATASTEP engine;
       %goto datastep;
     %end;
-    data &tempds;set &ds;
+    data &tempds;
+      set &ds;
+      &stmt_obs;
     %if &fmt=N %then format _numeric_ best32.;;
     /* PRETTY is necessary to avoid line truncation in large files */
     filename _sjs2 temp lrecl=131068 encoding='utf-8';
@@ -181,26 +189,99 @@
     %end;
 
     %if &fmt=Y %then %do;
-      data _data_;
+      /**
+        * Extract format definitions
+        * First, by getting library locations from dictionary.formats
+        * Then, by exporting the width using proc format
+        * Cannot use maxw from sashelp.vformat as not always populated
+        * Cannot use fmtinfo() as not supported in all flavours
+        */
+      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      proc sql noprint;
+      create table &tmpds1 as
+          select cats(libname,'.',memname) as FMTCAT,
+          FMTNAME
+        from dictionary.formats
+        where fmttype='F' and libname is not null
+          and fmtname in (select format from &colinfo where format is not null)
+        order by 1;
+      create table &tmpds2(
+          FMTNAME char(32),
+          LENGTH num
+      );
+      %local catlist cat fmtlist i;
+      select distinct fmtcat into: catlist separated by ' ' from &tmpds1;
+      %do i=1 %to %sysfunc(countw(&catlist,%str( )));
+        %let cat=%scan(&catlist,&i,%str( ));
+        proc sql;
+        select distinct fmtname into: fmtlist separated by ' '
+          from &tmpds1 where fmtcat="&cat";
+        proc format lib=&cat cntlout=&tmpds3(keep=fmtname length);
+          select &fmtlist;
+        run;
+        proc sql;
+        insert into &tmpds2 select distinct fmtname,length from &tmpds3;
+      %end;
+
+      proc sql;
+      create table &tmpds4 as
+        select a.*, b.length as MAXW
+        from &colinfo a
+        left join &tmpds2 b
+        on cats(a.format)=cats(upcase(b.fmtname))
+        order by a.varnum;
+      data _null_;
+        set &tmpds4;
+        if not missing(maxw);
+        call symputx(
+          cats('fmtlen',_n_),
+          /* vars need extra padding due to JSON escaping of special chars */
+          min(32767,ceil((max(length,maxw)+10)*1.5))
+          ,'l'
+        );
+      run;
+
+      /* configure varlenchk - as we are explicitly shortening the variables */
+      %let optval=%sysfunc(getoption(varlenchk));
+      options varlenchk=NOWARN;
+      data _data_(compress=char);
+        /* shorten the new vars */
+        length
+      %do i=1 %to &numcols;
+          &&name&i $&&fmtlen&i
+      %end;
+          ;
         /* rename on entry */
         set &ds(rename=(
       %do i=1 %to &numcols;
-        &&name&i=&&newname&i
+          &&name&i=&&newname&i
       %end;
         ));
+      &stmt_obs;
+
+      drop
+      %do i=1 %to &numcols;
+        &&newname&i
+      %end;
+        ;
       %do i=1 %to &numcols;
-        length &&name&i $&&len&i;
         %if &&typelong&i=num %then %do;
-          &&name&i=left(put(&&newname&i,&&fmt&i));
+          &&name&i=cats(put(&&newname&i,&&fmt&i));
         %end;
         %else %do;
           &&name&i=put(&&newname&i,&&fmt&i);
         %end;
-        drop &&newname&i;
       %end;
-        if _error_ then call symputx('syscc',1012);
+        if _error_ then do;
+          call symputx('syscc',1012);
+          stop;
+        end;
       run;
       %let fmtds=&syslast;
+      options varlenchk=&optval;
     %end;
 
     proc format; /* credit yabwon for special null removal */
@@ -219,8 +300,8 @@
       attrib _all_ label='';
       %do i=1 %to &numcols;
         %if &&typelong&i=char or &fmt=Y %then %do;
-          length &&name&i $32767;
-          format &&name&i $32767.;
+          length &&name&i $&&fmtlen&i...;
+          format &&name&i $&&fmtlen&i...;
         %end;
       %end;
       %if &fmt=Y %then %do;
@@ -229,10 +310,11 @@
       %else %do;
         set &ds;
       %end;
+      &stmt_obs;
       format _numeric_ bart.;
     %do i=1 %to &numcols;
       %if &&typelong&i=char or &fmt=Y %then %do;
-        if findc(&&name&i,'"\'!!'0A0D09000E0F01021011'x) then do;
+        if findc(&&name&i,'"\'!!'0A0D09000E0F010210111A'x) then do;
           &&name&i='"'!!trim(
             prxchange('s/"/\\"/',-1,        /* double quote */
             prxchange('s/\x0A/\n/',-1,      /* new line */
@@ -245,8 +327,9 @@
             prxchange('s/\x02/\\u0002/',-1, /* STX */
             prxchange('s/\x10/\\u0010/',-1, /* DLE */
             prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+            prxchange('s/\x1A/\\u001A/',-1, /* SUB */
             prxchange('s/\\/\\\\/',-1,&&name&i)
-          ))))))))))))!!'"';
+          )))))))))))))!!'"';
         end;
         else &&name&i=quote(cats(&&name&i));
       %end;
@@ -285,10 +368,11 @@
   proc sql;
   drop table &colinfo, &tempds;
 
-  %if &showmeta=YES %then %do;
+  %if %substr(&showmeta,1,1)=Y %then %do;
     filename _sjs4 temp lrecl=131068 encoding='utf-8';
     data _null_;
       file _sjs4;
+      length label $350;
       put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{";
       do i=1 to &numcols;
         name=quote(trim(symget(cats('name',i))));

base/mp_loadformat.sas

@@ -34,10 +34,10 @@
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
-  @li mddl_sas_cntlout.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
+  @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
   @li mp_storediffs.sas
@@ -45,7 +45,8 @@
   <h4> Related Macros </h4>
   @li mddl_dc_difftable.sas
   @li mddl_dc_locktable.sas
-  @li mp_loadformat.test.sas
+  @li mp_loadformat.test.1.sas
+  @li mp_loadformat.test.2.sas
   @li mp_lockanytable.sas
   @li mp_stackdiffs.sas
 
@@ -67,7 +68,7 @@
 );
 /* set up local macro variables and temporary tables (with a prefix) */
 %local err msg prefix dslist i var fmtlist ibufsize;
-%let dslist=base_fmts template inlibds ds1 stagedata storediffs;
+%let dslist=base_fmts template inlibds ds1 stagedata storediffs del1 del2;
 %if &outds_add=0 %then %let dslist=&dslist outds_add;
 %if &outds_del=0 %then %let dslist=&dslist outds_del;
 %if &outds_mod=0 %then %let dslist=&dslist outds_mod;
@@ -78,13 +79,6 @@
   %let &var=%upcase(&prefix._&var);
 %end;
 
-/*
-format values can be up to 32767 wide.  SQL joins on such a wide column can
-cause buffer issues.  Update ibufsize and reset at the end.
-*/
-%let ibufsize=%sysfunc(getoption(ibufsize));
-options ibufsize=32767 ;
-
 /* in DC, format catalogs maybe specified in the libds with a -FC extension */
 %let libcat=%scan(&libcat,1,-);
 
@@ -134,29 +128,62 @@ run;
   * First, extract only relevant formats from the catalog
   */
 proc sql noprint;
-select distinct upcase(fmtname) into: fmtlist separated by ' ' from &libds;
+select distinct
+  case
+    when type='N' then upcase(fmtname)
+    when type='C' then cats('$',upcase(fmtname))
+    when type='I' then cats('@',upcase(fmtname))
+    when type='J' then cats('@$',upcase(fmtname))
+    else "&sysmacroname:UNHANDLED"
+  end
+  into: fmtlist separated by ' '
+  from &libds;
 
 %mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
 
+/* get a hash of the row */
+%local cvars nvars;
+%let cvars=TYPE FMTNAME START END LABEL PREFIX FILL SEXCL EEXCL HLO DECSEP
+  DIG3SEP DATATYPE LANGUAGE;
+%let nvars=FMTROW MIN MAX DEFAULT LENGTH FUZZ MULT NOEDIT;
+data &base_fmts/note2err;
+  set &base_fmts;
+  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
+run;
 
 /**
   * Ensure input table and base_formats have consistent lengths and types
   */
-%mddl_sas_cntlout(libds=&template)
-data &inlibds;
-  length &delete_col $3;
-  if 0 then set &template;
+data &inlibds/nonote2err;
+  length &delete_col $3 FMTROW 8 start end label $32767;
+  if 0 then set &base_fmts;
   set &libds;
+  by type fmtname notsorted;
   if &delete_col='' then &delete_col='No';
   fmtname=upcase(fmtname);
+  type=upcase(type);
   if missing(type) then do;
-    if substr(fmtname,1,1)='$' then type='C';
-    else type='N';
+    if substr(fmtname,1,1)='@' then do;
+      if substr(fmtname,2,1)='$' then type='J';
+      else type='I';
+    end;
+    else do;
+      if substr(fmtname,1,1)='$' then type='C';
+      else type='N';
+    end;
   end;
-  if type='N' then do;
-    start=cats(start);
-    end=cats(end);
+  if type in ('N','I') then do;
+    %mp_aligndecimal(start,width=16)
+    %mp_aligndecimal(end,width=16)
   end;
+
+  /* update row marker - retain new var as fmtrow may already be in libds */
+  if first.fmtname then row=1;
+  else row+1;
+  drop row;
+  fmtrow=row;
+
+  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
 run;
 
 /**
@@ -167,23 +194,10 @@ create table &outds_add(drop=&delete_col) as
   select a.*
   from &inlibds a
   left join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where b.fmtname is null
     and upcase(a.&delete_col) ne "YES"
-  order by fmtname, start;;
-
-/**
-  * Identify deleted records
-  */
-create table &outds_del(drop=&delete_col) as
-  select a.*
-  from &inlibds a
-  inner join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
-  where upcase(a.&delete_col)="YES"
-  order by fmtname, start;
+  order by type, fmtname, fmtrow;
 
 /**
   * Identify modified records
@@ -192,12 +206,40 @@ create table &outds_mod (drop=&delete_col) as
   select a.*
   from &inlibds a
   inner join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where upcase(a.&delete_col) ne "YES"
-  order by fmtname, start;
+    and a.fmthash ne b.fmthash
+  order by type, fmtname, fmtrow;
+
+/**
+  * Identify deleted records
+  */
+create table &outds_del(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
+  where upcase(a.&delete_col)="YES"
+  order by type, fmtname, fmtrow;
+
+/**
+  * Identify fully deleted formats (where every record is removed)
+  * These require to be explicitly deleted in proc format
+  * del1 - identify _partial_ deletes
+  * del2 - exclude these, and also formats that come with _additions_
+  */
+create table &del1 as
+  select a.*
+  from &base_fmts a
+  left join &outds_del b
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
+  where b.fmtrow is null;
+
+create table &del2 as
+  select * from &outds_del
+  where cats(type,fmtname) not in (select cats(type,fmtname) from &outds_add)
+    and cats(type,fmtname) not in (select cats(type,fmtname) from &del1);
 
-options ibufsize=&ibufsize;
 
 %mp_abort(
   iftrue=(&syscc ne 0)
@@ -206,19 +248,21 @@ options ibufsize=&ibufsize;
 )
 
 %if &loadtarget=YES %then %do;
+  /* new records plus base records that are not deleted or modified */
   data &ds1;
     merge &base_fmts(in=base)
       &outds_mod(in=mod)
       &outds_add(in=add)
       &outds_del(in=del);
     if not del and not mod;
-    by fmtname start;
+    by type fmtname fmtrow;
   run;
+  /* add back the modified records */
   data &stagedata;
     set &ds1 &outds_mod;
   run;
   proc sort;
-    by fmtname start;
+    by type fmtname fmtrow;
   run;
 %end;
 /* mp abort needs to run outside of conditional blocks */
@@ -228,7 +272,7 @@ options ibufsize=&ibufsize;
   ,msg=%str(SYSCC=&syscc prior to actual load)
 )
 %if &loadtarget=YES %then %do;
-  %if %mf_nobs(&stagedata)=0 %then %do;
+  %if %mf_nobs(&stagedata)=0 and %mf_nobs(&del2)=0 %then %do;
     %put There are no changes to load in &libcat!;
     %return;
   %end;
@@ -244,6 +288,22 @@ options ibufsize=&ibufsize;
   /* do the actual load */
   proc format lib=&libcat cntlin=&stagedata;
   run;
+  /* apply any full deletes */
+  %if %mf_nobs(&del2)>0 %then %do;
+    %local delfmtlist;
+    proc sql noprint;
+    select distinct case when type='N' then cats(fmtname,'.FORMAT')
+        when type='C' then cats(fmtname,'.FORMATC')
+        when type='J' then cats(fmtname,'.INFMTC')
+        when type='I' then cats(fmtname,'.INFMT')
+        else cats(fmtname,'.BADENTRY!!!') end
+      into: delfmtlist
+      separated by ' '
+      from &del2;
+    proc catalog catalog=&libcat;
+      delete &delfmtlist;
+    quit;
+  %end;
   %if &locklibds ne 0 %then %do;
     /* unlock the table */
     %mp_lockanytable(UNLOCK
@@ -266,7 +326,7 @@ options ibufsize=&ibufsize;
 
     %mp_storediffs(&libcat-FC
       ,&base_fmts
-      ,FMTNAME START
+      ,TYPE FMTNAME FMTROW
       ,delds=&outds_del
       ,modds=&outds_mod
       ,appds=&outds_add

base/mp_lockanytable.sas

@@ -167,7 +167,7 @@ run;
         data _null_;
           putlog 'NOTE-' / 'NOTE-';
           putlog "NOTE- &sysmacroname: table locked, waiting "@;
-          putlog "%sysfunc(sleep(&loop_inc)) seconds.. ";
+          putlog "%sysfunc(sleep(&loop_secs)) seconds.. ";
           putlog "NOTE- (iteration &x of &loops)";
           putlog 'NOTE-' / 'NOTE-';
         run;
@@ -194,32 +194,39 @@ run;
   %end;
 %end;
 %else %if &ACTION=UNLOCK %then %do;
-  %local status;
+  %local status cnt;
+  %let cnt=0;
   proc sql noprint;
-  select LOCK_STATUS_CD into: status from &ctl_ds
-    where LOCK_LIB ="&lib" and LOCK_DS="&ds";
-  quit;
-  %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
-  %if &status=LOCKED %then %do;
-    data _null_;
-      putlog "&sysmacroname: unlocking &lib..&ds:";
-    run;
-    proc sql;
-    update &ctl_ds
-      set LOCK_STATUS_CD='UNLOCKED'
-        , LOCK_END_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
-        , LOCK_USER_NM="&user"
-        , LOCK_PID="&sysjobid"
-        , LOCK_REF="&ref"
-      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
-    quit;
-  %end;
-  %else %if &status=UNLOCKED %then %do;
-    %put %str(WAR)NING: &lib..&ds is already unlocked!;
+  select count(*) into: cnt from &ctl_ds where LOCK_LIB ="&lib" & LOCK_DS="&ds";
+  %if &cnt=0 %then %do;
+    %put %str(WAR)NING: &lib..&ds was not previously locked in &ctl_ds!;
   %end;
   %else %do;
-    %put NOTE: Unrecognised STATUS_CD (&status) in &ctl_ds;
-    %let abortme=1;
+    select LOCK_STATUS_CD into: status from &ctl_ds
+      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+    quit;
+    %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
+    %if &status=LOCKED %then %do;
+      data _null_;
+        putlog "&sysmacroname: unlocking &lib..&ds:";
+      run;
+      proc sql;
+      update &ctl_ds
+        set LOCK_STATUS_CD='UNLOCKED'
+          , LOCK_END_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
+          , LOCK_USER_NM="&user"
+          , LOCK_PID="&sysjobid"
+          , LOCK_REF="&ref"
+        where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+      quit;
+    %end;
+    %else %if &status=UNLOCKED %then %do;
+      %put %str(WAR)NING: &lib..&ds is already unlocked!;
+    %end;
+    %else %do;
+      %put NOTE: Unrecognised STATUS_CD (&status) in &ctl_ds;
+      %let abortme=1;
+    %end;
   %end;
 %end;
 %else %do;

base/mp_perflog.sas

@@ -1,13 +1,14 @@
 /**
   @file
-  @brief Logs the time the macro was executed in a control dataset.
-  @details If the dataset does not exist, it is created.  Usage:
+  @brief Logs a message in a dataset every time it is invoked
+  @details If the dataset does not exist, it is created.
+  Usage:
 
-    %mp_perflog(started)
-    %mp_perflog()
-    %mp_perflog(startanew,libds=work.newdataset)
-    %mp_perflog(finished,libds=work.newdataset)
-    %mp_perflog(finished)
+      %mp_perflog(started)
+      %mp_perflog()
+      %mp_perflog(startanew,libds=work.newdataset)
+      %mp_perflog(finished,libds=work.newdataset)
+      %mp_perflog(finished)
 
 
   @param label Provide label to go into the control dataset

base/mp_storediffs.sas

@@ -147,14 +147,16 @@ run;
 
 %if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do;
   /* this is a format catalog - cannot query cols directly */
-  %let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH"
-    ,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO"
-    ,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
+  %let vlist="TYPE","FMTNAME","FMTROW","START","END","LABEL","MIN","MAX"
+    ,"DEFAULT","LENGTH","FUZZ","PREFIX","MULT","FILL","NOEDIT","SEXCL"
+    ,"EEXCL","HLO","DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
 %end;
 %else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
 
 data &ds4;
-  length &inds_keep $41 tgtvar_nm $32;
+  length &inds_keep $41 tgtvar_nm $32 _label_ $256;
+  if _n_=1 then call missing(_label_);
+  drop _label_;
   set &ds2 &ds3 indsname=&inds_auto;
 
   tgtvar_nm=upcase(tgtvar_nm);
@@ -163,7 +165,7 @@ data &ds4;
   if upcase(&inds_auto)="&ds2" then tgtvar_type='N';
   else if upcase(&inds_auto)="&ds3" then tgtvar_type='C';
   else do;
-    putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
+    putlog 'ERR' +(-1) "OR: unidentified vartype input!" &inds_auto;
     call symputx('syscc',98);
   end;
 
@@ -172,7 +174,7 @@ data &ds4;
   else if &inds_keep="&modds" then move_type='M';
   else if &inds_keep="&origds" then move_type='O';
   else do;
-    putlog "%str(ERR)OR: unidentified movetype input!" &inds_keep;
+    putlog 'ERR' +(-1) "OR: unidentified movetype input!" &inds_keep;
     call symputx('syscc',99);
   end;
   tgtvar_nm=upcase(tgtvar_nm);

ddl/mddl_sas_cntlout.sas

@@ -1,24 +1,27 @@
 /**
   @file
   @brief The CNTLOUT table generated by proc format
-  @details This table will actually change format depending on the data values,
-  therefore the max possible lengths are described here to enable consistency
-  when dealing with format data.
+  @details The actual CNTLOUT table may have varying variable lengths,
+  depending on the data values, therefore the max possible lengths
+  (given various practical restrictions) are described here to enable
+  consistency when dealing with format data.
 
 **/
 
 
 %macro mddl_sas_cntlout(libds=WORK.CNTLOUT);
 
-proc sql;
-create table &libds(
-    FMTNAME char(32)      label='Format name'
+  proc sql;
+  create table &libds(
+    TYPE char(1) label='Type of format - either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
+    ,FMTNAME char(32)     label='Format name'
+    ,FMTROW num label='CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
+    ,START char(32767)    label='Starting value for format'
     /*
-      to accommodate larger START values, mp_loadformat.sas will need the
-      SQL dependency removed (proc sql needs to accommodate 3 index values in
-      a 32767 ibufsize limit)
+      Keep lengths of START and END the same to avoid this err:
+      "Start is greater than end:  -<."
+      Similar usage note: https://support.sas.com/kb/69/330.html
     */
-    ,START char(10000)    label='Starting value for format'
     ,END char(32767)      label='Ending value for format'
     ,LABEL char(32767)    label='Format value label'
     ,MIN num length=3     label='Minimum length'
@@ -30,14 +33,25 @@ create table &libds(
     ,MULT num             label='Multiplier'
     ,FILL char(1)         label='Fill character'
     ,NOEDIT num length=3  label='Is picture string noedit?'
-    ,TYPE char(1)         label='Type of format'
     ,SEXCL char(1)        label='Start exclusion'
     ,EEXCL char(1)        label='End exclusion'
-    ,HLO char(13)         label='Additional information'
+    ,HLO char(13)         label='Additional information. M=MultiLabel'
     ,DECSEP char(1)       label='Decimal separator'
     ,DIG3SEP char(1)      label='Three-digit separator'
     ,DATATYPE char(8)     label='Date/time/datetime?'
     ,LANGUAGE char(8)     label='Language for date strings'
-);
+  );
+
+  %local lib;
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create
+      pk_cntlout=(type fmtname fmtrow)
+      /nomiss unique;
+  quit;
 
 %mend mddl_sas_cntlout;

meta/mm_adduser2group.sas

@@ -81,7 +81,8 @@ run;
 filename __us2grp temp;
 
 proc metadata in= "<UpdateMetadata><Reposid>$METAREPOSITORY</Reposid><Metadata>
-    <Person Id='&uuri'><IdentityGroups><IdentityGroup ObjRef='&guri' />
+    <Person Id='%nrstr(&uuri)'>
+    <IdentityGroups><IdentityGroup ObjRef='%nrstr(&guri)' />
     </IdentityGroups></Person></Metadata>
     <NS>SAS</NS><Flags>268435456</Flags></UpdateMetadata>"
   out=__us2grp verbose;
@@ -98,4 +99,4 @@ run;
 
 filename __us2grp clear;
 
-%mend mm_adduser2group;
+%mend mm_adduser2group;

meta/mm_createwebservice.sas

@@ -93,14 +93,16 @@ data _null_;
   file sasjs lrecl=3000 ;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
-  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
-  put '  ,showmeta=NO ';
+  put '  ,showmeta=N ';
+  put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols; ';
+  put '%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval ';
+  put '  tmpds1 tmpds2 tmpds3 tmpds4; ';
   put '%let numcols=0; ';
+  put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
@@ -136,7 +138,7 @@ data _null_;
   put '    by varnum; ';
   put '  run; ';
   put '  /* move meta to mac vars */ ';
-  put '  data _null_; ';
+  put '  data &colinfo; ';
   put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
   put '    set &colinfo end=last nobs=nobs; ';
   put '    name=upcase(name); ';
@@ -147,7 +149,6 @@ data _null_;
   put '      if format='''' then fmt=cats(''$'',length,''.''); ';
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else fmt=cats(format,formatl,''.''); ';
-  put '      newlen=max(formatl,length); ';
   put '    end; ';
   put '    else do; ';
   put '      typelong=''num''; ';
@@ -155,23 +156,26 @@ data _null_;
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else if formatd=0 then fmt=cats(format,formatl,''.''); ';
   put '      else fmt=cats(format,formatl,''.'',formatd); ';
-  put '      /* needs to be wide, for datetimes etc */ ';
-  put '      newlen=max(length,formatl,24); ';
   put '    end; ';
   put '    /* 32 char unique name */ ';
   put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
   put ' ';
   put '    call symputx(cats(''name'',_n_),name,''l''); ';
   put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
-  put '    call symputx(cats(''len'',_n_),newlen,''l''); ';
   put '    call symputx(cats(''length'',_n_),length,''l''); ';
   put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
+  put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
+  put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
+  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '  proc sql; ';
+  put '  select count(*) into: lastobs from &ds; ';
+  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -179,7 +183,9 @@ data _null_;
   put '      %put &sysmacroname: Switching to DATASTEP engine; ';
   put '      %goto datastep; ';
   put '    %end; ';
-  put '    data &tempds;set &ds; ';
+  put '    data &tempds; ';
+  put '      set &ds; ';
+  put '      &stmt_obs; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
   put '    /* PRETTY is necessary to avoid line truncation in large files */ ';
   put '    filename _sjs2 temp lrecl=131068 encoding=''utf-8''; ';
@@ -206,26 +212,99 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      data _data_; ';
+  put '      /** ';
+  put '        * Extract format definitions ';
+  put '        * First, by getting library locations from dictionary.formats ';
+  put '        * Then, by exporting the width using proc format ';
+  put '        * Cannot use maxw from sashelp.vformat as not always populated ';
+  put '        * Cannot use fmtinfo() as not supported in all flavours ';
+  put '        */ ';
+  put '      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      proc sql noprint; ';
+  put '      create table &tmpds1 as ';
+  put '          select cats(libname,''.'',memname) as FMTCAT, ';
+  put '          FMTNAME ';
+  put '        from dictionary.formats ';
+  put '        where fmttype=''F'' and libname is not null ';
+  put '          and fmtname in (select format from &colinfo where format is not null) ';
+  put '        order by 1; ';
+  put '      create table &tmpds2( ';
+  put '          FMTNAME char(32), ';
+  put '          LENGTH num ';
+  put '      ); ';
+  put '      %local catlist cat fmtlist i; ';
+  put '      select distinct fmtcat into: catlist separated by '' '' from &tmpds1; ';
+  put '      %do i=1 %to %sysfunc(countw(&catlist,%str( ))); ';
+  put '        %let cat=%scan(&catlist,&i,%str( )); ';
+  put '        proc sql; ';
+  put '        select distinct fmtname into: fmtlist separated by '' '' ';
+  put '          from &tmpds1 where fmtcat="&cat"; ';
+  put '        proc format lib=&cat cntlout=&tmpds3(keep=fmtname length); ';
+  put '          select &fmtlist; ';
+  put '        run; ';
+  put '        proc sql; ';
+  put '        insert into &tmpds2 select distinct fmtname,length from &tmpds3; ';
+  put '      %end; ';
+  put ' ';
+  put '      proc sql; ';
+  put '      create table &tmpds4 as ';
+  put '        select a.*, b.length as MAXW ';
+  put '        from &colinfo a ';
+  put '        left join &tmpds2 b ';
+  put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
+  put '        order by a.varnum; ';
+  put '      data _null_; ';
+  put '        set &tmpds4; ';
+  put '        if not missing(maxw); ';
+  put '        call symputx( ';
+  put '          cats(''fmtlen'',_n_), ';
+  put '          /* vars need extra padding due to JSON escaping of special chars */ ';
+  put '          min(32767,ceil((max(length,maxw)+10)*1.5)) ';
+  put '          ,''l'' ';
+  put '        ); ';
+  put '      run; ';
+  put ' ';
+  put '      /* configure varlenchk - as we are explicitly shortening the variables */ ';
+  put '      %let optval=%sysfunc(getoption(varlenchk)); ';
+  put '      options varlenchk=NOWARN; ';
+  put '      data _data_(compress=char); ';
+  put '        /* shorten the new vars */ ';
+  put '        length ';
+  put '      %do i=1 %to &numcols; ';
+  put '          &&name&i $&&fmtlen&i ';
+  put '      %end; ';
+  put '          ; ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
-  put '        &&name&i=&&newname&i ';
+  put '          &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
+  put '      &stmt_obs; ';
+  put ' ';
+  put '      drop ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&newname&i ';
+  put '      %end; ';
+  put '        ; ';
   put '      %do i=1 %to &numcols; ';
-  put '        length &&name&i $&&len&i; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
-  put '        drop &&newname&i; ';
   put '      %end; ';
-  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '        if _error_ then do; ';
+  put '          call symputx(''syscc'',1012); ';
+  put '          stop; ';
+  put '        end; ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
+  put '      options varlenchk=&optval; ';
   put '    %end; ';
   put ' ';
   put '    proc format; /* credit yabwon for special null removal */ ';
@@ -244,8 +323,8 @@ data _null_;
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $32767; ';
-  put '          format &&name&i $32767.; ';
+  put '          length &&name&i $&&fmtlen&i...; ';
+  put '          format &&name&i $&&fmtlen&i...; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -254,10 +333,11 @@ data _null_;
   put '      %else %do; ';
   put '        set &ds; ';
   put '      %end; ';
+  put '      &stmt_obs; ';
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; ';
+  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
   put '          &&name&i=''"''!!trim( ';
   put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
   put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
@@ -270,8 +350,9 @@ data _null_;
   put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
   put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
   put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
   put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '          ))))))))))))!!''"''; ';
+  put '          )))))))))))))!!''"''; ';
   put '        end; ';
   put '        else &&name&i=quote(cats(&&name&i)); ';
   put '      %end; ';
@@ -310,10 +391,11 @@ data _null_;
   put '  proc sql; ';
   put '  drop table &colinfo, &tempds; ';
   put ' ';
-  put '  %if &showmeta=YES %then %do; ';
+  put '  %if %substr(&showmeta,1,1)=Y %then %do; ';
   put '    filename _sjs4 temp lrecl=131068 encoding=''utf-8''; ';
   put '    data _null_; ';
   put '      file _sjs4; ';
+  put '      length label $350; ';
   put '      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; ';
   put '      do i=1 to &numcols; ';
   put '        name=quote(trim(symget(cats(''name'',i)))); ';
@@ -365,8 +447,8 @@ data _null_;
   put '  %quote(&user) ';
   put ' ';
   put '%mend mf_getuser; ';
-  put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
-  put '  ,showmeta=NO ';
+  put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL ';
+  put '  ,showmeta=N,maxobs=MAX,workobs=0 ';
   put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
@@ -433,15 +515,14 @@ data _null_;
   put ' ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta ';
+  put '    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
   put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
   put '  /* To avoid issues with _webout on EBI we use a temporary file */ ';
   put '  filename _sjsref temp lrecl=131068; ';
-  put '  %if %str(&_debug) ge 131 %then %do; ';
-  put '    /* if debug mode, send back first 10 records of each work table also */ ';
-  put '    options obs=10; ';
+  put '  %if %str(&workobs) > 0 %then %do; ';
+  put '    /* if debug mode, send back first XX records of each work table also */ ';
   put '    data;run;%let tempds=%scan(&syslast,2,.); ';
   put '    ods output Members=&tempds; ';
   put '    proc datasets library=WORK memtype=data; ';
@@ -465,7 +546,9 @@ data _null_;
   put '        put " ""&wt"" : {"; ';
   put '        put ''"nlobs":'' nlobs; ';
   put '        put '',"nvars":'' nvars; ';
-  put '      %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=YES) ';
+  put '      %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=Y ';
+  put '        ,maxobs=&workobs ';
+  put '      ) ';
   put '      data _null_; file _sjsref mod encoding=''utf-8''; ';
   put '        put "}"; ';
   put '    %end; ';
@@ -475,26 +558,69 @@ data _null_;
   put '  %end; ';
   put '  /* close off json */ ';
   put '  data _null_;file _sjsref mod encoding=''utf-8''; ';
-  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
-  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
-  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
+  put '    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; ';
   put '    put ",""_DEBUG"" : ""&_debug"" "; ';
   put '    _METAUSER=quote(trim(symget(''_METAUSER''))); ';
   put '    put ",""_METAUSER"": " _METAUSER; ';
   put '    _METAPERSON=quote(trim(symget(''_METAPERSON''))); ';
   put '    put '',"_METAPERSON": '' _METAPERSON; ';
+  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
+  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
+  put '    put '',"AUTOEXEC" : '' autoexec; ';
+  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
   put '    put ",""SYSENCODING"" : ""&sysencoding"" "; ';
-  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    syserrortext=cats(symget(''syserrortext'')); ';
+  put '    if findc(syserrortext,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
+  put '      syserrortext=''"''!!trim( ';
+  put '        prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '        prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '        prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '        prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '        prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '        prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '        prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '        prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '        prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '        prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '        prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '        prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
+  put '        prxchange(''s/\\/\\\\/'',-1,syserrortext) ';
+  put '      )))))))))))))!!''"''; ';
+  put '    end; ';
+  put '    else syserrortext=cats(''"'',syserrortext,''"''); ';
   put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
+  put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
+  put '    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; ';
+  put '    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); ';
+  put '    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
+  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    syswarningtext=cats(symget(''syswarningtext'')); ';
+  put '    if findc(syswarningtext,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
+  put '      syswarningtext=''"''!!trim( ';
+  put '        prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '        prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '        prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '        prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '        prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '        prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '        prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '        prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '        prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '        prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '        prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '        prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
+  put '        prxchange(''s/\\/\\\\/'',-1,syswarningtext) ';
+  put '      )))))))))))))!!''"''; ';
+  put '    end; ';
+  put '    else syswarningtext=cats(''"'',syswarningtext,''"''); ';
   put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';
@@ -520,9 +646,11 @@ data _null_;
   put ' ';
   put '%mend mm_webout; ';
 /* WEBOUT END */
-  put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO);';
+  put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO';
+  put '    ,maxobs=MAX';
+  put ');';
   put '  %mm_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt,missing=&missing';
-  put '    ,showmeta=&showmeta';
+  put '    ,showmeta=&showmeta,maxobs=&maxobs';
   put '  )';
   put '%mend;';
 run;

meta/mm_getstpinfo.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Get the properties of a Stored Process
+  @details Extracts various properties and creates an output table in the
+  structure below:
+
+|STP_URI:$200.|SERVERCONTEXT:$200.|STOREDPROCESSCONFIGURATION:$1000.|SOURCECODE_FIRST32K:$32767.|PATH:$76.|
+|---|---|---|---|---|
+|`A5DN9TDQ.BH0000C8 `|`SASApp `|`<?xml version="1.0" encoding="UTF-8"?><StoredProcess><ServerContext LogicalServerType="Sps" OtherAllowed="false"/><ResultCapabilities Package="false" Streaming="true"/><OutputParameters/></StoredProcess> `|`%put first 32767 bytes of code; `|`/path/to/my/stp`|
+
+  @param [in] pgm The metadata path of the Stored Process
+  @param [out] outds= (work.mm_getstpinfo) The output table to create
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> Related Files </h4>
+  @li mm_getstpcode.sas
+  @li mm_getstps.sas
+  @li mm_createstp.sas
+  @li mm_deletestp.sas
+
+**/
+
+%macro mm_getstpinfo(pgm
+  ,outds=work.mm_getstpinfo
+  ,mDebug=0
+);
+
+%local mD;
+%if &mDebug=1 %then %let mD=;
+%else %let mD=%str(*);
+%&mD.put Executing &sysmacroname..sas;
+%&mD.put _local_;
+
+data &outds;
+  length type stp_uri tsuri servercontext value $200
+    StoredProcessConfiguration $1000 sourcecode_first32k $32767;
+  keep path stp_uri sourcecode_first32k StoredProcessConfiguration
+    servercontext;
+  call missing (of _all_);
+  path="&pgm(StoredProcess)";
+  /* first, find the STP ID */
+  if metadata_pathobj("",path,"StoredProcess",type,stp_uri)>0 then do;
+    /* get attributes */
+    cnt=1;
+    do while (metadata_getnasn(stp_uri,"Notes",cnt,tsuri)>0);
+      rc1=metadata_getattr(tsuri,"Name",value);
+      &mD.put tsuri= value=;
+      if value="SourceCode" then do;
+        rc2=metadata_getattr(tsuri,"StoredText",sourcecode_first32k);
+      end;
+      else if value="Stored Process" then do;
+        rc3=metadata_getattr(tsuri,"StoredText",StoredProcessConfiguration);
+      end;
+      cnt+1;
+    end;
+    /* get context (should only be one) */
+    rc4=metadata_getnasn(stp_uri,"ComputeLocations",1,tsuri);
+    rc5=metadata_getattr(tsuri,"Name",servercontext);
+  end;
+  else do;
+    put "%str(ERR)OR: could not find " path;
+    put (_all_)(=);
+  end;
+  &md.put (_all_)(=);
+run;
+
+%mend mm_getstpinfo ;

meta/mm_webout.sas

@@ -26,25 +26,33 @@
   @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE
   @param [in] ds The dataset to send back to the frontend
   @param [out] dslabel= Value to use instead of table name for sending to JSON
-  @param [in] fmt=(Y) Set to N to send back unformatted values
+  @param [in] fmt= (N) Setting Y converts all vars to their formatted values
   @param [out] fref= (_webout) The fileref to which to write the JSON
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
-  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+  @param [in] showmeta= (N) Set to Y to output metadata alongside each table,
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
-
+  @param [in] workobs= (0) When set to a positive integer, will create a new
+    output object (WORK) which contains this number of observations from all
+    tables in the WORK library.
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to output JSON
 
   <h4> SAS Macros </h4>
   @li mp_jsonout.sas
 
+  <h4> Related Macros </h4>
+  @li ms_webout.sas
+  @li mv_webout.sas
+
   @version 9.3
   @author Allan Bowe
 
 **/
-%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
-  ,showmeta=NO
+%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL
+  ,showmeta=N,maxobs=MAX,workobs=0
 );
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
@@ -111,15 +119,14 @@
 
 %else %if &action=ARR or &action=OBJ %then %do;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta
+    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta,maxobs=&maxobs
   )
 %end;
 %else %if &action=CLOSE %then %do;
   /* To avoid issues with _webout on EBI we use a temporary file */
   filename _sjsref temp lrecl=131068;
-  %if %str(&_debug) ge 131 %then %do;
-    /* if debug mode, send back first 10 records of each work table also */
-    options obs=10;
+  %if %str(&workobs) > 0 %then %do;
+    /* if debug mode, send back first XX records of each work table also */
     data;run;%let tempds=%scan(&syslast,2,.);
     ods output Members=&tempds;
     proc datasets library=WORK memtype=data;
@@ -143,7 +150,9 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=YES)
+      %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=Y
+        ,maxobs=&workobs
+      )
       data _null_; file _sjsref mod encoding='utf-8';
         put "}";
     %end;
@@ -153,26 +162,69 @@
   %end;
   /* close off json */
   data _null_;file _sjsref mod encoding='utf-8';
-    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
-    put ",""SYSUSERID"" : ""&sysuserid"" ";
-    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
+    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512;
     put ",""_DEBUG"" : ""&_debug"" ";
     _METAUSER=quote(trim(symget('_METAUSER')));
     put ",""_METAUSER"": " _METAUSER;
     _METAPERSON=quote(trim(symget('_METAPERSON')));
     put ',"_METAPERSON": ' _METAPERSON;
+    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ',"_PROGRAM" : ' _PROGRAM ;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
+    put ',"AUTOEXEC" : ' autoexec;
+    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""SYSCC"" : ""&syscc"" ";
     put ",""SYSENCODING"" : ""&sysencoding"" ";
-    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    syserrortext=cats(symget('syserrortext'));
+    if findc(syserrortext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syserrortext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syserrortext)
+      )))))))))))))!!'"';
+    end;
+    else syserrortext=cats('"',syserrortext,'"');
     put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
+    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
+    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
+    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    syswarningtext=cats(symget('syswarningtext'));
+    if findc(syswarningtext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syswarningtext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syswarningtext)
+      )))))))))))))!!'"';
+    end;
+    else syswarningtext=cats('"',syswarningtext,'"');
     put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;

metax/mmx_createmetafolder.sas

@@ -0,0 +1,50 @@
+/**
+  @file
+  @brief Creates a metadata folder
+  @details Creates a metadata folder using the batch tools
+
+  Usage:
+
+      %mmx_createmetafolder(loc=/some/meta/folder,user=sasdemo,pass=mars345)
+
+  <h4> SAS Macros </h4>
+  @li mf_loc.sas
+  @li mp_abort.sas
+
+  @param loc= the metadata folder to delete
+  @param user= username
+  @param pass= password
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mmx_createmetafolder(loc=,user=,pass=);
+
+%local host port path connx_string msg;
+%let host=%sysfunc(getoption(metaserver));
+%let port=%sysfunc(getoption(metaport));
+%let path=%mf_loc(POF)/tools;
+
+%let connx_string= -host &host -port &port -user '&user' -password '&pass';
+/* remove directory */
+data _null_;
+  infile " &path/sas-make-folder &connx_string ""&loc"" -makeFullPath 2>&1"
+    pipe lrecl=10000;
+  input;
+  putlog _infile_;
+run;
+
+data _null_; /* check tree exists */
+  length type uri $256;
+  rc=metadata_pathobj("","&loc","Folder",type,uri);
+  call symputx('foldertype',type,'l');
+run;
+%let msg=Location (&loc) was not created!!;
+%mp_abort(iftrue= (&foldertype ne Tree)
+  ,mac=&_program..sas
+  ,msg=%superq(msg)
+)
+
+%mend mmx_createmetafolder;

metax/mmx_deletemetafolder.sas

@@ -7,7 +7,7 @@
 
   Usage:
 
-    %mmx_deletemetafolder(loc=/some/meta/folder,user=sasdemo,pass=mars345)
+      %mmx_deletemetafolder(loc=/some/meta/folder,user=sasdemo,pass=mars345)
 
   <h4> SAS Macros </h4>
   @li mf_loc.sas
@@ -37,4 +37,4 @@ data _null_;
   putlog _infile_;
 run;
 
-%mend mmx_deletemetafolder;
+%mend mmx_deletemetafolder;

package.json

@@ -33,6 +33,6 @@
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "3.13.0"
+    "@sasjs/cli": "^4.4.1"
   }
-}
+}

sasjs/doxy/new_header.html

@@ -25,6 +25,21 @@
   <link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
   <link rel="shortcut icon" href="$relpath^favicon.ico" type="image/x-icon" />
   $extrastylesheet
+  <!-- Matomo -->
+  <script>
+    var _paq = window._paq = window._paq || [];
+    /* tracker methods like "setCustomDimension" should be called before "trackPageView" */
+    _paq.push(['trackPageView']);
+    _paq.push(['enableLinkTracking']);
+    (function () {
+      var u = "https://analytics.4gl.io/";
+      _paq.push(['setTrackerUrl', u + 'matomo.php']);
+      _paq.push(['setSiteId', '6']);
+      var d = document, g = d.createElement('script'), s = d.getElementsByTagName('script')[0];
+      g.async = true; g.src = u + 'matomo.js'; s.parentNode.insertBefore(g, s);
+    })();
+  </script>
+  <!-- End Matomo Code -->
 </head>
 
 <body>
@@ -68,4 +83,4 @@
   </div>
 </body>
 
-</html>
+</html>

sasjs/sasjsconfig.json

@@ -67,12 +67,16 @@
     },
     {
       "name": "server",
-      "serverUrl": "",
+      "serverUrl": "https://sas9.4gl.io",
       "serverType": "SASJS",
       "httpsAgentOptions": {
         "allowInsecureRequests": false
       },
       "appLoc": "/sasjs/core",
+      "deployConfig": {
+        "deployServicePack": true,
+        "deployScripts": []
+      },
       "macroFolders": [
         "server",
         "tests/serveronly"
@@ -81,7 +85,7 @@
     {
       "name": "docsonly",
       "serverType": "SASJS",
-      "appLoc": "dummy",
+      "appLoc": "/dummy",
       "macroFolders": [
         "meta",
         "metax",
@@ -105,6 +109,16 @@
         "deployServicePack": true
       },
       "contextName": "SAS Job Execution compute context"
+    },
+    {
+      "name": "sasjs9",
+      "serverUrl": "https://sas9.4gl.io",
+      "serverType": "SASJS",
+      "appLoc": "/Public/app/sasjs9",
+      "deployConfig": {
+        "deployServicePack": true,
+        "deployScripts": []
+      }
     }
   ]
-}
+}

sasjs/utils/create_sas_package.sas

@@ -0,0 +1,224 @@
+/**
+  @file
+  @brief Deploy repo as a SAS PACKAGES module
+  @details After every release, this program is executed to update the SASPAC
+    repo with the latest macros (and same version number).
+    The program is first compiled using sasjs compile, then executed using
+    sasjs run.
+
+    Requires the server to have SSH keys.
+
+  <h4> SAS Macros </h4>
+  @li mp_gitadd.sas
+  @li mp_gitreleaseinfo.sas
+  @li mp_gitstatus.sas
+
+**/
+
+
+/* get package version */
+%mp_gitreleaseinfo(GITHUB,sasjs/core,outlib=splib)
+data _null_;
+  set splib.root;
+  call symputx('version',substr(TAG_NAME,2));
+run;
+
+/* clone the source repo */
+%let dir = %sysfunc(pathname(work))/core;
+%put source clone rc=%sysfunc(GITFN_CLONE(https://github.com/sasjs/core,&dir));
+
+/*
+  clone the target repo.
+  If you have issues, see: https://stackoverflow.com/questions/74082874
+*/
+options dlcreatedir;
+%let dirOut = %sysfunc(pathname(work))/package;
+libname _ "&dirOut.";
+
+%put tgt clone rc=%sysfunc(GITFN_CLONE(
+  git@github.com:SASPAC/sasjscore.git,
+  &dirOut,
+  git,
+  %str( ),
+  /home/sasjssrv/.ssh/id_ecdsa.pub,
+  /home/sasjssrv/.ssh/id_ecdsa
+));
+
+
+/*
+  Prepare Package Metadata
+*/
+data _null_;
+  infile CARDS4;
+  file  "&dirOut./description.sas";
+  input;
+  if _infile_ =: 'Version:' then put "Version: &version.";
+                            else put _infile_;
+CARDS4;
+Type: Package
+Package: SASjsCore
+Title: SAS Macros for Application Development
+Version: $(PLACEHOLDER)
+Author: Allan Bowe
+Maintainer: 4GL Ltd
+License: MIT
+Encoding: UTF8
+
+DESCRIPTION START:
+
+The SASjs Macro Core library is a component of the SASjs framework, the
+source for which is avaible here: https://github.com/sasjs
+
+Macros are divided by:
+
+* Macro Functions (prefix mf_)
+* Macro Procedures (prefix mp_)
+* Macros for Metadata (prefix mm_)
+* Macros for SASjs Server (prefix ms_)
+* Macros for Viya (prefix mv_)
+
+DESCRIPTION END:
+;;;;
+run;
+
+/*
+  Prepare Package License
+*/
+data _null_;
+  file  "&dirOut./license.sas";
+  infile "&dir/LICENSE";
+  input;
+  put _infile_;
+run;
+
+/*
+  Extract Core files into MacroCore Package location
+*/
+data members(compress=char);
+  length dref dref2 $ 8 name name2 $ 32 path $ 2048;
+  rc = filename(dref, "&dir.");
+  put dref=;
+  did = dopen(dref);
+  if did then
+    do i = 1 to dnum(did);
+      name = dread(did, i);
+      if name in
+        ("base" "ddl" "fcmp" "lua" "meta" "metax" "server" "viya" "xplatform")
+      then do;
+          rc = filename(dref2,catx("/", "&dir.", name));
+          put dref2= name;
+          did2 = dopen(dref2);
+
+          if did2 then
+            do j = 1 to dnum(did2);
+              name2 = dread(did2, j);
+              path = catx("/", "&dir.", name, name2);
+              if "sas" = scan(name2, -1, ".") then output;
+            end;
+          rc = dclose(did2);
+          rc = filename(dref2);
+        end;
+    end;
+  rc = dclose(did);
+  rc = filename(dref);
+  keep name name2 path;
+run;
+
+%let temp_options = %sysfunc(getoption(source)) %sysfunc(getoption(notes));
+options nosource nonotes;
+data _null_;
+  set members;
+  by name notsorted;
+
+  ord + first.name;
+
+  if first.name then
+    do;
+      call execute('libname _ '
+        !! quote(catx("/", "&dirOut.", put(ord, z3.)!!"_macros"))
+        !! ";"
+      );
+      put @1 "./" ord z3. "_macros/";
+    end;
+
+  put @10 name2;
+  call execute("
+  data _null_;
+    infile " !! quote(strip(path)) !! ";
+    file " !! quote(catx("/", "&dirOut.", put(ord, z3.)!!"_macros", name2)) !!";
+    input;
+    select;
+      when (2 = trigger) put _infile_;
+      when (_infile_ = '/**') do; put '/*** HELP START ***//**'; trigger+1; end;
+      when (_infile_ = '**/') do; put '**//*** HELP END ***/';   trigger+1; end;
+      otherwise put _infile_;
+    end;
+  run;");
+
+run;
+options &temp_options.;
+
+/*
+  Generate SASjsCore Package
+*/
+%GeneratePackage(
+  filesLocation=&dirOut
+)
+
+/**
+  * apply new version in a github action
+  * 1. create folder
+  * 2. create template yaml
+  * 3. replace version number
+  */
+
+%mf_mkdir(&dirout/.github/workflows)
+
+%let desc=Version &version of sasjs/core is now on SAS PACKAGES :ok_hand:;
+data _null_;
+  file "&dirout/.github/workflows/release.yml";
+  put "name: SASjs Core Package Publish Tag";
+  put "on:";
+  put "  push:";
+  put "    branches:";
+  put "      - main";
+  put "jobs:";
+  put "  update:";
+  put "    runs-on: ubuntu-latest";
+  put "    steps:";
+  put "      - uses: actions/checkout@master";
+  put "      - name: Make Release";
+  put "        uses: alice-biometrics/release-creator/@v1.0.5";
+  put "        with:";
+  put "          github_token: ${{ secrets.GH_TOKEN }}";
+  put "          branch: main";
+  put "          draft: false";
+  put "          version: &version";
+  put "          description: '&desc'";
+run;
+
+
+/**
+  * Add, Commit & Push!
+  */
+%mp_gitstatus(&dirout,outds=work.gitstatus,mdebug=1)
+%mp_gitadd(&dirout,inds=work.gitstatus,mdebug=1)
+
+data _null_;
+  rc=gitfn_commit("&dirout"
+    ,"HEAD","&sysuserid","sasjs@core"
+    ,"FEAT: Releasing &version"
+  );
+  put rc=;
+  rc=git_push(
+    "&dirout"
+    ,"git"
+    ,""
+    ,"/home/sasjssrv/.ssh/id_ecdsa.pub"
+    ,"/home/sasjssrv/.ssh/id_ecdsa"
+  );
+run;
+
+
+
+

server/ms_createwebservice.sas

@@ -94,14 +94,16 @@ data _null_;
   file &sasjsref termstr=crlf lrecl=512;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
-  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
-  put '  ,showmeta=NO ';
+  put '  ,showmeta=N ';
+  put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols; ';
+  put '%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval ';
+  put '  tmpds1 tmpds2 tmpds3 tmpds4; ';
   put '%let numcols=0; ';
+  put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
@@ -137,7 +139,7 @@ data _null_;
   put '    by varnum; ';
   put '  run; ';
   put '  /* move meta to mac vars */ ';
-  put '  data _null_; ';
+  put '  data &colinfo; ';
   put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
   put '    set &colinfo end=last nobs=nobs; ';
   put '    name=upcase(name); ';
@@ -148,7 +150,6 @@ data _null_;
   put '      if format='''' then fmt=cats(''$'',length,''.''); ';
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else fmt=cats(format,formatl,''.''); ';
-  put '      newlen=max(formatl,length); ';
   put '    end; ';
   put '    else do; ';
   put '      typelong=''num''; ';
@@ -156,23 +157,26 @@ data _null_;
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else if formatd=0 then fmt=cats(format,formatl,''.''); ';
   put '      else fmt=cats(format,formatl,''.'',formatd); ';
-  put '      /* needs to be wide, for datetimes etc */ ';
-  put '      newlen=max(length,formatl,24); ';
   put '    end; ';
   put '    /* 32 char unique name */ ';
   put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
   put ' ';
   put '    call symputx(cats(''name'',_n_),name,''l''); ';
   put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
-  put '    call symputx(cats(''len'',_n_),newlen,''l''); ';
   put '    call symputx(cats(''length'',_n_),length,''l''); ';
   put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
+  put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
+  put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
+  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '  proc sql; ';
+  put '  select count(*) into: lastobs from &ds; ';
+  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -180,7 +184,9 @@ data _null_;
   put '      %put &sysmacroname: Switching to DATASTEP engine; ';
   put '      %goto datastep; ';
   put '    %end; ';
-  put '    data &tempds;set &ds; ';
+  put '    data &tempds; ';
+  put '      set &ds; ';
+  put '      &stmt_obs; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
   put '    /* PRETTY is necessary to avoid line truncation in large files */ ';
   put '    filename _sjs2 temp lrecl=131068 encoding=''utf-8''; ';
@@ -207,26 +213,99 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      data _data_; ';
+  put '      /** ';
+  put '        * Extract format definitions ';
+  put '        * First, by getting library locations from dictionary.formats ';
+  put '        * Then, by exporting the width using proc format ';
+  put '        * Cannot use maxw from sashelp.vformat as not always populated ';
+  put '        * Cannot use fmtinfo() as not supported in all flavours ';
+  put '        */ ';
+  put '      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      proc sql noprint; ';
+  put '      create table &tmpds1 as ';
+  put '          select cats(libname,''.'',memname) as FMTCAT, ';
+  put '          FMTNAME ';
+  put '        from dictionary.formats ';
+  put '        where fmttype=''F'' and libname is not null ';
+  put '          and fmtname in (select format from &colinfo where format is not null) ';
+  put '        order by 1; ';
+  put '      create table &tmpds2( ';
+  put '          FMTNAME char(32), ';
+  put '          LENGTH num ';
+  put '      ); ';
+  put '      %local catlist cat fmtlist i; ';
+  put '      select distinct fmtcat into: catlist separated by '' '' from &tmpds1; ';
+  put '      %do i=1 %to %sysfunc(countw(&catlist,%str( ))); ';
+  put '        %let cat=%scan(&catlist,&i,%str( )); ';
+  put '        proc sql; ';
+  put '        select distinct fmtname into: fmtlist separated by '' '' ';
+  put '          from &tmpds1 where fmtcat="&cat"; ';
+  put '        proc format lib=&cat cntlout=&tmpds3(keep=fmtname length); ';
+  put '          select &fmtlist; ';
+  put '        run; ';
+  put '        proc sql; ';
+  put '        insert into &tmpds2 select distinct fmtname,length from &tmpds3; ';
+  put '      %end; ';
+  put ' ';
+  put '      proc sql; ';
+  put '      create table &tmpds4 as ';
+  put '        select a.*, b.length as MAXW ';
+  put '        from &colinfo a ';
+  put '        left join &tmpds2 b ';
+  put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
+  put '        order by a.varnum; ';
+  put '      data _null_; ';
+  put '        set &tmpds4; ';
+  put '        if not missing(maxw); ';
+  put '        call symputx( ';
+  put '          cats(''fmtlen'',_n_), ';
+  put '          /* vars need extra padding due to JSON escaping of special chars */ ';
+  put '          min(32767,ceil((max(length,maxw)+10)*1.5)) ';
+  put '          ,''l'' ';
+  put '        ); ';
+  put '      run; ';
+  put ' ';
+  put '      /* configure varlenchk - as we are explicitly shortening the variables */ ';
+  put '      %let optval=%sysfunc(getoption(varlenchk)); ';
+  put '      options varlenchk=NOWARN; ';
+  put '      data _data_(compress=char); ';
+  put '        /* shorten the new vars */ ';
+  put '        length ';
+  put '      %do i=1 %to &numcols; ';
+  put '          &&name&i $&&fmtlen&i ';
+  put '      %end; ';
+  put '          ; ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
-  put '        &&name&i=&&newname&i ';
+  put '          &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
+  put '      &stmt_obs; ';
+  put ' ';
+  put '      drop ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&newname&i ';
+  put '      %end; ';
+  put '        ; ';
   put '      %do i=1 %to &numcols; ';
-  put '        length &&name&i $&&len&i; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
-  put '        drop &&newname&i; ';
   put '      %end; ';
-  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '        if _error_ then do; ';
+  put '          call symputx(''syscc'',1012); ';
+  put '          stop; ';
+  put '        end; ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
+  put '      options varlenchk=&optval; ';
   put '    %end; ';
   put ' ';
   put '    proc format; /* credit yabwon for special null removal */ ';
@@ -245,8 +324,8 @@ data _null_;
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $32767; ';
-  put '          format &&name&i $32767.; ';
+  put '          length &&name&i $&&fmtlen&i...; ';
+  put '          format &&name&i $&&fmtlen&i...; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -255,10 +334,11 @@ data _null_;
   put '      %else %do; ';
   put '        set &ds; ';
   put '      %end; ';
+  put '      &stmt_obs; ';
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; ';
+  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
   put '          &&name&i=''"''!!trim( ';
   put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
   put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
@@ -271,8 +351,9 @@ data _null_;
   put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
   put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
   put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
   put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '          ))))))))))))!!''"''; ';
+  put '          )))))))))))))!!''"''; ';
   put '        end; ';
   put '        else &&name&i=quote(cats(&&name&i)); ';
   put '      %end; ';
@@ -311,10 +392,11 @@ data _null_;
   put '  proc sql; ';
   put '  drop table &colinfo, &tempds; ';
   put ' ';
-  put '  %if &showmeta=YES %then %do; ';
+  put '  %if %substr(&showmeta,1,1)=Y %then %do; ';
   put '    filename _sjs4 temp lrecl=131068 encoding=''utf-8''; ';
   put '    data _null_; ';
   put '      file _sjs4; ';
+  put '      length label $350; ';
   put '      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; ';
   put '      do i=1 to &numcols; ';
   put '        name=quote(trim(symget(cats(''name'',i)))); ';
@@ -367,8 +449,8 @@ data _null_;
   put ' ';
   put '%mend mf_getuser; ';
   put ' ';
-  put '%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
-  put '  ,showmeta=NO ';
+  put '%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL ';
+  put '  ,showmeta=N,maxobs=MAX,workobs=0 ';
   put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
@@ -427,13 +509,12 @@ data _null_;
   put '    %let missing=NULL; ';
   put '  %end; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta ';
+  put '    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
   put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  %if %str(&_debug) ge 131 %then %do; ';
-  put '    /* if debug mode, send back first 10 records of each work table also */ ';
-  put '    options obs=10; ';
+  put '  %if %str(&workobs) > 0 %then %do; ';
+  put '    /* if debug mode, send back first XX records of each work table also */ ';
   put '    data;run;%let tempds=%scan(&syslast,2,.); ';
   put '    ods output Members=&tempds; ';
   put '    proc datasets library=WORK memtype=data; ';
@@ -458,7 +539,9 @@ data _null_;
   put '        put " ""&wt"" : {"; ';
   put '        put ''"nlobs":'' nlobs; ';
   put '        put '',"nvars":'' nvars; ';
-  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) ';
+  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=Y ';
+  put '        ,maxobs=&workobs ';
+  put '      ) ';
   put '      data _null_; file &fref mod encoding=''utf-8'' termstr=lf; ';
   put '        put "}"; ';
   put '    %end; ';
@@ -468,35 +551,70 @@ data _null_;
   put '  %end; ';
   put '  /* close off json */ ';
   put '  data _null_;file &fref mod encoding=''utf-8'' termstr=lf lrecl=32767; ';
-  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
-  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
-  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
+  put '    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; ';
   put '    put ",""_DEBUG"" : ""&_debug"" "; ';
+  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
+  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
+  put '    put '',"AUTOEXEC" : '' autoexec; ';
+  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
   put '    put ",""SYSENCODING"" : ""&sysencoding"" "; ';
-  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    syserrortext=cats(symget(''syserrortext'')); ';
+  put '    if findc(syserrortext,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
+  put '      syserrortext=''"''!!trim( ';
+  put '        prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '        prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '        prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '        prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '        prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '        prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '        prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '        prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '        prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '        prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '        prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '        prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
+  put '        prxchange(''s/\\/\\\\/'',-1,syserrortext) ';
+  put '      )))))))))))))!!''"''; ';
+  put '    end; ';
+  put '    else syserrortext=cats(''"'',syserrortext,''"''); ';
   put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    SYSHOSTINFOLONG=quote(trim(symget(''SYSHOSTINFOLONG''))); ';
   put '    put '',"SYSHOSTINFOLONG" : '' SYSHOSTINFOLONG; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
   put '    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; ';
-  put '    length SYSPROCESSNAME $512; ';
   put '    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); ';
   put '    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" "; ';
+  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    syswarningtext=cats(symget(''syswarningtext'')); ';
+  put '    if findc(syswarningtext,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
+  put '      syswarningtext=''"''!!trim( ';
+  put '        prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '        prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '        prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '        prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '        prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '        prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '        prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '        prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '        prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '        prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '        prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '        prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
+  put '        prxchange(''s/\\/\\\\/'',-1,syswarningtext) ';
+  put '      )))))))))))))!!''"''; ';
+  put '    end; ';
+  put '    else syswarningtext=cats(''"'',syswarningtext,''"''); ';
   put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
-  put '    length autoexec $512; ';
-  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
-  put '    put '',"AUTOEXEC" : '' autoexec; ';
   put '    length memsize $32; ';
   put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
   put '    memsize=quote(cats(memsize)); ';

server/ms_getgroups.sas

@@ -22,7 +22,7 @@
   @param [in] uid= (0) Provide the userid on which to filter
   @param [out] outds= (work.ms_getgroups) This output dataset will contain the
     list of groups. Format:
-|NAME:$32.|DESCRIPTION:$64.|GROUPID:best.|
+|NAME:$32.|DESCRIPTION:$256.|GROUPID:best.|
 |---|---|---|
 |`SomeGroup `|`A group `|`1`|
 |`Another Group`|`this is a different group`|`2`|
@@ -58,7 +58,7 @@
 %if %sysget(MODE)=desktop %then %do;
   /* groups api does not exist in desktop mode */
   data &outds;
-    length NAME $32 DESCRIPTION $64. GROUPID 8;
+    length NAME $32 DESCRIPTION $256. GROUPID 8;
     name="&sysuserid";
     description="&sysuserid (group - desktop mode)";
     groupid=1;
@@ -114,7 +114,7 @@ libname &libref JSON fileref=&fref1;
 
 %if "&user"="0" and "&uid"="0" %then %do;
   data &outds;
-    length NAME $32 DESCRIPTION $64. GROUPID 8;
+    length NAME $32 DESCRIPTION $256. GROUPID 8;
     if _n_=1 then call missing(of _all_);
     set &libref..root;
     drop ordinal_root;
@@ -122,7 +122,7 @@ libname &libref JSON fileref=&fref1;
 %end;
 %else %do;
   data &outds;
-    length NAME $32 DESCRIPTION $64. GROUPID 8;
+    length NAME $32 DESCRIPTION $256. GROUPID 8;
     if _n_=1 then call missing(of _all_);
     set &libref..groups;
     drop ordinal_:;

server/ms_getusers.sas

@@ -116,14 +116,16 @@ libname &libref JSON fileref=&fref1;
 %if "&group"="0" and "&gid"="0" %then %do;
   data &outds;
     length DISPLAYNAME $60 USERNAME:$30 ID 8;
-    set &libref..root;
+    if nobs=0 then call missing(of _all_);
+    set &libref..root nobs=nobs;
     drop ordinal_root;
   run;
 %end;
 %else %do;
   data &outds;
     length DISPLAYNAME $60 USERNAME:$30 ID 8;
-    set &libref..users;
+    if nobs=0 then call missing(of _all_);
+    set &libref..users nobs=nobs;
     drop ordinal_root ordinal_users;
   run;
 %end;

server/ms_runstp.sas

@@ -39,8 +39,9 @@
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
-  @li mf_getuniquelibref.sas
+  @li mf_getuniquename.sas
   @li mp_abort.sas
+  @li mp_chop.sas
 
 **/
 
@@ -153,7 +154,10 @@ run;
   run;
 %end;
 
-filename &outref temp lrecl=32767;
+%local resp_path;
+%let resp_path=%sysfunc(pathname(work))/%mf_getuniquename();
+filename &outref "&resp_path" lrecl=32767;
+
 /* prepare request*/
 proc http method='POST' headerin=&authref in=&mainref out=&outref
   url="&_sasjs_apiserverurl.&_sasjs_apipath?_program=&pgm%str(&)_debug=131";
@@ -161,6 +165,7 @@ proc http method='POST' headerin=&authref in=&mainref out=&outref
   debug level=2;
 %end;
 run;
+
 %if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
 or &mdebug=1
 %then %do;
@@ -176,11 +181,22 @@ or &mdebug=1
 options &optval;
 
 %if &outlogds ne _null_ or &mdebug=1 %then %do;
-  %local dumplib;
-  %let dumplib=%mf_getuniquelibref();
-  libname &dumplib json fileref=&outref;
+  %local matchstr chopout;
+  %let matchstr=SASJS_LOGS_SEPARATOR_163ee17b6ff24f028928972d80a26784;
+  %let chopout=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop);
+
+  %mp_chop("&resp_path"
+    ,matchvar=matchstr
+    ,keep=LAST
+    ,matchpoint=END
+    ,outfile="&chopout"
+    ,mdebug=&mdebug
+  )
+
   data &outlogds;
-    set &dumplib..log;
+    infile "&chopout" lrecl=2000;
+    length line $2000;
+    line=_infile_;
   %if &mdebug=1 %then %do;
     putlog line=;
   %end;

server/ms_testservice.sas

@@ -108,47 +108,34 @@ run;
 )
 
 
-/* SASjs services have the _webout embedded in wrapper JSON */
-/* Files can also be very large - so use a dedicated macro to chop it out */
-%local matchstr1 matchstr2 ;
-%let matchstr1={"status":"success","_webout":{;
-%let matchstr2=},"log":[{;
-%let chopout1=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop1);
-%let chopout2=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop2);
+/* chop out JSON section */
+%local matchstr chopout;
+%let matchstr=SASJS_LOGS_SEPARATOR_163ee17b6ff24f028928972d80a26784;
+%let chopout=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop);
 
 %mp_chop("%sysfunc(pathname(&fref1,F))"
-  ,matchvar=matchstr1
-  ,keep=LAST
-  ,matchpoint=END
-  ,offset=-1
-  ,outfile="&chopout1"
-  ,mdebug=&mdebug
-)
-
-%mp_chop("&chopout1"
-  ,matchvar=matchstr2
+  ,matchvar=matchstr
   ,keep=FIRST
   ,matchpoint=START
-  ,offset=1
-  ,outfile="&chopout2"
+  ,offset=-1
+  ,outfile="&chopout"
   ,mdebug=&mdebug
 )
 
 %if &outlib ne 0 %then %do;
-  libname &outlib json "&chopout2";
+  libname &outlib json "&chopout";
 %end;
 %if &outref ne 0 %then %do;
-  filename &outref "&chopout2";
+  filename &outref "&chopout";
 %end;
 
 %if &mdebug=0 %then %do;
   filename &webref clear;
   filename &fref1 clear;
-  filename &fref2 clear;
 %end;
 %else %do;
   %put &sysmacroname exit vars:;
   %put _local_;
 %end;
 
-%mend ms_testservice;
+%mend ms_testservice;

server/ms_webout.sas

@@ -23,14 +23,19 @@
   @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE
   @param [in] ds The dataset to send back to the frontend
   @param [out] dslabel= value to use instead of table name for sending to JSON
-  @param [in] fmt= (Y) Set to N to send back unformatted values
+  @param [in] fmt= (N) Setting Y converts all vars to their formatted values
   @param [out] fref= (_webout) The fileref to which to write the JSON
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
-  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+  @param [in] showmeta= (N) Set to Y to output metadata alongside each table,
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
+  @param [in] workobs= (0) When set to a positive integer, will create a new
+    output object (WORK) which contains this number of observations from all
+    tables in the WORK library.
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to output JSON
 
   <h4> SAS Macros </h4>
   @li mf_getuser.sas
@@ -46,8 +51,8 @@
 
 **/
 
-%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
-  ,showmeta=NO
+%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL
+  ,showmeta=N,maxobs=MAX,workobs=0
 );
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
@@ -106,13 +111,12 @@
     %let missing=NULL;
   %end;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta
+    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs
   )
 %end;
 %else %if &action=CLOSE %then %do;
-  %if %str(&_debug) ge 131 %then %do;
-    /* if debug mode, send back first 10 records of each work table also */
-    options obs=10;
+  %if %str(&workobs) > 0 %then %do;
+    /* if debug mode, send back first XX records of each work table also */
     data;run;%let tempds=%scan(&syslast,2,.);
     ods output Members=&tempds;
     proc datasets library=WORK memtype=data;
@@ -137,7 +141,9 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES)
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=Y
+        ,maxobs=&workobs
+      )
       data _null_; file &fref mod encoding='utf-8' termstr=lf;
         put "}";
     %end;
@@ -147,35 +153,70 @@
   %end;
   /* close off json */
   data _null_;file &fref mod encoding='utf-8' termstr=lf lrecl=32767;
-    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
-    put ",""SYSUSERID"" : ""&sysuserid"" ";
-    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
+    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512;
     put ",""_DEBUG"" : ""&_debug"" ";
+    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ',"_PROGRAM" : ' _PROGRAM ;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
+    put ',"AUTOEXEC" : ' autoexec;
+    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""SYSCC"" : ""&syscc"" ";
     put ",""SYSENCODING"" : ""&sysencoding"" ";
-    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    syserrortext=cats(symget('syserrortext'));
+    if findc(syserrortext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syserrortext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syserrortext)
+      )))))))))))))!!'"';
+    end;
+    else syserrortext=cats('"',syserrortext,'"');
     put ',"SYSERRORTEXT" : ' syserrortext;
     SYSHOSTINFOLONG=quote(trim(symget('SYSHOSTINFOLONG')));
     put ',"SYSHOSTINFOLONG" : ' SYSHOSTINFOLONG;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
     put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
-    length SYSPROCESSNAME $512;
     SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
     put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" ";
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    syswarningtext=cats(symget('syswarningtext'));
+    if findc(syswarningtext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syswarningtext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syswarningtext)
+      )))))))))))))!!'"';
+    end;
+    else syswarningtext=cats('"',syswarningtext,'"');
     put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
-    length autoexec $512;
-    autoexec=quote(urlencode(trim(getoption('autoexec'))));
-    put ',"AUTOEXEC" : ' autoexec;
     length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
     memsize=quote(cats(memsize));

tests/base/mf_existvar.test.sas

@@ -17,4 +17,24 @@
 %mp_assert(
   iftrue=(%mf_existvar(sashelp.class,isjustanumber)=0),
   desc=Checking non existing var does not exist
+)
+
+data work.lockcheck;
+  a=1;
+  output;
+  stop;
+run;
+
+%mp_assert(
+  iftrue=(%mf_existvar(work.lockcheck,)=0),
+  desc=Checking non-provided var does not exist
+)
+
+proc sql;
+update work.lockcheck set a=2;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking the lock was released,
+  outds=work.test_results
 )

tests/base/mf_getgitbranch.test.sas

@@ -0,0 +1,20 @@
+/**
+  @file
+  @brief Testing mf_getgitbranch.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getgitbranch.sas
+  @li mp_assert.sas
+
+**/
+
+/* grab core repo */
+%let gitdir=%sysfunc(pathname(work))/core;
+%let repo=https://github.com/sasjs/core;
+%put source clone rc=%sysfunc(GITFN_CLONE(&repo,&gitdir));
+
+%mp_assert(
+  iftrue=(%mf_getgitbranch(&gitdir)=main),
+  desc=Checking correct branch was obtained,
+  outds=work.test_results
+)

tests/base/mf_increment.test.sas

@@ -0,0 +1,35 @@
+/**
+  @file
+  @brief Testing mf_increment macro
+
+  <h4> SAS Macros </h4>
+  @li mf_increment.sas
+  @li mp_assert.sas
+
+**/
+
+%let var=0;
+
+%mp_assert(
+  iftrue=(
+    "%mf_increment(var)"="1"
+  ),
+  desc=Checking basic mf_increment usage 1,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_increment(var)"="2"
+  ),
+  desc=Checking basic mf_increment usage 2,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_increment(var,incr=2)"="4"
+  ),
+  desc=Checking incr option,
+  outds=work.test_results
+)

tests/base/mf_readfile.test.sas

@@ -0,0 +1,40 @@
+/**
+  @file
+  @brief Testing mf_readfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_readfile.sas
+  @li mf_writefile.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%let f=&sasjswork/myfile.txt;
+
+%mf_writefile(&f,l1=some content,l2=more content)
+data _null_;
+  infile "&f";
+  input;
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check code ran without errors,
+  outds=work.test_results
+)
+
+/* test for scope leakage */
+%global result;
+%mp_assertscope(SNAPSHOT)
+%put %mf_readfile(&f);
+%mp_assertscope(COMPARE)
+
+/* test result */
+%mp_assert(
+  iftrue=(%mf_readfile(&f)=some content),
+  desc=Checking first line was ingested successfully,
+  outds=work.test_results
+)
+

tests/base/mp_aligndecimal.test.sas

@@ -0,0 +1,46 @@
+/**
+  @file
+  @brief Testing mp_aligndecimal macro
+  @details Creates an aligned variable and checks the number of leading blanks
+
+  <h4> SAS Macros </h4>
+  @li mp_aligndecimal.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertscope.sas
+
+**/
+
+
+
+/* target values */
+data work.checkds;
+  do checkval='   0.56',' 123.45',' 123.4 ','   1.2 ','   0';
+    output;
+  end;
+run;
+
+/* raw values */
+data work.rawds;
+  set work.checkds;
+  tgtvar=cats(checkval);
+  drop checkval;
+run;
+%mp_assertcolvals(work.rawds.tgtvar,
+  checkvals=work.checkds.checkval,
+  desc=No values match (ready to align),
+  test=NOVAL
+)
+
+/* aligned values */
+%mp_assertscope(SNAPSHOT)
+data work.finalds;
+  set work.rawds;
+  %mp_aligndecimal(tgtvar,width=4)
+run;
+%mp_assertscope(COMPARE)
+
+%mp_assertcolvals(work.finalds.tgtvar,
+  checkvals=work.checkds.checkval,
+  desc=All values match (aligned),
+  test=ALLVALS
+)

tests/base/mp_cleancsv.test.sas

@@ -0,0 +1,43 @@
+/**
+  @file
+  @brief Testing mp_cleancsv.sas macro
+  @details Credit for test 1 goes to
+  [Tom](https://communities.sas.com/t5/user/viewprofilepage/user-id/159) from
+  SAS Communities:
+https://communities.sas.com/t5/SAS-Programming/Removing-embedded-carriage-returns/m-p/824790#M325761
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_cleancsv.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+/* test 1 - cope with empty rows on CR formatted file */
+
+filename crlf "%sysfunc(pathname(work))/crlf";
+filename cr "%sysfunc(pathname(work))/cr";
+data _null_;
+  file cr termstr=cr ;
+  put 'line 1'///'line 4'/'line 5';
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mp_cleancsv(in=cr,out=crlf)
+%mp_assertscope(COMPARE)
+
+/* 5 rows as all converted to OD0A */
+data test1;
+  infile "%sysfunc(pathname(work))/crlf" lrecl=100 termstr=crlf;
+  input;
+  list;
+run;
+
+%put test1=%mf_nobs(test1);
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.test1)=5),
+  desc=Checking blank rows on CR formatted file,
+  outds=work.test_results
+)

tests/base/mp_dictionary.test.sas

@@ -0,0 +1,26 @@
+/**
+  @file
+  @brief Testing mp_dictionary.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_dictionary.sas
+  @li mp_assert.sas
+
+**/
+
+libname test (work);
+%mp_dictionary(lib=test)
+
+proc sql;
+create table work.compare1 as select * from test.styles;
+create table work.compare2 as select * from dictionary.styles;
+
+proc compare base=compare1 compare=compare2;
+run;
+%put _all_;
+
+%mp_assert(
+  iftrue=(%mf_existds(&sysinfo)=0),
+  desc=Compare was exact,
+  outds=work.test_results
+)

tests/base/mp_dsmeta.test.sas

@@ -0,0 +1,32 @@
+/**
+  @file
+  @brief Testing mp_dsmeta.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mp_dsmeta.sas
+
+**/
+
+data work.Example;
+  set sashelp.vmacro;
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mp_dsmeta(work.example,outds=work.test)
+%mp_assertscope(COMPARE)
+
+proc sql noprint;
+select count(*) into: nobs from work.test;
+select count(distinct ods_table) into: tnobs from work.test;
+
+%mp_assert(
+  iftrue=(&tnobs=2),
+  desc=Check that both ATTRIBUTES and ENGINEHOST are provided
+)
+%mp_assert(
+  iftrue=(&nobs>10),
+  desc=Check that sufficient details are provided
+)
+

tests/base/mp_filtercheck.test.sas

@@ -38,7 +38,7 @@ William,M,15,66.5,112
 ;;;;
 run;
 
-/* valid filter conditions */
+/* VALID filter conditions */
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
@@ -51,6 +51,9 @@ AND,OR,2,Weight,>=,77.7
 AND,OR,2,Weight,NE,77.7
 AND,AND,1,age,=,.A
 AND,AND,1,height,<,.B
+AND,AND,1,age,IN,"(.a,.b,.)"
+AND,AND,1,age,IN,"(.A)"
+
 ;;;;
 run;
 
@@ -115,6 +118,28 @@ run;
   outds=work.test_results
 )
 
+/* invalid IN value */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,OR,2,age,IN,"(.,.a,X)"
+;;;;
+run;
+
+%mp_filtercheck(work.inds,
+  targetds=work.class,
+  outds=work.badrecords,
+  abort=NO
+)
+%let syscc=0;
+%mp_assertdsobs(work.badrecords,
+  desc=Invalid IN value,
+  test=HASOBS,
+  outds=work.test_results
+)
+
 /* Code injection - column name */
 data work.inds;
   infile datalines4 dsd;
@@ -163,7 +188,7 @@ run;
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:8;
+    OPERATOR_NM:$10. RAW_VALUE:8.;
 datalines4;
 AND,AND,1,age,=,0
 ;;;;

tests/base/mp_filterstore.test.1.sas

@@ -56,6 +56,9 @@ AND,AND,1,SEX,<=,"'M'"
 AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
 AND,OR,2,Weight,>=,77.7
 AND,OR,2,Weight,NE,77.7
+AND,AND,3,age,NOT IN,"(.a,.b,.)"
+AND,AND,3,age,NOT IN,"(.A)"
+AND,AND,4,Name,=,"'Jeremiah'"
 ;;;;
 run;
 

tests/base/mp_getcols.test.sas

@@ -7,23 +7,35 @@
   @li mp_assertcols.sas
   @li mp_assertcolvals.sas
   @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
 
 **/
 
 
-/* valid filter */
-%mp_getcols(sashelp.airline,outds=work.info)
+/* make some data */
+proc sql;
+create table work.src(
+  SOME_DATETIME float format=datetime19.,
+  SOME_CHAR char(16),
+  SOME_NUM num,
+  SOME_TIME num format=time8.,
+  SOME_DATE num format=date9.
+);
 
+/* run macro, checking for scope leakage */
+%mp_assertscope(SNAPSHOT)
+%mp_getcols(work.src,outds=work.info)
+%mp_assertscope(COMPARE)
 
 %mp_assertdsobs(work.info,
-  desc=Has 3 records,
-  test=EQUALS 3,
+  desc=Has 5 records,
+  test=EQUALS 5,
   outds=work.test_results
 )
 
 data work.check;
   length val $10;
-  do val='NUMERIC','DATE','CHARACTER';
+  do val='NUMERIC','DATE','CHARACTER','DATETIME','TIME';
     output;
   end;
 run;

tests/base/mp_gitadd.test.sas

@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Testing mp_gitadd.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_deletefile.sas
+  @li mf_writefile.sas
+  @li mp_gitadd.sas
+  @li mp_gitstatus.sas
+  @li mp_assert.sas
+
+**/
+
+/* clone the source repo */
+%let dir = %sysfunc(pathname(work))/core;
+%put source clone rc=%sysfunc(GITFN_CLONE(https://github.com/sasjs/core,&dir));
+
+/* add a file */
+%mf_writefile(&dir/somefile.txt,l1=some content)
+/* change a file */
+%mf_writefile(&dir/readme.md,l1=new readme)
+/* delete a file */
+%mf_deletefile(&dir/package.json)
+
+/* Run git status */
+%mp_gitstatus(&dir,outds=work.gitstatus)
+
+%let test1=0;
+proc sql noprint;
+select count(*) into: test1 from work.gitstatus where staged='FALSE';
+
+/* should be three unstaged changes now */
+%mp_assert(
+  iftrue=(&test1=3),
+  desc=3 changes are ready to add,
+  outds=work.test_results
+)
+
+/* add them */
+%mp_gitadd(&dir,inds=work.gitstatus,mdebug=&sasjs_mdebug)
+
+/* check status */
+%mp_gitstatus(&dir,outds=work.gitstatus2)
+%let test2=0;
+proc sql noprint;
+select count(*) into: test2 from work.gitstatus2 where staged='TRUE';
+
+/* should be three staged changes now */
+%mp_assert(
+  iftrue=(&test2=3),
+  desc=3 changes were added,
+  outds=work.test_results
+)

tests/base/mp_gitlog.test.sas

@@ -0,0 +1,32 @@
+/**
+  @file
+  @brief Testing mp_gitlog.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_gitlog.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+/* grab core repo */
+%let gitdir=%sysfunc(pathname(work))/core;
+%let repo=https://github.com/sasjs/core;
+%put source clone rc=%sysfunc(GITFN_CLONE(&repo,&gitdir));
+
+%mp_assertscope(SNAPSHOT)
+%mp_gitlog(&gitdir,outds=work.test1)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Regular test works,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.test1)>1000),
+  desc=output has gt 1000 rows,
+  outds=work.test_results
+)

tests/base/mp_gitreleaseinfo.test.sas

@@ -0,0 +1,30 @@
+/**
+  @file
+  @brief Testing mp_gitreleaseinfo.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_gitreleaseinfo.sas
+  @li mp_assert.sas
+
+**/
+
+
+%mp_gitreleaseinfo(github,sasjs/core,outlib=mylibref,mdebug=1)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=mp_gitreleaseinfo runs without errors,
+  outds=work.test_results
+)
+
+data _null_;
+  set mylibref.author;
+  putlog (_all_)(=);
+  call symputx('author',login);
+run;
+
+%mp_assert(
+  iftrue=(&author=sasjsbot),
+  desc=release info extracted successfully,
+  outds=work.test_results
+)

tests/base/mp_gitstatus.test.sas

@@ -0,0 +1,39 @@
+/**
+  @file
+  @brief Testing mp_gitstatus.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_deletefile.sas
+  @li mf_writefile.sas
+  @li mp_gitstatus.sas
+  @li mp_assertdsobs.sas
+
+**/
+
+/* clone the source repo */
+%let dir = %sysfunc(pathname(work))/core;
+%put source clone rc=%sysfunc(GITFN_CLONE(https://github.com/sasjs/core,&dir));
+
+%mp_gitstatus(&dir,outds=work.gitstatus)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Initial mp_gitstatus runs without errors,
+  outds=work.test_results
+)
+
+/* should be empty as there are no changes yet */
+%mp_assertdsobs(work.gitstatus,test=EMPTY)
+
+/* add a file */
+%mf_writefile(&dir/somefile.txt,l1=some content)
+/* change a file */
+%mf_writefile(&dir/readme.md,l1=new readme)
+/* delete a file */
+%mf_deletefile(&dir/package.json)
+
+/* re-run git status */
+%mp_gitstatus(&dir,outds=work.gitstatus)
+
+/* should be three changes now */
+%mp_assertdsobs(work.gitstatus,test=EQUALS 3)

tests/base/mp_hashdirectory.test.sas

@@ -0,0 +1,149 @@
+/**
+  @file
+  @brief Testing mp_hashdirectory.sas macro
+
+
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mf_nobs.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mp_hashdirectory.sas
+
+**/
+
+/* set up a directory to hash */
+%let fpath=%sysfunc(pathname(work))/testdir;
+
+%mf_mkdir(&fpath)
+%mf_mkdir(&fpath/sub1)
+%mf_mkdir(&fpath/sub2)
+%mf_mkdir(&fpath/sub1/subsub)
+
+/* note - the path in the file means the hash is different in each run */
+%macro makefile(path,name);
+  data _null_;
+    file "&path/&name" termstr=lf;
+    put "This file is located at:";
+    put "&path";
+    put "and it is called:";
+    put "&name";
+  run;
+%mend makefile;
+
+%macro spawner(path);
+  %do x=1 %to 5;
+    %makefile(&path,file&x..txt)
+  %end;
+%mend spawner;
+
+%spawner(&fpath)
+%spawner(&fpath/sub1)
+%spawner(&fpath/sub1/subsub)
+
+
+%mp_assertscope(SNAPSHOT)
+%mp_hashdirectory(&fpath,outds=work.hashes,maxdepth=MAX)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errors,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.hashes)=19),
+  desc=record created for each entry,
+  outds=work.test_results
+)
+
+proc sql;
+select count(*) into: misscheck
+  from work.hashes
+  where file_hash is missing;
+
+%mp_assert(
+  iftrue=(&misscheck=1),
+  desc=Only one missing hash - the empty directory,
+  outds=work.test_results
+)
+
+data _null_;
+  set work.hashes;
+  if directory=file_path then call symputx('tophash',file_hash);
+run;
+
+%mp_assert(
+  iftrue=(%length(&tophash)=32),
+  desc=ensure valid top level hash created,
+  outds=work.test_results
+)
+
+/* now change a file and re-hash */
+data _null_;
+  file "&fpath/sub1/subsub/file1.txt" termstr=lf;
+  put "This file has changed!";
+run;
+
+%mp_hashdirectory(&fpath,outds=work.hashes2,maxdepth=MAX)
+
+data _null_;
+  set work.hashes2;
+  if directory=file_path then call symputx('tophash2',file_hash);
+run;
+
+%mp_assert(
+  iftrue=(&tophash ne &tophash2),
+  desc=ensure the changing of the hash results in a new value,
+  outds=work.test_results
+)
+
+/* now change it back and see if it matches */
+data _null_;
+  file "&fpath/sub1/subsub/file1.txt" termstr=lf;
+    put "This file is located at:";
+    put "&fpath/sub1/subsub";
+    put "and it is called:";
+    put "file1.txt";
+  run;
+run;
+
+%mp_hashdirectory(&fpath,outds=work.hashes3,maxdepth=MAX)
+
+data _null_;
+  set work.hashes3;
+  if directory=file_path then call symputx('tophash3',file_hash);
+run;
+
+%mp_assert(
+  iftrue=(&tophash=&tophash3),
+  desc=ensure the same files result in the same hash,
+  outds=work.test_results
+)
+
+/* dump contents for debugging */
+data _null_;
+  set work.hashes;
+  put file_hash file_path;
+run;
+data _null_;
+  set work.hashes2;
+  put file_hash file_path;
+run;
+
+/* check that it works when the target directory is missing */
+
+%mp_hashdirectory(&fpath/doesnotexist,outds=work.hashes3,maxdepth=MAX)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errors when directory is missing,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.hashes3)=0),
+  desc=no records created when directory is missing,
+  outds=work.test_results
+)

tests/base/mp_loadformat.test.1.sas

@@ -0,0 +1,231 @@
+/**
+  @file
+  @brief Testing mp_loadformat.sas macro
+  @details first test regular formats, then informats
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mp_aligndecimal.sas
+  @li mp_cntlout.sas
+  @li mp_loadformat.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+/* prep format catalog */
+libname perm (work);
+
+%mddl_dc_difftable(libds=perm.audit)
+
+/* set up regular formats */
+data work.loadfmts;
+  /* matching start / end lengths (to baseds) are important */
+  length fmtname $32 start end $10000;
+  eexcl='Y';
+  type='N';
+  do i=1 to 10;
+    fmtname=cats('SASJS_',put(i,z4.),'X');
+    do j=1 to 20;
+      start=cats(j);
+      end=cats(j+1);
+      %mp_aligndecimal(start,width=16)
+      %mp_aligndecimal(end,width=16)
+      label= cats('Numeric Format ',start);
+      output;
+    end;
+  end;
+run;
+proc format cntlin=work.loadfmts library=perm.testcat;
+run;
+
+/*
+  use actual format data as test baseline, as proc format adds attributes eg
+  min/max etc
+*/
+%mp_cntlout(libcat=perm.testcat,cntlout=work.loadfmts2)
+
+/* make some test data */
+data work.stagedata;
+  set work.loadfmts2 end=lastobs;
+  by type fmtname;
+
+  if lastobs then do;
+    output;
+    fmtname='NEWFMT'!!cats(_n_,'x'); /* 1 new record */
+    start=cats(_n_);
+    end=cats(_n_+1);
+    %mp_aligndecimal(start,width=16)
+    %mp_aligndecimal(end,width=16)
+    label='newval'!!cats(_N_,'X');
+    output;
+    stop;
+  end;
+  else if last.fmtname then deleteme='Yes'; /* 9 deletions */
+  else if first.fmtname then label='modified '!!cats(_n_); /* 10 changes */
+
+  output;
+run;
+
+/* load the above */
+%mp_assertscope(SNAPSHOT)
+%mp_loadformat(perm.testcat
+  ,work.stagedata
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_test1
+  ,outds_del=del_test1
+  ,outds_mod=mod_test1
+  ,mdebug=1
+)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(%mf_nobs(del_test1)=9),
+  desc=Test 1 - delete obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(add_test1)=1),
+  desc=Test 1 - add obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(mod_test1)=10),
+  desc=Test 1 - mod obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(perm.audit)=440),
+  desc=Test 1 - audit table updated,
+  outds=work.test_results
+)
+data work.difftest;
+  set perm.audit;
+  where is_diff=1;
+run;
+%mp_assert(
+  iftrue=(%mf_nobs(work.difftest)>0),
+  desc=Test 1 - diffs were found,
+  outds=work.test_results
+)
+
+/* set up a mix of formats */
+data work.loadfmts3;
+  length fmtname $32 start end $10000;
+  eexcl='Y';
+  type='J';
+  do i=1 to 3;
+    fmtname=cats('SASJS_CI_',i,'X');
+    do j=1 to 4;
+      start=cats(j);
+      end=start;
+      label= cats('Char INFORMAT ',start);
+      output;
+    end;
+  end;
+  type='I';
+  do i=1 to 3;
+    fmtname=cats('SASJS_NI_',i,'X');
+    do j=1 to 4;
+      start=cats(j);
+      end=cats(j+1);
+      %mp_aligndecimal(start,width=16)
+      %mp_aligndecimal(end,width=16)
+      label= cats(ranuni(0));
+      output;
+    end;
+  end;
+  type='N';
+  do i=1 to 3;
+    fmtname=cats('SASJS_NF_',i,'X');
+    do j=1 to 4;
+      start=cats(j);
+      end=cats(j+1);
+      %mp_aligndecimal(start,width=16)
+      %mp_aligndecimal(end,width=16)
+      label= cats('Numeric Format ',start);
+      output;
+    end;
+  end;
+  type='C';
+  do i=1 to 3;
+    fmtname=cats('SASJS_CF_',i,'X');
+    do j=1 to 4;
+      start=cats(j);
+      end=start;
+      label= cats('Char Format ',start);
+      output;
+    end;
+  end;
+  drop i j;
+run;
+proc format cntlin=work.loadfmts3 library=perm.testcat3;
+run;
+%mp_cntlout(libcat=perm.testcat3,cntlout=work.loadfmts4)
+
+/* make some test data */
+data work.stagedata3;
+  set work.loadfmts4;
+  where type in ('I','J');
+  by type fmtname notsorted;
+  if type='I' then do;
+    if last.fmtname then do;
+      deleteme='Yes'; /* 3 deletions */
+      output;
+    end;
+    else if fmtrow le 3 then do; /* 9 changed values */
+      z=ranuni(0)*1000000;
+      start=cats(z);
+      end=cats(z+1);
+      %mp_aligndecimal(start,width=16)
+      %mp_aligndecimal(end,width=16)
+      output;
+    end;
+  end;
+  else do;
+    if last.fmtname then do;
+      output; /* 6 new records */
+      x=_n_;
+      x+1;start=cats("mod",x);end=start;label='newlabel1';output;
+      x+1;start=cats("mod",x);end=start;label='newlabel2';output;
+    end;
+    else if fmtrow le 3 then do; /* 9 more changed values */
+      start= cats("mod",_n_);
+      end=start;
+      label= "mod "||cats(ranuni(0)*100);
+      output;
+    end;
+  end;
+run;
+
+%mp_loadformat(perm.testcat3
+  ,work.stagedata3
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_test2
+  ,outds_del=del_test2
+  ,outds_mod=mod_test2
+  ,mdebug=1
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(del_test2)=3),
+  desc=Test 2 - delete obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(mod_test2)=18),
+  desc=Test 2 - mod obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(add_test2)=6),
+  desc=Test 2 - add obs,
+  outds=work.test_results
+)
+

tests/base/mp_loadformat.test.2.sas

@@ -0,0 +1,234 @@
+/**
+  @file
+  @brief Testing mp_loadformat.sas macro for multilabel formats
+  @details Multilabel records can be complete duplicates!!  Also, the order is
+  important.
+
+  The provided formats create a table as follows:
+
+
+|TYPE:$1.|FMTNAME:$32.|START:$10000.|END:$10000.|LABEL:$32767.|MIN:best.|MAX:best.|DEFAULT:best.|LENGTH:best.|FUZZ:best.|PREFIX:$2.|MULT:best.|FILL:$1.|NOEDIT:best.|SEXCL:$1.|EEXCL:$1.|HLO:$13.|DECSEP:$1.|DIG3SEP:$1.|DATATYPE:$8.|LANGUAGE:$8.|
+|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
+|`C `|`GENDERML `|` `|` `|`Total people `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`1 `|`1 `|`Male `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`1 `|`1 `|`Total people `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`2 `|`2 `|`Female `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`2 `|`2 `|`Female `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`2 `|`2 `|`Thormale `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`2 `|`2 `|`Total people `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`N `|`AGEMLA `|`1 `|`4 `|`Preschool `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLA `|`1 `|`18 `|`Children `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLA `|`19 `|`120 `|`Adults `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLB `|`1 `|`4 `|`Preschool `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLB `|`1 `|`18 `|`Children `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLB `|`19 `|`120 `|`Adults `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLC `|`1 `|`18 `|`Children `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLC `|`1 `|`4 `|`Preschool `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLC `|`19 `|`120 `|`Adults `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_cntlout.sas
+  @li mp_loadformat.sas
+  @li mp_assert.sas
+  @li mp_assertdsobs.sas
+  @li mp_getformats.sas
+  @li mp_ds2md.sas
+
+
+**/
+
+/* prep format catalog */
+libname perm (work);
+
+/* create some multilabel formats */
+%let cat1=perm.test1;
+proc format library=&cat1;
+  value $genderml (multilabel notsorted)
+    '1'='Male'
+    '2'='Female'
+    '2'='Female'
+    '2'='Farmale'
+    '1','2',' '='Total people';
+  value agemla (multilabel)
+    1-4='Preschool'
+    1-18='Children'
+    19-120='Adults';
+  value agemlb (multilabel)
+    19-120='Adults'
+    1-18='Children'
+    1-4='Preschool';
+  value agemlc (multilabel notsorted)
+    19-120='Adults'
+    1-18='Children'
+    1-4='Preschool';
+run;
+
+%mp_cntlout(libcat=&cat1,cntlout=work.cntlout1)
+%mp_assertdsobs(work.cntlout1,
+  desc=Has 16 records,
+  test=EQUALS 16
+)
+
+data work.stagedata3;
+  set work.cntlout1;
+  if fmtname='AGEMLA' and label ne 'Preschool' then deleteme='Yes';
+  if fmtname='AGEMLB' and label = 'Preschool' then label='Kids';
+  if fmtname='GENDERML' and label='Farmale' then output;
+  output;
+run;
+
+
+%mp_loadformat(&cat1
+  ,work.stagedata3
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_test1
+  ,outds_del=del_test1
+  ,outds_mod=mod_test1
+  ,mdebug=1
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(del_test1)=2),
+  desc=Test 1 - deleted obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(mod_test1)=4),
+  desc=Test 1 - mod obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(add_test1)=1),
+  desc=Test 1 - add obs,
+  outds=work.test_results
+)
+
+/* now check the order of the notsorted format */
+%mp_cntlout(libcat=&cat1,cntlout=work.cntlout2)
+
+%let check1=0;
+%let check2=0;
+data test;
+  set work.cntlout2;
+  where fmtname='GENDERML';
+  if _n_=4 and label='Farmale' then call symputx('check1',1);
+  if _n_=5 and label='Farmale' then call symputx('check2',1);
+run;
+%mp_assert(
+  iftrue=(&check1=1 and &check2=1),
+  desc=Ensuring Farmale values retain their order,
+  outds=work.test_results
+)
+
+/**
+  * completely delete a format and make sure it is removed
+  */
+
+/* first, make sure these three formats exist */
+options insert=(fmtsearch=(&cat1));
+%mp_getformats(fmtlist=AGEMLA AGEMLB AGEMLC $GENDERML,outsummary=work.fmtdels)
+
+%let fmtlist=NONE;
+proc sql;
+select distinct cats(fmtname) into: fmtlist separated by ' ' from work.fmtdels;
+
+%mp_assert(
+  iftrue=(%mf_nobs(fmtdels)=4),
+  desc=Deletion test 1 - ensure formats exist for deletion (&fmtlist found),
+  outds=work.test_results
+)
+
+/* deltest1 - deleting every record */
+%mp_cntlout(libcat=&cat1,cntlout=work.cntloutdel1)
+data work.stagedatadel1;
+  set work.cntloutdel1;
+  if fmtname='AGEMLA';
+  deleteme='Yes';
+run;
+%mp_loadformat(&cat1
+  ,work.stagedatadel1
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_testdel1
+  ,outds_del=del_testdel1
+  ,outds_mod=mod_testdel1
+  ,mdebug=1
+)
+%mp_getformats(fmtlist=AGEMLA,outsummary=work.fmtdel1)
+%mp_assert(
+  iftrue=(%mf_nobs(fmtdel1)=0),
+  desc=Deletion test 1 - ensure AGEMLA format was fully deleted,
+  outds=work.test_results
+)
+
+/* deltest2 - deleting every record except 1 */
+data work.stagedatadel2;
+  set work.cntloutdel1;
+  if fmtname='AGEMLB';
+  x+1;
+  if x>1 then deleteme='Yes';
+run;
+%mp_loadformat(&cat1
+  ,work.stagedatadel2
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_testdel2
+  ,outds_del=del_testdel2
+  ,outds_mod=mod_testdel2
+  ,mdebug=1
+)
+%mp_getformats(fmtlist=AGEMLB,outsummary=work.fmtdel2)
+%mp_assert(
+  iftrue=(%mf_nobs(fmtdel2)=1),
+  desc=Deletion test 2 - ensure AGEMLB format was not fully deleted,
+  outds=work.test_results
+)
+
+
+/* deltest3 - deleting every record, and adding a new one */
+data work.stagedatadel3;
+  set work.cntloutdel1;
+  if fmtname='GENDERML';
+  deleteme='Yes';
+run;
+data work.stagedatadel3;
+  set work.stagedatadel3 end=last;
+  output;
+  if last then do;
+    deleteme='No';
+    /* must be a new fmtrow (key value) if adding new row in same load! */
+    fmtrow=1000;
+    start='Mail';
+    end='Mail';
+    output;
+  end;
+run;
+
+%mp_loadformat(&cat1
+  ,work.stagedatadel3
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_testdel2
+  ,outds_del=del_testdel2
+  ,outds_mod=mod_testdel2
+  ,mdebug=1
+)
+%mp_getformats(fmtlist=$GENDERML,outsummary=work.fmtdel3)
+%mp_assert(
+  iftrue=(%mf_nobs(fmtdel3)=1),
+  desc=Deletion test 3 - ensure GENDERML format was not fully deleted,
+  outds=work.test_results
+)
+
+%mp_ds2md(work.fmtdel3)

tests/base/mp_loadformat.test.sas

@@ -1,93 +0,0 @@
-/**
-  @file
-  @brief Testing mp_loadformat.sas macro
-
-  <h4> SAS Macros </h4>
-  @li mddl_dc_difftable.sas
-  @li mp_loadformat.sas
-  @li mp_assert.sas
-  @li mp_assertscope.sas
-
-**/
-
-/* prep format catalog */
-libname perm (work);
-
-%mddl_dc_difftable(libds=perm.audit)
-
-data work.loadfmts;
-  length fmtname $32;
-  eexcl='Y';
-  type='N';
-  do i=1 to 100;
-    fmtname=cats('SASJS_',i,'X');
-    do j=1 to 100;
-      start=cats(j);
-      end=cats(j+1);
-      label= cats('Dummy ',start);
-      output;
-    end;
-  end;
-run;
-proc format cntlin=work.loadfmts library=perm.testcat;
-run;
-
-/* make some test data */
-data work.stagedata;
-  set work.loadfmts;
-  type='N';
-  eexcl='Y';
-  if _n_<150 then deleteme='Yes';
-  else if _n_<250 then label='mod'!!cats(_n_);
-  else if _n_<350 then do;
-    start=cats(_n_);
-    end=cats(_n_+1);
-    label='newval'!!cats(_N_);
-  end;
-  else stop;
-run;
-
-/* load the above */
-%mp_assertscope(SNAPSHOT)
-%mp_loadformat(perm.testcat
-  ,work.stagedata
-  ,loadtarget=YES
-  ,auditlibds=perm.audit
-  ,locklibds=0
-  ,delete_col=deleteme
-  ,outds_add=add_test1
-  ,outds_del=del_test1
-  ,outds_mod=mod_test1
-  ,mdebug=1
-)
-%mp_assertscope(COMPARE)
-
-%mp_assert(
-  iftrue=(%mf_nobs(del_test1)=149),
-  desc=Test 1 - delete obs,
-  outds=work.test_results
-)
-%mp_assert(
-  iftrue=(%mf_nobs(add_test1)=100),
-  desc=Test 1 - add obs,
-  outds=work.test_results
-)
-%mp_assert(
-  iftrue=(%mf_nobs(mod_test1)=100),
-  desc=Test 1 - mod obs,
-  outds=work.test_results
-)
-%mp_assert(
-  iftrue=(%mf_nobs(perm.audit)=7329),
-  desc=Test 1 - audit table updated,
-  outds=work.test_results
-)
-data work.difftest;
-  set perm.audit;
-  where is_diff=1;
-run;
-%mp_assert(
-  iftrue=(%mf_nobs(work.difftest)>0),
-  desc=Test 1 - diffs were found,
-  outds=work.test_results
-)

tests/base/mp_lockanytable.test.sas

@@ -4,8 +4,10 @@
 
   <h4> SAS Macros </h4>
   @li mp_lockanytable.sas
+  @li mp_assert.sas
   @li mp_assertcols.sas
   @li mp_assertcolvals.sas
+  @li mp_assertscope.sas
   @li mp_coretable.sas
 
 **/
@@ -61,3 +63,18 @@ run;
   desc=Ref is captured in unlock,
   test=ANYVAL
 )
+
+/* attempt unlock of a table that was never locked */
+
+%mp_lockanytable(UNLOCK,lib=no,ds=doesnotexist,ref=bye, ctl_ds=work.controller)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Ability to unlock a table that was never locked,
+  outds=work.test_results
+)
+
+/* test for macro variable scope leakage */
+%mp_assertscope(SNAPSHOT)
+%mp_lockanytable(LOCK,lib=tmp,ds=testscope,ref=This Ref, ctl_ds=work.controller)
+%mp_assertscope(COMPARE)

tests/serveronly/ms_adduser2group.test.sas

@@ -31,7 +31,7 @@ run;
 %mp_assertscope(SNAPSHOT)
 %ms_adduser2group(uid=1,gid=&groupid,mdebug=&sasjs_mdebug,outds=test1)
 %mp_assertscope(COMPARE
-  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN
+  ,ignorelist=MCLIB2_JADP1LEN MCLIB2_JADP2LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
 )
 
 /* check the user is in the output list */

tests/serveronly/ms_deletefile.test.sas

@@ -53,7 +53,7 @@ data _null_;
 run;
 
 %mp_assert(
-  iftrue=("&test2"="%str(Err)or: File doesn't exist."),
+  iftrue=("&test2"="File doesn't exist."),
   desc=Make sure the file was deleted,
   outds=work.test_results
 )

tests/serveronly/ms_getgroups.test.sas

@@ -21,7 +21,7 @@
 %mp_assertscope(SNAPSHOT)
 %ms_getgroups(outds=work.test1,mdebug=&sasjs_mdebug)
 %mp_assertscope(COMPARE
-  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN
+  ,ignorelist=MCLIB2_JADP1LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
 )
 
 /* check the group was created */

tests/serveronly/ms_runstp.test.sas

@@ -17,6 +17,7 @@ data _null_;
   file stpcode;
   put '%put hello world;';
   put '%put _all_;';
+  put 'data _null_; file _webout; put "runstptest";run;';
 run;
 
 options mprint;
@@ -34,25 +35,29 @@ options mprint;
 )
 %mp_assertscope(COMPARE)
 
-libname webeen json fileref=weboot;
-
+%let test1=0;
+%let test2=0;
 data _null_;
   infile weboot;
   input;
-  putlog _infile_;
+  if _n_=1 then call symputx('test1',_infile_);
+  if _n_=3 then do;
+    call symputx('test2',substr(_infile_,1,30));
+    putlog "SASJS_LOGS_SEPARATOR_xxx"; /* this marker affects the CLI parser */
+  end;
+  else putlog _infile_;
 run;
 
-%let test1=0;
-data work.log;
-  set webeen.log;
-  put (_all_)(=);
-  if _n_>10 then call symputx('test1',1);
-run;
+
 
 %mp_assert(
-  iftrue=("&test1"="1"),
-  desc=Checking log was returned,
+  iftrue=("&test1"="runstptest"),
+  desc=Checking webout was created,
   outds=work.test_results
 )
 
-
+%mp_assert(
+  iftrue=("&test2"="SASJS_LOGS_SEPARATOR_163ee17b6"),
+  desc=Checking debug was enabled,
+  outds=work.test_results
+)

tests/viyaonly/mv_jobflow.test.1.sas

@@ -4,7 +4,7 @@
   @brief Testing mv_jobflow macro
   @details One of the remote jobs aborts with syscc>0 - test to
   make sure this comes back to the calling session
-​
+
   <h4> SAS Macros </h4>
   @li mp_assert.sas
   @li mv_createjob.sas

tests/viyaonly/mv_jobflow.test.2.sas

@@ -3,7 +3,7 @@
   @brief Testing mv_jobflow macro
   @details All jobs complete successfully with syscc = 0 - test to
   make sure this comes back to the calling session
-​
+
   <h4> SAS Macros </h4>
   @li mp_assert.sas
   @li mv_createjob.sas

tests/viyaonly/mv_registerclient.test.1.sas

@@ -4,7 +4,7 @@
   @brief Testing mv_registerclient.sas macro
   @details Tests for successful registration.  For this to work, the test
   account must be an admin.
-​
+
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas
   @li mp_assertcolvals.sas

tests/viyaonly/mv_registerclient.test.2.sas

@@ -1,10 +1,9 @@
 /**
-
   @file
   @brief Testing mv_registerclient.sas macro
   @details Tests for unsuccessful registration.  To do this, overrides are
   applied for the mf_loc.sas and mp_abort.sas macros.
-​
+
   <h4> SAS Macros </h4>
   @li mp_assert.sas
   @li mv_registerclient.sas

viya/mfv_existsashdat.sas

@@ -0,0 +1,60 @@
+/**
+  @file mfv_existsashdat.sas
+  @brief Checks whether a CAS sashdat dataset exists in persistent storage.
+  @details Can be used in open code, eg as follows:
+
+      %if %mfv_existsashdat(libds=casuser.sometable) %then %put  yes it does!;
+
+  The function uses `dosubl()` to run the `table.fileinfo` action, for the
+  specified library, filtering for `*.sashdat` tables.  The results are stored
+  in a WORK table (&outprefix._&lib). If that table already exists, it is
+  queried instead, to avoid the dosubl() performance hit.
+
+  To force a rescan, just use a new `&outprefix` value, or delete the table(s)
+  before running the function.
+
+  @param libds library.dataset
+  @param outprefix= (work.mfv_existsashdat) Used to store the current HDATA
+    tables to improve subsequent query performance.  This reference is a prefix
+    and is converted to `&prefix._{libref}`
+
+  @return output returns 1 or 0
+
+  @version 0.2
+  @author Mathieu Blauw
+**/
+
+%macro mfv_existsashdat(libds,outprefix=work.mfv_existsashdat
+);
+%local rc dsid name lib ds;
+%let lib=%upcase(%scan(&libds,1,'.'));
+%let ds=%upcase(%scan(&libds,-1,'.'));
+
+/* if table does not exist, create it */
+%if %sysfunc(exist(&outprefix._&lib)) ne 1 %then %do;
+  %let rc=%sysfunc(dosubl(%nrstr(
+    /* Read in table list (once per &lib per session) */
+    proc cas;
+      table.fileinfo result=source_list /caslib="&lib";
+      val=findtable(source_list);
+      saveresult val dataout=&outprefix._&lib;
+    quit;
+    /* Only keep name, without file extension */
+    data &outprefix._&lib;
+      set &outprefix._&lib(where=(Name like '%.sashdat') keep=Name);
+      Name=upcase(scan(Name,1,'.'));
+    run;
+  )));
+%end;
+
+/* Scan table for hdat existence */
+%let dsid=%sysfunc(open(&outprefix._&lib(where=(name="&ds"))));
+%syscall set(dsid);
+%let rc = %sysfunc(fetch(&dsid));
+%let rc = %sysfunc(close(&dsid));
+
+/* Return result */
+%if "%trim(&name)"="%trim(&ds)" %then 1;
+%else 0;
+
+%mend mfv_existsashdat;

viya/mv_createfile.sas

@@ -24,7 +24,8 @@
   @param [in] contentdisp= (inline) Content Disposition. Example values:
     @li inline
     @li attachment
-
+  @param [in] ctype= (0) Set a default HTTP Content-Type header to be returned
+    with the file when the content is retrieved from the Files service.
   @param [in] access_token_var= The global macro variable to contain the access
     token, if using authorization_code grant type.
   @param [in] grant_type= (sas_services) Valid values are:
@@ -52,6 +53,7 @@
     ,inref=
     ,intype=BINARY
     ,contentdisp=inline
+    ,ctype=0
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
     ,mdebug=0
@@ -103,8 +105,10 @@ filename &fref filesrvc
   folderPath="&path"
   filename="&name"
   cdisp="&contentdisp"
+%if "&ctype" ne "0" %then %do;
+  ctype="&ctype"
+%end;
   lrecl=1048544;
-
 %if &intype=BINARY %then %do;
   %mp_binarycopy(inref=&inref, outref=&fref)
 %end;

viya/mv_createjob.sas

@@ -34,14 +34,25 @@
 
   @param path= The full path (on SAS Drive) where the job will be created
   @param name= The name of the job
-  @param desc= The description of the job
+  @param desc= (Created by the mv_createjob.sas macro) The job description
   @param precode= Space separated list of filerefs, pointing to the code that
     needs to be attached to the beginning of the job
-  @param code= Fileref(s) of the actual code to be added
-  @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values are "password" or "authorization_code"
-    (unquoted). The default is authorization_code.
-  @param replace= select NO to avoid replacing any existing job in that location
+  @param code= (ft15f001) Fileref(s) of the actual code to be added
+  @param access_token_var= (ACCESS_TOKEN) Global macro variable containing the
+    access token
+  @param grant_type= (sas_services) Valid values:
+    @li sas_services
+    @li detect
+    @li authorization_code
+    @li password
+  @param replace= (YES) select NO to avoid replacing any existing job
+  @param addjesbeginendmacros= (false) Relates to the `_addjesbeginendmacros`
+    setting.  Normally this would always be false however due to a Viya bug
+    (https://github.com/sasjs/cli/issues/1229) this is now configurable.  Valid
+    values:
+    @li true
+    @li false
+    @li 0 - this will prevent the flag from being set (job will default to true)
   @param contextname= Choose a specific context on which to run the Job.  Leave
     blank to use the default context.  From Viya 3.5 it is possible to configure
     a shared context - see
@@ -62,6 +73,7 @@ https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5p
     ,replace=YES
     ,debug=0
     ,contextname=
+    ,addjesbeginendmacros=false
   );
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
@@ -185,19 +197,29 @@ run;
 %end;
 
 /* set up the body of the request to create the service */
-%local fname3;
+%local fname3 comma;
 %let fname3=%mf_getuniquefileref();
 data _null_;
   file &fname3 TERMSTR=' ';
   length string $32767;
   string=cats('{"version": 0,"name":"'
     ,"&name"
-    ,'","type":"Compute","parameters":[{"name":"_addjesbeginendmacros"'
-    ,',"type":"CHARACTER","defaultValue":"false"}');
+    ,'","type":"Compute","parameters":['
+%if &addjesbeginendmacros ne 0 %then %do;
+    ,'{"name":"_addjesbeginendmacros"'
+    ,',"type":"CHARACTER","defaultValue":"'
+    ,"&addjesbeginendmacros"
+    ,'"}'
+  %let comma=%str(,);
+%end;
+  );
   context=quote(cats(symget('contextname')));
   if context ne '""' then do;
-    string=cats(string,',{"version": 1,"name": "_contextName","defaultValue":'
-      ,context,',"type":"CHARACTER","label":"Context Name","required": false}');
+    string=cats(string
+      ,"&comma"
+      ,'{"version": 1,"name": "_contextName","defaultValue":'
+      ,context,',"type":"CHARACTER","label":"Context Name","required": false}'
+    );
   end;
   string=cats(string,'],"code":"');
   put string;

viya/mv_createwebservice.sas

@@ -236,14 +236,16 @@ data _null_;
   file &adapter;
   put "/* Created on %sysfunc(datetime(),datetime19.) by &sysuserid */";
 /* WEBOUT BEGIN */
-  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
-  put '  ,showmeta=NO ';
+  put '  ,showmeta=N ';
+  put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols; ';
+  put '%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval ';
+  put '  tmpds1 tmpds2 tmpds3 tmpds4; ';
   put '%let numcols=0; ';
+  put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
@@ -279,7 +281,7 @@ data _null_;
   put '    by varnum; ';
   put '  run; ';
   put '  /* move meta to mac vars */ ';
-  put '  data _null_; ';
+  put '  data &colinfo; ';
   put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
   put '    set &colinfo end=last nobs=nobs; ';
   put '    name=upcase(name); ';
@@ -290,7 +292,6 @@ data _null_;
   put '      if format='''' then fmt=cats(''$'',length,''.''); ';
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else fmt=cats(format,formatl,''.''); ';
-  put '      newlen=max(formatl,length); ';
   put '    end; ';
   put '    else do; ';
   put '      typelong=''num''; ';
@@ -298,23 +299,26 @@ data _null_;
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else if formatd=0 then fmt=cats(format,formatl,''.''); ';
   put '      else fmt=cats(format,formatl,''.'',formatd); ';
-  put '      /* needs to be wide, for datetimes etc */ ';
-  put '      newlen=max(length,formatl,24); ';
   put '    end; ';
   put '    /* 32 char unique name */ ';
   put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
   put ' ';
   put '    call symputx(cats(''name'',_n_),name,''l''); ';
   put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
-  put '    call symputx(cats(''len'',_n_),newlen,''l''); ';
   put '    call symputx(cats(''length'',_n_),length,''l''); ';
   put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
+  put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
+  put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
+  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '  proc sql; ';
+  put '  select count(*) into: lastobs from &ds; ';
+  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -322,7 +326,9 @@ data _null_;
   put '      %put &sysmacroname: Switching to DATASTEP engine; ';
   put '      %goto datastep; ';
   put '    %end; ';
-  put '    data &tempds;set &ds; ';
+  put '    data &tempds; ';
+  put '      set &ds; ';
+  put '      &stmt_obs; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
   put '    /* PRETTY is necessary to avoid line truncation in large files */ ';
   put '    filename _sjs2 temp lrecl=131068 encoding=''utf-8''; ';
@@ -349,26 +355,99 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      data _data_; ';
+  put '      /** ';
+  put '        * Extract format definitions ';
+  put '        * First, by getting library locations from dictionary.formats ';
+  put '        * Then, by exporting the width using proc format ';
+  put '        * Cannot use maxw from sashelp.vformat as not always populated ';
+  put '        * Cannot use fmtinfo() as not supported in all flavours ';
+  put '        */ ';
+  put '      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      proc sql noprint; ';
+  put '      create table &tmpds1 as ';
+  put '          select cats(libname,''.'',memname) as FMTCAT, ';
+  put '          FMTNAME ';
+  put '        from dictionary.formats ';
+  put '        where fmttype=''F'' and libname is not null ';
+  put '          and fmtname in (select format from &colinfo where format is not null) ';
+  put '        order by 1; ';
+  put '      create table &tmpds2( ';
+  put '          FMTNAME char(32), ';
+  put '          LENGTH num ';
+  put '      ); ';
+  put '      %local catlist cat fmtlist i; ';
+  put '      select distinct fmtcat into: catlist separated by '' '' from &tmpds1; ';
+  put '      %do i=1 %to %sysfunc(countw(&catlist,%str( ))); ';
+  put '        %let cat=%scan(&catlist,&i,%str( )); ';
+  put '        proc sql; ';
+  put '        select distinct fmtname into: fmtlist separated by '' '' ';
+  put '          from &tmpds1 where fmtcat="&cat"; ';
+  put '        proc format lib=&cat cntlout=&tmpds3(keep=fmtname length); ';
+  put '          select &fmtlist; ';
+  put '        run; ';
+  put '        proc sql; ';
+  put '        insert into &tmpds2 select distinct fmtname,length from &tmpds3; ';
+  put '      %end; ';
+  put ' ';
+  put '      proc sql; ';
+  put '      create table &tmpds4 as ';
+  put '        select a.*, b.length as MAXW ';
+  put '        from &colinfo a ';
+  put '        left join &tmpds2 b ';
+  put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
+  put '        order by a.varnum; ';
+  put '      data _null_; ';
+  put '        set &tmpds4; ';
+  put '        if not missing(maxw); ';
+  put '        call symputx( ';
+  put '          cats(''fmtlen'',_n_), ';
+  put '          /* vars need extra padding due to JSON escaping of special chars */ ';
+  put '          min(32767,ceil((max(length,maxw)+10)*1.5)) ';
+  put '          ,''l'' ';
+  put '        ); ';
+  put '      run; ';
+  put ' ';
+  put '      /* configure varlenchk - as we are explicitly shortening the variables */ ';
+  put '      %let optval=%sysfunc(getoption(varlenchk)); ';
+  put '      options varlenchk=NOWARN; ';
+  put '      data _data_(compress=char); ';
+  put '        /* shorten the new vars */ ';
+  put '        length ';
+  put '      %do i=1 %to &numcols; ';
+  put '          &&name&i $&&fmtlen&i ';
+  put '      %end; ';
+  put '          ; ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
-  put '        &&name&i=&&newname&i ';
+  put '          &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
+  put '      &stmt_obs; ';
+  put ' ';
+  put '      drop ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&newname&i ';
+  put '      %end; ';
+  put '        ; ';
   put '      %do i=1 %to &numcols; ';
-  put '        length &&name&i $&&len&i; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
-  put '        drop &&newname&i; ';
   put '      %end; ';
-  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '        if _error_ then do; ';
+  put '          call symputx(''syscc'',1012); ';
+  put '          stop; ';
+  put '        end; ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
+  put '      options varlenchk=&optval; ';
   put '    %end; ';
   put ' ';
   put '    proc format; /* credit yabwon for special null removal */ ';
@@ -387,8 +466,8 @@ data _null_;
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $32767; ';
-  put '          format &&name&i $32767.; ';
+  put '          length &&name&i $&&fmtlen&i...; ';
+  put '          format &&name&i $&&fmtlen&i...; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -397,10 +476,11 @@ data _null_;
   put '      %else %do; ';
   put '        set &ds; ';
   put '      %end; ';
+  put '      &stmt_obs; ';
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; ';
+  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
   put '          &&name&i=''"''!!trim( ';
   put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
   put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
@@ -413,8 +493,9 @@ data _null_;
   put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
   put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
   put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
   put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '          ))))))))))))!!''"''; ';
+  put '          )))))))))))))!!''"''; ';
   put '        end; ';
   put '        else &&name&i=quote(cats(&&name&i)); ';
   put '      %end; ';
@@ -453,10 +534,11 @@ data _null_;
   put '  proc sql; ';
   put '  drop table &colinfo, &tempds; ';
   put ' ';
-  put '  %if &showmeta=YES %then %do; ';
+  put '  %if %substr(&showmeta,1,1)=Y %then %do; ';
   put '    filename _sjs4 temp lrecl=131068 encoding=''utf-8''; ';
   put '    data _null_; ';
   put '      file _sjs4; ';
+  put '      length label $350; ';
   put '      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; ';
   put '      do i=1 to &numcols; ';
   put '        name=quote(trim(symget(cats(''name'',i)))); ';
@@ -508,8 +590,8 @@ data _null_;
   put '  %quote(&user) ';
   put ' ';
   put '%mend mf_getuser; ';
-  put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL ';
-  put '  ,showmeta=NO ';
+  put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=N,stream=Y,missing=NULL ';
+  put '  ,showmeta=N,maxobs=MAX,workobs=0 ';
   put '); ';
   put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
   put '  sasjs_tables SYS_JES_JOB_URI; ';
@@ -601,8 +683,7 @@ data _null_;
   put ' ';
   put '  /* setup temp ref */ ';
   put '  %if %upcase(&fref) ne _WEBOUT %then %do; ';
-  put '    filename &fref temp lrecl=999999 permission=''A::u::rwx,A::g::rw-,A::o::---'' ';
-  put '      mod; ';
+  put '    filename &fref temp lrecl=999999 permission=''A::u::rwx,A::g::rw-,A::o::---''; ';
   put '  %end; ';
   put ' ';
   put '  /* setup json */ ';
@@ -612,14 +693,13 @@ data _null_;
   put '  run; ';
   put '%end; ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
-  put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
-  put '      ,jref=&fref,engine=DATASTEP,missing=&missing,showmeta=&showmeta ';
+  put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
+  put '      ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
   put '    ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  %if %str(&_debug) ge 131 %then %do; ';
-  put '    /* send back first 10 records of each work table for debugging */ ';
-  put '    options obs=10; ';
+  put '  %if %str(&workobs) > 0 %then %do; ';
+  put '    /* send back first XX records of each work table for debugging */ ';
   put '    data;run;%let tempds=%scan(&syslast,2,.); ';
   put '    ods output Members=&tempds; ';
   put '    proc datasets library=WORK memtype=data; ';
@@ -642,7 +722,9 @@ data _null_;
   put '        put " ""&wt"" : {"; ';
   put '        put ''"nlobs":'' nlobs; ';
   put '        put '',"nvars":'' nvars; ';
-  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) ';
+  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=Y ';
+  put '        ,maxobs=&workobs ';
+  put '      ) ';
   put '      data _null_; file &fref mod;put "}"; ';
   put '    %end; ';
   put '    data _null_; file &fref mod;put "}";run; ';
@@ -650,23 +732,67 @@ data _null_;
   put ' ';
   put '  /* close off json */ ';
   put '  data _null_;file &fref mod; ';
+  put '    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; ';
+  put '    put ",""_DEBUG"" : ""&_debug"" "; ';
   put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
-  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
+  put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
+  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
+  put '    put '',"AUTOEXEC" : '' autoexec; ';
   put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    SYS_JES_JOB_URI=quote(trim(resolve(symget(''SYS_JES_JOB_URI'')))); ';
   put '    put '',"SYS_JES_JOB_URI" : '' SYS_JES_JOB_URI ; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
-  put '    put ",""_DEBUG"" : ""&_debug"" "; ';
-  put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    syserrortext=cats(symget(''syserrortext'')); ';
+  put '    if findc(syserrortext,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
+  put '      syserrortext=''"''!!trim( ';
+  put '        prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '        prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '        prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '        prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '        prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '        prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '        prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '        prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '        prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '        prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '        prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '        prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
+  put '        prxchange(''s/\\/\\\\/'',-1,syserrortext) ';
+  put '      )))))))))))))!!''"''; ';
+  put '    end; ';
+  put '    else syserrortext=cats(''"'',syserrortext,''"''); ';
   put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
+  put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
+  put '    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; ';
+  put '    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); ';
+  put '    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; ';
+  put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
+  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    syswarningtext=cats(symget(''syswarningtext'')); ';
+  put '    if findc(syswarningtext,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
+  put '      syswarningtext=''"''!!trim( ';
+  put '        prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '        prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '        prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '        prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '        prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '        prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '        prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '        prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '        prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '        prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '        prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '        prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
+  put '        prxchange(''s/\\/\\\\/'',-1,syswarningtext) ';
+  put '      )))))))))))))!!''"''; ';
+  put '    end; ';
+  put '    else syswarningtext=cats(''"'',syswarningtext,''"''); ';
   put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';