build.sh build on 2022-02-24:21:33:11

.all-contributorsrc

@@ -117,24 +117,6 @@
       "contributions": [
         "bug"
       ]
-    },
-    {
-      "login": "yabwon",
-      "name": "Bart Jablonski",
-      "avatar_url": "https://avatars.githubusercontent.com/u/9314894?v=4",
-      "profile": "https://github.com/yabwon",
-      "contributions": [
-        "code"
-      ]
-    },
-    {
-      "login": "eltociear",
-      "name": "Ikko Ashimine",
-      "avatar_url": "https://avatars.githubusercontent.com/u/22633385?v=4",
-      "profile": "https://bandism.net/",
-      "contributions": [
-        "code"
-      ]
     }
   ],
   "contributorsPerLine": 7,

.devcontainer/Dockerfile

@@ -1,8 +0,0 @@
-# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/javascript-node/.devcontainer/base.Dockerfile
-
-# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
-ARG VARIANT="18-bullseye"
-FROM mcr.microsoft.com/vscode/devcontainers/javascript-node:0-${VARIANT}
-
-RUN apt-get update \
-    && apt-get install -y doxygen

.devcontainer/devcontainer.json

@@ -1,26 +0,0 @@
-// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
-// https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/typescript-node
-{
-	"name": "Node.js & TypeScript",
-	"build": {
-		"dockerfile": "Dockerfile",
-		// Update 'VARIANT' to pick a Node version: 18, 16, 14.
-		// Append -bullseye or -buster to pin to an OS version.
-		// Use -bullseye variants on local on arm64/Apple Silicon.
-		"args": {
-			"VARIANT": "16-bullseye"
-		}
-	},
-	// Set *default* container specific settings.json values on container create.
-	"settings": {},
-	// Add the IDs of extensions you want installed when the container is created.
-	"extensions": [
-		"SASjs.sasjs-for-vscode"
-	],
-	// Use 'forwardPorts' to make a list of ports inside the container available locally.
-	// "forwardPorts": [],
-	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "npm i && npm i -g @sasjs/cli",
-	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
-	"remoteUser": "node"
-}

.git-hooks/pre-commit

@@ -1,12 +1,5 @@
 #!/bin/bash
-
+sasjs lint
-# Ensure lint is passing
-LINT=`sasjs lint`
-if [[ "$LINT" != "✔ All matched files use @sasjs/lint code style!" ]]; then
-  echo "$LINT"
-  echo "To commit in spite of these warnings, use the -n parameter."
-  exit 1
-fi
 
 # Avoid commits to the master branch
 BRANCH=`git rev-parse --abbrev-ref HEAD`

.github/FUNDING.yml

@@ -1,3 +0,0 @@
-# These are supported funding model platforms
-
-github: [sasjs]

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,17 +0,0 @@
-## Issue
-
-Link any related issue(s) in this section.
-
-## Intent
-
-What this PR intends to achieve.
-
-## Implementation
-
-What code changes have been made to achieve the intent.
-
-## Checks
-
-- [ ] Code is formatted correctly (`sasjs lint`).
-- [ ] Any new functionality has been unit tested.
-- [ ] All unit tests are passing (`sasjs test`).

.github/vpn/config.ovpn

@@ -0,0 +1,30 @@
+cipher AES-256-CBC
+setenv FORWARD_COMPATIBLE 1
+client
+server-poll-timeout 4
+nobind
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 443 tcp
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 1194 udp
+dev tun
+dev-type tun
+ns-cert-type server
+setenv opt tls-version-min 1.0 or-highest
+reneg-sec 604800
+sndbuf 0
+rcvbuf 0
+# NOTE: LZO commands are pushed by the Access Server at connect time.
+# NOTE: The below line doesn't disable LZO.
+comp-lzo no
+verb 3
+setenv PUSH_PEER_INFO
+
+ca ca.crt
+cert user.crt
+key user.key
+tls-auth tls.key 1

.github/workflows/main.yml

@@ -15,14 +15,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v3
+        uses: cycjimmy/semantic-release-action@v2
         env:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-      - name: SAS Packages Release
-        run: |
-          npx @sasjs/cli compile job -s sasjs/utils/create_sas_package.sas -o sasjsbuild
-          # this part depends on https://github.com/sasjs/server/issues/307
-          # sasjs run sasjsbuild/jobs/utils/create_sas_package.sas -t sas9
-
-

.github/workflows/run-tests.yml

@@ -21,6 +21,31 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
 
+      - name: Write VPN Files
+        run: |
+          echo "$CA_CRT" > .github/vpn/ca.crt
+          echo "$USER_CRT" > .github/vpn/user.crt
+          echo "$USER_KEY" > .github/vpn/user.key
+          echo "$TLS_KEY" > .github/vpn/tls.key
+        shell: bash
+        env:
+          CA_CRT: ${{ secrets.CA_CRT}}
+          USER_CRT: ${{ secrets.USER_CRT }}
+          USER_KEY: ${{ secrets.USER_KEY }}
+          TLS_KEY: ${{ secrets.TLS_KEY }}
+
+      - name: Install Open VPN
+        run: |
+          sudo apt install apt-transport-https
+          sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
+          sudo apt-key add openvpn-repo-pkg-key.pub
+          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-focal.list
+          sudo apt update
+          sudo apt install openvpn3
+
+      - name: Start Open VPN 3
+        run: openvpn3 session-start --config .github/vpn/config.ovpn
+
       - name: Install Doxygen
         run: sudo apt-get install doxygen
 
@@ -53,5 +78,7 @@ jobs:
           SECRET: ${{secrets.SECRET}}
           SAS_USERNAME: ${{secrets.SAS_USERNAME}}
           SAS_PASSWORD: ${{secrets.SAS_PASSWORD}}
+          SERVER_URL: ${{secrets.SERVER_URL}}
+          SERVER_TYPE: ${{secrets.SERVER_TYPE}}
           ACCESS_TOKEN: ${{secrets.ACCESS_TOKEN}}
           REFRESH_TOKEN: ${{secrets.REFRESH_TOKEN}}

.gitignore

@@ -10,6 +10,4 @@ sasjsresults/
 mc_*
 
 # ignore .env files as they can contain sasjs access tokens
-*.env*
+*.env*
-
-~

.gitpod.yml

@@ -1,7 +1,7 @@
 tasks:
-  - init: npm install -g npm
+  - init: |
-  - command: npm i
+      nvm install --lts
-  - command: npm i -g @sasjs/cli
+      npm i -g @sasjs/cli
 
 image:
   file: .gitpod.dockerfile
@@ -24,4 +24,4 @@ github:
     # add a "Review in Gitpod" button to pull requests (defaults to false)
     addBadge: false
     # add a label once the prebuild is ready to pull requests (defaults to false)
-    addLabel: prebuilt-in-gitpod
+    addLabel: prebuilt-in-gitpod

.npmignore

@@ -6,6 +6,7 @@ sasjs/
 .github/
 .git-hooks/
 .vscode/
+main.dox
 make_singlefile.sh
 *.md
 .all-contributorsrc

.sasjslint

@@ -1,14 +1,13 @@
 {
-  "noTrailingSpaces": true,
+    "noTrailingSpaces": true,
-  "noEncodedPasswords": true,
+    "noEncodedPasswords": true,
-  "hasDoxygenHeader": true,
+    "hasDoxygenHeader": true,
-  "hasMacroNameInMend": true,
+    "hasMacroNameInMend": true,
-  "hasMacroParentheses": true,
+    "hasMacroParentheses": true,
-  "noGremlins": true,
+    "noNestedMacros": false,
-  "noNestedMacros": false,
+    "noSpacesInFileNames": true,
-  "noSpacesInFileNames": true,
+    "maxLineLength": 300,
-  "maxLineLength": 300,
+    "lowerCaseFileNames": true,
-  "lowerCaseFileNames": true,
+    "noTabIndentation": true,
-  "noTabs": true,
+    "indentationMultiple": 2
-  "indentationMultiple": 2
+}
-}

.vscode/settings.json

@@ -6,7 +6,5 @@
   "editor.rulers": [
     80
   ],
-  "files.trimTrailingWhitespace": true,
+  "files.trimTrailingWhitespace": true
-  "sasjs-for-vscode.target": "docsonly",
-  "sasjs-for-vscode.isLocal": true
 }

CNAME

@@ -0,0 +1 @@
+core.sasjs.io

README.md

@@ -2,6 +2,8 @@
 [![npm package][npm-image]][npm-url]
 [![Github Workflow][githubworkflow-image]][githubworkflow-url]
 ![npm](https://img.shields.io/npm/dt/@sasjs/core)
+![Snyk Vulnerabilities for npm package](https://img.shields.io/snyk/vulnerabilities/npm/@sasjs/core)
+[![License](https://img.shields.io/apm/l/atomic-design-ui.svg)](/LICENSE)
 ![GitHub top language](https://img.shields.io/github/languages/top/sasjs/core)
 [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sasjs/core)](https://github.com/sasjs/core/issues?q=is%3Aissue+is%3Aclosed)
 [![GitHub issues](https://img.shields.io/github/issues-raw/sasjs/core)](https://github.com/sasjs/core/issues)
@@ -16,7 +18,7 @@
 [dependency-url]:https://github.com/sasjs/core/blob/main/package.json
 
 
-Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/.github/CONTRIBUTING.md) are welcome.
+Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
 
 You can download and compile them all in just two lines of SAS code:
 
@@ -29,30 +31,62 @@ Documentation: https://core.sasjs.io
 
 ## Components
 
-### BASE folder (All Platforms)
+### BASE library (All Platforms)
 
 - OS independent
 - Works on all SAS Platforms
 - No X command
-- Prefixes: `mf_`, `mp_`
+- Prefixes: _mf_, _mp_
 
-### DDL folder (All Platforms)
+### DDL library (All Platforms)
 
 - OS independent
 - Works on all SAS Platforms
 - No X command
-- Prefixes: `mddl_(lib)_` -> where lib can be "SAS" (in relation to a SAS component) or "DC" (in relation to a Data Controller component)
+- Prefixes: _mddl_(lib)_ -> where lib can be "SAS" (in relation to a SAS component) or "DC" (in relation to a Data Controller component)
 
 This library will not be used for storing data entries (such as formats or datalines).  Where this becomes necessary in the future, a new repo will be created, in order to keep the NPM bundle size down (for the benefit of those looking to embed purely macros in their applications).
 
-### FCMP folder (All Platforms)
+#### FCMP library (All Platforms)
 
 - Function and macro names are identical, except for special cases
-- Prefixes: `mcf_`
+- Prefixes: _mcf_
 
 The fcmp macros are used to generate fcmp functions, and can be used with or without the `proc fcmp` wrapper.
 
-### LUA folder
+### META library (SAS9 only)
+
+Macros used in SAS EBI, which connect to the metadata server.
+
+- OS independent
+- Metadata aware
+- No X command
+- Prefixes: _mm_
+
+### SERVER library (@sasjs/server only)
+These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
+
+- OS independent
+- @sasjs/server aware
+- No X command
+- Prefixes: _ms_
+
+### VIYA library (Viya only)
+
+Macros used for interfacing with SAS Viya.
+
+- OS independent
+- No X command
+- Prefixes: _mv_, _mvf_
+
+### METAX library (SAS9 only)
+
+- OS specific
+- Metadata aware
+- X command enabled
+- Prefixes: _mmw_,_mmu_,_mmx_
+
+### LUA library
 
 Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
 
@@ -70,63 +104,15 @@ endsubmit;
 run;
 ```
 
-- Prefixes: `ml_`
+- Prefixes: _ml_
-
-### META folder (SAS9 only)
-
-Macros used in SAS EBI, which connect to the metadata server.
-
-- OS independent
-- Metadata aware
-- No X command
-- Prefixes: `mm_`
-
-### METAX folder (SAS9 only)
-
-- OS specific
-- Metadata aware
-- X command enabled
-- Prefixes: `mmx_`
-
-### SERVER folder (@sasjs/server only)
-These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
-
-- OS independent
-- @sasjs/server aware
-- No X command
-- Prefixes: `ms_`
-
-### VIYA folder (Viya only)
-
-Macros used for interfacing with SAS Viya.
-
-- OS independent
-- No X command
-- Prefixes: `mv_`, `mvf_`
-
-### XPLATFORM folder (Viya, Meta, and Server)
-
-Sometimes it is helpful to use a macro that can be used interchangeably regardless of the server type on which is is running (SASVIYA, SAS9, SASJS).
-
-- OS independent
-- No X command
-- Prefixes: `mx_`
 
 ## Installation
 
 First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available, eg:
 
 ```sas
-%let repoloc=/your/path/core;
+options insert=(sasautos="/your/path/macrocore/base");
-options insert=(sasautos="&repoloc/base");
+options insert=(sasautos="/your/path/macrocore/meta");
-options insert=(sasautos="&repoloc/ddl");
-options insert=(sasautos="&repoloc/fcmp");
-options insert=(sasautos="&repoloc/lua");
-options insert=(sasautos="&repoloc/meta");
-options insert=(sasautos="&repoloc/metax");
-options insert=(sasautos="&repoloc/server");
-options insert=(sasautos="&repoloc/viya");
-options insert=(sasautos="&repoloc/xplatform");
 ```
 
 The above can be done directly in your sas program, via an autoexec, or an initialisation program.
@@ -156,7 +142,7 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
   - _mp_ for macro procedures (which generate sas code)
   - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
   - _mv_ for macro procedures that will only work in Viya
-  - _mx_ for macros that work on Viya, SAS 9 EBI and SASjs Server
+  - _mx_ for macros that are XCMD enabled (working on both windows and unix)
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
@@ -218,7 +204,6 @@ When contributing to this library, it is therefore important to ensure that all
 
 We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major/breaking release (v5) becomes necessary:
 
-* mf_getuniquelibref.sas to have the deprecated maxtried parameter removed (no longer needed)
 * mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
 * `insert_cmplib` option of mcf_xxx macros will be deprecated (the option is now checked automatically with value inserted only if needed)
 * mcf_xxx macros to have `wrap=` option defaulted to YES for convenience.  Set this option explicitly to avoid issues.
@@ -231,22 +216,11 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 ![](https://starchart.cc/sasjs/core.svg)
 
-## Other SAS Repositories
 
-The following repositories are also worth checking out:
-
-* [xieliaing/SAS](https://github.com/xieliaing/SAS)
-* [SASJedi/sas-macros](https://github.com/SASJedi/sas-macros)
-* [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
-* [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)
-* [KatjaGlassConsulting/SMILE-SmartSASMacros](https://github.com/KatjaGlassConsulting/SMILE-SmartSASMacros)
-* [rogerjdeangelis](https://github.com/rogerjdeangelis)
-* [scottbass/sas](https://github.com/scottbass/SAS)
-* [yabwon/sas_packages](https://github.com/yabwon/SAS_PACKAGES)
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-12-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-10-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -267,8 +241,6 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
     <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
     <td align="center"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
-    <td align="center"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
-    <td align="center"><a href="https://bandism.net/"><img src="https://avatars.githubusercontent.com/u/22633385?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Ikko Ashimine</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=eltociear" title="Code">💻</a></td>
   </tr>
 </table>
 

base/mf_abort.sas

@@ -12,7 +12,7 @@
 **/
 
 %macro mf_abort(mac=mf_abort.sas, msg=, iftrue=%str(1=1)
-)/des='ungraceful abort' /*STORE SOURCE*/;
+)/*/STORE SOURCE*/;
 
   %if not(%eval(%unquote(&iftrue))) %then %return;
 
@@ -24,4 +24,4 @@
 
 %mend mf_abort;
 
-/** @endcond */
+/** @endcond */

base/mf_deletefile.sas

@@ -1,31 +0,0 @@
-/**
-  @file
-  @brief Deletes a physical file, if it exists
-  @details Usage:
-
-      %mf_writefile(&sasjswork/myfile.txt,l1=some content)
-
-      %mf_deletefile(&sasjswork/myfile.txt)
-
-      %mf_deletefile(&sasjswork/myfile.txt)
-
-
-  @param filepath Full path to the target file
-
-  @returns The return code from the fdelete() invocation
-
-  <h4> Related Macros </h4>
-  @li mf_deletefile.test.sas
-  @li mf_writefile.sas
-
-  @version 9.2
-  @author Allan Bowe
-**/
-
-%macro mf_deletefile(file
-)/*/STORE SOURCE*/;
-  %local rc fref;
-  %let rc= %sysfunc(filename(fref,&file));
-  %if %sysfunc(fdelete(&fref)) ne 0 %then %put %sysfunc(sysmsg());
-  %let rc= %sysfunc(filename(fref));
-%mend mf_deletefile;

base/mf_existfeature.sas

@@ -9,17 +9,19 @@
 
         %put %mf_existfeature(PROCLUA);
 
-  @param [in] feature The feature to detect.
+  @param feature the feature to detect.  Leave blank to list all in log.
 
   @return output returns 1 or 0 (or -1 if not found)
 
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
 
+
   @version 8
   @author Allan Bowe
 **/
 /** @cond */
+
 %macro mf_existfeature(feature
 )/*/STORE SOURCE*/;
   %let feature=%upcase(&feature);
@@ -27,11 +29,7 @@
   %let platform=%mf_getplatform();
 
   %if &feature= %then %do;
-    %put No feature was requested for detection;
+    %put Supported features:  PROCLUA;
-  %end;
-  %else %if &feature=COLCONSTRAINTS %then %do;
-    %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 0;
-    %else 1;
   %end;
   %else %if &feature=PROCLUA %then %do;
     /* https://blogs.sas.com/content/sasdummy/2015/08/03/using-lua-within-your-sas-programs */
@@ -40,20 +38,10 @@
     %else %if "&SYSVLONG" < "9.04.01M3" %then 0;
     %else 1;
   %end;
-  %else %if &feature=DBMS_MEMTYPE %then %do;
-    /* does dbms_memtype exist in dictionary.tables? */
-    %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 0;
-    %else 1;
-  %end;
-  %else %if &feature=EXPORTXLS %then %do;
-    /* is it possible to PROC EXPORT an excel file? */
-    %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 1;
-    %else %if %sysfunc(sysprod(SAS/ACCESS Interface to PC Files)) = 1 %then 1;
-    %else 0;
-  %end;
   %else %do;
     -1
     %put &sysmacroname: &feature not found;
   %end;
 %mend mf_existfeature;
-/** @endcond */
+
+/** @endcond */

base/mf_existfileref.sas

@@ -30,4 +30,4 @@
     0
   %end;
 
-%mend mf_existfileref;
+%mend mf_existfileref;

base/mf_existvarlist.sas

@@ -6,6 +6,9 @@
 
       %put %mf_existVarList(sashelp.class, age sex name dummyvar);
 
+  <h4> SAS Macros </h4>
+  @li mf_abort.sas
+
   @param libds 2 part dataset or view reference
   @param varlist space separated variable names
 

base/mf_fmtdttm.sas

@@ -1,42 +0,0 @@
-/**
-  @file
-  @brief Returns E8601DT26.6 if compatible else DATETIME19.3
-  @details From our experience in [Data Controller for SAS]
-  (https://datacontroller.io) deployments, the E8601DT26.6 datetime format has
-  the widest support when it comes to pass-through SQL queries.
-
-  However, it is not supported in WPS or early versions of SAS 9 (M3 and below)
-  when used as a datetime literal, eg:
-
-      data _null_;
-        demo="%sysfunc(datetime(),E8601DT26.6)"dt;
-        demo=;
-      run;
-
-  This macro will therefore return DATEITME19.3 as an alternative format
-  based on the runtime environment so that it can be used in such cases, eg:
-
-      data _null_;
-        demo="%sysfunc(datetime(),%mf_fmtdttm())"dt;
-        demo=;
-      run;
-
-  <h4> Related Macros </h4>
-  @li mf_fmtdttm.test.sas
-
-  @author Allan Bowe
-**/
-
-%macro mf_fmtdttm(
-)/*/STORE SOURCE*/;
-
-%if "&sysver"="9.2" or "&sysver"="9.3"
-  or ("&sysver"="9.4" and "%substr(&SYSVLONG,9,1)" le "3")
-  or "%substr(&sysver,1,1)"="4"
-  or "%substr(&sysver,1,1)"="5"
-%then %do;DATETIME19.3%end;
-%else %do;E8601DT26.6%end;
-
-%mend mf_fmtdttm;
-
-

base/mf_getgitbranch.sas

@@ -1,37 +0,0 @@
-/**
-  @file
-  @brief Retrieves the current branch from a local GIT repo
-  @details In a local git repository, the current branch is always available in
-  the `.git/HEAD` file in a format like this:  `ref: refs/heads/master`
-
-  This macro simply reads the file and returns the last word (eg `master`).
-
-  Example usage:
-
-      %let gitdir=%sysfunc(pathname(work))/core;
-      %let repo=https://github.com/sasjs/core;
-      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&gitdir));
-
-      %put The current branch is %mf_getgitbranch(&gitdir);
-
-  @param [in] gitdir The directory containing the GIT repository
-
-  <h4> SAS Macros </h4>
-  @li mf_readfile.sas
-
-  <h4> Related Macros </h4>
-  @li mp_gitadd.sas
-  @li mp_gitlog.sas
-  @li mp_gitreleaseinfo.sas
-  @li mp_gitstatus.sas
-
-  @version 9.2
-  @author Allan Bowe
-**/
-
-%macro mf_getgitbranch(gitdir
-)/*/STORE SOURCE*/;
-
-  %scan(%mf_readfile(&gitdir/.git/HEAD),-1)
-
-%mend mf_getgitbranch;

base/mf_getplatform.sas

@@ -5,12 +5,8 @@
 
       %put %mf_getplatform();
 
-  returns one of:
+    returns:
-
+      SASMETA  (or SASVIYA)
-  @li SASMETA
-  @li SASVIYA
-  @li SASJS
-  @li BASESAS
 
   @param switch the param for which to return a platform specific variable
 
@@ -72,4 +68,4 @@
 %else %if &switch=VIYARESTAPI %then %do;
   %mf_trimstr(%sysfunc(getoption(servicesbaseurl)),/)
 %end;
-%mend mf_getplatform;
+%mend mf_getplatform;

base/mf_getuniquefileref.sas

@@ -28,17 +28,15 @@
     be 8 characters, so a 7 letter prefix would mean `maxtries` should be 10.
     if using zero (0) as the prefix, a native assignment is used.
   @param [in] maxtries= (1000) the last part of the libref. Must be an integer.
-  @param [in] lrecl= (32767) Provide a default lrecl with which to initialise
-    the generated fileref.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-%macro mf_getuniquefileref(prefix=_,maxtries=1000,lrecl=32767);
+%macro mf_getuniquefileref(prefix=_,maxtries=1000);
   %local rc fname;
   %if &prefix=0 %then %do;
-    %let rc=%sysfunc(filename(fname,,temp,lrecl=&lrecl));
+    %let rc=%sysfunc(filename(fname,,temp));
     %if &rc %then %put %sysfunc(sysmsg());
     &fname
   %end;
@@ -49,7 +47,7 @@
     %do x=0 %to &maxtries;
       %let fname=&prefix%substr(%sysfunc(ranuni(0)),3,&len);
       %if %sysfunc(fileref(&fname)) > 0 %then %do;
-        %let rc=%sysfunc(filename(fname,,temp,lrecl=&lrecl));
+        %let rc=%sysfunc(filename(fname,,temp));
         %if &rc %then %put %sysfunc(sysmsg());
         &fname
         %return;

base/mf_getuniquelibref.sas

@@ -3,55 +3,38 @@
   @brief Returns an unused libref
   @details Use as follows:
 
-      libname mclib0 (work);
+    libname mclib0 (work);
-      libname mclib1 (work);
+    libname mclib1 (work);
-      libname mclib2 (work);
+    libname mclib2 (work);
 
-      %let libref=%mf_getuniquelibref();
+    %let libref=%mf_getuniquelibref();
-      %put &=libref;
+    %put &=libref;
 
   which returns:
 
 > mclib3
 
-  A blank value is returned if no usable libname is determined.
+  @param prefix= first part of libref.  Remember that librefs can only be 8 characters,
-
+    so a 7 letter prefix would mean that maxtries should be 10.
-  @param [in] prefix= (mclib) first part of the returned libref. As librefs can
+  @param maxtries= the last part of the libref.  Provide an integer value.
-    be as long as 8 characters, a maximum length of 7 characters is premitted
-    for this prefix.
-  @param [in] maxtries= Deprecated parameter. Remains here to ensure a
-    non-breaking change.  Will be removed in v5.
 
   @version 9.2
   @author Allan Bowe
 **/
 
+
 %macro mf_getuniquelibref(prefix=mclib,maxtries=1000);
-  %local x;
+  %local x libref;
-
+  %let x=0;
-  %if ( %length(&prefix) gt 7 ) %then %do;
+  %do x=0 %to &maxtries;
-    %put %str(ERR)OR: The prefix parameter cannot exceed 7 characters.;
+  %if %sysfunc(libref(&prefix&x)) ne 0 %then %do;
-    0
+    %let libref=&prefix&x;
+    %let rc=%sysfunc(libname(&libref,%sysfunc(pathname(work))));
+    %if &rc %then %put %sysfunc(sysmsg());
+    &prefix&x
+    %*put &sysmacroname: Libref &libref assigned as WORK and returned;
     %return;
   %end;
-  %else %if (%sysfunc(NVALID(&prefix,v7))=0) %then %do;
-    %put %str(ERR)OR: Invalid prefix (&prefix);
-    0
-    %return;
   %end;
-
+  %put unable to find available libref in range &prefix.0-&maxtries;
-  /* Set maxtries equal to '10 to the power of [# unused characters] - 1' */
-  %let maxtries=%eval(10**(8-%length(&prefix))-1);
-
-  %do x = 0 %to &maxtries;
-    %if %sysfunc(libref(&prefix&x)) ne 0 %then %do;
-      &prefix&x
-      %return;
-    %end;
-    %let x = %eval(&x + 1);
-  %end;
-
-  %put %str(ERR)OR: No usable libref in range &prefix.0-&maxtries;
-  %put %str(ERR)OR- Try reducing the prefix or deleting some libraries!;
-  0
 %mend mf_getuniquelibref;

base/mf_getuser.sas

@@ -23,19 +23,17 @@
   @author Allan Bowe
 **/
 
-%macro mf_getuser(
+%macro mf_getuser(type=META
 )/*/STORE SOURCE*/;
-  %local user;
+  %local user metavar;
+  %if &type=OS %then %let metavar=_secureusername;
+  %else %let metavar=_metaperson;
 
-  %if %symexist(_sasjs_username) %then %let user=&_sasjs_username;
+  %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %let user=&SYS_COMPUTE_SESSION_OWNER;
-  %else %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %do;
+  %else %if %symexist(&metavar) %then %do;
-    %let user=&SYS_COMPUTE_SESSION_OWNER;
+    %if %length(&&&metavar)=0 %then %let user=&sysuserid;
-  %end;
-  %else %if %symexist(_metaperson) %then %do;
-    %if %length(&_metaperson)=0 %then %let user=&sysuserid;
     /* sometimes SAS will add @domain extension - remove for consistency */
-    /* but be sure to quote in case of usernames with commas */
+    %else %let user=%scan(&&&metavar,1,@);
-    %else %let user=%unquote(%scan(%quote(&_metaperson),1,@));
   %end;
   %else %let user=&sysuserid;
 

base/mf_getvarcount.sas

@@ -2,48 +2,31 @@
   @file
   @brief Returns number of variables in a dataset
   @details Useful to identify those renagade datasets that have no columns!
-  Can also be used to count for numeric, or character columns
 
-      %put Number of Variables=%mf_getvarcount(sashelp.class);
+        %put Number of Variables=%mf_getvarcount(sashelp.class);
-      %put Character Variables=%mf_getvarcount(sashelp.class,typefilter=C);
-      %put Numeric Variables = %mf_getvarcount(sashelp.class,typefilter=N);
 
   returns:
   > Number of Variables=4
 
-
+  @param libds Two part dataset (or view) reference.
-  @param [in] libds Two part dataset (or view) reference.
-  @param [in] typefilter= (A) Filter for certain types of column.  Valid values:
-    @li A Count All columns
-    @li C Count Character columns only
-    @li N Count Numeric columns only
 
   @version 9.2
   @author Allan Bowe
 
 **/
 
-%macro mf_getvarcount(libds,typefilter=A
+%macro mf_getvarcount(libds
 )/*/STORE SOURCE*/;
-  %local dsid nvars rc outcnt x;
+  %local dsid nvars rc ;
   %let dsid=%sysfunc(open(&libds));
   %let nvars=.;
-  %let outcnt=0;
-  %let typefilter=%upcase(&typefilter);
   %if &dsid %then %do;
     %let nvars=%sysfunc(attrn(&dsid,NVARS));
-    %if &typefilter=A %then %let outcnt=&nvars;
-    %else %if &nvars>0 %then %do x=1 %to &nvars;
-      /* increment based on variable type */
-      %if %sysfunc(vartype(&dsid,&x))=&typefilter %then %do;
-        %let outcnt=%eval(&outcnt+1);
-      %end;
-    %end;
     %let rc=%sysfunc(close(&dsid));
   %end;
   %else %do;
     %put unable to open &libds (rc=&dsid);
     %let rc=%sysfunc(close(&dsid));
   %end;
-  &outcnt
+  &nvars
 %mend mf_getvarcount;

base/mf_increment.sas

@@ -1,29 +0,0 @@
-/**
-  @file
-  @brief Increments a macro variable
-  @details Useful outside of do-loops - will increment a macro variable every
-  time it is called.
-
-  Example:
-
-      %let cnt=1;
-      %put We have run %mf_increment(cnt) lines;
-      %put Now we have run %mf_increment(cnt) lines;
-      %put There are %mf_increment(cnt) lines in total;
-
-  @param [in] MACRO_NAME the name of the macro variable to increment
-  @param [in] ITER= The amount to add or subtract to the macro
-
-  <h4> Related Files </h4>
-  @li mf_increment.test.sas
-
-**/
-
-%macro mf_increment(macro_name,incr=1);
-
-  /* iterate the value */
-  %let &macro_name=%eval(&&&macro_name+&incr);
-  /* return the value */
-  &&&macro_name
-
-%mend mf_increment;

base/mf_isint.sas

@@ -20,11 +20,8 @@
 
 %macro mf_isint(arg
 )/*/STORE SOURCE*/;
-
-  /* blank val is not an integer */
-  %if "&arg"="" %then %do;0%return;%end;
-
   /* remove minus sign if exists */
+
   %local val;
   %if "%substr(%str(&arg),1,1)"="-" %then %let val=%substr(%str(&arg),2);
   %else %let val=&arg;
@@ -33,4 +30,4 @@
   %if %sysfunc(findc(%str(&val),,kd)) %then %do;0%end;
   %else %do;1%end;
 
-%mend mf_isint;
+%mend mf_isint;

base/mf_readfile.sas

@@ -1,63 +0,0 @@
-/**
-  @file
-  @brief Reads the first line of a file using pure macro
-  @details Reads the first line of a file and returns it.  Future versions may
-  read each line into a macro variable array.
-
-  Generally, reading data into macro variables is not great as certain
-  nonprintable characters (such as CR, LF) may be dropped in the conversion.
-
-    Usage:
-
-        %mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=more content)
-
-        %put %mf_readfile(&sasjswork/myfile.txt);
-
-
-  @param [in] fpath Full path to file to be read
-
-  <h4> Related Macros </h4>
-  @li mf_deletefile.sas
-  @li mf_writefile.sas
-  @li mf_readfile.test.sas
-
-  @version 9.2
-  @author Allan Bowe
-**/
-/** @cond */
-
-%macro mf_readfile(fpath
-)/*/STORE SOURCE*/;
-%local fref rc fid fcontent;
-
-/* check file exists */
-%if %sysfunc(filename(fref,&fpath)) ne 0 %then %do;
-  %put &=fref &=fpath;
-  %put %str(ERR)OR: %sysfunc(sysmsg());
-  %return;
-%end;
-
-%let fid=%sysfunc(fopen(&fref,I));
-
-%if &fid=0 %then %do;
-  %put %str(ERR)OR: %sysfunc(sysmsg());
-  %return;
-%end;
-
-%if %sysfunc(fread(&fid)) = 0 %then %do;
-  %let rc=%sysfunc(fget(&fid,fcontent,65534));
-  &fcontent
-%end;
-
-/*
-%do %while(%sysfunc(fread(&fid)) = 0);
-  %let rc=%sysfunc(fget(&fid,fcontent,65534));
-  &fcontent
-%end;
-*/
-
-%let rc=%sysfunc(fclose(&fid));
-%let rc=%sysfunc(filename(&fref));
-
-%mend mf_readfile;
-/** @endcond */

base/mf_verifymacvars.sas

@@ -59,7 +59,7 @@
 
   %goto exit_success;
   %exit_err:
-    %put &abortmsg;
+    %put %str(ERR)OR: &abortmsg;
     %mf_abort(iftrue=(&mabort ne SOFT),
       mac=mf_verifymacvars,
       msg=%str(&abortmsg)

base/mp_abort.sas

@@ -8,29 +8,23 @@
 
   The method used varies according to the context.  Important points:
 
-  @li should not use endsas or abort cancel in 9.4m3 WIN environments as this
+  @li should not use endsas or abort cancel in 9.4m3 environments as this can
-    can cause hung multibridge sessions and result in a frozen STP server
+    cause hung multibridge sessions and result in a frozen STP server
-  @li The use of endsas in 9.4m6+ windows environments for POST requests to the
-    STP server can result in an empty response body
   @li should not use endsas in viya 3.5 as this destroys the session and cannot
     fetch results (although both mv_getjoblog.sas and the @sasjs/adapter will
     recognise this and fetch the log of the parent session instead)
   @li STP environments must finish cleanly to avoid the log being sent to
     _webout.  To assist with this, we also run stpsrvset('program error', 0)
-    and set SYSCC=0.
+    and set SYSCC=0.  We take a unique "soft abort" approach - we open a macro
-    Where possible, we take a unique "soft abort" approach - we open a macro
     but don't close it!  This works everywhere EXCEPT inside a \%include inside
     a macro.  For that, we recommend you use mp_include.sas to perform the
     include, and then call \%mp_abort(mode=INCLUDE) from the source program (ie,
     OUTSIDE of the top-parent macro).
-    The soft abort has become ineffective in 9.4m6 WINDOWS environments.  We are
-    currently investigating approaches to deal with this.
 
 
-  @param mac= (mp_abort.sas) To contain the name of the calling macro. Do not
+  @param mac= to contain the name of the calling macro
-    use &sysmacroname as this will always resolve to MP_ABORT.
   @param msg= message to be returned
-  @param iftrue= (1=1) Supply a condition for which the macro should be executed
+  @param iftrue= supply a condition under which the macro should be executed.
   @param errds= (work.mp_abort_errds) There is no clean way to end a process
     within a %include called within a macro.  Furthermore, there is no way to
     test if a macro is called within a %include.  To handle this particular
@@ -51,12 +45,11 @@
     @li REGULAR (default)
     @li INCLUDE
 
-  @version 9.4
-  @author Allan Bowe
-
   <h4> Related Macros </h4>
   @li mp_include.sas
 
+  @version 9.4
+  @author Allan Bowe
   @cond
 **/
 
@@ -65,233 +58,167 @@
   , mode=REGULAR
 )/*/STORE SOURCE*/;
 
-%global sysprocessmode sysprocessname sasjs_stpsrv_header_loc sasjsprocessmode;
+  %global sysprocessmode sysprocessname;
-%local fref fid i;
 
-%if not(%eval(%unquote(&iftrue))) %then %return;
+  %if not(%eval(%unquote(&iftrue))) %then %return;
 
-%put NOTE: ///  mp_abort macro executing //;
+  %put NOTE: ///  mp_abort macro executing //;
-%if %length(&mac)>0 %then %put NOTE- called by &mac;
+  %if %length(&mac)>0 %then %put NOTE- called by &mac;
-%put NOTE - &msg;
+  %put NOTE - &msg;
 
-%if %symexist(_SYSINCLUDEFILEDEVICE)
+  %if %symexist(_SYSINCLUDEFILEDEVICE)
-/* abort cancel FILE does not restart outside the INCLUDE on Viya 3.5 */
+  /* abort cancel FILE does not restart outside the INCLUDE on Viya 3.5 */
-and %superq(SYSPROCESSNAME) ne %str(Compute Server)
+  and "&SYSPROCESSNAME " ne "Compute Server "
-%then %do;
-  %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do;
-    data &errds;
-      iftrue='1=1';
-      length mac $100 msg $5000;
-      mac=symget('mac');
-      msg=symget('msg');
-    run;
-    data _null_;
-      abort cancel FILE;
-    run;
-    %return;
-  %end;
-%end;
-
-/* Web App Context */
-%if %symexist(_PROGRAM)
-  or %superq(SYSPROCESSNAME) = %str(Compute Server)
-  or &mode=INCLUDE
-%then %do;
-  options obs=max replace mprint;
-  %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5"
   %then %do;
-    options nosyntaxcheck;
+    %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do;
-  %end;
+      data &errds;
-
+        iftrue='1=1';
-  %if &mode=INCLUDE %then %do;
+        length mac $100 msg $5000;
-    %if %sysfunc(exist(&errds))=1 %then %do;
+        mac=symget('mac');
-      data _null_;
+        msg=symget('msg');
-        set &errds;
+      run;
-        call symputx('iftrue',iftrue,'l');
+      data _null_;
-        call symputx('mac',mac,'l');
+        abort cancel FILE;
-        call symputx('msg',msg,'l');
-        putlog (_all_)(=);
       run;
-      %if (&iftrue)=0 %then %return;
-    %end;
-    %else %do;
-      %put &sysmacroname: No include errors found;
       %return;
     %end;
   %end;
 
-  /* extract log errs / warns, if exist */
+  /* Stored Process Server web app context */
-  %local logloc logline;
+  %if %symexist(_metaperson)
-  %global logmsg; /* capture global messages */
+    or "&SYSPROCESSNAME "="Compute Server "
-  %if %symexist(SYSPRINTTOLOG) %then %let logloc=&SYSPRINTTOLOG;
+    or &mode=INCLUDE
-  %else %let logloc=%qsysfunc(getoption(LOG));
+  %then %do;
-  proc printto log=log;run;
+    options obs=max replace nosyntaxcheck mprint;
-  %let logline=0;
+    %if &mode=INCLUDE %then %do;
-  %if %length(&logloc)>0 %then %do;
+      %if %sysfunc(exist(&errds))=1 %then %do;
-    data _null_;
+        data _null_;
-      infile &logloc lrecl=5000;
+          set &errds;
-      input; putlog _infile_;
+          call symputx('iftrue',iftrue,'l');
-      i=1;
+          call symputx('mac',mac,'l');
-      retain logonce 0;
+          call symputx('msg',msg,'l');
-      if (
+          putlog (_all_)(=);
-          _infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR"
+        run;
-        ) and logonce=0 then
+        %if (&iftrue)=0 %then %return;
-      do;
+      %end;
-        call symputx('logline',_n_);
+      %else %do;
-        logonce+1;
+        %put &sysmacroname: No include errors found;
-      end;
+        %return;
-    run;
+      %end;
-    /* capture log including lines BEFORE the err */
+    %end;
-    %if &logline>0 %then %do;
+
+    /* extract log errs / warns, if exist */
+    %local logloc logline;
+    %global logmsg; /* capture global messages */
+    %if %symexist(SYSPRINTTOLOG) %then %let logloc=&SYSPRINTTOLOG;
+    %else %let logloc=%qsysfunc(getoption(LOG));
+    proc printto log=log;run;
+    %let logline=0;
+    %if %length(&logloc)>0 %then %do;
       data _null_;
         infile &logloc lrecl=5000;
-        input;
+        input; putlog _infile_;
         i=1;
-        stoploop=0;
+        retain logonce 0;
-        if _n_ ge &logline-15 and stoploop=0 then do until (i>22);
+        if (
-          call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
+            _infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR"
-          input;
+          ) and logonce=0 then
-          i+1;
+        do;
-          stoploop=1;
+          call symputx('logline',_n_);
+          logonce+1;
         end;
-        if stoploop=1 then stop;
       run;
+      /* capture log including lines BEFORE the err */
+      %if &logline>0 %then %do;
+        data _null_;
+          infile &logloc lrecl=5000;
+          input;
+          i=1;
+          stoploop=0;
+          if _n_ ge &logline-15 and stoploop=0 then do until (i>22);
+            call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
+            input;
+            i+1;
+            stoploop=1;
+          end;
+          if stoploop=1 then stop;
+        run;
+      %end;
     %end;
-  %end;
 
-  %if %symexist(SYS_JES_JOB_URI) %then %do;
+    %if %symexist(SYS_JES_JOB_URI) %then %do;
-    /* setup webout for Viya */
+      /* setup webout */
-    options nobomfile;
+      OPTIONS NOBOMFILE;
-    %if "X&SYS_JES_JOB_URI.X"="XX" %then %do;
+      %if "X&SYS_JES_JOB_URI.X"="XX" %then %do;
-        filename _webout temp lrecl=999999 mod;
+          filename _webout temp lrecl=999999 mod;
+      %end;
+      %else %do;
+        filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
+          name="_webout.json" lrecl=999999 mod;
+      %end;
     %end;
-    %else %do;
-      filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
-        name="_webout.json" lrecl=999999 mod;
-    %end;
-  %end;
-  %else %if %sysfunc(filename(fref,&sasjs_stpsrv_header_loc))=0 %then %do;
-    options nobomfile;
-    /* set up http header for SASjs Server */
-    %let fid=%sysfunc(fopen(&fref,A));
-    %if &fid=0 %then %do;
-      %put %str(ERR)OR: %sysfunc(sysmsg());
-      %return;
-    %end;
-    %let rc=%sysfunc(fput(&fid,%str(Content-Type: application/json)));
-    %let rc=%sysfunc(fwrite(&fid));
-    %let rc=%sysfunc(fclose(&fid));
-    %let rc=%sysfunc(filename(&fref));
-  %end;
 
-  /* send response in SASjs JSON format */
+    /* send response in SASjs JSON format */
-  data _null_;
-    file _webout mod lrecl=32000 encoding='utf-8';
-    length msg syswarningtext syserrortext $32767 mode $10 ;
-    sasdatetime=datetime();
-    msg=symget('msg');
-  %if &logline>0 %then %do;
-    msg=cats(msg,'\n\nLog Extract:\n',symget('logmsg'));
-  %end;
-    /* escape the escapes */
-    msg=tranwrd(msg,'\','\\');
-    /* escape the quotes */
-    msg=tranwrd(msg,'"','\"');
-    /* ditch the CRLFs as chrome complains */
-    msg=compress(msg,,'kw');
-    /* quote without quoting the quotes (which are escaped instead) */
-    msg=cats('"',msg,'"');
-    if symexist('_debug') then debug=quote(trim(symget('_debug')));
-    else debug='""';
-    if symget('sasjsprocessmode')='Stored Program' then mode='SASJS';
-    if mode ne 'SASJS' then put '>>weboutBEGIN<<';
-    put '{"SYSDATE" : "' "&SYSDATE" '"';
-    put ',"SYSTIME" : "' "&SYSTIME" '"';
-    put ',"sasjsAbort" : [{';
-    put ' "MSG":' msg ;
-    put ' ,"MAC": "' "&mac" '"}]';
-    put ",""SYSUSERID"" : ""&sysuserid"" ";
-    put ',"_DEBUG":' debug ;
-    if symexist('_metauser') then do;
-      _METAUSER=quote(trim(symget('_METAUSER')));
-      put ",""_METAUSER"": " _METAUSER;
-      _METAPERSON=quote(trim(symget('_METAPERSON')));
-      put ',"_METAPERSON": ' _METAPERSON;
-    end;
-    if symexist('SYS_JES_JOB_URI') then do;
-      SYS_JES_JOB_URI=quote(trim(symget('SYS_JES_JOB_URI')));
-      put ',"SYS_JES_JOB_URI": ' SYS_JES_JOB_URI;
-    end;
-    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
-    put ',"_PROGRAM" : ' _PROGRAM ;
-    put ",""SYSCC"" : ""&syscc"" ";
-    syserrortext=cats(symget('syserrortext'));
-    if findc(syserrortext,'"\'!!'0A0D09000E0F010210111A'x) then do;
-      syserrortext='"'!!trim(
-        prxchange('s/"/\\"/',-1,        /* double quote */
-        prxchange('s/\x0A/\n/',-1,      /* new line */
-        prxchange('s/\x0D/\r/',-1,      /* carriage return */
-        prxchange('s/\x09/\\t/',-1,     /* tab */
-        prxchange('s/\x00/\\u0000/',-1, /* NUL */
-        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
-        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
-        prxchange('s/\x01/\\u0001/',-1, /* SOH */
-        prxchange('s/\x02/\\u0002/',-1, /* STX */
-        prxchange('s/\x10/\\u0010/',-1, /* DLE */
-        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
-        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
-        prxchange('s/\\/\\\\/',-1,syserrortext)
-      )))))))))))))!!'"';
-    end;
-    else syserrortext=cats('"',syserrortext,'"');
-    put ',"SYSERRORTEXT" : ' syserrortext;
-    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
-    put ",""SYSJOBID"" : ""&sysjobid"" ";
-    put ",""SYSSCPL"" : ""&sysscpl"" ";
-    put ",""SYSSITE"" : ""&syssite"" ";
-    sysvlong=quote(trim(symget('sysvlong')));
-    put ',"SYSVLONG" : ' sysvlong;
-    syswarningtext=cats(symget('syswarningtext'));
-    if findc(syswarningtext,'"\'!!'0A0D09000E0F010210111A'x) then do;
-      syswarningtext='"'!!trim(
-        prxchange('s/"/\\"/',-1,        /* double quote */
-        prxchange('s/\x0A/\n/',-1,      /* new line */
-        prxchange('s/\x0D/\r/',-1,      /* carriage return */
-        prxchange('s/\x09/\\t/',-1,     /* tab */
-        prxchange('s/\x00/\\u0000/',-1, /* NUL */
-        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
-        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
-        prxchange('s/\x01/\\u0001/',-1, /* SOH */
-        prxchange('s/\x02/\\u0002/',-1, /* STX */
-        prxchange('s/\x10/\\u0010/',-1, /* DLE */
-        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
-        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
-        prxchange('s/\\/\\\\/',-1,syswarningtext)
-      )))))))))))))!!'"';
-    end;
-    else syswarningtext=cats('"',syswarningtext,'"');
-    put ",""SYSWARNINGTEXT"" : " syswarningtext;
-    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
-    put "}" ;
-    if mode ne 'SASJS' then put '>>weboutEND<<';
-  run;
-
-  %put _all_;
-
-  %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
     data _null_;
-      putlog 'stpsrvset program err and syscc';
+      file _webout mod lrecl=32000 encoding='utf-8';
-      rc=stpsrvset('program error', 0);
+      length msg $32767 ;
-      call symputx("syscc",0,"g");
+      sasdatetime=datetime();
-    run;
+      msg=symget('msg');
-    %if &sysscp=WIN
+    %if &logline>0 %then %do;
-    and 1=0 /* deprecating this logic until we figure out a consistent abort */
+      msg=cats(msg,'\n\nLog Extract:\n',symget('logmsg'));
-    and "%substr(%str(&sysvlong         ),1,8)"="9.04.01M"
-    and "%substr(%str(&sysvlong         ),9,1)">"5" %then %do;
-      /* skip approach (below) does not work in windows m6+ envs */
-      endsas;
     %end;
-    %else %do;
+      /* escape the quotes */
+      msg=tranwrd(msg,'"','\"');
+      /* ditch the CRLFs as chrome complains */
+      msg=compress(msg,,'kw');
+      /* quote without quoting the quotes (which are escaped instead) */
+      msg=cats('"',msg,'"');
+      if symexist('_debug') then debug=quote(trim(symget('_debug')));
+      else debug='""';
+      put '>>weboutBEGIN<<';
+      put '{"SYSDATE" : "' "&SYSDATE" '"';
+      put ',"SYSTIME" : "' "&SYSTIME" '"';
+      put ',"sasjsAbort" : [{';
+      put ' "MSG":' msg ;
+      put ' ,"MAC": "' "&mac" '"}]';
+      put ",""SYSUSERID"" : ""&sysuserid"" ";
+      put ',"_DEBUG":' debug ;
+      if symexist('_metauser') then do;
+        _METAUSER=quote(trim(symget('_METAUSER')));
+        put ",""_METAUSER"": " _METAUSER;
+        _METAPERSON=quote(trim(symget('_METAPERSON')));
+        put ',"_METAPERSON": ' _METAPERSON;
+      end;
+      if symexist('SYS_JES_JOB_URI') then do;
+        SYS_JES_JOB_URI=quote(trim(symget('SYS_JES_JOB_URI')));
+        put ',"SYS_JES_JOB_URI": ' SYS_JES_JOB_URI;
+      end;
+      _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
+      put ',"_PROGRAM" : ' _PROGRAM ;
+      put ",""SYSCC"" : ""&syscc"" ";
+      syserrortext=quote(trim(symget('syserrortext')));
+      put ",""SYSERRORTEXT"" : " syserrortext;
+      put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+      put ",""SYSJOBID"" : ""&sysjobid"" ";
+      put ",""SYSSCPL"" : ""&sysscpl"" ";
+      put ",""SYSSITE"" : ""&syssite"" ";
+      sysvlong=quote(trim(symget('sysvlong')));
+      put ',"SYSVLONG" : ' sysvlong;
+      syswarningtext=quote(trim(symget('syswarningtext')));
+      put ",""SYSWARNINGTEXT"" : " syswarningtext;
+      put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
+      put "}" @;
+      put '>>weboutEND<<';
+    run;
+
+    %put _all_;
+
+    %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
+      data _null_;
+        putlog 'stpsrvset program err and syscc';
+        rc=stpsrvset('program error', 0);
+        call symputx("syscc",0,"g");
+      run;
       /**
         * endsas kills 9.4m3 deployments by orphaning multibridges.
         * Abort variants are ungraceful (non zero return code)
@@ -309,29 +236,28 @@ and %superq(SYSPROCESSNAME) ne %str(Compute Server)
       run;
       %inc skip;
     %end;
-  %end;
+    %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
-  %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
+      /* endsas kills the session making it harder to fetch results */
-    /* endsas kills the session making it harder to fetch results */
+      data _null_;
-    data _null_;
+        syswarningtext=symget('syswarningtext');
-      syswarningtext=symget('syswarningtext');
+        syserrortext=symget('syserrortext');
-      syserrortext=symget('syserrortext');
+        abort_msg=symget('msg');
-      abort_msg=symget('msg');
+        syscc=symget('syscc');
-      syscc=symget('syscc');
+        sysuserid=symget('sysuserid');
-      sysuserid=symget('sysuserid');
+        iftrue=symget('iftrue');
-      iftrue=symget('iftrue');
+        put (_all_)(/=);
-      put (_all_)(/=);
+        call symputx('syscc',0);
-      call symputx('syscc',0);
+        abort cancel nolist;
-      abort cancel nolist;
+      run;
-    run;
+    %end;
+    %else %do;
+      %abort cancel;
+    %end;
   %end;
   %else %do;
+    %put _all_;
     %abort cancel;
   %end;
-%end;
-%else %do;
-  %put _all_;
-  %abort cancel;
-%end;
 %mend mp_abort;
 
-/** @endcond */
+/** @endcond */

base/mp_assertscope.sas

@@ -74,8 +74,7 @@
   outds=work.test_results
 )/*/STORE SOURCE*/;
 %local ds test_result test_comments del add mod ilist;
-%let ilist=%upcase(&sasjs_prefix._FUNCTIONS SYS_PROCHTTP_STATUS_CODE
+%let ilist=%upcase(&sasjs_prefix._FUNCTIONS &ignorelist);
-  SYS_PROCHTTP_STATUS_CODE SYS_PROCHTTP_STATUS_PHRASE &ignorelist);
 
 /**
   * this sets up the global vars, it will also enter STRICT mode.  If this
@@ -90,7 +89,7 @@
   create table &scopeds as
     select name,offset,value
     from dictionary.macros
-    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
+    where scope="&scope" and name not in (%mf_getquotedstr(&ilist))
     order by name,offset;
 %end;
 %else %if &action=COMPARE %then %do;
@@ -104,9 +103,7 @@
 
   %let ds=&syslast;
 
-  proc compare
+  proc compare base=&scopeds compare=&ds;
-    base=&scopeds(where=(upcase(name) not in (%mf_getquotedstr(&ilist))))
-    compare=&ds noprint;
   run;
 
   %if &sysinfo=0 %then %do;
@@ -144,4 +141,4 @@
   drop table &ds;
 %end;
 
-%mend mp_assertscope;
+%mend mp_assertscope;

base/mp_binarycopy.sas

@@ -4,8 +4,8 @@
   @details Reads in a file byte by byte and writes it back out.  Is an
   os-independent method to copy files.  In case of naming collision, the
   default filerefs can be modified.
-  Note that if you have a new enough version of SAS, and you don't need features
+  Based on:
-  such as APPEND, you may be better of using the fcopy() function instead.
+  https://stackoverflow.com/questions/13046116/using-sas-to-copy-a-text-file
 
         %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
 
@@ -31,7 +31,6 @@
   @param [in] mode (CREATE) Valid values:
     @li CREATE - Create the file (even if it already exists)
     @li APPEND - Append to the file (don't overwrite)
-  @param iftrue= (1=1) Supply a condition for which the macro should be executed
 
   @returns nothing
 
@@ -45,14 +44,15 @@
     ,inref=____in   /* override default to use own filerefs */
     ,outref=____out /* override default to use own filerefs */
     ,mode=CREATE
-    ,iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
-  %local mod;
+  %local mod outmode;
-
+  %if &mode=APPEND %then %do;
-  %if not(%eval(%unquote(&iftrue))) %then %return;
+    %let mod=mod;
-
+    %let outmode='a';
-  %if &mode=APPEND %then %let mod=mod;
+  %end;
-
+  %else %do;
+    %let outmode='o';
+  %end;
   /* these IN and OUT filerefs can point to anything */
   %if &inref = ____in %then %do;
     filename &inref &inloc lrecl=1048576 ;
@@ -63,17 +63,22 @@
 
   /* copy the file byte-for-byte  */
   data _null_;
-    infile &inref lrecl=1 recfm=n;
+    length filein 8 fileid 8;
-    file &outref &mod recfm=n;
+    filein = fopen("&inref",'I',1,'B');
-    input sourcechar $char1. @@;
+    fileid = fopen("&outref",&outmode,1,'B');
-    format sourcechar hex2.;
+    rec = '20'x;
-    put sourcechar char1. @@;
+    do while(fread(filein)=0);
+      rc = fget(filein,rec,1);
+      rc = fput(fileid, rec);
+      rc =fwrite(fileid);
+    end;
+    rc = fclose(filein);
+    rc = fclose(fileid);
   run;
-
   %if &inref = ____in %then %do;
     filename &inref clear;
   %end;
   %if &outref=____out %then %do;
     filename &outref clear;
   %end;
-%mend mp_binarycopy;
+%mend mp_binarycopy;

base/mp_chop.sas

@@ -1,194 +0,0 @@
-/**
-  @file
-  @brief Splits a file of ANY SIZE by reference to a search string.
-  @details Provide a fileref and a search string to chop off part of a file.
-
-  Works by reading in the file byte by byte, then marking the beginning and end
-  of each matched string, before finally doing the chop.
-
-  Choose whether to keep the FIRST or the LAST section of the file.  Optionally,
-  use an OFFSET to fix the precise chop point.
-
-  Usage:
-
-      %let src="%sysfunc(pathname(work))/file.txt";
-      %let str=Chop here!;
-      %let out1="%sysfunc(pathname(work))/file1.txt";
-      %let out2="%sysfunc(pathname(work))/file2.txt";
-      %let out3="%sysfunc(pathname(work))/file3.txt";
-      %let out4="%sysfunc(pathname(work))/file4.txt";
-
-      data _null_;
-        file &src;
-        put "startsection&str.endsection";
-      run;
-
-      %mp_chop(&src, matchvar=str, keep=FIRST, outfile=&out1)
-      %mp_chop(&src, matchvar=str, keep=LAST, outfile=&out2)
-      %mp_chop(&src, matchvar=str, keep=FIRST, matchpoint=END, outfile=&out3)
-      %mp_chop(&src, matchvar=str, keep=LAST, matchpoint=END, outfile=&out4)
-
-      filename results (&out1 &out2 &out3 &out4);
-      data _null_;
-        infile results;
-        input;
-        list;
-      run;
-
-  Results:
-    @li `startsection`
-    @li `Chop here!endsection`
-    @li `startsectionChop here!`
-    @li `endsection`
-
-  For more examples, see mp_chop.test.sas
-
-  @param [in] infile The QUOTED path to the file on which to perform the chop
-  @param [in] matchvar= Macro variable NAME containing the string to split by
-  @param [in] matchpoint= (START) Valid values:
-    @li START - chop at the beginning of the string in `matchvar`.
-    @li END - chop at the end of the string in `matchvar`.
-  @param [in] offset= (0) An adjustment to the precise chop location, by
-    by reference to the `matchpoint`. Should be a positive or negative integer.
-  @param [in] keep= (FIRST) Valid values:
-    @li FIRST - keep the section of the file before the chop
-    @li LAST - keep the section of the file after the chop
-  @param [in] mdebug= (0) Set to 1 to provide macro debugging
-  @param outfile= (0) Optional QUOTED path to the adjusted output file (avoids
-    overwriting the first file).
-
-  <h4> SAS Macros </h4>
-  @li mf_getuniquefileref.sas
-  @li mf_getuniquename.sas
-
-  <h4> Related Macros </h4>
-  @li mp_abort.sas
-  @li mp_gsubfile.sas
-  @li mp_replace.sas
-  @li mp_chop.test.sas
-
-  @version 9.4
-  @author Allan Bowe
-
-**/
-
-%macro mp_chop(infile,
-  matchvar=,
-  matchpoint=START,
-  keep=FIRST,
-  offset=0,
-  mdebug=0,
-  outfile=0
-)/*/STORE SOURCE*/;
-
-%local fref0 dttm ds1 outref;
-%let fref0=%mf_getuniquefileref();
-%let ds1=%mf_getuniquename(prefix=allchars);
-%let ds2=%mf_getuniquename(prefix=startmark);
-
-%if &outfile=0 %then %let outfile=&infile;
-
-%mp_abort(iftrue= (%length(%superq(&matchvar))=0)
-  ,mac=mp_chop.sas
-  ,msg=%str(&matchvar is an empty variable)
-)
-
-/* START */
-%let dttm=%sysfunc(datetime());
-
-filename &fref0 &infile lrecl=1 recfm=n;
-
-/* create dataset with one char per row */
-data &ds1;
-  infile &fref0;
-  input sourcechar $char1. @@;
-  format sourcechar hex2.;
-run;
-
-/* get start & stop position of first matchvar string (one row, two vars) */
-data &ds2;
-  /* set find string to length in bytes to cover trailing spaces */
-  length string $ %length(%superq(&matchvar));
-  string =symget("&matchvar");
-  drop string;
-
-  firstchar=char(string,1);
-  findlen=lengthm(string); /* <- for trailing bytes */
-
-  do _N_=1 to nobs;
-    set &ds1 nobs=nobs point=_N_;
-    if sourcechar=firstchar then do;
-      pos=1;
-      s=0;
-      do point=_N_ to min(_N_ + findlen -1,nobs);
-        set &ds1 point=point;
-        if sourcechar=char(string, pos) then s + 1;
-        else goto _leave_;
-        pos+1;
-      end;
-      _leave_:
-      if s=findlen then do;
-        START =_N_;
-        _N_ =_N_+ s - 1;
-        STOP =_N_;
-        output;
-        /* matched! */
-        stop;
-      end;
-    end;
-  end;
-  stop;
-  keep START STOP;
-run;
-
-%local split;
-%let split=0;
-data _null_;
-  set &ds2;
-  if "&matchpoint"='START' then do;
-    if "&keep"='FIRST' then mp=start;
-    else if "&keep"='LAST' then mp=start-1;
-  end;
-  else if "&matchpoint"='END' then do;
-    if "&keep"='FIRST' then mp=stop+1;
-    else if "&keep"='LAST' then mp=stop;
-  end;
-  split=mp+&offset;
-  call symputx('split',split,'l');
-%if &mdebug=1 %then %do;
-  put (_all_)(=);
-  %put &=offset;
-%end;
-run;
-%if &split=0 %then %do;
-  %put &sysmacroname: No match found in &infile for string %superq(&matchvar);
-  %return;
-%end;
-
-data _null_;
-  file &outfile recfm=n;
-  set &ds1;
-%if &keep=FIRST %then %do;
-  if _n_ ge &split then stop;
-%end;
-%else %do;
-  if _n_ gt &split;
-%end;
-  put sourcechar char1.;
-run;
-
-%if &mdebug=0 %then %do;
-  filename &fref0 clear;
-%end;
-%else %do;
-  data _null_;
-    infile &outfile lrecl=32767;
-    input;
-    list;
-    if _n_>50 then stop;
-  run;
-%end;
-/* END */
-%put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) seconds to run;
-
-%mend mp_chop;

base/mp_cleancsv.sas

@@ -1,5 +1,5 @@
 /**
-  @file
+  @file mp_cleancsv.sas
   @brief Fixes embedded cr / lf / crlf in CSV
   @details CSVs will sometimes contain lf or crlf within quotes (eg when
   saved by excel).  When the termstr is ALSO lf or crlf that can be tricky
@@ -7,16 +7,14 @@
   This macro converts any csv to follow the convention of a windows excel file,
   applying CRLF line endings and converting embedded cr and crlf to lf.
 
-  Usage:
+  usage:
 
       fileref mycsv "/path/your/csv";
       %mp_cleancsv(in=mycsv,out=/path/new.csv)
 
-  @param in= (NOTPROVIDED) Provide path or fileref to input csv. If a period is
+  @param in= provide path or fileref to input csv
-    found, it is assumed to be a file.
+  @param out= output path or fileref to output csv
-  @param out= (NOTPROVIDED) Output path or fileref to output csv.  If a period
+  @param qchar= quote char - hex code 22 is the double quote.
-    is found, it is assumed to be a file.
-  @param qchar= ('22'x) Quote char - hex code 22 is the double quote.
 
   @version 9.2
   @author Allan Bowe
@@ -58,14 +56,9 @@
     else do;
       /* outside a quote, change cr and lf to crlf */
       if inchar='0D'x then do;
-        crblank:
         put '0D0A'x;
         input inchar $char1.;
-        if inchar='0D'x then do;
+        if inchar ne '0A'x then do;
-          /* multiple CR indicates CR formatted file with blank lines */
-          goto crblank;
-        end;
-        else if inchar ne '0A'x then do;
           put inchar $char1.;
           if inchar=qchar then isq = mod(isq+1,2);
         end;

base/mp_copyfolder.sas

@@ -16,11 +16,8 @@
 
       %mp_copyfolder(&rootdir,&copydir)
 
-  @param [in] source Unquoted path to the folder to copy from.
+  @param source Unquoted path to the folder to copy from.
-  @param [out] target Unquoted path to the folder to copy to.
+  @param target Unquoted path to the folder to copy to.
-  @param [in] copymax=(MAX) Set to a positive integer to indicate the level of
-    subdirectory copy recursion - eg 3, to go `./3/levels/deep`.  For unlimited
-    recursion, set to MAX.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas
@@ -34,7 +31,7 @@
 
 **/
 
-%macro mp_copyfolder(source,target,copymax=MAX);
+%macro mp_copyfolder(source,target);
 
   %mp_abort(iftrue=(%mf_isdir(&source)=0)
     ,mac=&sysmacroname
@@ -53,7 +50,7 @@
   %let tempds=%mf_getuniquename();
 
   /* recursive directory listing */
-  %mp_dirlist(path=&source,outds=work.&tempds,maxdepth=&copymax)
+  %mp_dirlist(path=&source,outds=work.&tempds, maxdepth=MAX)
 
   /* create folders and copy content */
   data _null_;
@@ -81,4 +78,4 @@
   proc sql;
   drop table work.&tempds;
 
-%mend mp_copyfolder;
+%mend mp_copyfolder;

base/mp_coretable.sas

@@ -30,7 +30,6 @@
   @li mddl_dc_filtersummary.sas
   @li mddl_dc_locktable.sas
   @li mddl_dc_maxkeytable.sas
-  @li mf_getuniquename.sas
 
   <h4> Related Macros </h4>
   @li mp_filterstore.sas
@@ -47,7 +46,7 @@
 %macro mp_coretable(table_ref,libds=0
 )/*/STORE SOURCE*/;
 %local outds ;
-%let outds=%sysfunc(ifc(&libds=0,%mf_getuniquename(),&libds));
+%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
 proc sql;
 %if &table_ref=DIFFTABLE %then %do;
   %mddl_dc_difftable(libds=&outds)
@@ -66,8 +65,7 @@ proc sql;
 %end;
 
 %if &libds=0 %then %do;
-  proc sql;
   describe table &syslast;
   drop table &syslast;
 %end;
-%mend mp_coretable;
+%mend mp_coretable;

base/mp_createconstraints.sas

@@ -18,14 +18,11 @@
       %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
       %mp_createconstraints(inds=work.constraints,outds=created,execute=YES)
 
-  @param inds= (work.mp_getconstraints) The input table containing the
+  @param inds= The input table containing the constraint info
-    constraint info
+  @param outds= a table containing the create statements (create_statement column)
-  @param outds= (work.mp_createconstraints) A table containing the create
+  @param execute= `YES|NO` - default is NO. To actually create, use YES.
-    statements (create_statement column)
-  @param execute= (NO) To actually create, use YES.
 
-  <h4> Related Files </h4>
+  <h4> SAS Macros </h4>
-  @li mp_getconstraints.sas
 
   @version 9.2
   @author Allan Bowe
@@ -33,7 +30,7 @@
 **/
 
 %macro mp_createconstraints(inds=mp_getconstraints
-  ,outds=work.mp_createconstraints
+  ,outds=mp_createconstraints
   ,execute=NO
 )/*/STORE SOURCE*/;
 
@@ -67,4 +64,4 @@ data &outds;
   output;
 run;
 
-%mend mp_createconstraints;
+%mend mp_createconstraints;

base/mp_createwebservice.sas

@@ -1,13 +1,49 @@
 /**
   @file mp_createwebservice.sas
-  @brief Create a web service in SAS 9, Viya or SASjs Server
+  @brief Create a web service in SAS 9 or Viya
-  @details This is actually a wrapper for mx_createwebservice.sas, remaining
+  @details Creates a SASJS ready Stored Process in SAS 9 or Job Execution
-  for legacy purposes.  For new apps, use mx_createwebservice.sas.
+  Service in SAS Viya
 
+Usage:
+
+    %* compile macros ;
+    filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+    %inc mc;
+
+    %* write some code;
+    filename ft15f001 temp;
+    parmcards4;
+        %* fetch any data from frontend ;
+        %webout(FETCH)
+        data example1 example2;
+          set sashelp.class;
+        run;
+        %* send data back;
+        %webout(OPEN)
+        %webout(ARR,example1) * Array format, fast, suitable for large tables ;
+        %webout(OBJ,example2) * Object format, easier to work with ;
+        %webout(CLOSE)
+    ;;;;
+    %mp_createwebservice(path=/Public/app/common,name=appInit,replace=YES)
 
   <h4> SAS Macros </h4>
-  @li mx_createwebservice.sas
+  @li mf_getplatform.sas
+  @li mm_createwebservice.sas
+  @li mv_createwebservice.sas
 
+  @param [in,out] path= The full folder path where the service will be created
+  @param [in,out] name= Service name.  Avoid spaces.
+  @param [in] desc= The description of the service (optional)
+  @param [in] precode= Space separated list of filerefs, pointing to the code
+    that needs to be attached to the beginning of the service (optional)
+  @param [in] code= (ft15f001) Space seperated fileref(s) of the actual code to
+    be added
+  @param [in] replace= (YES) Select YES to replace any existing service in that
+    location
+
+
+  @version 9.2
+  @author Allan Bowe
 
 **/
 
@@ -17,16 +53,33 @@
     ,code=ft15f001
     ,desc=This service was created by the mp_createwebservice macro
     ,replace=YES
-    ,mdebug=0
 )/*/STORE SOURCE*/;
 
-  %mx_createwebservice(path=&path
+%if &syscc ge 4 %then %do;
+  %put syscc=&syscc - &sysmacroname will not execute in this state;
+  %return;
+%end;
+
+%local platform; %let platform=%mf_getplatform();
+%if &platform=SASVIYA %then %do;
+  %if "&path"="HOME" %then %let path=/Users/&sysuserid/My Folder;
+  %mv_createwebservice(path=&path
     ,name=&name
-    ,precode=&precode
     ,code=&code
+    ,precode=&precode
     ,desc=&desc
     ,replace=&replace
-    ,mdebug=&mdebug
   )
+%end;
+%else %do;
+  %if "&path"="HOME" %then %let path=/User Folders/&sysuserid/My Folder;
+  %mm_createwebservice(path=&path
+    ,name=&name
+    ,code=&code
+    ,precode=&precode
+    ,desc=&desc
+    ,replace=&replace
+  )
+%end;
 
 %mend mp_createwebservice;

base/mp_dictionary.sas

@@ -1,52 +0,0 @@
-/**
-  @file mp_dictionary.sas
-  @brief Creates a portal (libref) into the SQL Dictionary Views
-  @details Provide a libref and the macro will create a series of views against
-  each view in the special PROC SQL dictionary libref.
-
-  This is useful if you would like to visualise (navigate) the views in a SAS
-  client such as Base SAS, Enterprise Guide, or Studio (or [Data Controller](
-  https://datacontroller.io)).
-
-  It works by extracting the dictionary.dictionaries view into
-  YOURLIB.dictionaries, then uses that to create a YOURLIB.{viewName} for every
-  other dictionary.view, eg:
-
-      proc sql;
-      create view YOURLIB.columns as select * from dictionary.columns;
-
-  Usage:
-
-      libname demo "/lib/directory";
-      %mp_dictionary(lib=demo)
-
-  Or, to just create them in WORK:
-
-      %mp_dictionary()
-
-  If you'd just like to browse the dictionary data model, you can also check
-  out [this article](https://rawsas.com/dictionary-of-dictionaries/).
-
-  ![](https://user-images.githubusercontent.com/4420615/188278365-2987db97-0594-4a39-ac81-dbacdef5cdc8.png)
-
-  @param lib= (WORK) The libref in which to create the views
-
-  <h4> Related Files </h4>
-  @li mp_dictionary.test.sas
-
-  @version 9.2
-  @author Allan Bowe
-
-**/
-
-%macro mp_dictionary(lib=WORK)/*/STORE SOURCE*/;
-  %local list i mem;
-  proc sql noprint;
-  create view &lib..dictionaries as select * from dictionary.dictionaries;
-  select distinct memname into: list separated by ' '  from &lib..dictionaries;
-  %do i=1 %to %sysfunc(countw(&list,%str( )));
-    %let mem=%scan(&list,&i,%str( ));
-    create view &lib..&mem as select * from dictionary.&mem;
-  %end;
-  quit;
-%mend mp_dictionary;

base/mp_dirlist.sas

@@ -27,9 +27,6 @@
   @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
     subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
     recursion, set to MAX.
-  @param [in] showparent= (NO) By default, the initial parent directory is not
-    part of the results.  Set to YES to include it.  For this record only,
-    directory=filepath.
   @param [out] outds= (work.mp_dirlist) The output dataset to create
   @param [out] getattrs= (NO)  If getattrs=YES then the doptname / foptname
     functions are used to scan all properties - any characters that are not
@@ -66,7 +63,6 @@
     , fref=0
     , outds=work.mp_dirlist
     , getattrs=NO
-    , showparent=NO
     , maxdepth=0
     , level=0 /* The level of recursion to perform.  For internal use only. */
 )/*/STORE SOURCE*/;
@@ -98,12 +94,6 @@ data &out_ds(compress=no
   %end;
   if rc = 0 then do;
     did = dopen(fref);
-    if did=0 then do;
-      putlog "NOTE: This directory is empty, or does not exist - &path";
-      msg=sysmsg();
-      put (_all_)(=);
-      stop;
-    end;
     /* attribute is OS-dependent - could be "Directory" or "Directory Name" */
     numopts=doptnum(did);
     do i=1 to numopts;
@@ -111,6 +101,12 @@ data &out_ds(compress=no
       if foption=:'Directory' then i=numopts;
     end;
     directory=dinfo(did,foption);
+    if did=0 then do;
+      putlog "NOTE: This directory is empty - " directory;
+      msg=sysmsg();
+      put _all_;
+      stop;
+    end;
     rc = filename(fref);
   end;
   else do;
@@ -148,15 +144,6 @@ data &out_ds(compress=no
     output;
   end;
   rc = dclose(did);
-  %if &showparent=YES and &level=0 %then %do;
-    filepath=directory;
-    file_or_folder='folder';
-    ext='';
-    filename=scan(directory,-1,'/\');
-    msg='';
-    level=&level;
-    output;
-  %end;
   stop;
 run;
 
@@ -164,7 +151,6 @@ run;
   data &out_ds;
     set &out_ds;
     length infoname infoval $60 fref $8;
-    if _n_=1 then call missing(fref);
     rc=filename(fref,filepath);
     drop rc infoname fid i close fref;
     if file_or_folder='file' then do;
@@ -244,9 +230,6 @@ run;
   data _null_;
     set &out_ds;
     where file_or_folder='folder';
-  %if &showparent=YES and &level=0 %then %do;
-    if filepath ne directory;
-  %end;
     length code $10000;
     code=cats('%nrstr(%mp_dirlist(path=',filepath,",outds=&outds"
       ,",getattrs=&getattrs,level=%eval(&level+1),maxdepth=&maxdepth))");
@@ -259,4 +242,4 @@ run;
 proc sql;
 drop table &out_ds;
 
-%mend mp_dirlist;
+%mend mp_dirlist;

base/mp_ds2cards.sas

@@ -11,7 +11,7 @@
 
   Usage:
 
-      %mp_ds2cards(sashelp.class
+      %mp_ds2cards(base_ds=sashelp.class
         , tgt_ds=work.class
         , cards_file= "C:\temp\class.sas"
         , showlog=NO
@@ -23,7 +23,7 @@
     - explicity setting a unix LF
     - constraints / indexes etc
 
-  @param [in] base_ds Should be two level - eg work.blah.  This is the table
+  @param [in] base_ds= Should be two level - eg work.blah.  This is the table
                   that is converted to a cards file.
   @param [in] tgt_ds= Table that the generated cards file would create.
     Optional - if omitted, will be same as BASE_DS.
@@ -48,10 +48,9 @@
 
   @version 9.2
   @author Allan Bowe
-  @cond
 **/
 
-%macro mp_ds2cards(base_ds, tgt_ds=
+%macro mp_ds2cards(base_ds=, tgt_ds=
     ,cards_file="%sysfunc(pathname(work))/cardgen.sas"
     ,maxobs=max
     ,random_sample=NO
@@ -220,8 +219,7 @@ data _null_;
     put '  @file';
     put "  @brief Datalines for %upcase(%scan(&base_ds,2)) dataset";
     put "  @details Generated by %nrstr(%%)mp_ds2cards()";
-    put "    Source: https://github.com/sasjs/core";
+    put "  Available on github.com/sasjs/core";
-    put '  @cond ';
     put '**/';
     put "data &tgt_ds &indexes;";
     put "attrib ";
@@ -254,7 +252,6 @@ data _null_;
         ;
       %end;
       put ";";
-      put 'missing a b c d e f g h i j k l m n o p q r s t u v w x y z _;';
       put "datalines4;";
     end;
   end;
@@ -267,7 +264,6 @@ data _null_;
   if __lastobs then do;
     put ';;;;';
     put 'run;';
-    put '/** @endcond **/';
     stop;
   end;
 run;
@@ -287,5 +283,4 @@ quit;
 %put NOTE-;%put NOTE-;
 %put NOTE- %sysfunc(dequote(&cards_file.));
 %put NOTE-;%put NOTE-;
-%mend mp_ds2cards;
+%mend mp_ds2cards;
-/** @endcond **/

base/mp_ds2fmtds.sas

@@ -17,7 +17,7 @@
   <h4> SAS Macros </h4>
   @li mf_existds.sas
 
-  <h4> Related Macros </h4>
+  <h4> Related Macros <h4>
   @li mp_jsonout.sas
 
   @version 9.2

base/mp_ds2md.sas

@@ -57,11 +57,6 @@
 %local vars;
 %let vars=%upcase(%mf_getvarlist(&libds));
 
-%if %trim(X&vars)=X %then %do;
-  %put &sysmacroname: Table &libds has no columns!!;
-  %return;
-%end;
-
 /* create the header row */
 data _null_;
   file &outref;
@@ -92,7 +87,7 @@ data _null_;
 run;
 
 %if %upcase(&showlog)=YES %then %do;
-  options ps=max lrecl=max;
+  options ps=max;
   data _null_;
     infile &outref;
     input;
@@ -100,4 +95,4 @@ run;
   run;
 %end;
 
-%mend mp_ds2md;
+%mend mp_ds2md;

base/mp_dsmeta.sas

@@ -1,110 +0,0 @@
-/**
-  @file
-  @brief Export dataset metadata to a single output table
-  @details Exports the dataset attributes and enginehost information, then
-  converts the datasets into a single output table in the following format:
-
-|ODS_TABLE:$10.|NAME:$100.|VALUE:$1000.|
-|---|---|---|
-|`ATTRIBUTES `|`Data Set Name `|`SASHELP.CLASS `|
-|`ATTRIBUTES `|`Observations `|`19 `|
-|`ATTRIBUTES `|`Member Type `|`DATA `|
-|`ATTRIBUTES `|`Variables `|`5 `|
-|`ATTRIBUTES `|`Engine `|`V9 `|
-|`ATTRIBUTES `|`Indexes `|`0 `|
-|`ATTRIBUTES `|`Created `|`06/08/2020 00:59:14 `|
-|`ATTRIBUTES `|`Observation Length `|`40 `|
-|`ATTRIBUTES `|`Last Modified `|`06/08/2020 00:59:14 `|
-|`ATTRIBUTES `|`Deleted Observations `|`0 `|
-|`ATTRIBUTES `|`Protection `|`. `|
-|`ATTRIBUTES `|`Compressed `|`NO `|
-|`ATTRIBUTES `|`Data Set Type `|`. `|
-|`ATTRIBUTES `|`Sorted `|`NO `|
-|`ATTRIBUTES `|`Label `|`Student Data `|
-|`ATTRIBUTES `|`Data Representation `|`SOLARIS_X86_64, LINUX_X86_64, ALPHA_TRU64, LINUX_IA64 `|
-|`ATTRIBUTES `|`Encoding `|`us-ascii  ASCII (ANSI) `|
-|`ENGINEHOST `|`Data Set Page Size `|`65536 `|
-|`ENGINEHOST `|`Number of Data Set Pages `|`1 `|
-|`ENGINEHOST `|`First Data Page `|`1 `|
-|`ENGINEHOST `|`Max Obs per Page `|`1632 `|
-|`ENGINEHOST `|`Obs in First Data Page `|`19 `|
-|`ENGINEHOST `|`Number of Data Set Repairs `|`0 `|
-|`ENGINEHOST `|`Filename `|`/opt/sas/sas9/SASHome/SASFoundation/9.4/sashelp/class.sas7bdat `|
-|`ENGINEHOST `|`Release Created `|`9.0401M7 `|
-|`ENGINEHOST `|`Host Created `|`Linux `|
-|`ENGINEHOST `|`Inode Number `|`28314616 `|
-|`ENGINEHOST `|`Access Permission `|`rw-r--r-- `|
-|`ENGINEHOST `|`Owner Name `|`sas `|
-|`ENGINEHOST `|`File Size `|`128KB `|
-|`ENGINEHOST `|`File Size (bytes) `|`131072 `|
-
-  Example usage:
-
-      %mp_dsmeta(sashelp.class,outds=work.mymeta)
-      proc print data=work.mymeta;
-      run;
-
-  For more details on creating datasets from PROC CONTENTS check out this
-  excellent [paper](
-  https://support.sas.com/resources/papers/proceedings14/1549-2014.pdf) by
-  [Louise Hadden](https://www.linkedin.com/in/louisehadden/).
-
-  @param libds The library.dataset to export the metadata for
-  @param outds= (work.dsmeta) The output table to contain the metadata
-
-  <h4> Related Files </h4>
-  @li mp_dsmeta.test.sas
-  @li mp_getcols.sas
-  @li mp_getdbml.sas
-  @li mp_getddl.sas
-  @li mp_getformats.sas
-  @li mp_getpk.sas
-  @li mp_guesspk.sas
-
-**/
-
-%macro mp_dsmeta(libds,outds=work.dsmeta);
-
-%local ds1 ds2;
-data;run; %let ds1=&syslast;
-data;run; %let ds2=&syslast;
-
-/* setup the ODS capture */
-ods output attributes=&ds1 enginehost=&ds2;
-
-/* export the metadata */
-proc contents data=&libds;
-run;
-
-/* load it into a single table */
-data &outds (keep=ods_table name value);
-  length ods_table $10 name label2 label1 label $100
-    value cvalue cvalue1 cvalue2 $1000
-    nvalue nvalue1 nvalue2 8;
-  if _n_=1 then call missing (of _all_);
-  * putlog (_all_)(=);
-  set &ds1 (in=atrs) &ds2 (in=eng);
-  if atrs then do;
-    ods_table='ATTRIBUTES';
-    name=coalescec(label1,label);
-    value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.));
-    output;
-    if label2 ne '' then do;
-    name=label2;
-    value=coalescec(cvalue2,put(nvalue2,best.));
-    output;
-    end;
-  end;
-  else if eng then do;
-    ods_table='ENGINEHOST';
-    name=coalescec(label1,label);
-    value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.));
-    output;
-  end;
-run;
-
-proc sql;
-drop table &ds1, &ds2;
-
-%mend mp_dsmeta;
-

base/mp_filtercheck.sas

@@ -92,37 +92,7 @@ data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
-  length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000;
+  length reason_cd $4032;
-  drop tmp;
-
-  /* quick check to ensure column exists */
-  if upcase(VARIABLE_NM) not in
-  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
-  then do;
-    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
-    putlog REASON_CD= VARIABLE_NM=;
-    call symputx('reason_cd',reason_cd,'l');
-    call symputx('nobs',_n_,'l');
-    output;
-    return;
-  end;
-
-  /* need to open the dataset to get the column type */
-  dsid=open("&targetds","i");
-  if dsid>0 then do;
-    vnum=varnum(dsid,VARIABLE_NM);
-    if vnum<1 then do;
-      /* should not happen as was also tested for above */
-      REASON_CD=cats("Variable (",VARIABLE_NM,") not found in &targetds");
-      putlog REASON_CD= dsid=;
-      call symputx('reason_cd',reason_cd,'l');
-      call symputx('nobs',_n_,'l');
-      output;
-      return;
-    end;
-    /* now we can get the type */
-    else vtype=vartype(dsid,vnum);
-  end;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
@@ -146,8 +116,17 @@ data &outds;
     call symputx('nobs',_n_,'l');
     output;
   end;
+  if upcase(VARIABLE_NM) not in
+  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
+  then do;
+    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
+    putlog REASON_CD= VARIABLE_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
+    output;
+  end;
   if OPERATOR_NM not in
-  ('=','>','<','<=','>=','NE','GE','LE','BETWEEN','IN','NOT IN','CONTAINS')
+  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE')
   then do;
     REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
@@ -156,45 +135,19 @@ data &outds;
     output;
   end;
 
-  /* special missing logic */
-  if vtype='N'
-    and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE')
-    and cats(upcase(raw_value)) in (
-      '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N'
-      '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._'
-    )
-  then do;
-    /* valid numeric - exit data step loop */
-    return;
-  end;
-
   /* special logic */
-  if OPERATOR_NM in ('IN','NOT IN','BETWEEN') then do;
+  if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ','');
-    if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ',',');
+  else if OPERATOR_NM in ('IN','NOT IN') then do;
-    else do;
+    if substr(raw_value,1,1) ne '('
-      if substr(raw_value,1,1) ne '('
+    or substr(cats(reverse(raw_value)),1,1) ne ')'
-      or substr(cats(reverse(raw_value)),1,1) ne ')'
+    then do;
-      then do;
+      REASON_CD='Missing start/end bracket in RAW_VALUE';
-        REASON_CD='Missing start/end bracket in RAW_VALUE';
+      putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
-        putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+      call symputx('reason_cd',reason_cd,'l');
-        call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
-        call symputx('nobs',_n_,'l');
+      output;
-        output;
-      end;
-      else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
-    end;
-    /* we now have a comma seperated list of values */
-    if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
-      tmp=scan(raw_value1,i,',');
-      if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
-        REASON_CD='Non Numeric value provided';
-        putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
-        call symputx('reason_cd',reason_cd,'l');
-        call symputx('nobs',_n_,'l');
-        output;
-      end;
-      return;
     end;
+    else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
   end;
   else raw_value1=raw_value;
 

base/mp_filtergenerate.sas

@@ -84,9 +84,6 @@ filename &outref temp;
   run;
 %end;
 %else %do;
-  proc sort data=&inds;
-    by SUBGROUP_ID;
-  run;
   data _null_;
     file &outref lrecl=32800;
     set &inds end=last;

base/mp_getconstraints.sas

@@ -40,22 +40,6 @@
 %let lib=%upcase(&lib);
 %let ds=%upcase(&ds);
 
-/**
-  * Cater for environments where sashelp.vcncolu is not available
-  */
-%if %sysfunc(exist(sashelp.vcncolu,view))=0 %then %do;
-  proc sql;
-  create table &outds(
-      libref char(8)
-    ,TABLE_NAME char(32)
-    ,constraint_type char(8)     label='Constraint Type'
-    ,constraint_name char(32)     label='Constraint Name'
-    ,column_name char(32)     label='Column'
-    ,constraint_order num
-  );
-  %return;
-%end;
-
 /**
   * Neither dictionary tables nor sashelp provides a constraint order column,
   * however they DO arrive in the correct order.  So, create the col.
@@ -94,11 +78,8 @@ create table &outds as
 /**
   * We cannot apply this clause to the underlying dictionary table.  See:
   * https://communities.sas.com/t5/SAS-Programming/Unexpected-Where-Clause-behaviour-in-dictionary-TABLE/m-p/771554#M244867
-  * cannot use`where calculated libref="&lib"` either as it will STILL execute
-  * all the underlying constraint queries, causing exception errors in some
-  * cases: https://github.com/sasjs/core/issues/283
   */
-  where a.TABLE_CATALOG="&lib"
+  where calculated libref="&lib"
   %if "&ds" ne "" %then %do;
     and upcase(a.TABLE_NAME)="&ds"
     and upcase(b.TABLE_NAME)="&ds"

base/mp_getddl.sas

@@ -130,13 +130,13 @@ run;
 
 %local x curds;
 %if &flavour=SAS %then %do;
+  data _null_;
+    file &fref mod;
+    put "/* SAS Flavour DDL for %upcase(&libref).&curds */";
+    put "proc sql;";
+  run;
   %do x=1 %to %sysfunc(countw(&dsnlist));
     %let curds=%scan(&dsnlist,&x);
-    data _null_;
-      file &fref mod;
-      put "/* SAS Flavour DDL for %upcase(&libref).&curds */";
-      put "proc sql;";
-    run;
     data _null_;
       file &fref mod;
       length lab $1024 typ $20;

base/mp_getmaxvarlengths.sas

@@ -41,7 +41,6 @@
   <h4> SAS Macros </h4>
   @li mcf_length.sas
   @li mf_getuniquename.sas
-  @li mf_getvarcount.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
   @li mf_getvarformat.sas
@@ -61,7 +60,7 @@
   ,outds=work.mp_getmaxvarlengths
 )/*/STORE SOURCE*/;
 
-%local vars prefix x var fmt srcds;
+%local vars prefix x var fmt;
 %let vars=%mf_getvarlist(libds=&libds);
 %let prefix=%substr(%mf_getuniquename(),1,25);
 %let num2char=%upcase(&num2char);
@@ -71,24 +70,6 @@
   %mcf_length(wrap=YES, insert_cmplib=YES)
 %end;
 
-%if &num2char=NO
-  and ("%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5")
-  and %mf_getvarcount(&libds,typefilter=N) gt 0
-%then %do;
-  /* custom functions not supported in summary operations */
-  %let srcds=%mf_getuniquename();
-  data &srcds/view=&srcds;
-    set &libds;
-  %do x=1 %to %sysfunc(countw(&vars,%str( )));
-    %let var=%scan(&vars,&x);
-    %if %mf_getvartype(&libds,&var)=N %then %do;
-      &prefix.&x=mcf_length(&var);
-    %end;
-  %end;
-  run;
-%end;
-%else %let srcds=&libds;
-
 proc sql;
 create table &outds (rename=(
     %do x=1 %to %sysfunc(countw(&vars,%str( )));
@@ -113,15 +94,10 @@ create table &outds (rename=(
         %end;
       %end;
       %else %do;
-        %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do;
+        max(mcf_length(&var)) as &prefix.&x
-          max(&prefix.&x) as &prefix.&x
-        %end;
-        %else %do;
-          max(mcf_length(&var)) as &prefix.&x
-        %end;
       %end;
     %end;
-  from &srcds;
+  from &libds;
 
   proc transpose data=&outds
     out=&outds(rename=(_name_=NAME COL1=MAXLEN));

base/mp_getpk.sas

@@ -34,7 +34,6 @@
   @param [out] outds= (work.mp_getpk) The name of the output table to create.
 
   <h4> SAS Macros </h4>
-  @li mf_existfeature.sas
   @li mf_getengine.sas
   @li mf_getschema.sas
   @li mp_dropmembers.sas
@@ -56,8 +55,7 @@
 )/*/STORE SOURCE*/;
 
 
-%local engine schema ds1 ds2 ds3 dsn tabs1 tabs2 sum pk4sure pkdefault finalpks
+%local engine schema ds1 ds2 ds3 dsn tabs1 tabs2 sum pk4sure pkdefault finalpks;
-  pkfromindex;
 
 %let lib=%upcase(&lib);
 %let ds=%upcase(&ds);
@@ -72,7 +70,6 @@
 %let sum=%mf_getuniquename(prefix=getpk_sum);
 %let pk4sure=%mf_getuniquename(prefix=getpk_pk4sure);
 %let pkdefault=%mf_getuniquename(prefix=getpk_pkdefault);
-%let pkfromindex=%mf_getuniquename(prefix=getpk_pkfromindex);
 %let finalpks=%mf_getuniquename(prefix=getpk_finalpks);
 
 %local dbg;
@@ -183,23 +180,9 @@ create table &ds1 as
       and a.constraint_name=b.constraint_name
     order by 1,2,3,4;
 
-  /* extract cols from the relevant unique INDEXES */
-  create table &pkfromindex as
-    select libname as libref
-      ,memname as table_name
-      ,indxname as constraint_name
-      ,indxpos as constraint_order
-      ,name
-    from dictionary.indexes
-    where nomiss='yes' and unique='yes' and upcase(libname)="&lib"
-  %if &ds ne 0 %then %do;
-      and upcase(memname)="&ds"
-  %end;
-    order by 1,2,3,4;
-
   /* create one table */
   data &finalpks;
-    set &pkdefault &pk4sure &pkfromindex;
+    set &pkdefault &pk4sure ;
     pk_ind=1;
     /* if there are multiple unique constraints, take the first */
     by libref table_name constraint_name;
@@ -230,12 +213,7 @@ create table work.&tabs1 as select
   libname as libref
   ,upcase(memname) as dsn
   ,memtype
-%if %mf_existfeature(DBMS_MEMTYPE)=1 %then %do;
   ,dbms_memtype
-%end;
-%else %do;
-  ,'n/a' as dbms_memtype format=$32.
-%end;
   ,typemem
   ,memlabel
   ,nvar
@@ -278,4 +256,4 @@ create table &outds as
   iftrue=(&mdebug=0)
 )
 
-%mend mp_getpk;
+%mend mp_getpk;

base/mp_gitadd.sas

@@ -1,45 +0,0 @@
-/**
-  @file
-  @brief Stages files in a GIT repo
-  @details Uses the output dataset from mp_gitstatus.sas to determine the files
-  that should be staged.
-
-  If `STAGED ne "TRUE"` then the file is staged.
-
-  Usage:
-
-      %let dir=%sysfunc(pathname(work))/core;
-      %let repo=https://github.com/sasjs/core;
-      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir));
-      %mf_writefile(&dir/somefile.txt,l1=some content)
-      %mf_deletefile(&dir/package.json)
-      %mp_gitstatus(&dir,outds=work.gitstatus)
-
-      %mp_gitadd(&dir,inds=work.gitstatus)
-
-  @param [in] gitdir The directory containing the GIT repository
-  @param [in] inds= (work.mp_gitadd) The input dataset with the list of files
-  to stage.  Will accept the output from mp_gitstatus(), else just use a table
-  with the following columns:
-    @li path $1024 - relative path to the file in the repo
-    @li staged $32 - whether the file is staged (TRUE or FALSE)
-    @li status $64 - either new, deleted, or modified
-
-  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
-
-  <h4> Related Files </h4>
-  @li mp_gitadd.test.sas
-  @li mp_gitstatus.sas
-
-**/
-
-%macro mp_gitadd(gitdir,inds=work.mp_gitadd,mdebug=0);
-
-data _null_;
-  set &inds;
-  if STAGED ne "TRUE";
-  rc=git_index_add("&gitdir",cats(path),status);
-  if rc ne 0 or &mdebug=1 then put rc=;
-run;
-
-%mend mp_gitadd;

base/mp_gitlog.sas

@@ -1,104 +0,0 @@
-/**
-  @file
-  @brief Creates a dataset with the commit history of a local repository
-  @details Returns the commit history from a local repository. The name of the
-  branch is also returned.
-
-  More details here:
-https://documentation.sas.com/doc/ko/pgmsascdc/v_033/lefunctionsref/n1qo5miyvry1nen111js203hlwrh.htm
-
-  Usage:
-
-      %let gitdir=%sysfunc(pathname(work))/core;
-      %let repo=https://github.com/sasjs/core;
-      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir));
-
-      %mp_gitlog(&gitdir,outds=work.mp_gitlog)
-
-  @param [in] gitdir The directory containing the GIT repository
-  @param [in] filter= (BRANCHONLY) To return only the commits for the current
-  branch, use BRANCHONLY (the default).  Anything else will return the entire
-  commit history.
-  @param [out] outds= (work.mp_gitlog) The output dataset to create.
-  All vars are $128 except `message` which is $4000.
-    @li author returns the author who submitted the commit.
-    @li children_ids returns a list of the children commit IDs
-    @li committer returns the name of the committer.
-    @li committer_email returns the email of the committer.
-    @li email returns the email of the commit author.
-    @li id returns the commit ID of the commit object.
-    @li in_current_branch returns "TRUE" or "FALSE" to indicate if the commit is
-      in the current branch.
-    @li message returns the commit message.
-    @li parent_ids returns a list of the parent commit IDs.
-    @li stash returns "TRUE" or "FALSE" to indicate if the commit is a stash
-      commit.
-    @li time returns the time of the commit as numeric string
-    @li commit_time_num time of the commit as numeric SAS datetime
-    @li commit_time_str the commit_time_num variable cast as string
-
-  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
-  @param [in] nobs= (0) Set to an integer greater than 0 to restrict the number
-  of rows returned
-
-  <h4> SAS Macros </h4>
-  @li mf_getgitbranch.sas
-
-  <h4> Related Files </h4>
-  @li mp_gitadd.sas
-  @li mp_gitreleaseinfo.sas
-  @li mp_gitstatus.sas
-
-**/
-
-%macro mp_gitlog(gitdir,outds=work.mp_gitlog,mdebug=0,filter=BRANCHONLY,nobs=0);
-
-%local varlist i var;
-%let varlist=author children_ids committer committer_email email id
-  in_current_branch parent_ids stash time ;
-
-data &outds;
-  LENGTH gitdir branch $ 1024 message $4000 &varlist $128 commit_time_num 8.
-    commit_time_str $32;
-  call missing (of _all_);
-  branch="%mf_getgitbranch(&gitdir)";
-  gitdir=symget('gitdir');
-  rc=git_status_free(trim(gitdir));
-  if rc=-1 then do;
-    put "The libgit2 library is unavailable and no Git operations can be used.";
-    put "See: https://stackoverflow.com/questions/74082874";
-    stop;
-  end;
-  else if rc=-2 then do;
-    put "The libgit2 library is available, but the status function failed.";
-    put "See the log for details.";
-    stop;
-  end;
-  entries=git_commit_log(trim(gitdir));
-  do n=1 to entries;
-
-  %do i=1 %to %sysfunc(countw(&varlist message));
-    %let var=%scan(&varlist message,&i,%str( ));
-    rc=git_commit_get(n,trim(gitdir),"&var",&var);
-  %end;
-    /* convert unix time to SAS time - https://4gl.uk/corelink0 */
-    /* Number of seconds between 01JAN1960 and 01JAN1970: 315619200 */
-    format commit_time_num datetime19.;
-    commit_time_num=sum(input(cats(time),best.),315619200);
-    commit_time_str=put(commit_time_num,datetime19.);
-  %if &mdebug=1 %then %do;
-    putlog (_all_)(=);
-  %end;
-    if "&filter"="BRANCHONLY" then do;
-      if cats(in_current_branch)='TRUE' then output;
-    end;
-    else output;
-  %if &nobs>0 %then %do;
-    if n ge &nobs then stop;
-  %end;
-  end;
-  rc=git_commit_free(trim(gitdir));
-  keep gitdir branch &varlist message time commit_time_num commit_time_str;
-run;
-
-%mend mp_gitlog;

base/mp_gitreleaseinfo.sas

@@ -1,74 +0,0 @@
-/**
-  @file
-  @brief Pulls latest release info from a GIT repository
-  @details Useful for grabbing the latest version number or other attributes
-  from a GIT server.  Supported providers are GitLab and GitHub. Pull requests
-  are welcome if you'd like to see additional providers!
-
-  Note that each provider provides slightly different JSON output.  Therefore
-  the macro simply extracts the JSON and assigns the libname (using the JSON
-  engine).
-
-  Example usage (eg, to grab latest release version from github):
-
-      %mp_gitreleaseinfo(GITHUB,sasjs/core,outlib=mylibref)
-
-      data _null_;
-        set mylibref.root;
-        putlog TAG_NAME=;
-      run;
-
-  @param [in] provider The GIT provider for the release info.  Accepted values:
-    @li GITLAB
-    @li GITHUB - Tables include root, assets, author, alldata
-  @param [in] project The link to the repository.  This has different formats
-    depending on the vendor:
-    @li GITHUB - org/repo, eg sasjs/core
-    @li GITLAB - project, eg 1343223
-  @param [in] server= (0) If your repo is self-hosted, then provide the domain
-    here.  Otherwise it will default to the provider domain (eg gitlab.com).
-  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
-  @param [out] outlib= (GITREL) The JSON-engine libref to be created, which will
-    point at the returned JSON
-
-  <h4> SAS Macros </h4>
-  @li mf_getuniquefileref.sas
-
-  <h4> Related Files </h4>
-  @li mp_gitreleaseinfo.test.sas
-
-**/
-
-%macro mp_gitreleaseinfo(provider,project,server=0,outlib=GITREL,mdebug=0);
-%local url fref;
-
-%let provider=%upcase(&provider);
-
-%if &provider=GITHUB %then %do;
-  %if "&server"="0" %then %let server=https://api.github.com;
-  %let url=&server/repos/&project/releases/latest;
-%end;
-%else %if &provider=GITLAB %then %do;
-  %if "&server"="0" %then %let server=https://gitlab.com;
-  %let url=&server/api/v4/projects/&project/releases;
-%end;
-
-%let fref=%mf_getuniquefileref();
-
-proc http method='GET' out=&fref url="&url";
-%if &mdebug=1 %then %do;
-  debug level = 3;
-%end;
-run;
-
-libname &outlib JSON fileref=&fref;
-
-%if &mdebug=1 %then %do;
-  data _null_;
-    infile &fref;
-    input;
-    putlog _infile_;
-  run;
-%end;
-
-%mend mp_gitreleaseinfo;

base/mp_gitstatus.sas

@@ -1,67 +0,0 @@
-/**
-  @file
-  @brief Creates a dataset with the output from `GIT_STATUS()`
-  @details Uses `git_status()` to fetch the number of changed files, then
-  iterates with `git_status_get()`, inserting all attributes into an output
-  dataset.
-
-  Usage:
-
-      %let dir=%sysfunc(pathname(work))/core;
-      %let repo=https://github.com/sasjs/core;
-      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir));
-      %mf_writefile(&dir/somefile.txt,l1=some content)
-      %mf_deletefile(&dir/package.json)
-
-      %mp_gitstatus(&dir,outds=work.gitstatus)
-
-  More info on these functions is in this [helpful paper]
-(https://www.sas.com/content/dam/SAS/support/en/sas-global-forum-proceedings/2019/3057-2019.pdf)
-  by Danny Zimmerman.
-
-  @param [in] gitdir The directory containing the GIT repository
-  @param [out] outds= (work.git_status) The output dataset to create.  Vars:
-    @li gitdir $1024 - directory of repo
-    @li path $1024 - relative path to the file in the repo
-    @li staged $32 - whether the file is staged (TRUE or FALSE)
-    @li status $64 - either new, deleted, or modified
-    @li cnt - number of files
-    @li n - the "nth" file in the list from git_status()
-
-  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
-
-  <h4> Related Files </h4>
-  @li mp_gitstatus.test.sas
-  @li mp_gitadd.sas
-
-**/
-
-%macro mp_gitstatus(gitdir,outds=work.mp_gitstatus,mdebug=0);
-
-data &outds;
-  LENGTH gitdir path $ 1024 STATUS $ 64 STAGED $ 32;
-  call missing (of _all_);
-  gitdir=symget('gitdir');
-  cnt=git_status(trim(gitdir));
-  if cnt=-1 then do;
-    put "The libgit2 library is unavailable and no Git operations can be used.";
-    put "See: https://stackoverflow.com/questions/74082874";
-  end;
-  else if cnt=-2 then do;
-    put "The libgit2 library is available, but the status function failed.";
-    put "See the log for details.";
-  end;
-  else do n=1 to cnt;
-    rc=GIT_STATUS_GET(n,gitdir,'PATH',path);
-    rc=GIT_STATUS_GET(n,gitdir,'STAGED',staged);
-    rc=GIT_STATUS_GET(n,gitdir,'STATUS',status);
-    output;
-  %if &mdebug=1 %then %do;
-    putlog (_all_)(=);
-  %end;
-  end;
-  rc=git_status_free(trim(gitdir));
-  drop rc cnt;
-run;
-
-%mend mp_gitstatus;

base/mp_gsubfile.sas

@@ -48,11 +48,6 @@
   outfile=0
 )/*/STORE SOURCE*/;
 
-  %if "%substr(&sysver.XX,1,4)"="V.04" %then %do;
-    %put %str(ERR)OR: Viya 4 does not support the IO library in lua;
-    %return;
-  %end;
-
   %ml_gsubfile()
 
 %mend mp_gsubfile;

base/mp_hashdataset.sas

@@ -1,8 +1,8 @@
 /**
   @file
   @brief Returns a unique hash for a dataset
-  @details Ignores metadata attributes, used only to hash values. If used to
+  @details Ignores metadata attributes, used only to hash values. Compared
-  compare datasets, they must have their columns and rows in the same order.
+  datasets must be in the same order.
 
       %mp_hashdataset(sashelp.class,outds=myhash)
 
@@ -11,23 +11,19 @@
         put hashkey=;
       run;
 
-  ![sas md5 hash dataset log results](https://i.4gl.io/1/KorUKoyE05.png/raw)
+  ![sas md5 hash dataset log results](https://i.imgur.com/MqF98vk.png)
 
   <h4> SAS Macros </h4>
   @li mf_getattrn.sas
   @li mf_getuniquename.sas
   @li mf_getvarlist.sas
-  @li mp_md5.sas
+  @li mf_getvartype.sas
-
-  <h4> Related Files </h4>
-  @li mp_hashdataset.test.sas
-  @li mp_hashdirectory.sas
 
   @param [in] libds dataset to hash
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
-  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [in] iftrue= A condition under which the macro should be executed.
-  @param [out] outds= (work._data_) The output dataset to create. This
+  @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
-  will contain one column (hashkey) with one observation (a $hex32.
+  will contain one column (hashkey) with one observation (a hex32.
   representation of the input hash)
   |hashkey:$32.|
   |---|
@@ -39,53 +35,48 @@
 
 %macro mp_hashdataset(
   libds,
-  outds=work._data_,
+  outds=,
   salt=,
   iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
-%local keyvar /* roll up the md5 */
+  %if not(%eval(%unquote(&iftrue))) %then %return;
-  prevkeyvar /* retain prev record md5 */
-  lastvar /* last var in input ds */
-  cvars nvars;
 
-%if not(%eval(%unquote(&iftrue))) %then %return;
+  %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
-
+    %put %str(WARN)ING: Dataset &libds is empty, or is not a dataset;
-/* avoid naming conflict for hash key vars */
-%let keyvar=%mf_getuniquename();
-%let prevkeyvar=%mf_getuniquename();
-%let lastvar=%mf_getuniquename();
-
-%if %mf_getattrn(&libds,NLOBS)=0 %then %do;
-  data &outds;
-    length hashkey $32;
-    hashkey=put(md5("&salt"),$hex32.);
-    output;
-    stop;
-  run;
-  %put &sysmacroname: Dataset &libds is empty, or is not a dataset;
-  %put &sysmacroname: hashkey of &outds is based on salt (&salt) only;
-%end;
-%else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
-  %put %str(ERR)OR: Dataset &libds is not a dataset;
-%end;
-%else %do;
-  data &outds(rename=(&keyvar=hashkey) keep=&keyvar)
-  %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
-    /nonote2err
   %end;
-  ;
+  %else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
-    length &prevkeyvar &keyvar $32;
+    %put %str(ERR)OR: Dataset &libds is not a dataset;
-    retain &prevkeyvar;
+  %end;
-    if _n_=1 then &prevkeyvar=put(md5("&salt"),$hex32.);
+  %else %do;
-    set &libds end=&lastvar;
+    %local keyvar /* roll up the md5 */
-    /* hash should include previous row */
+      prevkeyvar /* retain prev record md5 */
-    &keyvar=%mp_md5(
+      lastvar /* last var in input ds */
-      cvars=%mf_getvarlist(&libds,typefilter=C) &prevkeyvar,
+      varlist var i;
-      nvars=%mf_getvarlist(&libds,typefilter=N)
+    /* avoid naming conflict for hash key vars */
-    );
+    %let keyvar=%mf_getuniquename();
-    &prevkeyvar=&keyvar;
+    %let prevkeyvar=%mf_getuniquename();
-    if &lastvar then output;
+    %let lastvar=%mf_getuniquename();
-  run;
+    %let varlist=%mf_getvarlist(&libds);
-%end;
+    data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
-%mend mp_hashdataset;
+      length &prevkeyvar &keyvar $32;
+      retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
+      set &libds end=&lastvar;
+      /* hash should include previous row */
+      &keyvar=put(md5(&prevkeyvar
+      /* loop every column, hashing every individual value */
+    %do i=1 %to %sysfunc(countw(&varlist));
+      %let var=%scan(&varlist,&i,%str( ));
+      %if %mf_getvartype(&libds,&var)=C %then %do;
+          !!put(md5(trim(&var)),$hex32.)
+      %end;
+      %else %do;
+          !!put(md5(trim(put(&var*1,binary64.))),$hex32.)
+      %end;
+    %end;
+      ),$hex32.);
+      &prevkeyvar=&keyvar;
+      if &lastvar then output;
+    run;
+  %end;
+%mend mp_hashdataset;

base/mp_hashdirectory.sas

@@ -1,164 +0,0 @@
-/**
-  @file
-  @brief Returns a unique hash for each file in a directory
-  @details Hashes each file in each directory, and then hashes the hashes to
-  create a hash for each directory also.
-
-  This makes use of the new `hashing_file()` and `hashing` functions, available
-  since 9.4m6. Interestingly, those functions can be used in pure macro, eg:
-
-      %put %sysfunc(hashing_file(md5,/path/to/file.blob,0));
-
-  Actual usage:
-
-      %let fpath=/some/directory;
-
-      %mp_hashdirectory(&fpath,outds=myhash,maxdepth=2)
-
-      data _null_;
-        set work.myhash;
-        put (_all_)(=);
-      run;
-
-  Whilst files are hashed in their entirety, the logic for creating a folder
-  hash is as follows:
-
-  @li Sort the files by filename (case sensitive, uppercase then lower)
-  @li Take the first 100 hashes, concatenate and hash
-  @li Concatenate this hash with another 100 hashes and hash again
-  @li Continue until the end of the folder.  This is the folder hash
-  @li If a folder contains other folders, start from the bottom of the tree -
-    the folder hashes cascade upwards so you know immediately if there is a
-    change in a sub/sub directory
-  @li If a subfolder has no content (empty) then it is ignored. No hash created.
-  @li If the file is empty, it is also ignored / no hash created.
-  @li If the target directory (&inloc) is empty, &outds will also be empty
-
-  <h4> SAS Macros </h4>
-  @li mp_dirlist.sas
-
-  <h4> Related Files </h4>
-  @li mp_hashdataset.sas
-  @li mp_hashdirectory.test.sas
-  @li mp_md5.sas
-
-  @param [in] inloc Full filepath of the file to be hashed (unquoted)
-  @param [in] iftrue= (1=1) A condition under which the macro should be executed
-  @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
-    subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
-    recursion, set to MAX.
-  @param [in] method= (MD5) the hashing method to use.  Available options:
-    @li MD5
-    @li SH1
-    @li SHA256
-    @li SHA384
-    @li SHA512
-    @li CRC32
-  @param [out] outds= (work.mp_hashdirectory) The output dataset.  Contains:
-    @li directory - the parent folder
-    @li file_hash - the hash output
-    @li hash_duration - how long the hash took (first hash always takes longer)
-    @li file_path - /full/path/to/each/file.ext
-    @li file_or_folder - contains either "file" or "folder"
-    @li level - the depth of the directory (top level is 0)
-
-  @version 9.4m6
-  @author Allan Bowe
-**/
-
-%macro mp_hashdirectory(inloc,
-  outds=work.mp_hashdirectory,
-  method=MD5,
-  maxdepth=0,
-  iftrue=%str(1=1)
-)/*/STORE SOURCE*/;
-
-%local curlevel tempds maxlevel;
-
-%if not(%eval(%unquote(&iftrue))) %then %return;
-
-/* get the directory listing */
-%mp_dirlist(path=&inloc, outds=&outds, maxdepth=&maxdepth, showparent=YES)
-
-/* create the hashes */
-data &outds;
-  set &outds (rename=(filepath=file_path));
-  length FILE_HASH $32 HASH_DURATION 8;
-  keep directory file_hash hash_duration file_path file_or_folder level;
-
-  ts=datetime();
-  if file_or_folder='file' then do;
-    /* if file is empty, hashing_file will break - so ignore / delete */
-    length fname val $8;
-    drop fname val fid is_empty;
-    rc=filename(fname,file_path);
-    fid=fopen(fname);
-    if fid > 0 then do;
-      rc=fread(fid);
-      is_empty=fget(fid,val);
-    end;
-    rc=fclose(fid);
-    rc=filename(fname);
-    if is_empty ne 0 then delete;
-    else file_hash=hashing_file("&method",cats(file_path),0);
-  end;
-  hash_duration=datetime()-ts;
-run;
-
-proc sort data=&outds ;
-  by descending level directory file_path;
-run;
-
-%let maxlevel=0;
-data _null_;
-  set &outds;
-  call symputx('maxlevel',level,'l');
-  stop;
-run;
-
-/* now hash the hashes to populate folder hashes, starting from the bottom */
-%do curlevel=&maxlevel %to 0 %by -1;
-  data work._data_ (keep=directory file_hash);
-    set &outds;
-    where level=&curlevel;
-    by descending level directory file_path;
-    length str $32767 tmp_hash $32;
-    retain str tmp_hash ;
-    /* reset vars when starting a new directory */
-    if first.directory then do;
-      str='';
-      tmp_hash='';
-      i=0;
-    end;
-    /* hash each chunk of 100 file paths */
-    i+1;
-    str=cats(str,file_hash);
-    if mod(i,100)=0 or last.directory then do;
-      tmp_hash=hashing("&method",cats(tmp_hash,str));
-      str='';
-    end;
-    /* output the hash at directory level */
-    if last.directory then do;
-      file_hash=tmp_hash;
-      output;
-    end;
-    if last.level then stop;
-  run;
-  %let tempds=&syslast;
-  /* join the hash back into the main table */
-  proc sql undo_policy=none;
-  create table &outds as
-    select a.directory
-      ,coalesce(b.file_hash,a.file_hash) as file_hash
-      ,a.hash_duration
-      ,a.file_path
-      ,a.file_or_folder
-      ,a.level
-    from &outds a
-    left join &tempds b
-    on a.file_path=b.directory
-    order by level desc, directory, file_path;
-  drop table &tempds;
-%end;
-
-%mend mp_hashdirectory;

base/mp_init.sas

@@ -67,10 +67,10 @@ options
   validvarname=V7         /* avoid special characters etc in variable names */
   varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
   varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
-%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+%if %substr(&sysver,1,1) ne 4 %then %do;
   noautocorrect           /* disallow misspelled procedure names            */
   dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
 %end;
 ;
 
-%mend mp_init;
+%mend mp_init;

base/mp_jsonout.sas

@@ -1,12 +1,7 @@
 /**
   @file mp_jsonout.sas
   @brief Writes JSON in SASjs format to a fileref
-  @details This macro can be used to OPEN a JSON stream and send one or more
+  @details PROC JSON is faster but will produce errs like the ones below if
-  tables as arrays of rows, where each row can be an object or a nested array.
-
-  There are two engines available - DATASTEP or PROCJSON.
-
-  PROC JSON is fast but will produce errs like the ones below if
   special chars are encountered.
 
   > (ERR)OR: Some code points did not transcode.
@@ -17,10 +12,6 @@
 
   If this happens, try running with ENGINE=DATASTEP.
 
-  The DATASTEP engine is used to handle special SAS missing numerics, and
-  can also convert entire datasets to formatted values.  Output JSON is always
-  in UTF-8.
-
   Usage:
 
         filename tmp temp;
@@ -28,7 +19,7 @@
 
         %mp_jsonout(OPEN,jref=tmp)
         %mp_jsonout(OBJ,class,jref=tmp)
-        %mp_jsonout(OBJ,class,dslabel=class2,jref=tmp,showmeta=Y)
+        %mp_jsonout(OBJ,class,dslabel=class2,jref=tmp,showmeta=YES)
         %mp_jsonout(CLOSE,jref=tmp)
 
         data _null_;
@@ -55,14 +46,12 @@
     @li DATASTEP (more reliable when data has non standard characters)
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
-  @param [in] showmeta= (N) Set to Y to output metadata alongside each table,
+  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
-  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
-    that should be converted to JSON
 
-  <h4> Related Files </h4>
+  <h4> Related Macros <h4>
   @li mp_ds2fmtds.sas
 
   @version 9.2
@@ -70,41 +59,25 @@
   @source https://github.com/sasjs/core
 
 **/
+
 %macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y
   ,engine=DATASTEP
   ,missing=NULL
-  ,showmeta=N
+  ,showmeta=NO
-  ,maxobs=MAX
 )/*/STORE SOURCE*/;
-%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval
+%local tempds colinfo fmtds i numcols;
-  tmpds1 tmpds2 tmpds3 tmpds4;
 %let numcols=0;
-%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;);
 
 %if &action=OPEN %then %do;
   options nobomfile;
-  data _null_;file &jref encoding='utf-8' lrecl=200;
+  data _null_;file &jref encoding='utf-8' ;
     put '{"PROCESSED_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '"';
   run;
 %end;
 %else %if (&action=ARR or &action=OBJ) %then %do;
-  /* force variable names to always be uppercase in the JSON */
   options validvarname=upcase;
-  /* To avoid issues with _webout on EBI - such as encoding diffs and truncation
+  data _null_; file &jref encoding='utf-8' mod;
-    (https://support.sas.com/kb/49/325.html) we use temporary files */
-  filename _sjs1 temp lrecl=200 ;
-  data _null_; file _sjs1 encoding='utf-8';
     put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":";
-  run;
-  /* now write to _webout 1 char at a time */
-  data _null_;
-    infile _sjs1 lrecl=1 recfm=n;
-    file &jref mod lrecl=1 recfm=n;
-    input sourcechar $char1. @@;
-    format sourcechar hex2.;
-    put sourcechar char1. @@;
-  run;
-  filename _sjs1 clear;
 
   /* grab col defs */
   proc contents noprint data=&ds
@@ -115,7 +88,7 @@
     by varnum;
   run;
   /* move meta to mac vars */
-  data &colinfo;
+  data _null_;
     if _n_=1 then call symputx('numcols',nobs,'l');
     set &colinfo end=last nobs=nobs;
     name=upcase(name);
@@ -126,6 +99,7 @@
       if format='' then fmt=cats('$',length,'.');
       else if formatl=0 then fmt=cats(format,'.');
       else fmt=cats(format,formatl,'.');
+      newlen=max(formatl,length);
     end;
     else do;
       typelong='num';
@@ -133,26 +107,23 @@
       else if formatl=0 then fmt=cats(format,'.');
       else if formatd=0 then fmt=cats(format,formatl,'.');
       else fmt=cats(format,formatl,'.',formatd);
+      /* needs to be wide, for datetimes etc */
+      newlen=max(length,formatl,24);
     end;
     /* 32 char unique name */
     newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
 
     call symputx(cats('name',_n_),name,'l');
     call symputx(cats('newname',_n_),newname,'l');
+    call symputx(cats('len',_n_),newlen,'l');
     call symputx(cats('length',_n_),length,'l');
     call symputx(cats('fmt',_n_),fmt,'l');
     call symputx(cats('type',_n_),type,'l');
     call symputx(cats('typelong',_n_),typelong,'l');
     call symputx(cats('label',_n_),coalescec(label,name),'l');
-    /* overwritten when fmt=Y and a custom format exists in catalog */
-    if typelong='num' then call symputx(cats('fmtlen',_n_),200,'l');
-    else call symputx(cats('fmtlen',_n_),min(32767,ceil((length+10)*1.5)),'l');
   run;
 
   %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
-  proc sql;
-  select count(*) into: lastobs from &ds;
-  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs));
 
   %if &engine=PROCJSON %then %do;
     %if &missing=STRING %then %do;
@@ -160,25 +131,13 @@
       %put &sysmacroname: Switching to DATASTEP engine;
       %goto datastep;
     %end;
-    data &tempds;
+    data &tempds;set &ds;
-      set &ds;
-      &stmt_obs;
     %if &fmt=N %then format _numeric_ best32.;;
     /* PRETTY is necessary to avoid line truncation in large files */
-    filename _sjs2 temp lrecl=131068 encoding='utf-8';
+    proc json out=&jref pretty
-    proc json out=_sjs2 pretty
         %if &action=ARR %then nokeys ;
         ;export &tempds / nosastags fmtnumeric;
     run;
-    /* send back to webout */
-    data _null_;
-      infile _sjs2 lrecl=1 recfm=n;
-      file &jref mod lrecl=1 recfm=n;
-      input sourcechar $char1. @@;
-      format sourcechar hex2.;
-      put sourcechar char1. @@;
-    run;
-    filename _sjs2 clear;
   %end;
   %else %if &engine=DATASTEP %then %do;
     %datastep:
@@ -189,99 +148,26 @@
     %end;
 
     %if &fmt=Y %then %do;
-      /**
+      data _data_;
-        * Extract format definitions
-        * First, by getting library locations from dictionary.formats
-        * Then, by exporting the width using proc format
-        * Cannot use maxw from sashelp.vformat as not always populated
-        * Cannot use fmtinfo() as not supported in all flavours
-        */
-      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
-      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
-      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
-      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
-      proc sql noprint;
-      create table &tmpds1 as
-          select cats(libname,'.',memname) as FMTCAT,
-          FMTNAME
-        from dictionary.formats
-        where fmttype='F' and libname is not null
-          and fmtname in (select format from &colinfo where format is not null)
-        order by 1;
-      create table &tmpds2(
-          FMTNAME char(32),
-          LENGTH num
-      );
-      %local catlist cat fmtlist i;
-      select distinct fmtcat into: catlist separated by ' ' from &tmpds1;
-      %do i=1 %to %sysfunc(countw(&catlist,%str( )));
-        %let cat=%scan(&catlist,&i,%str( ));
-        proc sql;
-        select distinct fmtname into: fmtlist separated by ' '
-          from &tmpds1 where fmtcat="&cat";
-        proc format lib=&cat cntlout=&tmpds3(keep=fmtname length);
-          select &fmtlist;
-        run;
-        proc sql;
-        insert into &tmpds2 select distinct fmtname,length from &tmpds3;
-      %end;
-
-      proc sql;
-      create table &tmpds4 as
-        select a.*, b.length as MAXW
-        from &colinfo a
-        left join &tmpds2 b
-        on cats(a.format)=cats(upcase(b.fmtname))
-        order by a.varnum;
-      data _null_;
-        set &tmpds4;
-        if not missing(maxw);
-        call symputx(
-          cats('fmtlen',_n_),
-          /* vars need extra padding due to JSON escaping of special chars */
-          min(32767,ceil((max(length,maxw)+10)*1.5))
-          ,'l'
-        );
-      run;
-
-      /* configure varlenchk - as we are explicitly shortening the variables */
-      %let optval=%sysfunc(getoption(varlenchk));
-      options varlenchk=NOWARN;
-      data _data_(compress=char);
-        /* shorten the new vars */
-        length
-      %do i=1 %to &numcols;
-          &&name&i $&&fmtlen&i
-      %end;
-          ;
         /* rename on entry */
         set &ds(rename=(
       %do i=1 %to &numcols;
-          &&name&i=&&newname&i
+        &&name&i=&&newname&i
       %end;
         ));
-      &stmt_obs;
-
-      drop
-      %do i=1 %to &numcols;
-        &&newname&i
-      %end;
-        ;
       %do i=1 %to &numcols;
+        length &&name&i $&&len&i;
         %if &&typelong&i=num %then %do;
-          &&name&i=cats(put(&&newname&i,&&fmt&i));
+          &&name&i=left(put(&&newname&i,&&fmt&i));
         %end;
         %else %do;
           &&name&i=put(&&newname&i,&&fmt&i);
         %end;
+        drop &&newname&i;
       %end;
-        if _error_ then do;
+        if _error_ then call symputx('syscc',1012);
-          call symputx('syscc',1012);
-          stop;
-        end;
       run;
       %let fmtds=&syslast;
-      options varlenchk=&optval;
     %end;
 
     proc format; /* credit yabwon for special null removal */
@@ -300,8 +186,8 @@
       attrib _all_ label='';
       %do i=1 %to &numcols;
         %if &&typelong&i=char or &fmt=Y %then %do;
-          length &&name&i $&&fmtlen&i...;
+          length &&name&i $32767;
-          format &&name&i $&&fmtlen&i...;
+          format &&name&i $32767.;
         %end;
       %end;
       %if &fmt=Y %then %do;
@@ -310,35 +196,23 @@
       %else %do;
         set &ds;
       %end;
-      &stmt_obs;
       format _numeric_ bart.;
     %do i=1 %to &numcols;
       %if &&typelong&i=char or &fmt=Y %then %do;
-        if findc(&&name&i,'"\'!!'0A0D09000E0F010210111A'x) then do;
+        &&name&i='"'!!trim(prxchange('s/"/\"/',-1,
-          &&name&i='"'!!trim(
+                    prxchange('s/'!!'0A'x!!'/\n/',-1,
-            prxchange('s/"/\\"/',-1,        /* double quote */
+                    prxchange('s/'!!'0D'x!!'/\r/',-1,
-            prxchange('s/\x0A/\n/',-1,      /* new line */
+                    prxchange('s/'!!'09'x!!'/\t/',-1,
-            prxchange('s/\x0D/\r/',-1,      /* carriage return */
+                    prxchange('s/\\/\\\\/',-1,&&name&i)
-            prxchange('s/\x09/\\t/',-1,     /* tab */
+        )))))!!'"';
-            prxchange('s/\x00/\\u0000/',-1, /* NUL */
-            prxchange('s/\x0E/\\u000E/',-1, /* SS  */
-            prxchange('s/\x0F/\\u000F/',-1, /* SF  */
-            prxchange('s/\x01/\\u0001/',-1, /* SOH */
-            prxchange('s/\x02/\\u0002/',-1, /* STX */
-            prxchange('s/\x10/\\u0010/',-1, /* DLE */
-            prxchange('s/\x11/\\u0011/',-1, /* DC1 */
-            prxchange('s/\x1A/\\u001A/',-1, /* SUB */
-            prxchange('s/\\/\\\\/',-1,&&name&i)
-          )))))))))))))!!'"';
-        end;
-        else &&name&i=quote(cats(&&name&i));
       %end;
     %end;
     run;
 
-    filename _sjs3 temp lrecl=131068 ;
+    /* write to temp loc to avoid _webout truncation
-    data _null_;
+      - https://support.sas.com/kb/49/325.html */
-      file _sjs3 encoding='utf-8';
+    filename _sjs temp lrecl=131068 encoding='utf-8';
+    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod ;
       if _n_=1 then put "[";
       set &tempds;
       if _n_>1 then put "," @; put
@@ -346,38 +220,39 @@
       %do i=1 %to &numcols;
         %if &i>1 %then  "," ;
         %if &action=OBJ %then """&&name&i"":" ;
-        "&&name&i"n /* name literal for reserved variable names */
+        &&name&i
       %end;
       %if &action=ARR %then "]" ; %else "}" ; ;
-
+    /* now write the long strings to _webout 1 byte at a time */
-    /* close out the table */
     data _null_;
-      file _sjs3 mod encoding='utf-8';
+      length filein 8 fileid 8;
-      put ']';
+      filein=fopen("_sjs",'I',1,'B');
+      fileid=fopen("&jref",'A',1,'B');
+      rec='20'x;
+      do while(fread(filein)=0);
+        rc=fget(filein,rec,1);
+        rc=fput(fileid, rec);
+        rc=fwrite(fileid);
+      end;
+      /* close out the table */
+      rc=fput(fileid, "]");
+      rc=fwrite(fileid);
+      rc=fclose(filein);
+      rc=fclose(fileid);
     run;
-    data _null_;
+    filename _sjs clear;
-      infile _sjs3 lrecl=1 recfm=n;
-      file &jref mod lrecl=1 recfm=n;
-      input sourcechar $char1. @@;
-      format sourcechar hex2.;
-      put sourcechar char1. @@;
-    run;
-    filename _sjs3 clear;
   %end;
 
   proc sql;
   drop table &colinfo, &tempds;
 
-  %if %substr(&showmeta,1,1)=Y %then %do;
+  %if &showmeta=YES %then %do;
-    filename _sjs4 temp lrecl=131068 encoding='utf-8';
+    data _null_; file &jref encoding='utf-8' mod;
-    data _null_;
-      file _sjs4;
-      length label $350;
       put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{";
       do i=1 to &numcols;
         name=quote(trim(symget(cats('name',i))));
         format=quote(trim(symget(cats('fmt',i))));
-        label=quote(prxchange('s/\\/\\\\/',-1,trim(symget(cats('label',i)))));
+        label=quote(trim(symget(cats('label',i))));
         length=quote(trim(symget(cats('length',i))));
         type=quote(trim(symget(cats('typelong',i))));
         if i>1 then put "," @@;
@@ -386,15 +261,6 @@
       end;
       put '}}';
     run;
-    /* send back to webout */
-    data _null_;
-      infile _sjs4 lrecl=1 recfm=n;
-      file &jref mod lrecl=1 recfm=n;
-      input sourcechar $char1. @@;
-      format sourcechar hex2.;
-      put sourcechar char1. @@;
-    run;
-    filename _sjs4 clear;
   %end;
 %end;
 

base/mp_loadformat.sas

@@ -40,13 +40,13 @@
   @li mp_abort.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
-  @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
   @li mddl_dc_difftable.sas
   @li mddl_dc_locktable.sas
   @li mp_loadformat.test.sas
   @li mp_lockanytable.sas
+  @li mp_storediffs.sas
   @li mp_stackdiffs.sas
 
 
@@ -134,7 +134,7 @@ run;
   * First, extract only relevant formats from the catalog
   */
 proc sql noprint;
-select distinct upcase(fmtname) into: fmtlist separated by ' ' from &libds;
+select distinct fmtname into: fmtlist separated by ' ' from &libds;
 
 %mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
 
@@ -144,11 +144,8 @@ select distinct upcase(fmtname) into: fmtlist separated by ' ' from &libds;
   */
 %mddl_sas_cntlout(libds=&template)
 data &inlibds;
-  length &delete_col $3;
   if 0 then set &template;
   set &libds;
-  if &delete_col='' then &delete_col='No';
-  fmtname=upcase(fmtname);
   if missing(type) then do;
     if substr(fmtname,1,1)='$' then type='C';
     else type='N';
@@ -159,6 +156,7 @@ data &inlibds;
   end;
 run;
 
+
 /**
   * Identify new records
   */
@@ -265,7 +263,7 @@ options ibufsize=&ibufsize;
     %end;
 
     %mp_storediffs(&libcat-FC
-      ,&base_fmts
+      ,&inlibds
       ,FMTNAME START
       ,delds=&outds_del
       ,modds=&outds_mod
@@ -274,9 +272,6 @@ options ibufsize=&ibufsize;
       ,mdebug=&mdebug
     )
 
-    proc append base=&auditlibds data=&storediffs;
-    run;
-
     %if &locklibds ne 0 %then %do;
       %mp_lockanytable(UNLOCK
         ,lib=%scan(&auditlibds,1,.)
@@ -300,4 +295,4 @@ options ibufsize=&ibufsize;
   %put &sysmacroname exit vars:;
   %put _local_;
 %end;
-%mend mp_loadformat;
+%mend mp_loadformat;

base/mp_lockanytable.sas

@@ -1,8 +1,7 @@
 /**
   @file
   @brief Mechanism for locking tables to prevent parallel modifications
-  @details Uses a control table to enable ANY table to be locked for updates
+  @details Uses a control table to enable ANY table to be locked for updates.
-  (not just SAS datasets).
   Only useful if every update uses the macro!   Used heavily within
   [Data Controller for SAS](https://datacontroller.io).
 
@@ -16,13 +15,12 @@
     length is 200 characters.
   @param [out] ctl_ds= (0) The control table which controls the actual locking.
     Should already be assigned and available.  The definition is available by
-    running the mddl_dc_locktable.sas macro.
+    running mp_coretable.sas as follows:  `%mp_coretable(LOCKTABLE)`.
 
   @param [in] loops= (25) Number of times to check for a lock.
   @param [in] loop_secs= (1) Seconds to wait between each lock attempt
 
   <h4> SAS Macros </h4>
-  @li mf_fmtdttm.sas
   @li mp_abort.sas
   @li mp_lockfilecheck.sas
   @li mf_getuser.sas
@@ -112,7 +110,7 @@ run;
       LOCK_LIB ="&lib";
       LOCK_DS="&ds";
       LOCK_STATUS_CD='LOCKED';
-      LOCK_START_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt;
+      LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt;
       LOCK_USER_NM="&user";
       LOCK_PID="&sysjobid";
       LOCK_REF="&ref";
@@ -132,7 +130,7 @@ run;
     proc sql;
     update &ctl_ds
       set LOCK_STATUS_CD='LOCKED'
-        , LOCK_START_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
+        , LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
         , LOCK_USER_NM="&user"
         , LOCK_PID="&sysjobid"
         , LOCK_REF="&ref"
@@ -207,7 +205,7 @@ run;
     proc sql;
     update &ctl_ds
       set LOCK_STATUS_CD='UNLOCKED'
-        , LOCK_END_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
+        , LOCK_END_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
         , LOCK_USER_NM="&user"
         , LOCK_PID="&sysjobid"
         , LOCK_REF="&ref"

base/mp_md5.sas

@@ -54,5 +54,5 @@ put(md5(
     &sep put(md5(trim(put(ifn(missing(&var),&var,&var*1),binary64.))),$hex32.)
     %let sep=!!;
   %end;
-),$hex32.)
+),hex32.)
 %mend mp_md5;

base/mp_perflog.sas

@@ -1,14 +1,13 @@
 /**
   @file
-  @brief Logs a message in a dataset every time it is invoked
+  @brief Logs the time the macro was executed in a control dataset.
-  @details If the dataset does not exist, it is created.
+  @details If the dataset does not exist, it is created.  Usage:
-  Usage:
 
-      %mp_perflog(started)
+    %mp_perflog(started)
-      %mp_perflog()
+    %mp_perflog()
-      %mp_perflog(startanew,libds=work.newdataset)
+    %mp_perflog(startanew,libds=work.newdataset)
-      %mp_perflog(finished,libds=work.newdataset)
+    %mp_perflog(finished,libds=work.newdataset)
-      %mp_perflog(finished)
+    %mp_perflog(finished)
 
 
   @param label Provide label to go into the control dataset

base/mp_replace.sas

@@ -1,151 +0,0 @@
-/**
-  @file
-  @brief Performs a text substitution on a file
-  @details Performs a find and replace on a file, either in place or to a new
-  file. Can be used on files where lines are longer than 32767.
-
-  Works by reading in the file byte by byte, then marking the beginning and end
-  of each matched string, before finally doing the replace.
-
-  Full credit for this highly efficient and syntactically satisfying SAS logic
-  goes to [Bartosz Jabłoński](https://www.linkedin.com/in/yabwon), founder of
-  the [SAS Packages](https://github.com/yabwon/SAS_PACKAGES) framework.
-
-  Usage:
-
-      %let file="%sysfunc(pathname(work))/file.txt";
-      %let str=replace/me;
-      %let rep=with/this;
-      data _null_;
-        file &file;
-        put 'blahblah';
-        put "blahblah&str.blah";
-        put 'blahblahblah';
-      run;
-      %mp_replace(&file, findvar=str, replacevar=rep)
-      data _null_;
-        infile &file;
-        input;
-        list;
-      run;
-
-  Note - if you are running a version of SAS that will allow the io package in
-  LUA, you can also use this macro: mp_gsubfile.sas
-
-  @param infile The QUOTED path to the file on which to perform the substitution
-  @param findvar= Macro variable NAME containing the string to search for
-  @param replacevar= Macro variable NAME containing the replacement string
-  @param outfile= (0) Optional QUOTED path to the adjusted output file (to
-    avoid overwriting the first file).
-
-  <h4> SAS Macros </h4>
-  @li mf_getuniquefileref.sas
-  @li mf_getuniquename.sas
-
-  <h4> Related Macros </h4>
-  @li mp_chop.sas
-  @li mp_gsubfile.sas
-  @li mp_replace.test.sas
-
-  @version 9.4
-  @author Bartosz Jabłoński
-  @author Allan Bowe
-**/
-
-%macro mp_replace(infile,
-  findvar=,
-  replacevar=,
-  outfile=0
-)/*/STORE SOURCE*/;
-
-%local inref dttm ds1;
-%let inref=%mf_getuniquefileref();
-%let outref=%mf_getuniquefileref();
-%if &outfile=0 %then %let outfile=&infile;
-%let ds1=%mf_getuniquename(prefix=allchars);
-%let ds2=%mf_getuniquename(prefix=startmark);
-
-/* START */
-%let dttm=%sysfunc(datetime());
-
-filename &inref &infile lrecl=1 recfm=n;
-
-data &ds1;
-  infile &inref;
-  input sourcechar $char1. @@;
-  format sourcechar hex2.;
-run;
-
-data &ds2;
-  /* set find string to length in bytes to cover trailing spaces */
-  length string $ %length(%superq(&findvar));
-  string =symget("&findvar");
-  drop string;
-
-  firstchar=char(string,1);
-  findlen=lengthm(string); /* <- for trailing bytes */
-
-  do _N_=1 to nobs;
-    set &ds1 nobs=nobs point=_N_;
-    if sourcechar=firstchar then do;
-      pos=1;
-      s=0;
-      do point=_N_ to min(_N_ + findlen -1,nobs);
-        set &ds1 point=point;
-        if sourcechar=char(string, pos) then s + 1;
-        else goto _leave_;
-        pos+1;
-      end;
-      _leave_:
-      if s=findlen then do;
-        START =_N_;
-        _N_ =_N_+ s - 1;
-        STOP =_N_;
-        output;
-      end;
-    end;
-  end;
-  stop;
-  keep START STOP;
-run;
-
-data &ds1;
-  declare hash HS(dataset:"&ds2(keep=start)");
-  HS.defineKey("start");
-  HS.defineDone();
-  declare hash HE(dataset:"&ds2(keep=stop)");
-  HE.defineKey("stop");
-  HE.defineDone();
-  do until(eof);
-    set &ds1 end=eof curobs =n;
-    start = ^HS.check(key:n);
-    stop  = ^HE.check(key:n);
-    length strt $ 1;
-    strt =put(start,best. -L);
-    retain out 1;
-    if out   then output;
-    if start then out=0;
-    if stop  then out=1;
-  end;
-  stop;
-  keep sourcechar strt;
-run;
-
-filename &outref &outfile recfm=n;
-
-data _null_;
-  length replace $ %length(%superq(&replacevar));
-  replace=symget("&replacevar");
-  file &outref;
-  do until(eof);
-    set &ds1 end=eof;
-    if strt ="1" then put replace char.;
-    else put sourcechar char1.;
-  end;
-  stop;
-run;
-
-/* END */
-%put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) seconds to run;
-
-%mend mp_replace;

base/mp_resetoption.sas

@@ -21,19 +21,15 @@ https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-s
 %macro mp_resetoption(option /* the option to reset */
 )/*/STORE SOURCE*/;
 
-%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+data _null_;
-  data _null_;
+  length code  $1500;
-    length code  $1500;
+  startup=getoption("&option",'startupvalue');
-    startup=getoption("&option",'startupvalue');
+  current=getoption("&option");
-    current=getoption("&option");
+  if startup ne current then do;
-    if startup ne current then do;
+    code =cat('OPTIONS ',getoption("&option",'keyword','startupvalue'),';');
-      code =cat('OPTIONS ',getoption("&option",'keyword','startupvalue'),';');
+    putlog "NOTE: Resetting system option: " code ;
-      putlog "NOTE: Resetting system option: " code ;
+    call execute(code );
-      call execute(code );
+  end;
-    end;
+run;
-  run;
+
-%end;
-%else %do;
-  %put &sysmacroname: reset option feature unavailable on &sysvlong;
-%end;
 %mend mp_resetoption;

base/mp_retainedkey.sas

@@ -58,7 +58,6 @@
 
   <h4> SAS Macros </h4>
   @li mf_existvar.sas
-  @li mf_fmtdttm.sas
   @li mf_getquotedstr.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
@@ -218,12 +217,12 @@ quit;
     set keytable="&base_libds"
       ,keycolumn="&retained_key"
       ,max_key=%eval(&maxkey+&newkey_cnt)
-      ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt;
+      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt;
   %end;
   %else %do;
   update &maxkeytable
     set max_key=%eval(&maxkey+&newkey_cnt)
-      ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt
+      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt
     where keytable="&base_libds";
   %end;
   %mp_lockanytable(UNLOCK

base/mp_storediffs.sas

@@ -125,7 +125,7 @@
 data &ds1;
   set &dslist indsname=&inds_auto;
   &hashkey=put(md5(catx('|',%mf_getquotedstr(&key,quote=N))),$hex32.);
-  &inds_keep=upcase(&inds_auto);
+  &inds_keep=&inds_auto;
 proc sort;
   by &inds_keep &hashkey;
 run;
@@ -154,16 +154,14 @@ run;
 %else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
 
 data &ds4;
-  length &inds_keep $41 tgtvar_nm $32 _label_ $256;
+  length &inds_keep $41 tgtvar_nm $32;
-  if _n_=1 then call missing(_label_);
-  drop _label_;
   set &ds2 &ds3 indsname=&inds_auto;
 
   tgtvar_nm=upcase(tgtvar_nm);
   if tgtvar_nm in (%upcase(&vlist));
 
-  if upcase(&inds_auto)="&ds2" then tgtvar_type='N';
+  if &inds_auto="&ds2" then tgtvar_type='N';
-  else if upcase(&inds_auto)="&ds3" then tgtvar_type='C';
+  else if &inds_auto="&ds3" then tgtvar_type='C';
   else do;
     putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
     call symputx('syscc',98);

base/mp_streamfile.sas

@@ -1,9 +1,8 @@
 /**
   @file
   @brief Streams a file to _webout according to content type
-  @details Will set headers using appropriate functions per the server type
+  @details Will set headers using appropriate functions (SAS 9 vs Viya) and send
-  (Viya, EBI, [SASjs Server](https://github.com/sasjs/server)) and stream
+  content as a binary stream.
-  content using mp_binarycopy().
 
   Usage:
 
@@ -13,14 +12,7 @@
 
       %mp_streamfile(contenttype=csv,inloc=/some/where.txt,outname=myfile.txt)
 
-  @param [in] contenttype= (TEXT) Supported:
+  @param [in] contenttype= (TEXTS) Either TEXT, ZIP, CSV, EXCEL
-    @li CSV
-    @li EXCEL
-    @li MARKDOWN
-    @li TEXT
-    @li ZIP
-    Feel free to submit PRs to support more mime types!  The official list is
-    here:  https://www.iana.org/assignments/media-types/media-types.xhtml
   @param [in] inloc= /path/to/file.ext to be sent
   @param [in] inref= fileref of file to be sent (if provided, overrides `inloc`)
   @param [in] iftrue= (1=1) Provide a condition under which to execute.
@@ -66,7 +58,7 @@ run;
 %if &contentype=CSV %then %do;
   %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-Type','application/csv');
+      rc=stpsrv_header('Content-type','application/csv');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
@@ -76,7 +68,7 @@ run;
       contentdisp="attachment; filename=&outname";
   %end;
   %else %if &platform=SASJS %then %do;
-    %mfs_httpheader(Content-Type,application/csv)
+    %mfs_httpheader(Content-type,application/csv)
     %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
   %end;
 %end;
@@ -84,7 +76,7 @@ run;
   /* suitable for XLS format */
   %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-Type','application/vnd.ms-excel');
+      rc=stpsrv_header('Content-type','application/vnd.ms-excel');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
@@ -94,14 +86,14 @@ run;
       contentdisp="attachment; filename=&outname";
   %end;
   %else %if &platform=SASJS %then %do;
-    %mfs_httpheader(Content-Type,application/vnd.ms-excel)
+    %mfs_httpheader(Content-type,application/vnd.ms-excel)
     %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
   %end;
 %end;
 %else %if &contentype=GIF or &contentype=JPEG or &contentype=PNG %then %do;
   %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-Type',"image/%lowcase(&contenttype)");
+      rc=stpsrv_header('Content-type',"image/%lowcase(&contenttype)");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
@@ -109,30 +101,22 @@ run;
       contenttype="image/%lowcase(&contenttype)";
   %end;
   %else %if &platform=SASJS %then %do;
-    %mfs_httpheader(Content-Type,image/%lowcase(&contenttype))
+    %mfs_httpheader(Content-type,image/%lowcase(&contenttype))
   %end;
 %end;
-%else %if &contentype=HTML or &contenttype=MARKDOWN %then %do;
+%else %if &contentype=HTML %then %do;
-  %if (&platform=SASMETA and &streamweb=1) %then %do;
+  %if &platform=SASVIYA %then %do;
-    data _null_;
-      rc=stpsrv_header('Content-Type',"text/%lowcase(&contenttype)");
-      rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
-    run;
-  %end;
-  %else %if &platform=SASVIYA %then %do;
     filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
-      contenttype="text/%lowcase(&contenttype)"
+      contenttype="text/html";
-      contentdisp="attachment; filename=&outname";
   %end;
   %else %if &platform=SASJS %then %do;
-    %mfs_httpheader(Content-Type,text/%lowcase(&contenttype))
+    %mfs_httpheader(Content-type,text/html)
-    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
   %end;
 %end;
 %else %if &contentype=TEXT %then %do;
   %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-Type','application/text');
+      rc=stpsrv_header('Content-type','application/text');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
@@ -142,14 +126,14 @@ run;
       contentdisp="attachment; filename=&outname";
   %end;
   %else %if &platform=SASJS %then %do;
-    %mfs_httpheader(Content-Type,application/text)
+    %mfs_httpheader(Content-type,application/text)
     %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
   %end;
 %end;
 %else %if &contentype=WOFF or &contentype=WOFF2 or &contentype=TTF %then %do;
   %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-Type',"font/%lowcase(&contenttype)");
+      rc=stpsrv_header('Content-type',"font/%lowcase(&contenttype)");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
@@ -157,13 +141,13 @@ run;
       contenttype="font/%lowcase(&contenttype)";
   %end;
   %else %if &platform=SASJS %then %do;
-    %mfs_httpheader(Content-Type,font/%lowcase(&contenttype))
+    %mfs_httpheader(Content-type,font/%lowcase(&contenttype))
   %end;
 %end;
 %else %if &contentype=XLSX %then %do;
   %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-Type',
+      rc=stpsrv_header('Content-type',
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
@@ -175,7 +159,7 @@ run;
       contentdisp="attachment; filename=&outname";
   %end;
   %else %if &platform=SASJS %then %do;
-    %mfs_httpheader(Content-Type
+    %mfs_httpheader(Content-type
       ,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
     )
     %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
@@ -184,7 +168,7 @@ run;
 %else %if &contentype=ZIP %then %do;
   %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-Type','application/zip');
+      rc=stpsrv_header('Content-type','application/zip');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
@@ -194,7 +178,7 @@ run;
       contentdisp="attachment; filename=&outname";
   %end;
   %else %if &platform=SASJS %then %do;
-    %mfs_httpheader(Content-Type,application/zip)
+    %mfs_httpheader(Content-type,application/zip)
     %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
   %end;
 %end;

base/mp_testservice.sas

@@ -1,11 +1,51 @@
 /**
   @file
-  @brief To be deprecated.  Will execute a SASjs web service on SAS 9 or Viya
+  @brief Will execute a SASjs web service on SAS 9 or Viya
-  @details Use the mx_testservice.sas macro instead (documentation can be
+  @details Prepares the input files and retrieves the resulting datasets from
-  found there)
+  the response JSON.
+
+
+
+  Note - the _webout fileref should NOT be assigned prior to running this macro.
+
+  @param [in] program The _PROGRAM endpoint to test
+  @param [in] inputfiles=(0) A list of space seperated fileref:filename pairs as
+    follows:
+        inputfiles=inref:filename inref2:filename2
+  @param [in] inputdatasets= (0) All datasets in this space seperated list are
+    converted into SASJS-formatted CSVs (see mp_ds2csv.sas) files and added to
+    the list of `inputfiles` for ingestion.  The dataset will be sent with the
+    same name (no need for a colon modifier).
+  @param [in] inputparams=(0) A dataset containing name/value pairs in the
+    following format:
+    |name:$32|value:$1000|
+    |---|---|
+    |stpmacname|some value|
+    |mustbevalidname|can be anything, oops, %abort!!|
+
+  @param [in] debug= (log) Provide the _debug value
+  @param [in] mdebug= (0) Set to 1 to provide macro debugging
+  @param [in] viyaresult= (WEBOUT_JSON) The Viya result type to return.  For
+    more info, see mv_getjobresult.sas
+  @param [in] viyacontext= (SAS Job Execution compute context) The Viya compute
+    context on which to run the service
+  @param [out] outlib= (0) Output libref to contain the final tables.  Set to
+    0 if the service output is not in JSON format.
+  @param [out] outref= (0) Output fileref to create, to contain the full _webout
+    response.
 
   <h4> SAS Macros </h4>
-  @li mx_testservice.sas
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+  @li mp_ds2csv.sas
+  @li mv_getjobresult.sas
+  @li mv_jobflow.sas
+
+  <h4> Related Programs </h4>
+  @li mp_testservice.test.sas
 
   @version 9.4
   @author Allan Bowe
@@ -23,17 +63,216 @@
   viyaresult=WEBOUT_JSON,
   viyacontext=SAS Job Execution compute context
 )/*/STORE SOURCE*/;
+%local dbg pcnt fref1 webref i webcount var platform;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
 
-%mx_testservice(&program,
+/* sanitise inputparams */
-  inputfiles=&inputfiles,
+%let pcnt=0;
-  inputdatasets=&inputdatasets,
+%if &inputparams ne 0 %then %do;
-  inputparams=&inputparams,
+  data _null_;
-  debug=&debug,
+    set &inputparams;
-  mdebug=&mdebug,
+    if not nvalid(name,'v7') then putlog (_all_)(=);
-  outlib=&outlib,
+    else if name in (
-  outref=&outref,
+      'program','inputfiles','inputparams','debug','outlib','outref'
-  viyaresult=&viyaresult,
+    ) then putlog (_all_)(=);
-  viyacontext=&viyacontext
+    else do;
-)
+      x+1;
+      call symputx(name,quote(cats(value)),'l');
+      call symputx(cats('pval',x),name,'l');
+      call symputx('pcnt',x,'l');
+    end;
+  run;
+  %mp_abort(iftrue= (%mf_nobs(&inputparams) ne &pcnt)
+    ,mac=&sysmacroname
+    ,msg=%str(Invalid values in &inputparams)
+  )
+%end;
 
-%mend mp_testservice;
+/* convert inputdatasets to filerefs */
+%if "&inputdatasets" ne "0" %then %do;
+  %if %quote(&inputfiles)=0 %then %let inputfiles=;
+  %do i=1 %to %sysfunc(countw(&inputdatasets,%str( )));
+    %let var=%scan(&inputdatasets,&i,%str( ));
+    %local dsref&i;
+    %let dsref&i=%mf_getuniquefileref();
+    %mp_ds2csv(&var,outref=&&dsref&i,headerformat=SASJS)
+    %let inputfiles=&inputfiles &&dsref&i:%scan(&var,-1,.);
+  %end;
+%end;
+
+
+%let fref1=%mf_getuniquefileref();
+%let webref=%mf_getuniquefileref();
+%let platform=%mf_getplatform();
+%if &platform=SASMETA %then %do;
+
+  /* parse the input files */
+  %if %quote(&inputfiles) ne 0 %then %do;
+    %let webcount=%sysfunc(countw(&inputfiles));
+    %put &=webcount;
+    %do i=1 %to &webcount;
+      %let var=%scan(&inputfiles,&i,%str( ));
+      %local webfref&i webname&i;
+      %let webref&i=%scan(&var,1,%str(:));
+      %let webname&i=%scan(&var,2,%str(:));
+      %put webref&i=&&webref&i;
+      %put webname&i=&&webname&i;
+    %end;
+  %end;
+  %else %let webcount=0;
+
+  proc stp program="&program";
+    inputparam _program="&program"
+  %do i=1 %to &webcount;
+    %if &webcount=1 %then %do;
+      _webin_fileref="&&webref&i"
+      _webin_name="&&webname&i"
+    %end;
+    %else %do;
+      _webin_fileref&i="&&webref&i"
+      _webin_name&i="&&webname&i"
+    %end;
+  %end;
+    _webin_file_count="&webcount"
+    _debug="&debug"
+  %do i=1 %to &pcnt;
+    /* resolve name only, proc stp fetches value */
+    &&pval&i=&&&&&&pval&i
+  %end;
+    ;
+  %do i=1 %to &webcount;
+    inputfile &&webref&i;
+  %end;
+    outputfile _webout=&webref;
+  run;
+
+  data _null_;
+    infile &webref;
+    file &fref1;
+    input;
+    length line $10000;
+    if index(_infile_,'>>weboutBEGIN<<') then do;
+        line=tranwrd(_infile_,'>>weboutBEGIN<<','');
+        put line;
+    end;
+    else if index(_infile_,'>>weboutEND<<') then do;
+        line=tranwrd(_infile_,'>>weboutEND<<','');
+        put line;
+        stop;
+    end;
+    else put _infile_;
+  run;
+  data _null_;
+    infile &fref1;
+    input;
+    put _infile_;
+  run;
+  %if &outlib ne 0 %then %do;
+    libname &outlib json (&fref1);
+  %end;
+  %if &outref ne 0 %then %do;
+    filename &outref temp;
+    %mp_binarycopy(inref=&webref,outref=&outref)
+  %end;
+
+%end;
+%else %if &platform=SASVIYA %then %do;
+
+  /* prepare inputparams */
+  %local ds1;
+  %let ds1=%mf_getuniquename();
+  %if "&inputparams" ne "0" %then %do;
+    proc transpose data=&inputparams out=&ds1;
+      id name;
+      var value;
+    run;
+  %end;
+  %else %do;
+    data &ds1;run;
+  %end;
+
+  /* parse the input files - convert to sasjs params */
+  %local webcount i var sasjs_tables;
+  %if %quote(&inputfiles) ne 0 %then %do;
+    %let webcount=%sysfunc(countw(&inputfiles));
+    %put &=webcount;
+    %do i=1 %to &webcount;
+      %let var=%scan(&inputfiles,&i,%str( ));
+      %local webfref&i webname&i sasjs&i.data;
+      %let webref&i=%scan(&var,1,%str(:));
+      %let webname&i=%scan(&var,2,%str(:));
+      %put webref&i=&&webref&i;
+      %put webname&i=&&webname&i;
+
+      %let sasjs_tables=&sasjs_tables &&webname&i;
+      data _null_;
+        infile &&webref&i lrecl=32767;
+        input;
+        if _n_=1 then call symputx("sasjs&i.data",_infile_);
+        else call symputx(
+          "sasjs&i.data",cats(symget("sasjs&i.data"),'0D0A'x,_infile_)
+        );
+        putlog "&sysmacroname infile: " _infile_;
+      run;
+      data &ds1;
+        set &ds1;
+        length sasjs&i.data $32767 sasjs_tables $1000;
+        sasjs&i.data=symget("sasjs&i.data");
+        sasjs_tables=symget("sasjs_tables");
+      run;
+    %end;
+  %end;
+  %else %let webcount=0;
+
+  data &ds1;
+    retain _program "&program";
+    retain _contextname "&viyacontext";
+    set &ds1;
+    putlog "&sysmacroname inputparams:";
+    putlog (_all_)(=);
+  run;
+
+  %mv_jobflow(inds=&ds1
+    ,maxconcurrency=1
+    ,outds=work.results
+    ,outref=&fref1
+    ,mdebug=&mdebug
+  )
+  /* show the log */
+  data _null_;
+    infile &fref1;
+    input;
+    putlog _infile_;
+  run;
+  /* get the uri to fetch results */
+  data _null_;
+    set work.results;
+    call symputx('uri',uri);
+    putlog "&sysmacroname: fetching results for " uri;
+  run;
+  /* fetch results from webout.json */
+  %mv_getjobresult(uri=&uri,
+    result=&viyaresult,
+    outref=&outref,
+    outlib=&outlib,
+    mdebug=&mdebug
+  )
+
+%end;
+%else %do;
+  %put %str(ERR)OR: Unrecognised platform:  &platform;
+%end;
+
+%if &mdebug=0 %then %do;
+  filename &webref clear;
+%end;
+%else %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+
+%mend mp_testservice;

build.py

@@ -4,8 +4,8 @@ from pathlib import Path
 # Prepare Lua Macros
 files = [f for f in Path('lua').iterdir() if f.match("*.lua")]
 for file in files:
-    basename = os.path.basename(file)
+    basename=os.path.basename(file)
-    name = 'ml_' + os.path.splitext(basename)[0]
+    name='ml_' + os.path.splitext(basename)[0]
     ml = open('lua/' + name + '.sas', "w")
     ml.write("/**\n")
     ml.write("  @file " + name + '.sas\n')
@@ -20,68 +20,50 @@ for file in files:
     ml.write("  file \"%sysfunc(pathname(work))/" + name + ".lua\";\n")
     with open(file) as infile:
         for line in infile:
-            ml.write("  put '" + line.rstrip().replace("'", "''") + " ';\n")
+          ml.write("  put '" + line.rstrip().replace("'","''") + " ';\n")
     ml.write("run;\n\n")
-
+    ml.write("%inc \"%sysfunc(pathname(work))/" + name + ".lua\" /source2;\n\n")
-    ml.write("/* ensure big enough lrecl to avoid lua compilation issues */\n")
-    ml.write("%local optval;\n")
-    ml.write("%let optval=%sysfunc(getoption(lrecl));\n")
-    ml.write("options lrecl=1024;\n\n")
-    ml.write("/* execute the lua code by using a .lua extension */\n")
-    ml.write("%inc \"%sysfunc(pathname(work))/" +
-             name + ".lua\" /source2;\n\n")
-    ml.write("options lrecl=&optval;\n\n")
     ml.write("%mend " + name + ";\n")
 
 ml.close()
 
 # prepare web files
-files = ['viya/mv_createwebservice.sas',
+files=['viya/mv_createwebservice.sas','meta/mm_createwebservice.sas']
-         'meta/mm_createwebservice.sas', 'server/ms_createwebservice.sas']
 for file in files:
-    webout0 = open('base/mp_jsonout.sas', 'r')
+    webout0=open('base/mp_jsonout.sas','r')
-    webout1 = open('base/mf_getuser.sas', 'r')
+    if file=='viya/mv_createwebservice.sas':
-
+        webout1=open('viya/mv_webout.sas',"r")
-    if file == 'viya/mv_createwebservice.sas':
-        webout2 = open('viya/mv_webout.sas', "r")
-        weboutfiles = [webout0, webout1, webout2]
-    elif file == 'server/ms_createwebservice.sas':
-        webout2 = open('server/ms_webout.sas', "r")
-        webout3 = open('server/mfs_httpheader.sas', 'r')
-        weboutfiles = [webout0, webout1, webout2, webout3]
     else:
-        webout2 = open('meta/mm_webout.sas', 'r')
+        webout1=open('meta/mm_webout.sas','r')
-        weboutfiles = [webout0, webout1, webout2]
+    webout2=open('base/mf_getuser.sas','r')
-    outfile = open(file + 'TEMP', 'w')
+    outfile=open(file + 'TEMP','w')
-    infile = open(file, 'r')
+    infile=open(file,'r')
-    delrow = 0
+    delrow=0
     for line in infile:
-        if line == '/* WEBOUT BEGIN */\n':
+        if line=='/* WEBOUT BEGIN */\n':
-            delrow = 1
+            delrow=1
             outfile.write('/* WEBOUT BEGIN */\n')
+            weboutfiles=[webout0,webout1,webout2]
             for weboutfile in weboutfiles:
-                stripcomment = 1
+                stripcomment=1
                 for w in weboutfile:
-                    if w == '**/\n':
+                    if w=='**/\n': stripcomment=0
-                        stripcomment = 0
+                    elif stripcomment==0:
-                    elif stripcomment == 0:
+                        outfile.write("  put '" + w.rstrip().replace("'","''") + " ';\n")
-                        outfile.write(
+        elif delrow==1 and line=='/* WEBOUT END */\n':
-                            "  put '" + w.rstrip().replace("'", "''") + " ';\n")
+                delrow=0
-        elif delrow == 1 and line == '/* WEBOUT END */\n':
+                outfile.write('/* WEBOUT END */\n')
-            delrow = 0
+        elif delrow==0:
-            outfile.write('/* WEBOUT END */\n')
-        elif delrow == 0:
             outfile.write(line.rstrip() + "\n")
     webout0.close()
     webout1.close()
-    webout2.close()
     outfile.close()
     infile.close()
     os.remove(file)
-    os.rename(file + 'TEMP', file)
+    os.rename(file + 'TEMP',file)
 
 # Concatenate all macros into a single file
-header = """
+header="""
 /**
   @file
   @brief Auto-generated file
@@ -102,15 +84,14 @@ options noquotelenmax;
 """
 f = open('all.sas', "w")             # r / r+ / rb / rb+ / w / wb
 f.write(header)
-folders = ['base', 'ddl', 'meta', 'metax', 'server', 'viya', 'lua', 'fcmp', 'xplatform']
+folders=['base','ddl','meta','metax','server','viya','lua','fcmp']
 for folder in folders:
-    filenames = [fn for fn in Path(
+    filenames = [fn for fn in Path('./' + folder).iterdir() if fn.match("*.sas")]
-        './' + folder).iterdir() if fn.match("*.sas")]
     filenames.sort()
     with open('mc_' + folder + '.sas', 'w') as outfile:
         for fname in filenames:
             with open(fname) as infile:
                 outfile.write(infile.read())
-    with open('mc_' + folder + '.sas', 'r') as c:
+    with open('mc_' + folder + '.sas','r') as c:
         f.write(c.read())
 f.close()

ddl/mddl_dc_difftable.sas

@@ -26,19 +26,9 @@
       oldval_num num format=best32. label='Old (numeric) value',
       newval_num num format=best32. label='New (numeric) value',
       oldval_char char(32765) label='Old (character) value',
-      newval_char char(32765) label='New (character) value'
+      newval_char char(32765) label='New (character) value',
+    constraint pk_mpe_audit
+      primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
   );
 
-  %local lib;
-  %let libds=%upcase(&libds);
-  %if %index(&libds,.)=0 %then %let lib=WORK;
-  %else %let lib=%scan(&libds,1,.);
-
-  proc datasets lib=&lib noprint;
-    modify %scan(&libds,-1,.);
-    index create
-      pk_mpe_audit=(load_ref libref dsn key_hash tgtvar_nm)
-      /nomiss unique;
-  quit;
-
 %mend mddl_dc_difftable;

ddl/mddl_dc_filterdetail.sas

@@ -9,32 +9,19 @@
 
 %macro mddl_dc_filterdetail(libds=WORK.FILTER_DETAIL);
 
-%local nn lib;
-%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
-  %let nn=not null;
-%end;
-%else %let nn=;
-
   proc sql;
   create table &libds(
-      filter_hash char(32) &nn,
+      filter_hash char(32) not null,
-      filter_line num &nn,
+      filter_line num not null,
-      group_logic char(3) &nn,
+      group_logic char(3) not null,
-      subgroup_logic char(3) &nn,
+      subgroup_logic char(3) not null,
-      subgroup_id num &nn,
+      subgroup_id num not null,
-      variable_nm varchar(32) &nn,
+      variable_nm varchar(32) not null,
-      operator_nm varchar(12) &nn,
+      operator_nm varchar(12) not null,
-      raw_value varchar(4000) &nn,
+      raw_value varchar(4000) not null,
-      processed_dttm num &nn format=E8601DT26.6
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_hash,filter_line)
   );
 
-  %let libds=%upcase(&libds);
-  %if %index(&libds,.)=0 %then %let lib=WORK;
-  %else %let lib=%scan(&libds,1,.);
-
-  proc datasets lib=&lib noprint;
-    modify %scan(&libds,-1,.);
-    index create pk_mpe_filterdetail=(filter_hash filter_line)/nomiss unique;
-  quit;
-
 %mend mddl_dc_filterdetail;

ddl/mddl_dc_filtersummary.sas

@@ -9,27 +9,14 @@
 
 %macro mddl_dc_filtersummary(libds=WORK.FILTER_SUMMARY);
 
-%local nn lib;
-%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
-  %let nn=not null;
-%end;
-%else %let nn=;
-
   proc sql;
   create table &libds(
-      filter_rk num &nn,
+      filter_rk num not null,
-      filter_hash char(32) &nn,
+      filter_hash char(32) not null,
-      filter_table char(41) &nn,
+      filter_table char(41) not null,
-      processed_dttm num &nn format=E8601DT26.6
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_rk)
   );
 
-  %let libds=%upcase(&libds);
-  %if %index(&libds,.)=0 %then %let lib=WORK;
-  %else %let lib=%scan(&libds,1,.);
-
-  proc datasets lib=&lib noprint;
-    modify %scan(&libds,-1,.);
-    index create filter_rk /nomiss unique;
-  quit;
-
 %mend mddl_dc_filtersummary;

ddl/mddl_dc_locktable.sas

@@ -9,33 +9,17 @@
 
 %macro mddl_dc_locktable(libds=WORK.LOCKTABLE);
 
-%local nn lib;
-%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
-  %let nn=not null;
-%end;
-%else %let nn=;
-
   proc sql;
   create table &libds(
       lock_lib char(8),
       lock_ds char(32),
-      lock_status_cd char(10) &nn,
+      lock_status_cd char(10) not null,
-      lock_user_nm char(100) &nn ,
+      lock_user_nm char(100) not null ,
       lock_ref char(200),
       lock_pid char(10),
       lock_start_dttm num format=E8601DT26.6,
-      lock_end_dttm num format=E8601DT26.6
+      lock_end_dttm num format=E8601DT26.6,
+    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds)
   );
 
-  %let libds=%upcase(&libds);
-  %if %index(&libds,.)=0 %then %let lib=WORK;
-  %else %let lib=%scan(&libds,1,.);
-
-  proc datasets lib=&lib noprint;
-    modify %scan(&libds,-1,.);
-    index create
-      pk_mp_lockanytable=(lock_lib lock_ds)
-      /nomiss unique;
-  quit;
-
 %mend mddl_dc_locktable;

ddl/mddl_dc_maxkeytable.sas

@@ -17,17 +17,8 @@
       max_key num label=
         'Integer representing current max RK or SK value in the KEYTABLE',
       processed_dttm num format=E8601DT26.6
-        label='Datetime this value was last updated'
+        label='Datetime this value was last updated',
-  );
+    constraint pk_mpe_maxkeyvalues
-
+        primary key(keytable));
-  %local lib;
-  %let libds=%upcase(&libds);
-  %if %index(&libds,.)=0 %then %let lib=WORK;
-  %else %let lib=%scan(&libds,1,.);
-
-  proc datasets lib=&lib noprint;
-    modify %scan(&libds,-1,.);
-    index create keytable /nomiss unique;
-  quit;
 
 %mend mddl_dc_maxkeytable;

ddl/mddl_sas_cntlout.sas

@@ -14,8 +14,8 @@ proc sql;
 create table &libds(
     FMTNAME char(32)      label='Format name'
     /*
-      to accommodate larger START values, mp_loadformat.sas will need the
+      to accomodate larger START values, mp_loadformat.sas will need the
-      SQL dependency removed (proc sql needs to accommodate 3 index values in
+      SQL dependency removed (proc sql needs to accomodate 3 index values in
       a 32767 ibufsize limit)
     */
     ,START char(10000)    label='Starting value for format'
@@ -40,4 +40,4 @@ create table &libds(
     ,LANGUAGE char(8)     label='Language for date strings'
 );
 
-%mend mddl_sas_cntlout;
+%mend mddl_sas_cntlout;

fcmp/mcf_stpsrv_header.sas

@@ -0,0 +1,110 @@
+/**
+  @file
+  @brief Provides a replacement for the stpsrv_header function
+  @details The stpsrv_header is normally a built-in function, used to set the
+  headers for SAS 9 Stored Processes as documented here:
+  https://go.documentation.sas.com/doc/en/itechcdc/9.4/stpug/srvhead.htm
+
+  The purpose of this custom function is to provide a replacement when running
+  similar code as a web service against
+  [sasjs/server](https://github.com/sasjs/server).  It operates by creating a
+  text file with the headers.  The location of this text file is determined by
+  a macro variable (`sasjs_stpsrv_header_loc`) which needs to be injected into
+  each service by the calling process, eg:
+
+      %let sasjs_stpsrv_header_loc = C:/temp/some_uuid/stpsrv_header.txt;
+
+  Note - the function works by appending headers to the file.  If multiple same-
+  named headers are provided, they will all be appended - the calling process
+  needs to pick up the last one.  This will mean removing the attribute if the
+  final record has an empty value.
+
+  The function takes the following (positional) parameters:
+
+  | PARAMETER | DESCRIPTION |
+  |------------|-------------|
+  | name $  | name of the header attribute to create|
+  | value  $  | value of the header attribute|
+
+  It returns 0 if successful, or -1 if an error occured.
+
+  Usage:
+
+      %let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/stpsrv_header.txt;
+
+      %mcf_stpsrv_header(wrap=YES, insert_cmplib=YES)
+
+      data _null_;
+        rc=stpsrv_header('Content-type','application/text');
+        rc=stpsrv_header('Content-disposition',"attachment; filename=file.txt");
+      run;
+
+      data _null_;
+        infile "&sasjs_stpsrv_header_loc";
+        input;
+        putlog _infile_;
+      run;
+
+
+  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
+  @param [out] lib= (work) The output library in which to create the catalog.
+  @param [out] cat= (sasjs) The output catalog in which to create the package.
+  @param [out] pkg= (utils) The output package in which to create the function.
+    Uses a 3 part format:  libref.catalog.package
+  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
+    values inserted only if needed.
+
+  <h4> SAS Macros </h4>
+  @li mcf_init.sas
+
+  <h4> Related Programs </h4>
+  @li mcf_stpsrv_header.test.sas
+  @li mp_init.sas
+
+**/
+
+%macro mcf_stpsrv_header(wrap=NO
+  ,insert_cmplib=DEPRECATED
+  ,lib=WORK
+  ,cat=SASJS
+  ,pkg=UTILS
+)/*/STORE SOURCE*/;
+%local i var cmpval found;
+%if %mcf_init(stpsrv_header)=1 %then %return;
+
+%if &wrap=YES  %then %do;
+  proc fcmp outlib=&lib..&cat..&pkg;
+%end;
+
+function stpsrv_header(name $, value $);
+  length loc $128 val $512;
+  loc=symget('sasjs_stpsrv_header_loc');
+  val=trim(name)!!': '!!value;
+  length fref $8;
+  rc=filename(fref,loc);
+  if (rc ne 0) then return( -1 );
+  fid = fopen(fref,'a');
+  if (fid = 0) then return( -1 );
+  rc=fput(fid, val);
+  rc=fwrite(fid);
+  rc=fclose(fid);
+  rc=filename(fref);
+  return(0);
+endsub;
+
+%if &wrap=YES %then %do;
+  quit;
+%end;
+
+/* insert the CMPLIB if not already there */
+%let cmpval=%sysfunc(getoption(cmplib));
+%let found=0;
+%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
+  %let var=%scan(&cmpval,&i,%str( %(%)));
+  %if &var=&lib..&cat %then %let found=1;
+%end;
+%if &found=0 %then %do;
+  options insert=(CMPLIB=(&lib..&cat));
+%end;
+
+%mend mcf_stpsrv_header;

lua/ml_gsubfile.sas

@@ -39,14 +39,6 @@ data _null_;
   put 'io.close(file) ';
 run;
 
-/* ensure big enough lrecl to avoid lua compilation issues */
-%local optval;
-%let optval=%sysfunc(getoption(lrecl));
-options lrecl=1024;
-
-/* execute the lua code by using a .lua extension */
 %inc "%sysfunc(pathname(work))/ml_gsubfile.lua" /source2;
 
-options lrecl=&optval;
-
 %mend ml_gsubfile;

lua/ml_json.sas

@@ -389,14 +389,6 @@ data _null_;
   put '-- JSON.LUA ENDS HERE ';
 run;
 
-/* ensure big enough lrecl to avoid lua compilation issues */
-%local optval;
-%let optval=%sysfunc(getoption(lrecl));
-options lrecl=1024;
-
-/* execute the lua code by using a .lua extension */
 %inc "%sysfunc(pathname(work))/ml_json.lua" /source2;
 
-options lrecl=&optval;
-
 %mend ml_json;

main.dox

@@ -18,17 +18,7 @@
     statements.  Those starting `mp_` are macro _procedures_, which generate
     SAS statements, and must therefore be applied accordingly.
 
-*/
+ */
-
-/*! \dir ddl
- *  \brief Data Definition Language files
- *  \details Provides templates for commonly used tables in sasjs/core.
-    Attributes:
-
-        * OS independent
-        * No X command
-        * Prefixes: _mddl_
-*/
 
 /*! \dir fcmp
  *  \brief Macros for generating FCMP functions
@@ -102,39 +92,4 @@
         * Auto-generated from the plain source `.lua` files in the same directory
         * Prefixes: _ml_
 
-*/
+ */
-
-/*! \dir tests/base
- *  \brief Tests for Base macros
- */
-
-/*! \dir tests/ddlonly
- *  \brief Tests for DDL macros
- */
-
-/*! \dir tests/sas9only
- *  \brief Tests for SAS Metadata macros
- */
-
-/*! \dir tests/serveronly
- *  \brief Tests for SASjs Server macros
- */
-
-/*! \dir tests/viyaonly
- *  \brief Tests for Viya macros
- */
-
-/*! \dir tests/x-platform
- *  \brief Tests for cross-platform macros
- */
-
-/*! \dir xplatform
- *  \brief Cross Platform, works on all SAS servers (Viya, EBI, SASjs)
- *  \details Useful when you need to run a single piece of code against Viya,
-     SAS 9 with metadata, or SASjs on Base SAS.
-
-        * OS independent
-        * No X command
-        * Prefixes: _mx_
-
- */

meta/mm_adduser2group.sas

@@ -2,10 +2,7 @@
   @file mm_adduser2group.sas
   @brief Adds a user to a group
   @details Adds a user to a metadata group.  The macro first checks whether the
-  user is in that group, and if not, the user is added.
+    user is in that group, and if not, the user is added.
-
-  Note that the macro does not check inherited group memberships - it looks at
-    direct members only.
 
   Usage:
 
@@ -15,10 +12,10 @@
 
   @param user= the user name (not displayname)
   @param group= the group to which to add the user
-  @param mdebug= (0) set to 1 to show debug info in log
+  @param mdebug= set to 1 to show debug info in log
 
-  <h4> Related Files </h4>
+  @warning the macro does not check inherited group memberships - it looks at
-  @li ms_adduser2group.sas
+    direct members only
 
   @version 9.3
   @author Allan Bowe
@@ -81,8 +78,7 @@ run;
 filename __us2grp temp;
 
 proc metadata in= "<UpdateMetadata><Reposid>$METAREPOSITORY</Reposid><Metadata>
-    <Person Id='%nrstr(&uuri)'>
+    <Person Id='&uuri'><IdentityGroups><IdentityGroup ObjRef='&guri' />
-    <IdentityGroups><IdentityGroup ObjRef='%nrstr(&guri)' />
     </IdentityGroups></Person></Metadata>
     <NS>SAS</NS><Flags>268435456</Flags></UpdateMetadata>"
   out=__us2grp verbose;
@@ -99,4 +95,4 @@ run;
 
 filename __us2grp clear;
 
-%mend mm_adduser2group;
+%mend mm_adduser2group;

meta/mm_assignlib.sas

@@ -13,8 +13,8 @@
   <h4> SAS Macros </h4>
   @li mp_abort.sas
 
-  @param [in] libref The libref (not name) of the metadata library
+  @param libref the libref (not name) of the metadata library
-  @param [in] mAbort= If not assigned, HARD will call %mp_abort(), SOFT will
+  @param mAbort= If not assigned, HARD will call %mp_abort(), SOFT will
     silently return
 
   @returns libname statement
@@ -28,11 +28,11 @@
     libref
     ,mAbort=HARD
 )/*/STORE SOURCE*/;
-%local mp_abort msg;
+
-%let mp_abort=0;
 %if %sysfunc(libref(&libref)) %then %do;
+  %local mp_abort msg; %let mp_abort=0;
   data _null_;
-    length liburi LibName msg $200;
+    length liburi LibName $200;
     call missing(of _all_);
     nobj=metadata_getnobj("omsobj:SASLibrary?@Libref='&libref'",1,liburi);
     if nobj=1 then do;
@@ -40,30 +40,7 @@
       /* now try and assign it */
       if libname("&libref",,'meta',cats('liburi="',liburi,'";')) ne 0 then do;
         putlog "&libref could not be assigned";
-        putlog liburi=;
+        call symputx('msg',sysmsg(),'l');
-        /**
-          * Fetch the system message for display in the abort modal.  This is
-          * not always helpful though.  One example, previously received:
-          * NOTE: Libref XX refers to the same library metadata as libref XX.
-          */
-        msg=sysmsg();
-        if msg=:'ERROR: Libref SAVE is not assigned.' then do;
-          msg=catx(" ",
-            "Could not assign %upcase(&libref).",
-            "Please check metadata permissions!  Libname:",libname,
-            "Liburi:",liburi
-          );
-        end;
-        else if msg="ERROR: User does not have appropriate authorization "!!
-          "level for library SAVE."
-        then do;
-          msg=catx(" ",
-            "ERROR: User does not have appropriate authorization level",
-            "for library %upcase(&libref), libname:",libname,
-            "Liburi:",liburi
-          );
-        end;
-        call symputx('msg',msg,'l');
         if "&mabort"='HARD' then call symputx('mp_abort',1,'l');
       end;
       else do;
@@ -82,16 +59,20 @@
     end;
   run;
 
-  %put NOTE: &msg;
+  %if &mp_abort=1 %then %do;
+    %mp_abort(iftrue= (&mp_abort=1)
+      ,mac=&sysmacroname
+      ,msg=&msg
+    )
+    %return;
+  %end;
+  %else %if %length(&msg)>2 %then %do;
+    %put NOTE: &msg;
+    %return;
+  %end;
 
 %end;
 %else %do;
   %put NOTE: Library &libref is already assigned;
 %end;
-
-%mp_abort(iftrue= (&mp_abort=1)
-  ,mac=mm_assignlib.sas
-  ,msg=%superq(msg)
-)
-
 %mend mm_assignlib;

meta/mm_createdataset.sas

@@ -1,21 +1,22 @@
 /**
-  @file
+  @file mm_createdataset.sas
-  @brief Create an empty dataset from a metadata definition
+  @brief Create a dataset from a metadata definition
   @details This macro was built to support viewing empty tables in
-  https://datacontroller.io
+    https://datacontroller.io - a free evaluation copy is available by
+    contacting the author (Allan Bowe).
 
-  The table can be retrieved using LIBRARY.DATASET reference, or directly
+    The table can be retrieved using LIBRARY.DATASET reference, or directly
-  using the metadata URI.
+    using the metadata URI.
 
-  The dataset is written to the WORK library.
+    The dataset is written to the WORK library.
 
-  Usage:
+  usage:
 
-      %mm_createdataset(libds=metlib.some_dataset)
+    %mm_createdataset(libds=metlib.some_dataset)
 
-  or
+    or
 
-      %mm_createdataset(tableuri=G5X8AFW1.BE00015Y)
+    %mm_createdataset(tableuri=G5X8AFW1.BE00015Y)
 
   <h4> SAS Macros </h4>
   @li mm_getlibs.sas
@@ -25,9 +26,9 @@
   @param libds= library.dataset metadata source.  Note - table names in metadata
     can be longer than 32 chars (just fyi, not an issue here)
   @param tableuri= Metadata URI of the table to be created
-  @param outds= (work.mm_createdataset) The dataset to create.  The table name
+  @param outds= The dataset to create, default is `work.mm_createdataset`.
-    needs to be 32 chars or less as per SAS naming rules.
+    The table name needs to be 32 chars or less as per SAS naming rules.
-  @param mdebug= (0) Set to 1 to enable DEBUG messages
+  @param mdebug= set DBG to 1 to disable DEBUG messages
 
   @version 9.4
   @author Allan Bowe
@@ -53,23 +54,14 @@
   %mm_gettables(uri=&liburi,outds=&tempds2)
   data _null_;
     set &tempds2;
-    where upcase(tablename)="%upcase(%scan(&libds,2,.))";
+    if upcase(tablename)="%upcase(%scan(&libds,2,.))";
-    &dbg putlog tableuri=;
     call symputx('tableuri',tableuri);
   run;
 %end;
 
-data;run;
+data;run;%let tempds3=&syslast;
-%let tempds3=&syslast;
 %mm_getcols(tableuri=&tableuri,outds=&tempds3)
 
-%if %mf_nobs(&tempds3)=0 %then %do;
-  %put &libds (&tableuri) has no columns defined!!;
-  data &outds;
-  run;
-  %return;
-%end;
-
 data _null_;
   set &tempds3 end=last;
   if _n_=1 then call execute('data &outds;');

meta/mm_createfolder.sas

@@ -17,7 +17,7 @@
       %mm_createfolder(path=/some/meta/folder)
 
   @param [in] path= Name of the folder to create.
-  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] mdebug= set DBG to 1 to disable DEBUG messages
 
 
   @version 9.4

meta/mm_createwebservice.sas

@@ -2,8 +2,7 @@
   @file mm_createwebservice.sas
   @brief Create a Web Ready Stored Process
   @details This macro creates a Type 2 Stored Process with the mm_webout macro
-  (and dependencies) included as pre-code.
+    included as pre-code.
-
 Usage:
 
     %* compile macros ;
@@ -93,41 +92,25 @@ data _null_;
   file sasjs lrecl=3000 ;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
+  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
-  put '  ,showmeta=N ';
+  put '  ,showmeta=NO ';
-  put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval ';
+  put '%local tempds colinfo fmtds i numcols; ';
-  put '  tmpds1 tmpds2 tmpds3 tmpds4; ';
   put '%let numcols=0; ';
-  put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
-  put '  data _null_;file &jref encoding=''utf-8'' lrecl=200; ';
+  put '  data _null_;file &jref encoding=''utf-8'' ; ';
   put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
   put '  run; ';
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
-  put '  /* force variable names to always be uppercase in the JSON */ ';
   put '  options validvarname=upcase; ';
-  put '  /* To avoid issues with _webout on EBI - such as encoding diffs and truncation ';
+  put '  data _null_; file &jref encoding=''utf-8'' mod; ';
-  put '    (https://support.sas.com/kb/49/325.html) we use temporary files */ ';
-  put '  filename _sjs1 temp lrecl=200 ; ';
-  put '  data _null_; file _sjs1 encoding=''utf-8''; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
-  put '  run; ';
-  put '  /* now write to _webout 1 char at a time */ ';
-  put '  data _null_; ';
-  put '    infile _sjs1 lrecl=1 recfm=n; ';
-  put '    file &jref mod lrecl=1 recfm=n; ';
-  put '    input sourcechar $char1. @@; ';
-  put '    format sourcechar hex2.; ';
-  put '    put sourcechar char1. @@; ';
-  put '  run; ';
-  put '  filename _sjs1 clear; ';
   put ' ';
   put '  /* grab col defs */ ';
   put '  proc contents noprint data=&ds ';
@@ -138,7 +121,7 @@ data _null_;
   put '    by varnum; ';
   put '  run; ';
   put '  /* move meta to mac vars */ ';
-  put '  data &colinfo; ';
+  put '  data _null_; ';
   put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
   put '    set &colinfo end=last nobs=nobs; ';
   put '    name=upcase(name); ';
@@ -149,6 +132,7 @@ data _null_;
   put '      if format='''' then fmt=cats(''$'',length,''.''); ';
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else fmt=cats(format,formatl,''.''); ';
+  put '      newlen=max(formatl,length); ';
   put '    end; ';
   put '    else do; ';
   put '      typelong=''num''; ';
@@ -156,26 +140,23 @@ data _null_;
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else if formatd=0 then fmt=cats(format,formatl,''.''); ';
   put '      else fmt=cats(format,formatl,''.'',formatd); ';
+  put '      /* needs to be wide, for datetimes etc */ ';
+  put '      newlen=max(length,formatl,24); ';
   put '    end; ';
   put '    /* 32 char unique name */ ';
   put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
   put ' ';
   put '    call symputx(cats(''name'',_n_),name,''l''); ';
   put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '    call symputx(cats(''len'',_n_),newlen,''l''); ';
   put '    call symputx(cats(''length'',_n_),length,''l''); ';
   put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
-  put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
-  put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
-  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
-  put '  proc sql; ';
-  put '  select count(*) into: lastobs from &ds; ';
-  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -183,25 +164,13 @@ data _null_;
   put '      %put &sysmacroname: Switching to DATASTEP engine; ';
   put '      %goto datastep; ';
   put '    %end; ';
-  put '    data &tempds; ';
+  put '    data &tempds;set &ds; ';
-  put '      set &ds; ';
-  put '      &stmt_obs; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
   put '    /* PRETTY is necessary to avoid line truncation in large files */ ';
-  put '    filename _sjs2 temp lrecl=131068 encoding=''utf-8''; ';
+  put '    proc json out=&jref pretty ';
-  put '    proc json out=_sjs2 pretty ';
   put '        %if &action=ARR %then nokeys ; ';
   put '        ;export &tempds / nosastags fmtnumeric; ';
   put '    run; ';
-  put '    /* send back to webout */ ';
-  put '    data _null_; ';
-  put '      infile _sjs2 lrecl=1 recfm=n; ';
-  put '      file &jref mod lrecl=1 recfm=n; ';
-  put '      input sourcechar $char1. @@; ';
-  put '      format sourcechar hex2.; ';
-  put '      put sourcechar char1. @@; ';
-  put '    run; ';
-  put '    filename _sjs2 clear; ';
   put '  %end; ';
   put '  %else %if &engine=DATASTEP %then %do; ';
   put '    %datastep: ';
@@ -212,99 +181,26 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      /** ';
+  put '      data _data_; ';
-  put '        * Extract format definitions ';
-  put '        * First, by getting library locations from dictionary.formats ';
-  put '        * Then, by exporting the width using proc format ';
-  put '        * Cannot use maxw from sashelp.vformat as not always populated ';
-  put '        * Cannot use fmtinfo() as not supported in all flavours ';
-  put '        */ ';
-  put '      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
-  put '      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
-  put '      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
-  put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
-  put '      proc sql noprint; ';
-  put '      create table &tmpds1 as ';
-  put '          select cats(libname,''.'',memname) as FMTCAT, ';
-  put '          FMTNAME ';
-  put '        from dictionary.formats ';
-  put '        where fmttype=''F'' and libname is not null ';
-  put '          and fmtname in (select format from &colinfo where format is not null) ';
-  put '        order by 1; ';
-  put '      create table &tmpds2( ';
-  put '          FMTNAME char(32), ';
-  put '          LENGTH num ';
-  put '      ); ';
-  put '      %local catlist cat fmtlist i; ';
-  put '      select distinct fmtcat into: catlist separated by '' '' from &tmpds1; ';
-  put '      %do i=1 %to %sysfunc(countw(&catlist,%str( ))); ';
-  put '        %let cat=%scan(&catlist,&i,%str( )); ';
-  put '        proc sql; ';
-  put '        select distinct fmtname into: fmtlist separated by '' '' ';
-  put '          from &tmpds1 where fmtcat="&cat"; ';
-  put '        proc format lib=&cat cntlout=&tmpds3(keep=fmtname length); ';
-  put '          select &fmtlist; ';
-  put '        run; ';
-  put '        proc sql; ';
-  put '        insert into &tmpds2 select distinct fmtname,length from &tmpds3; ';
-  put '      %end; ';
-  put ' ';
-  put '      proc sql; ';
-  put '      create table &tmpds4 as ';
-  put '        select a.*, b.length as MAXW ';
-  put '        from &colinfo a ';
-  put '        left join &tmpds2 b ';
-  put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
-  put '        order by a.varnum; ';
-  put '      data _null_; ';
-  put '        set &tmpds4; ';
-  put '        if not missing(maxw); ';
-  put '        call symputx( ';
-  put '          cats(''fmtlen'',_n_), ';
-  put '          /* vars need extra padding due to JSON escaping of special chars */ ';
-  put '          min(32767,ceil((max(length,maxw)+10)*1.5)) ';
-  put '          ,''l'' ';
-  put '        ); ';
-  put '      run; ';
-  put ' ';
-  put '      /* configure varlenchk - as we are explicitly shortening the variables */ ';
-  put '      %let optval=%sysfunc(getoption(varlenchk)); ';
-  put '      options varlenchk=NOWARN; ';
-  put '      data _data_(compress=char); ';
-  put '        /* shorten the new vars */ ';
-  put '        length ';
-  put '      %do i=1 %to &numcols; ';
-  put '          &&name&i $&&fmtlen&i ';
-  put '      %end; ';
-  put '          ; ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
-  put '          &&name&i=&&newname&i ';
+  put '        &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
-  put '      &stmt_obs; ';
-  put ' ';
-  put '      drop ';
-  put '      %do i=1 %to &numcols; ';
-  put '        &&newname&i ';
-  put '      %end; ';
-  put '        ; ';
   put '      %do i=1 %to &numcols; ';
+  put '        length &&name&i $&&len&i; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
+  put '        drop &&newname&i; ';
   put '      %end; ';
-  put '        if _error_ then do; ';
+  put '        if _error_ then call symputx(''syscc'',1012); ';
-  put '          call symputx(''syscc'',1012); ';
-  put '          stop; ';
-  put '        end; ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
-  put '      options varlenchk=&optval; ';
   put '    %end; ';
   put ' ';
   put '    proc format; /* credit yabwon for special null removal */ ';
@@ -323,8 +219,8 @@ data _null_;
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $&&fmtlen&i...; ';
+  put '          length &&name&i $32767; ';
-  put '          format &&name&i $&&fmtlen&i...; ';
+  put '          format &&name&i $32767.; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -333,35 +229,23 @@ data _null_;
   put '      %else %do; ';
   put '        set &ds; ';
   put '      %end; ';
-  put '      &stmt_obs; ';
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
+  put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
-  put '          &&name&i=''"''!!trim( ';
+  put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
-  put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
-  put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
-  put '            prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '            prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '        )))))!!''"''; ';
-  put '            prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
-  put '            prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
-  put '            prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
-  put '            prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
-  put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
-  put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
-  put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
-  put '            prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
-  put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '          )))))))))))))!!''"''; ';
-  put '        end; ';
-  put '        else &&name&i=quote(cats(&&name&i)); ';
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
   put ' ';
-  put '    filename _sjs3 temp lrecl=131068 ; ';
+  put '    /* write to temp loc to avoid _webout truncation ';
-  put '    data _null_; ';
+  put '      - https://support.sas.com/kb/49/325.html */ ';
-  put '      file _sjs3 encoding=''utf-8''; ';
+  put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
+  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
   put '      if _n_=1 then put "["; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
@@ -369,38 +253,39 @@ data _null_;
   put '      %do i=1 %to &numcols; ';
   put '        %if &i>1 %then  "," ; ';
   put '        %if &action=OBJ %then """&&name&i"":" ; ';
-  put '        "&&name&i"n /* name literal for reserved variable names */ ';
+  put '        &&name&i ';
   put '      %end; ';
   put '      %if &action=ARR %then "]" ; %else "}" ; ; ';
-  put ' ';
+  put '    /* now write the long strings to _webout 1 byte at a time */ ';
-  put '    /* close out the table */ ';
   put '    data _null_; ';
-  put '      file _sjs3 mod encoding=''utf-8''; ';
+  put '      length filein 8 fileid 8; ';
-  put '      put '']''; ';
+  put '      filein=fopen("_sjs",''I'',1,''B''); ';
+  put '      fileid=fopen("&jref",''A'',1,''B''); ';
+  put '      rec=''20''x; ';
+  put '      do while(fread(filein)=0); ';
+  put '        rc=fget(filein,rec,1); ';
+  put '        rc=fput(fileid, rec); ';
+  put '        rc=fwrite(fileid); ';
+  put '      end; ';
+  put '      /* close out the table */ ';
+  put '      rc=fput(fileid, "]"); ';
+  put '      rc=fwrite(fileid); ';
+  put '      rc=fclose(filein); ';
+  put '      rc=fclose(fileid); ';
   put '    run; ';
-  put '    data _null_; ';
+  put '    filename _sjs clear; ';
-  put '      infile _sjs3 lrecl=1 recfm=n; ';
-  put '      file &jref mod lrecl=1 recfm=n; ';
-  put '      input sourcechar $char1. @@; ';
-  put '      format sourcechar hex2.; ';
-  put '      put sourcechar char1. @@; ';
-  put '    run; ';
-  put '    filename _sjs3 clear; ';
   put '  %end; ';
   put ' ';
   put '  proc sql; ';
   put '  drop table &colinfo, &tempds; ';
   put ' ';
-  put '  %if %substr(&showmeta,1,1)=Y %then %do; ';
+  put '  %if &showmeta=YES %then %do; ';
-  put '    filename _sjs4 temp lrecl=131068 encoding=''utf-8''; ';
+  put '    data _null_; file &jref encoding=''utf-8'' mod; ';
-  put '    data _null_; ';
-  put '      file _sjs4; ';
-  put '      length label $350; ';
   put '      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; ';
   put '      do i=1 to &numcols; ';
   put '        name=quote(trim(symget(cats(''name'',i)))); ';
   put '        format=quote(trim(symget(cats(''fmt'',i)))); ';
-  put '        label=quote(prxchange(''s/\\/\\\\/'',-1,trim(symget(cats(''label'',i))))); ';
+  put '        label=quote(trim(symget(cats(''label'',i)))); ';
   put '        length=quote(trim(symget(cats(''length'',i)))); ';
   put '        type=quote(trim(symget(cats(''typelong'',i)))); ';
   put '        if i>1 then put "," @@; ';
@@ -409,15 +294,6 @@ data _null_;
   put '      end; ';
   put '      put ''}}''; ';
   put '    run; ';
-  put '    /* send back to webout */ ';
-  put '    data _null_; ';
-  put '      infile _sjs4 lrecl=1 recfm=n; ';
-  put '      file &jref mod lrecl=1 recfm=n; ';
-  put '      input sourcechar $char1. @@; ';
-  put '      format sourcechar hex2.; ';
-  put '      put sourcechar char1. @@; ';
-  put '    run; ';
-  put '    filename _sjs4 clear; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
@@ -427,28 +303,8 @@ data _null_;
   put '  run; ';
   put '%end; ';
   put '%mend mp_jsonout; ';
-  put ' ';
+  put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
-  put '%macro mf_getuser( ';
+  put '  ,showmeta=NO ';
-  put ')/*/STORE SOURCE*/; ';
-  put '  %local user; ';
-  put ' ';
-  put '  %if %symexist(_sasjs_username) %then %let user=&_sasjs_username; ';
-  put '  %else %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %do; ';
-  put '    %let user=&SYS_COMPUTE_SESSION_OWNER; ';
-  put '  %end; ';
-  put '  %else %if %symexist(_metaperson) %then %do; ';
-  put '    %if %length(&_metaperson)=0 %then %let user=&sysuserid; ';
-  put '    /* sometimes SAS will add @domain extension - remove for consistency */ ';
-  put '    /* but be sure to quote in case of usernames with commas */ ';
-  put '    %else %let user=%unquote(%scan(%quote(&_metaperson),1,@)); ';
-  put '  %end; ';
-  put '  %else %let user=&sysuserid; ';
-  put ' ';
-  put '  %quote(&user) ';
-  put ' ';
-  put '%mend mf_getuser; ';
-  put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL ';
-  put '  ,showmeta=N,maxobs=MAX,workobs=0 ';
   put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
@@ -515,14 +371,13 @@ data _null_;
   put ' ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
+  put '    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta ';
   put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  /* To avoid issues with _webout on EBI we use a temporary file */ ';
+  put '  %if %str(&_debug) ge 131 %then %do; ';
-  put '  filename _sjsref temp lrecl=131068; ';
+  put '    /* if debug mode, send back first 10 records of each work table also */ ';
-  put '  %if %str(&workobs) > 0 %then %do; ';
+  put '    options obs=10; ';
-  put '    /* if debug mode, send back first XX records of each work table also */ ';
   put '    data;run;%let tempds=%scan(&syslast,2,.); ';
   put '    ods output Members=&tempds; ';
   put '    proc datasets library=WORK memtype=data; ';
@@ -533,11 +388,11 @@ data _null_;
   put '      i+1; ';
   put '      call symputx(cats(''wt'',i),name,''l''); ';
   put '      call symputx(''wtcnt'',i,''l''); ';
-  put '    data _null_; file _sjsref mod encoding=''utf-8''; ';
+  put '    data _null_; file &fref mod encoding=''utf-8''; ';
   put '      put ",""WORK"":{"; ';
   put '    %do i=1 %to &wtcnt; ';
   put '      %let wt=&&wt&i; ';
-  put '      data _null_; file _sjsref mod encoding=''utf-8''; ';
+  put '      data _null_; file &fref mod encoding=''utf-8''; ';
   put '        dsid=open("WORK.&wt",''is''); ';
   put '        nlobs=attrn(dsid,''NLOBS''); ';
   put '        nvars=attrn(dsid,''NVARS''); ';
@@ -546,82 +401,34 @@ data _null_;
   put '        put " ""&wt"" : {"; ';
   put '        put ''"nlobs":'' nlobs; ';
   put '        put '',"nvars":'' nvars; ';
-  put '      %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=Y ';
+  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) ';
-  put '        ,maxobs=&workobs ';
+  put '      data _null_; file &fref mod encoding=''utf-8''; ';
-  put '      ) ';
-  put '      data _null_; file _sjsref mod encoding=''utf-8''; ';
   put '        put "}"; ';
   put '    %end; ';
-  put '    data _null_; file _sjsref mod encoding=''utf-8''; ';
+  put '    data _null_; file &fref mod encoding=''utf-8''; ';
   put '      put "}"; ';
   put '    run; ';
   put '  %end; ';
   put '  /* close off json */ ';
-  put '  data _null_;file _sjsref mod encoding=''utf-8''; ';
+  put '  data _null_;file &fref mod encoding=''utf-8''; ';
-  put '    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; ';
+  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
+  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
+  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""_DEBUG"" : ""&_debug"" "; ';
   put '    _METAUSER=quote(trim(symget(''_METAUSER''))); ';
   put '    put ",""_METAUSER"": " _METAUSER; ';
   put '    _METAPERSON=quote(trim(symget(''_METAPERSON''))); ';
   put '    put '',"_METAPERSON": '' _METAPERSON; ';
-  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
-  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
-  put '    put '',"AUTOEXEC" : '' autoexec; ';
-  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    put ",""SYSENCODING"" : ""&sysencoding"" "; ';
+  put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
-  put '    syserrortext=cats(symget(''syserrortext'')); ';
-  put '    if findc(syserrortext,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
-  put '      syserrortext=''"''!!trim( ';
-  put '        prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
-  put '        prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
-  put '        prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
-  put '        prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
-  put '        prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
-  put '        prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
-  put '        prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
-  put '        prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
-  put '        prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
-  put '        prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
-  put '        prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
-  put '        prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
-  put '        prxchange(''s/\\/\\\\/'',-1,syserrortext) ';
-  put '      )))))))))))))!!''"''; ';
-  put '    end; ';
-  put '    else syserrortext=cats(''"'',syserrortext,''"''); ';
-  put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
-  put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
-  put '    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; ';
-  put '    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); ';
-  put '    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
-  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    syswarningtext=cats(symget(''syswarningtext'')); ';
+  put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
-  put '    if findc(syswarningtext,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
-  put '      syswarningtext=''"''!!trim( ';
-  put '        prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
-  put '        prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
-  put '        prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
-  put '        prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
-  put '        prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
-  put '        prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
-  put '        prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
-  put '        prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
-  put '        prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
-  put '        prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
-  put '        prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
-  put '        prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
-  put '        prxchange(''s/\\/\\\\/'',-1,syswarningtext) ';
-  put '      )))))))))))))!!''"''; ';
-  put '    end; ';
-  put '    else syswarningtext=cats(''"'',syswarningtext,''"''); ';
-  put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';
   put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
@@ -632,25 +439,31 @@ data _null_;
   put '    put ''>>weboutEND<<''; ';
   put '  %end; ';
   put '  run; ';
-  put '  /* now write to _webout 1 char at a time */ ';
-  put '  data _null_; ';
-  put '    infile _sjsref lrecl=1 recfm=n; ';
-  put '    file &fref mod lrecl=1 recfm=n; ';
-  put '    input sourcechar $char1. @@; ';
-  put '    format sourcechar hex2.; ';
-  put '    put sourcechar char1. @@; ';
-  put '  run; ';
-  put '  filename _sjsref clear; ';
-  put ' ';
   put '%end; ';
   put ' ';
   put '%mend mm_webout; ';
+  put ' ';
+  put '%macro mf_getuser(type=META ';
+  put ')/*/STORE SOURCE*/; ';
+  put '  %local user metavar; ';
+  put '  %if &type=OS %then %let metavar=_secureusername; ';
+  put '  %else %let metavar=_metaperson; ';
+  put ' ';
+  put '  %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %let user=&SYS_COMPUTE_SESSION_OWNER; ';
+  put '  %else %if %symexist(&metavar) %then %do; ';
+  put '    %if %length(&&&metavar)=0 %then %let user=&sysuserid; ';
+  put '    /* sometimes SAS will add @domain extension - remove for consistency */ ';
+  put '    %else %let user=%scan(&&&metavar,1,@); ';
+  put '  %end; ';
+  put '  %else %let user=&sysuserid; ';
+  put ' ';
+  put '  %quote(&user) ';
+  put ' ';
+  put '%mend mf_getuser; ';
 /* WEBOUT END */
-  put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO';
+  put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO);';
-  put '    ,maxobs=MAX';
-  put ');';
   put '  %mm_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt,missing=&missing';
-  put '    ,showmeta=&showmeta,maxobs=&maxobs';
+  put '    ,showmeta=&showmeta';
   put '  )';
   put '%mend;';
 run;
@@ -665,7 +478,7 @@ run;
   %if &x>1 %then %let mod=mod;
 
   %let fref=%scan(&freflist,&x);
-  %&mD.put &sysmacroname: adding &fref;
+  %put &sysmacroname: adding &fref;
   data _null_;
     file "&work/&tmpfile" lrecl=3000 &mod;
     infile &fref;
@@ -701,10 +514,12 @@ data _null_;
   if rc=0 then call symputx('url',url,'l');
 run;
 
+%put ;%put ;%put ;%put ;%put ;%put ;
 %put &sysmacroname: STP &name successfully created in &path;
+%put ;%put ;%put ;
 %put Check it out here:;
 %put ;%put ;%put ;
 %put &url?_PROGRAM=&path/&name;
-%put ;%put ;%put ;
+%put ;%put ;%put ;%put ;%put ;%put ;
 
 %mend mm_createwebservice;

meta/mm_deletelibrary.sas

@@ -78,7 +78,6 @@ filename &fname2 clear;
 %local isgone;
 data _null_;
   length type uri $256;
-  call missing (of _all_);
   rc=metadata_resolve("omsobj:SASLibrary?@Id='&liburi'",type,uri);
   call symputx('isgone',type,'l');
 run;

meta/mm_getfoldertree.sas

@@ -8,15 +8,17 @@
 
   Usage:
 
-      options ps=max nonotes nosource;
+    options ps=max nonotes nosource;
-      %mm_getfoldertree(root=/My/Meta/Path, outds=iwantthisdataset)
+    %mm_getfoldertree(root=/My/Meta/Path, outds=iwantthisdataset)
-      options notes source;
+    options notes source;
 
   @param [in] root= the parent folder under which to return all contents
   @param [out] outds= the dataset to create that contains the list of
     directories
   @param [in] mDebug= set to 1 to show debug messages in the log
 
+  <h4> SAS Macros </h4>
+
   @version 9.4
   @author Allan Bowe
 

meta/mm_getstpcode.sas

@@ -21,7 +21,6 @@
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
-  @li mp_abort.sas
 
   @author Allan Bowe
 
@@ -58,7 +57,7 @@ data _null_;
     cnt=1;
     do while (metadata_getnasn(uri,"Notes",cnt,tsuri)>0);
       rc=metadata_getattr(tsuri,"Name",value);
-      &mD.put tsuri= value=;
+      put tsuri= value=;
       if value="SourceCode" then do;
         /* found it! */
         rc=metadata_getattr(tsuri,"Id",value);
@@ -71,10 +70,11 @@ data _null_;
   else put (_all_)(=);
 run;
 
-%mp_abort(iftrue= (&tsuri=stopifempty)
+%if &tsuri=stopifempty %then %do;
-  ,mac=mm_getstpcode
+  %put %str(WARN)ING:  &tree&name.(StoredProcess) not found!;
-  ,msg=%str(&tree&name.(StoredProcess) not found!)
+  %return;
-)
+%end;
+
 
 /**
   * Now we can extract the textstore

meta/mm_getstpinfo.sas

@@ -1,67 +0,0 @@
-/**
-  @file
-  @brief Get the properties of a Stored Process
-  @details Extracts various properties and creates an output table in the
-  structure below:
-
-|STP_URI:$200.|SERVERCONTEXT:$200.|STOREDPROCESSCONFIGURATION:$1000.|SOURCECODE_FIRST32K:$32767.|PATH:$76.|
-|---|---|---|---|---|
-|`A5DN9TDQ.BH0000C8 `|`SASApp `|`<?xml version="1.0" encoding="UTF-8"?><StoredProcess><ServerContext LogicalServerType="Sps" OtherAllowed="false"/><ResultCapabilities Package="false" Streaming="true"/><OutputParameters/></StoredProcess> `|`%put first 32767 bytes of code; `|`/path/to/my/stp`|
-
-  @param [in] pgm The metadata path of the Stored Process
-  @param [out] outds= (work.mm_getstpinfo) The output table to create
-  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
-
-  <h4> Related Files </h4>
-  @li mm_getstpcode.sas
-  @li mm_getstps.sas
-  @li mm_createstp.sas
-  @li mm_deletestp.sas
-
-**/
-
-%macro mm_getstpinfo(pgm
-  ,outds=work.mm_getstpinfo
-  ,mDebug=0
-);
-
-%local mD;
-%if &mDebug=1 %then %let mD=;
-%else %let mD=%str(*);
-%&mD.put Executing &sysmacroname..sas;
-%&mD.put _local_;
-
-data &outds;
-  length type stp_uri tsuri servercontext value $200
-    StoredProcessConfiguration $1000 sourcecode_first32k $32767;
-  keep path stp_uri sourcecode_first32k StoredProcessConfiguration
-    servercontext;
-  call missing (of _all_);
-  path="&pgm(StoredProcess)";
-  /* first, find the STP ID */
-  if metadata_pathobj("",path,"StoredProcess",type,stp_uri)>0 then do;
-    /* get attributes */
-    cnt=1;
-    do while (metadata_getnasn(stp_uri,"Notes",cnt,tsuri)>0);
-      rc1=metadata_getattr(tsuri,"Name",value);
-      &mD.put tsuri= value=;
-      if value="SourceCode" then do;
-        rc2=metadata_getattr(tsuri,"StoredText",sourcecode_first32k);
-      end;
-      else if value="Stored Process" then do;
-        rc3=metadata_getattr(tsuri,"StoredText",StoredProcessConfiguration);
-      end;
-      cnt+1;
-    end;
-    /* get context (should only be one) */
-    rc4=metadata_getnasn(stp_uri,"ComputeLocations",1,tsuri);
-    rc5=metadata_getattr(tsuri,"Name",servercontext);
-  end;
-  else do;
-    put "%str(ERR)OR: could not find " path;
-    put (_all_)(=);
-  end;
-  &md.put (_all_)(=);
-run;
-
-%mend mm_getstpinfo ;

meta/mm_getusers.sas

@@ -8,18 +8,17 @@
 
       %mm_getusers()
 
-  Optionally, filter for a user (useful to get the uri):
-
-      %mm_getusers(user=&_metaperson)
-
   @param outds the dataset to create that contains the list of libraries
 
   @returns outds  dataset containing all users, with the following columns:
     - uri
     - name
 
-  @param user= (0) Set to a metadata user to filter on that user
+  @warning The following filenames are created and then de-assigned:
-  @param outds= (work.mm_getusers) The output table to create
+
+      filename sxlemap clear;
+      filename response clear;
+      libname _XML_ clear;
 
   @version 9.3
   @author Allan Bowe
@@ -27,49 +26,23 @@
 **/
 
 %macro mm_getusers(
-    outds=work.mm_getusers,
+    outds=work.mm_getusers
-    user=0
 )/*/STORE SOURCE*/;
 
 filename response temp;
-%if %superq(user)=0 %then %do;
+proc metadata in= '<GetMetadataObjects>
-  proc metadata in= '<GetMetadataObjects>
+  <Reposid>$METAREPOSITORY</Reposid>
-    <Reposid>$METAREPOSITORY</Reposid>
+  <Type>Person</Type>
-    <Type>Person</Type>
+  <NS>SAS</NS>
-    <NS>SAS</NS>
+  <Flags>0</Flags>
-    <Flags>0</Flags>
+  <Options>
-    <Options>
+  <Templates>
-    <Templates>
+  <Person Name=""/>
-    <Person Name=""/>
+  </Templates>
-    </Templates>
+  </Options>
-    </Options>
+  </GetMetadataObjects>'
-    </GetMetadataObjects>'
+  out=response;
-    out=response;
+run;
-  run;
-%end;
-%else %do;
-  filename inref temp;
-  data _null_;
-    file inref;
-    put "<GetMetadataObjects>";
-    put "<Reposid>$METAREPOSITORY</Reposid>";
-    put "<Type>Person</Type>";
-    put "<NS>SAS</NS>";
-    put "<!-- Specify the OMI_XMLSELECT (128) flag  -->";
-    put "<Flags>128</Flags>";
-    put "<Options>";
-    put "<Templates>";
-    put '<Person Name=""/>';
-    put "</Templates>";
-    length string $10000;
-    string=cats('<XMLSELECT search="Person[@Name=',"'&user'",']"/>');
-    put string;
-    put "</Options>";
-    put "</GetMetadataObjects>";
-  run;
-  proc metadata in=inref out=response;
-  run;
-%end;
 
 filename sxlemap temp;
 data _null_;

meta/mm_spkexport.sas

@@ -49,25 +49,20 @@
 
 
   <h4> SAS Macros </h4>
-  @li mf_getuniquefileref.sas
-  @li mf_getuniquename.sas
-  @li mf_isblank.sas
   @li mf_loc.sas
   @li mm_tree.sas
+  @li mf_getuniquefileref.sas
+  @li mf_isblank.sas
   @li mp_abort.sas
 
-
+  @param metaloc= the metadata folder to export
-  @param [in] metaloc= the metadata folder to export
+  @param secureref= fileref containing the username / password (should point to
-  @param [in] secureref= fileref containing the username / password (should
+    a file in a secure location).  Leave blank to substitute $bash type vars.
-    point to a file in a secure location). Leave blank to substitute $bash vars.
+  @param outref= fileref to which to write the command
-  @param [in] excludevars= (0) A space seperated list of macro variable names,
+  @param cmdoutloc= the directory to which the command will write the SPK
-    each of which contains a value that should be used to filter the output
+    (default=WORK)
-    objects.
+  @param cmdoutname= the name of the spk / log files to create (will be
-  @param [out] outref= fileref to which to write the command
+    identical just with .spk or .log extension)
-  @param [out] cmdoutloc= (%sysfunc(pathname(work))) The directory to which the
-    command will write the SPK
-  @param [out] cmdoutname= (mmxport) The name of the spk / log files to create
-    (will be identical just with .spk or .log extension)
 
   @version 9.4
   @author Allan Bowe
@@ -76,7 +71,6 @@
 
 %macro mm_spkexport(metaloc=
   ,secureref=
-  ,excludevars=0
   ,outref=
   ,cmdoutloc=%sysfunc(pathname(work))
   ,cmdoutname=mmxport
@@ -88,7 +82,7 @@
 %end;
 
 /* set creds */
-%local mmxuser mmxpath i var;
+%local mmxuser mmxpath;
 %let mmxuser=$1;
 %let mmxpass=$2;
 %if %mf_isblank(&secureref)=0 %then %do;
@@ -96,51 +90,35 @@
 %end;
 
 /* setup metadata connection options */
-%local host port platform_object_path ds;
+%local host port platform_object_path connx_string;
 %let host=%sysfunc(getoption(metaserver));
 %let port=%sysfunc(getoption(metaport));
 %let platform_object_path=%mf_loc(POF);
-%let ds=%mf_getuniquename(prefix=spkexportable);
 
-%mm_tree(root=%str(&metaloc),types=EXPORTABLE ,outds=&ds)
+%let connx_string=%str(-host &host -port &port -user &mmxuser %trim(
+  )-password &mmxpass);
+
+%mm_tree(root=%str(&metaloc) ,types=EXPORTABLE ,outds=exportable)
 
 %if %mf_isblank(&outref)=1 %then %let outref=%mf_getuniquefileref();
 
 data _null_;
-  set &ds end=last;
+  set exportable end=last;
   file &outref lrecl=32767;
   length str $32767;
   if _n_=1 then do;
-    put "# Script generated by &sysuserid on %sysfunc(datetime(),datetime19.)";
     put "cd ""&platform_object_path"" \";
-    put "; ./ExportPackage -host &host -port &port -user &mmxuser \";
+    put "; ./ExportPackage &connx_string -disableX11 \";
-    put "  -disableX11 -password &mmxpass \";
+    put " -package ""&cmdoutloc/&cmdoutname..spk"" \";
-    put "  -package ""&cmdoutloc/&cmdoutname..spk"" \";
   end;
-/* exclude particular patterns from the exported SPK */
-%if "&excludevars" ne "0" %then %do;
-  %do i=1 %to %sysfunc(countw(&excludevars));
-    %let var=%scan(&excludevars,&i);
-    if _n_=1 then do;
-      length excludestr&i $1000;
-      retain excludestr&i;
-      excludestr&i=symget("&var");
-      putlog excludestr&i=;
-      putlog path=;
-    end;
-    if index(path,cats(excludestr&i))=0 and index(name,cats(excludestr&i))=0;
-  %end;
-  /* ignore top level folder else all subcontent will be exported regardless */
-  if _n_>1;
-%end;
   str=' -objects '!!cats('"',path,'/',name,"(",publictype,')" \');
   put str;
   if last then put " -log ""&cmdoutloc/&cmdoutname..log"" 2>&1 ";
 run;
 
 %mp_abort(iftrue= (&syscc ne 0)
-  ,mac=mm_spkexport
+  ,mac=&sysmacroname
   ,msg=%str(syscc=&syscc)
 )
 
-%mend mm_spkexport;
+%mend mm_spkexport;

meta/mm_updatestpsourcecode.sas

@@ -43,9 +43,7 @@ data _null_;
     cnt=1;
     do while (metadata_getnasn(uri,"Notes",cnt,tsuri)>0);
       rc=metadata_getattr(tsuri,"Name",value);
-    %if &mdebug=1 %then %do;
       put tsuri= value=;
-    %end;
       if value="SourceCode" then do;
         /* found it! */
         rc=metadata_getattr(tsuri,"Id",value);
@@ -128,4 +126,4 @@ run;
   filename &frefout clear;
 %end;
 
-%mend mm_updatestpsourcecode;
+%mend mm_updatestpsourcecode;

meta/mm_webout.sas

@@ -4,55 +4,43 @@
   @details This macro should be added to the start of each Stored Process,
   **immediately** followed by a call to:
 
-      %mm_webout(FETCH)
+        %mm_webout(FETCH)
 
-  This will read all the input data and create same-named SAS datasets in the
+    This will read all the input data and create same-named SAS datasets in the
-  WORK library.  You can then insert your code, and send data back using the
+    WORK library.  You can then insert your code, and send data back using the
-  following syntax:
+    following syntax:
 
-      data some datasets; * make some data ;
+        data some datasets; * make some data ;
-      retain some columns;
+        retain some columns;
-      run;
+        run;
 
-      %mm_webout(OPEN)
+        %mm_webout(OPEN)
-      %mm_webout(ARR,some)  * Array format, fast, suitable for large tables ;
+        %mm_webout(ARR,some)  * Array format, fast, suitable for large tables ;
-      %mm_webout(OBJ,datasets) * Object format, easier to work with ;
+        %mm_webout(OBJ,datasets) * Object format, easier to work with ;
 
-  Finally, wrap everything up send some helpful system variables too
+    Finally, wrap everything up send some helpful system variables too
 
-      %mm_webout(CLOSE)
+        %mm_webout(CLOSE)
 
 
   @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE
   @param [in] ds The dataset to send back to the frontend
   @param [out] dslabel= Value to use instead of table name for sending to JSON
-  @param [in] fmt= (N) Setting Y converts all vars to their formatted values
+  @param [in] fmt=(Y) Set to N to send back unformatted values
   @param [out] fref= (_webout) The fileref to which to write the JSON
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
-  @param [in] showmeta= (N) Set to Y to output metadata alongside each table,
+  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
-  @param [in] workobs= (0) When set to a positive integer, will create a new
-    output object (WORK) which contains this number of observations from all
-    tables in the WORK library.
-  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
-    that should be converted to output JSON
-
-  <h4> SAS Macros </h4>
-  @li mp_jsonout.sas
-
-  <h4> Related Macros </h4>
-  @li ms_webout.sas
-  @li mv_webout.sas
 
   @version 9.3
   @author Allan Bowe
 
 **/
-%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL
+%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
-  ,showmeta=N,maxobs=MAX,workobs=0
+  ,showmeta=NO
 );
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
@@ -119,14 +107,13 @@
 
 %else %if &action=ARR or &action=OBJ %then %do;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta,maxobs=&maxobs
+    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta
   )
 %end;
 %else %if &action=CLOSE %then %do;
-  /* To avoid issues with _webout on EBI we use a temporary file */
+  %if %str(&_debug) ge 131 %then %do;
-  filename _sjsref temp lrecl=131068;
+    /* if debug mode, send back first 10 records of each work table also */
-  %if %str(&workobs) > 0 %then %do;
+    options obs=10;
-    /* if debug mode, send back first XX records of each work table also */
     data;run;%let tempds=%scan(&syslast,2,.);
     ods output Members=&tempds;
     proc datasets library=WORK memtype=data;
@@ -137,11 +124,11 @@
       i+1;
       call symputx(cats('wt',i),name,'l');
       call symputx('wtcnt',i,'l');
-    data _null_; file _sjsref mod encoding='utf-8';
+    data _null_; file &fref mod encoding='utf-8';
       put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
-      data _null_; file _sjsref mod encoding='utf-8';
+      data _null_; file &fref mod encoding='utf-8';
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
         nvars=attrn(dsid,'NVARS');
@@ -150,82 +137,34 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=Y
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES)
-        ,maxobs=&workobs
+      data _null_; file &fref mod encoding='utf-8';
-      )
-      data _null_; file _sjsref mod encoding='utf-8';
         put "}";
     %end;
-    data _null_; file _sjsref mod encoding='utf-8';
+    data _null_; file &fref mod encoding='utf-8';
       put "}";
     run;
   %end;
   /* close off json */
-  data _null_;file _sjsref mod encoding='utf-8';
+  data _null_;file &fref mod encoding='utf-8';
-    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512;
+    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
+    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""_DEBUG"" : ""&_debug"" ";
     _METAUSER=quote(trim(symget('_METAUSER')));
     put ",""_METAUSER"": " _METAUSER;
     _METAPERSON=quote(trim(symget('_METAPERSON')));
     put ',"_METAPERSON": ' _METAPERSON;
-    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ',"_PROGRAM" : ' _PROGRAM ;
-    autoexec=quote(urlencode(trim(getoption('autoexec'))));
-    put ',"AUTOEXEC" : ' autoexec;
-    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSENCODING"" : ""&sysencoding"" ";
+    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
-    syserrortext=cats(symget('syserrortext'));
-    if findc(syserrortext,'"\'!!'0A0D09000E0F010210111A'x) then do;
-      syserrortext='"'!!trim(
-        prxchange('s/"/\\"/',-1,        /* double quote */
-        prxchange('s/\x0A/\n/',-1,      /* new line */
-        prxchange('s/\x0D/\r/',-1,      /* carriage return */
-        prxchange('s/\x09/\\t/',-1,     /* tab */
-        prxchange('s/\x00/\\u0000/',-1, /* NUL */
-        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
-        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
-        prxchange('s/\x01/\\u0001/',-1, /* SOH */
-        prxchange('s/\x02/\\u0002/',-1, /* STX */
-        prxchange('s/\x10/\\u0010/',-1, /* DLE */
-        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
-        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
-        prxchange('s/\\/\\\\/',-1,syserrortext)
-      )))))))))))))!!'"';
-    end;
-    else syserrortext=cats('"',syserrortext,'"');
-    put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
-    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
-    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
-    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
-    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
-    put ",""SYSUSERID"" : ""&sysuserid"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    syswarningtext=cats(symget('syswarningtext'));
+    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
-    if findc(syswarningtext,'"\'!!'0A0D09000E0F010210111A'x) then do;
-      syswarningtext='"'!!trim(
-        prxchange('s/"/\\"/',-1,        /* double quote */
-        prxchange('s/\x0A/\n/',-1,      /* new line */
-        prxchange('s/\x0D/\r/',-1,      /* carriage return */
-        prxchange('s/\x09/\\t/',-1,     /* tab */
-        prxchange('s/\x00/\\u0000/',-1, /* NUL */
-        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
-        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
-        prxchange('s/\x01/\\u0001/',-1, /* SOH */
-        prxchange('s/\x02/\\u0002/',-1, /* STX */
-        prxchange('s/\x10/\\u0010/',-1, /* DLE */
-        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
-        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
-        prxchange('s/\\/\\\\/',-1,syswarningtext)
-      )))))))))))))!!'"';
-    end;
-    else syswarningtext=cats('"',syswarningtext,'"');
-    put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
@@ -236,16 +175,6 @@
     put '>>weboutEND<<';
   %end;
   run;
-  /* now write to _webout 1 char at a time */
-  data _null_;
-    infile _sjsref lrecl=1 recfm=n;
-    file &fref mod lrecl=1 recfm=n;
-    input sourcechar $char1. @@;
-    format sourcechar hex2.;
-    put sourcechar char1. @@;
-  run;
-  filename _sjsref clear;
-
 %end;
 
 %mend mm_webout;

metax/mmx_createmetafolder.sas

@@ -1,50 +0,0 @@
-/**
-  @file
-  @brief Creates a metadata folder
-  @details Creates a metadata folder using the batch tools
-
-  Usage:
-
-      %mmx_createmetafolder(loc=/some/meta/folder,user=sasdemo,pass=mars345)
-
-  <h4> SAS Macros </h4>
-  @li mf_loc.sas
-  @li mp_abort.sas
-
-  @param loc= the metadata folder to delete
-  @param user= username
-  @param pass= password
-
-  @version 9.4
-  @author Allan Bowe
-
-**/
-
-%macro mmx_createmetafolder(loc=,user=,pass=);
-
-%local host port path connx_string msg;
-%let host=%sysfunc(getoption(metaserver));
-%let port=%sysfunc(getoption(metaport));
-%let path=%mf_loc(POF)/tools;
-
-%let connx_string= -host &host -port &port -user '&user' -password '&pass';
-/* remove directory */
-data _null_;
-  infile " &path/sas-make-folder &connx_string ""&loc"" -makeFullPath 2>&1"
-    pipe lrecl=10000;
-  input;
-  putlog _infile_;
-run;
-
-data _null_; /* check tree exists */
-  length type uri $256;
-  rc=metadata_pathobj("","&loc","Folder",type,uri);
-  call symputx('foldertype',type,'l');
-run;
-%let msg=Location (&loc) was not created!!;
-%mp_abort(iftrue= (&foldertype ne Tree)
-  ,mac=&_program..sas
-  ,msg=%superq(msg)
-)
-
-%mend mmx_createmetafolder;

metax/mmx_deletemetafolder.sas

@@ -7,7 +7,7 @@
 
   Usage:
 
-      %mmx_deletemetafolder(loc=/some/meta/folder,user=sasdemo,pass=mars345)
+    %mmx_deletemetafolder(loc=/some/meta/folder,user=sasdemo,pass=mars345)
 
   <h4> SAS Macros </h4>
   @li mf_loc.sas
@@ -37,4 +37,4 @@ data _null_;
   putlog _infile_;
 run;
 
-%mend mmx_deletemetafolder;
+%mend mmx_deletemetafolder;