Merge pull request #432 from sasjs/mp_rowhash

feat: mp_rowhash macro and fixed failing tests

.all-contributorsrc

@@ -144,8 +144,27 @@
       "contributions": [
         "doc"
       ]
+    },
+    {
+      "login": "rudvfaden",
+      "name": "Rud Faden",
+      "avatar_url": "https://avatars.githubusercontent.com/u/2445577?v=4",
+      "profile": "http://rudvfaden.github.io/",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "andyjessen",
+      "name": "andyjessen",
+      "avatar_url": "https://avatars.githubusercontent.com/u/62343929?v=4",
+      "profile": "https://github.com/andyjessen",
+      "contributions": [
+        "doc"
+      ]
     }
   ],
   "contributorsPerLine": 7,
-  "skipCi": true
+  "skipCi": true,
+  "commitType": "docs"
 }

.git-hooks/pre-commit

@@ -2,7 +2,7 @@
 
 # Ensure lint is passing
 LINT=`sasjs lint`
-if [[ "$LINT" != *"✔ All matched files use @sasjs/lint code style!" ]]; then
+if [[ "$LINT" != *"All matched files use @sasjs/lint code style!" ]]; then
   echo "$LINT"
   echo "To commit in spite of these warnings, use the -n parameter."
   exit 1

.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

.github/CONTRIBUTING.md

@@ -27,5 +27,6 @@ To contribute:
 1.  Create your feature branch (`git checkout -b myfeature`)
 2.  Make your change
 3.  Update the `all.sas` file (`python3 build.py`)
-4.  Push and make a PR
+4.  Commit using a [Conventional Commit](https://www.conventionalcommits.org)
+5.  Push and make a PR
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,4 +15,4 @@ What code changes have been made to achieve the intent.
 - [ ] Code is formatted correctly (`sasjs lint`).
 - [ ] Any new functionality has been unit tested.
 - [ ] All unit tests are passing (`sasjs test`).
-- [ ] `all.sas` has been regenerated (`python3 build.py`)
+- [ ] The PR desc or underlying commits follow the [Conventional Commit](https://www.conventionalcommits.org) standard

.github/vpn/config.ovpn

@@ -3,10 +3,12 @@ client
 tls-client
 dev tun
 # this will connect with whatever proto DNS tells us (https://community.openvpn.net/openvpn/ticket/934)
-proto tcp
+proto udp
-remote vpn.4gl.io 7494
+remote vpn.4gl.io 7194
 resolv-retry infinite
-cipher AES-256-CBC
+# this will fallback from udp6 to udp4 as well
+connect-timeout 5
+data-ciphers AES-256-CBC:AES-256-GCM
 auth SHA256
 script-security 2
 keepalive 10 120

.github/workflows/main.yml

@@ -11,12 +11,27 @@ on:
 jobs:
   release:
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      issues: write
+      pull-requests: write
+      id-token: write # required for npm Trusted Publishing (OIDC)
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v6
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 'lts/*'
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Ensure npm supports Trusted Publishing
+        run: npm install -g npm@latest
 
       - name: Install dependencies
-        run: npm ci
+        run: |
+          npm ci
 
       - name: Check code style (aborts if errors found)
         run: npx @sasjs/cli lint
@@ -36,14 +51,15 @@ jobs:
 
       - name: Install Open VPN
         run: |
-          sudo apt install apt-transport-https
+          #sudo apt install apt-transport-https
-          sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
+          #sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
-          sudo apt-key add openvpn-repo-pkg-key.pub
+          #sudo apt-key add openvpn-repo-pkg-key.pub
-          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
+          #sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
-          sudo apt update
+          #sudo apt update
-          sudo apt install openvpn3=17~betaUb22042+jammy
+          #sudo apt install openvpn3=17~betaUb22042+jammy
       - name: Start Open VPN 3
-        run: openvpn3 session-start --config .github/vpn/config.ovpn
+        run: |
+          # openvpn3 session-start --config .github/vpn/config.ovpn
 
       - name: Add credentials
         run: |
@@ -52,13 +68,40 @@ jobs:
           echo "REFRESH_TOKEN=${{secrets.SAS9_4GL_IO_REFRESH_TOKEN}}" >> .env.server
 
       - name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v3
+        id: makerelease
+        uses: cycjimmy/semantic-release-action@v6
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+          # NPM_TOKEN no longer required: package is published via npm
+          # Trusted Publishing (OIDC). The id-token: write permission above
+          # allows npm CLI to authenticate to the registry automatically.
 
       - name: SAS Packages Release
         run: |
           npx @sasjs/cli compile job -s sasjs/utils/create_sas_package.sas -o sasjsbuild -t server
           # need long duration token per https://github.com/sasjs/server/issues/307
           # npx @sasjs/cli run sasjsbuild/jobs/utils/create_sas_package.sas -t server
+
+      - name: Update Docs Site
+        if: steps.makerelease.outputs.new_release_published == 'true'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y doxygen
+          npx @sasjs/cli doc -t docsonly 
+          git clone https://x-access-token:${{ secrets.CORESASJSIO_PAT }}@github.com/sasjs/core.github.io.git
+          cd core.github.io
+          rm -rf *.html
+          rm -rf *.js
+          rm -rf *.png
+          rm -rf *.dot
+          rm -rf *.css
+          rm -rf *.svg
+          rm -rf search
+          cp -R ../sasjsbuild/docs/* .
+          ls
+          git config user.name sasjs
+          echo 'core.sasjs.io' > CNAME
+          git add .
+          git commit -m "core pipeline build on $(date +%F:%H:%M:%S)"
+          git push
+          echo "check it out:  https://sasjs.github.io/core.github.io/files.html"

.github/workflows/notmain.yml

@@ -0,0 +1,33 @@
+# This workflow will run tests using node and then publish a package to GitHub Packages when a release is created
+# For more information see: https://help.github.com/actions/language-and-framework-guides/publishing-nodejs-packages
+
+name: SASjs Core - Update all.sas
+
+on:
+  push:
+    branches-ignore:
+      - main
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install dependencies
+        run: |
+          npm ci
+          npm i -g @sasjs/cli@latest
+
+      - name: Ensure all.sas is always up to date
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          python3 build.py
+          git add all.sas
+          git add viya/mv_createwebservice.sas
+          git commit -m "chore: updating all.sas" --allow-empty
+          git push
+
+

.github/workflows/run-tests.yml

@@ -8,14 +8,14 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     strategy:
       matrix:
-        node-version: [lts/hydrogen]
+        node-version: [lts/iron]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v6
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v2
         with:
@@ -34,6 +34,10 @@ jobs:
           USER_KEY: ${{ secrets.USER_KEY }}
           TLS_KEY: ${{ secrets.TLS_KEY }}
 
+      - name: Chmod VPN files
+        run: |
+          chmod 600 .github/vpn/ca.crt .github/vpn/user.crt .github/vpn/user.key .github/vpn/tls.key
+
       - name: Install Open VPN
         run: |
           sudo apt install apt-transport-https
@@ -42,8 +46,13 @@ jobs:
           sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
           sudo apt update
           sudo apt install openvpn3=17~betaUb22042+jammy
+
       - name: Start Open VPN 3
         run: openvpn3 session-start --config .github/vpn/config.ovpn
+
+      - name: Fetch SASJS server
+        run: curl ${{ secrets.SASJS_SERVER_URL }}/SASjsApi/info
+
       - name: Install Doxygen
         run: sudo apt-get install doxygen
 
@@ -64,6 +73,8 @@ jobs:
 
       - name: Build & Deploy Project to SAS server
         run: npx @sasjs/cli cbd -t server
+        env:
+          macroCorePath: .
 
       - name: Run all tests
         run: npx @sasjs/cli test -t server
@@ -75,3 +86,4 @@ jobs:
           SAS_PASSWORD: ${{secrets.SAS_PASSWORD}}
           ACCESS_TOKEN: ${{secrets.ACCESS_TOKEN}}
           REFRESH_TOKEN: ${{secrets.REFRESH_TOKEN}}
+

.gitignore

@@ -13,3 +13,5 @@ mc_*
 *.env*
 
 ~
+
+.claude

.gitpod.dockerfile

@@ -1,6 +0,0 @@
-FROM gitpod/workspace-full
-
-RUN sudo apt-get update \
- && sudo apt-get install -y doxygen \
- && sudo apt-get install -y graphviz \
- && sudo rm -rf /var/lib/apt/lists/*

.gitpod.yml

@@ -1,27 +0,0 @@
-tasks:
-  - init: npm install -g npm
-  - command: npm i
-  - command: npm i -g @sasjs/cli
-
-image:
-  file: .gitpod.dockerfile
-vscode:
-  extensions:
-    - sasjs.sasjs-for-vscode
-
-github:
-  prebuilds:
-    # enable for the master/default branch (defaults to true)
-    master: true
-    # enable for all branches in this repo (defaults to false)
-    branches: false
-    # enable for pull requests coming from this repo (defaults to true)
-    pullRequests: true
-    # enable for pull requests coming from forks (defaults to false)
-    pullRequestsFromForks: true
-    # add a "Review in Gitpod" button as a comment to pull requests (defaults to true)
-    addComment: true
-    # add a "Review in Gitpod" button to pull requests (defaults to false)
-    addBadge: false
-    # add a label once the prebuild is ready to pull requests (defaults to false)
-    addLabel: prebuilt-in-gitpod

.npmignore

@@ -1,6 +1,5 @@
 all.sas
 build.py
-.gitpod*
 tests/
 sasjs/
 .github/

.npmrc

@@ -0,0 +1 @@
+ignore-scripts=true

AGENTS.md

@@ -0,0 +1,31 @@
+# Agent Instructions for @sasjs/core
+
+Follow these rules when editing or generating code for the @sasjs/core SAS macro library.
+
+## Project Context
+This repo is the SASjs Macro Core library — a collection of MIT-licensed, production-quality SAS macros for SAS application development.
+
+## Versioning
+- NEVER bump or modify the version in `package.json`.
+- Versioning is handled entirely by the CI/CD pipeline using semantic-release.
+
+## SAS Style & Standards
+- Read and follow the standards documented in `README.md` (Sections: Components, Standards, File Properties, Header Properties, Coding Standards).
+- Read and follow `.sasjslint`:
+  - No trailing spaces.
+  - Requires a Doxygen header on every macro (`@file`, `@brief`, etc.).
+  - Lowercase file names without spaces.
+  - Lowercase macro names.
+  - Macro definitions must use parentheses, e.g. `%macro x();` not `%macro x;`.
+  - Indentation = 2 spaces (or multiple thereof); no tabs.
+  - Max line length 300.
+  - No gremlins / invisible characters.
+- One macro per file; filename must match macro name.
+- Macro *calls* should NOT be terminated with a semicolon. Use `%my_macro()` not `%my_macro();`.
+- Macro variables must always be local, to prevent scope leakage.
+
+
+## Build / Generated Files
+- Do not run the build script locally; it is executed in the CI/CD pipeline.
+- Generated files, including the consolidated `all.sas`, the per-folder `mc_*.sas` files, and the LUA macro wrappers in the `lua` folder, can generally be ignored unless the pipeline requires an update. Do not edit generated files by hand.
+- run sasjs lint after each change

README.md

@@ -1,19 +1,10 @@
 # Macro Core
-[![npm package][npm-image]][npm-url]
+
-[![Github Workflow][githubworkflow-image]][githubworkflow-url]
+[![npm package](https://img.shields.io/npm/v/@sasjs/core.svg)](http://npmjs.org/package/@sasjs/core)
+[![Github Workflow](https://github.com/sasjs/core/actions/workflows/main.yml/badge.svg)](https://github.com/sasjs/core/blob/main/.github/workflows/main.yml)
 ![npm](https://img.shields.io/npm/dt/@sasjs/core)
 ![GitHub top language](https://img.shields.io/github/languages/top/sasjs/core)
 [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sasjs/core)](https://github.com/sasjs/core/issues?q=is%3Aissue+is%3Aclosed)
-[![GitHub issues](https://img.shields.io/github/issues-raw/sasjs/core)](https://github.com/sasjs/core/issues)
-![total lines](https://tokei.rs/b1/github/sasjs/core)
-[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-908a85?logo=gitpod)](https://gitpod.io/#https://github.com/sasjs/core)
-
-
-[npm-image]:https://img.shields.io/npm/v/@sasjs/core.svg
-[npm-url]:http://npmjs.org/package/@sasjs/core
-[githubworkflow-image]:https://github.com/sasjs/core/actions/workflows/main.yml/badge.svg
-[githubworkflow-url]:https://github.com/sasjs/core/blob/main/.github/workflows/main.yml
-[dependency-url]:https://github.com/sasjs/core/blob/main/package.json
 
 
 Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/.github/CONTRIBUTING.md) are welcome.
@@ -147,16 +138,17 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 - macro names must be lowercase
 - one macro per file
 - prefixes:
-  - _mcf_ for macro compiled functions (proc fcmp)
+  - _mcf__: macro compiled functions (proc fcmp)
-  - _mddl_ for macros containing DDL (Data Definition Language)
+  - _mddl__: macros containing DDL (Data Definition Language)
-  - _mf_ for macro functions (can be used in open code).
+  - _mf__: macro functions (can be used in open code).
-  - _ml_ for macros that are used to compile LUA modules
+  - _mfv__: macro functions that work only in Viya
-  - _mm_ for metadata macros (interface with the metadata server).
+  - _ml__: macros that are used to compile LUA modules
-  - _mmx_ for macros that use metadata and are XCMD enabled (working on both windows and unix)
+  - _mm__: metadata macros (interface with the metadata server).
-  - _mp_ for macro procedures (which generate sas code)
+  - _mmx__: macros that use metadata and are XCMD enabled (working on both windows and unix)
-  - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
+  - _mp__: macro procedures (which generate sas code)
-  - _mv_ for macro procedures that will only work in Viya
+  - _ms__: macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
-  - _mx_ for macros that work on Viya, SAS 9 EBI and SASjs Server
+  - _mv__: macro procedures that will only work in Viya
+  - _mx__: macros that work on Viya, SAS 9 EBI and SASjs Server
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
@@ -207,23 +199,25 @@ When contributing to this library, it is therefore important to ensure that all
 - All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect, or the [USER](https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/lrcon/n18m1vkqmeo4esn1moikt23zhp8s.htm) library is active.
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
 - Where global macro variables are absolutely necessary, they should make use of `&sasjs_prefix` - see mp_init.sas
-- The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
+- The use of `quit;` for `proc sql` is essential, to avoid `WARNING: You cannot disconnect or terminate session XXXX until the procedure completes.` when terminating CAS sessions in Viya.
 - Use [sasjs lint](https://github.com/sasjs/lint)!
 
 ## General Notes
 
-- All macros should be compatible with SAS versions from support level B and above (so currently 9.2 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully (eg `%if %sysevalf(&sysver<9.3) %then %return`).
+- All macros should be compatible with SAS versions from support level B and above (so currently 9.3 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully.
+- It's [best to avoid](https://git.datacontroller.io/dc/dc/issues/50) special / non-ASCII characters for compatibility with the widest variety of SAS installations.
 
 ## Breaking Changes
 
 We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major/breaking release (v5) becomes necessary:
 
-* mf_getuniquelibref.sas to have the deprecated maxtried parameter removed (no longer needed)
+* mcf_xxx macros to have `insert_cmplib` option deprecated (the option is now checked automatically with value inserted only if needed)
-* mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
-* `insert_cmplib` option of mcf_xxx macros will be deprecated (the option is now checked automatically with value inserted only if needed)
 * mcf_xxx macros to have `wrap=` option defaulted to YES for convenience.  Set this option explicitly to avoid issues.
-* mp_getddl.sas to be renamed to mp_ds2ddl.sas (consistent with other ds2xxx macros).  A wrapper macro is already in place, and you are able to use this immediately.  The default for SHOWLOG will also be YES instead of NO.
+* mf_getuniquelibref.sas to have the deprecated maxtries parameter removed (no longer needed)
+* mp_abort.sas will have the redundant type= parameter removed.
 * mp_coretable.sas will be replaced by the standalone macros in the `ddl` folder (which are already available)
+* mp_getddl.sas to be renamed to mp_ds2ddl.sas (consistent with other ds2xxx macros).  A wrapper macro is already in place, and you are able to use this immediately.  The default for SHOWLOG will also be YES instead of NO.
+* mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
 
 ## Star Gazing
 
@@ -236,6 +230,7 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 The following repositories are also worth checking out:
 
 * [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
+* [Criptic/sas_snippets](https://github.com/Criptic/sas_snippets)
 * [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)
 * [KatjaGlassConsulting/SMILE-SmartSASMacros](https://github.com/KatjaGlassConsulting/SMILE-SmartSASMacros)
 * [paul-canals/toolbox](https://github.com/paul-canals/toolbox)
@@ -247,7 +242,7 @@ The following repositories are also worth checking out:
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-13-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-15-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -272,6 +267,10 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt="Bart Jablonski"/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://bandism.net/"><img src="https://avatars.githubusercontent.com/u/22633385?v=4?s=100" width="100px;" alt="Ikko Ashimine"/><br /><sub><b>Ikko Ashimine</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=eltociear" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/henrik-forsell"><img src="https://avatars.githubusercontent.com/u/109935936?v=4?s=100" width="100px;" alt="Henrik Forsell"/><br /><sub><b>Henrik Forsell</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=henrik-forsell" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://rudvfaden.github.io/"><img src="https://avatars.githubusercontent.com/u/2445577?v=4?s=100" width="100px;" alt="Rud Faden"/><br /><sub><b>Rud Faden</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rudvfaden" title="Code">💻</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/andyjessen"><img src="https://avatars.githubusercontent.com/u/62343929?v=4?s=100" width="100px;" alt="andyjessen"/><br /><sub><b>andyjessen</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=andyjessen" title="Documentation">📖</a></td>
     </tr>
   </tbody>
 </table>

base/mf_getschema.sas

@@ -25,8 +25,9 @@
   %local dsid vnum rc schema;
   /* in case the parameter is a libref.tablename, pull off just the libref */
   %let libref = %upcase(%scan(&libref, 1, %str(.)));
+  /* sysname can be 'Schema/Owner' or just 'Schema' (eg snowflake) */
   %let dsid=%sysfunc(open(sashelp.vlibnam(where=(
-    libname="%upcase(&libref)" and sysname='Schema/Owner'
+    libname="%upcase(&libref)" and sysname=:'Schema'
   )),i));
   %if (&dsid ^= 0) %then %do;
     %let vnum=%sysfunc(varnum(&dsid,SYSVALUE));

base/mf_getuniquelibref.sas

@@ -26,7 +26,7 @@
   @author Allan Bowe
 **/
 
-%macro mf_getuniquelibref(prefix=mclib,maxtries=1000);
+%macro mf_getuniquelibref(prefix=mc,maxtries=1000);
   %local x;
 
   %if ( %length(&prefix) gt 7 ) %then %do;

base/mf_getvalue.sas

@@ -1,7 +1,8 @@
 /**
   @file
-  @brief Retrieves a value from a dataset.  If no filter supplied, then first
+  @brief Retrieves a value from a dataset. Returned value is fetched from the
-    record is used.
+  'fetchobs=' record (row 1 by default), after applying the optional filter.
+
   @details Be sure to <code>%quote()</code> your where clause.  Example usage:
 
       %put %mf_getvalue(sashelp.class,name,filter=%quote(age=15));
@@ -16,21 +17,39 @@
   @param [in] libds dataset to query
   @param [in] variable the variable which contains the value to return.
   @param [in] filter= (1) contents of where clause
+  @param [in] fetchobs= (1) observation to fetch. NB: Filter applies first.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-%macro mf_getvalue(libds,variable,filter=1
+%macro mf_getvalue(libds,variable,filter=1,fetchobs=1
 )/*/STORE SOURCE*/;
-  %if %mf_getattrn(&libds,NLOBS)>0 %then %do;
+  %local dsid;
-    %local dsid rc &variable;
+
   %let dsid=%sysfunc(open(&libds(where=(&filter))));
+  %if (&dsid) %then %do;
+    %local rc &variable;
     %syscall set(dsid);
-    %let rc = %sysfunc(fetch(&dsid));
+    %let rc = %sysfunc(fetchobs(&dsid,&fetchobs));
+    %if (&rc ne 0) %then %do;
+      %put NOTE: Problem reading obs &fetchobs from &libds..;
+      %put %sysfunc(sysmsg());
+      /* Coerce an rc value of -1 (read past end of data) to a 4
+      that, in SAS condition code terms, represents the sysmsg
+      w@rning it generates. */
+      %if &rc eq -1 %then %let rc = 4;
+      /* And update SYSCC if the &rc value is higher */
+      %let syscc = %sysfunc(max(&syscc,&rc));
+    %end;
     %let rc = %sysfunc(close(&dsid));
 
     %trim(&&&variable)
 
   %end;
+  %else %do;
+    %put %sysfunc(sysmsg());
+    %let syscc = %sysfunc(max(&syscc,%sysfunc(sysrc())));
+  %end;
+
 %mend mf_getvalue;

base/mf_isblank.sas

@@ -7,7 +7,7 @@
 
   Usage:
 
-      %put mf_isblank(&var);
+      %put %mf_isblank(&var);
 
   inspiration:
   https://support.sas.com/resources/papers/proceedings09/022-2009.pdf

base/mf_mimetype.sas

@@ -0,0 +1,436 @@
+/**
+  @file
+  @brief Returns a mime type from a file extension
+  @details Provide a file extension and get a mime type.
+    The mappings were derived from this source:
+    https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
+
+
+
+  @param [in] ext The file extension (unquoted)
+  @return output The mime type
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_mimetype(
+    ext
+)/*/STORE SOURCE*/ /minoperator mindelimiter=' ';
+
+%let ext=%lowcase(&ext);
+
+%if &ext in (sas txt text conf def list log)
+%then %do;%str(text/plain)%end;
+%else %if &ext=xlsx %then %do;
+%str(application/vnd.openxmlformats-officedocument.spreadsheetml.sheet)%end;
+%else %if &ext in (xls xlm xla xlc xlt xlw)
+%then %do;%str(application/vnd.ms-excel)%end;
+%else %if &ext=xlsm
+%then %do;%str(application/vnd.ms-excel.sheet.macroenabled.12)%end;
+%else %if &ext=xlsb
+%then %do;%str(application/vnd.ms-excel.sheet.binary.macroenabled.12)%end;
+%else %if &ext in (css csv html n3 sgml vcard)
+%then %do;%str(text/&ext)%end;
+%else %if &ext in (avif bmp cgm gif ief jxl ktx png sgi tiff webp)
+%then %do;%str(image/&ext)%end;
+%else %if &ext in (exi gxf ipfix json mbox mp21 mxf oda oxps pdf rtf sdp wasm
+  xml yang zip)
+%then %do;%str(application/&ext)%end;
+%else %if &ext in (jpeg jpg jpe) %then %do;%str(image/jpeg)%end;
+%else %if &ext in (mp4 mp4v mpg4) %then %do;%str(video/mp4)%end;
+%else %if &ext in (otf ttf woff woff2) %then %do;%str(font/&ext)%end;
+%else %if &ext in (mpeg mpg mpe m1v m2v)
+%then %do;%str(video/mpeg)%end;
+%else %if &ext in (h261 h263 h264 jpm mj2 webm)
+%then %do;%str(video/&ext)%end;
+%else %if &ext in (f4v fli flv m4v mng smv)
+%then %do;%str(video/x-&ext)%end;
+%else %if &ext in (3ds cmx pcx rgb tga)
+%then %do;%str(image/x-&ext)%end;
+%else %if &ext in (asm nfo opml sfv)
+%then %do;%str(text/x-&ext)%end;
+%else %if &ext in (aac caf flac wav)
+%then %do;%str(audio/x-&ext)%end;
+%else %if &ext in (ts m2t m2ts mts)
+%then %do;%str(video/mp2t)%end;
+%else %if &ext in (pfa pfb pfm afm)
+%then %do;%str(application/x-font-type1)%end;
+%else %if &ext in (oga ogg spx opus)
+%then %do;%str(audio/ogg)%end;
+%else %if &ext in (mid midi kar rmi)
+%then %do;%str(audio/midi)%end;
+%else %if &ext in (onetoc onetoc2 onetmp onepkg)
+%then %do;%str(application/onenote)%end;
+%else %if &ext in (mxml xhvml xvml xvm)
+%then %do;%str(application/xv+xml)%end;
+%else %if &ext in (f for f77 f90)
+%then %do;%str(text/x-fortran)%end;
+%else %if &ext in (wmf wmz emf emz)
+%then %do;%str(application/x-msmetafile)%end;
+%else %if &ext in (exe dll com bat msi)
+%then %do;%str(application/x-msdownload)%end;
+%else %if &ext in (bin dms lrf mar so dist distz pkg bpk dump elc deploy)
+%then %do;%str(application/octet-stream)%end;
+%else %if &ext in (atom atomcat atomsvc ccxml davmount emma gml gpx inkml mads
+  mathml metalink mets mods omdoc pls rdf rsd rss sbml shf smil sru ssdl ssml
+  tei wsdl wspolicy xaml xenc xhtml xop xslt xspf yin)
+%then %do;%str(application/&ext+xml)%end;
+%else %if &ext in (dir dcr dxr cst cct cxt w3d fgd swa)
+%then %do;%str(application/x-director)%end;
+%else %if &ext in (z1 z2 z3 z4 z5 z6 z7 z8)
+%then %do;%str(application/x-zmachine)%end;
+%else %if &ext in (c cc cxx cpp h hh dic)
+%then %do;%str(text/x-c)%end;
+%else %if &ext in (mpga mp2 mp2a mp3 m2a m3a)
+%then %do;%str(audio/mpeg)%end;
+%else %if &ext in (t tr roff man me ms)
+%then %do;%str(text/troff)%end;
+%else %if &ext in (cbr cba cbt cbz cb7)
+%then %do;%str(application/x-cbr)%end;
+%else %if &ext in (fh fhc fh4 fh5 fh7)
+%then %do;%str(image/x-freehand)%end;
+%else %if &ext in (aab x32 u32 vox)
+%then %do;%str(application/x-authorware-bin)%end;
+%else %if &ext in (uvi uvvi uvg uvvg)
+%then %do;%str(image/vnd.dece.graphic)%end;
+%else %if &ext in (cdx cif cmdf cml csml xyz)
+%then %do;%str(chemical/x-&ext)%end;
+%else %if &ext in (aif aiff aifc) %then %do;%str(audio/x-aiff)%end;
+%else %if &ext in (ma nb mb) %then %do;%str(application/mathematica)%end;
+%else %if &ext in (mvb m13 m14) %then %do;%str(application/x-msmediaview)%end;
+%else %if &ext in (msh mesh silo) %then %do;%str(model/mesh)%end;
+%else %if &ext in (uri uris urls) %then %do;%str(text/uri-list)%end;
+%else %if &ext in (mkv mk3d mks) %then %do;%str(video/x-matroska)%end;
+%else %if &ext=ez %then %do;%str(application/andrew-inset)%end;
+%else %if &ext=aw %then %do;%str(application/applixware)%end;
+%else %if &ext=cdmia %then %do;%str(application/cdmi-capability)%end;
+%else %if &ext=cdmic %then %do;%str(application/cdmi-container)%end;
+%else %if &ext=cdmid %then %do;%str(application/cdmi-domain)%end;
+%else %if &ext=cdmio %then %do;%str(application/cdmi-object)%end;
+%else %if &ext=cdmiq %then %do;%str(application/cdmi-queue)%end;
+%else %if &ext=cu %then %do;%str(application/cu-seeme)%end;
+%else %if &ext=dssc %then %do;%str(application/dssc+der)%end;
+%else %if &ext=xdssc %then %do;%str(application/dssc+xml)%end;
+%else %if &ext=ecma %then %do;%str(application/ecmascript)%end;
+%else %if &ext=epub %then %do;%str(application/epub+zip)%end;
+%else %if &ext=pfr %then %do;%str(application/font-tdpfr)%end;
+%else %if &ext=stk %then %do;%str(application/hyperstudio)%end;
+%else %if &ext=ink %then %do;%str(application/inkml+xml)%end;
+%else %if &ext=jar %then %do;%str(application/java-archive)%end;
+%else %if &ext=ser %then %do;%str(application/java-serialized-object)%end;
+%else %if &ext=class %then %do;%str(application/java-vm)%end;
+%else %if &ext=jsonml %then %do;%str(application/jsonml+json)%end;
+%else %if &ext=lostxml %then %do;%str(application/lost+xml)%end;
+%else %if &ext=hqx %then %do;%str(application/mac-binhex40)%end;
+%else %if &ext=cpt %then %do;%str(application/mac-compactpro)%end;
+%else %if &ext=mrc %then %do;%str(application/marc)%end;
+%else %if &ext=mrcx %then %do;%str(application/marcxml+xml)%end;
+%else %if &ext=mscml %then %do;%str(application/mediaservercontrol+xml)%end;
+%else %if &ext=meta4 %then %do;%str(application/metalink4+xml)%end;
+%else %if &ext=m21 %then %do;%str(application/mp21)%end;
+%else %if &ext=mp4s %then %do;%str(application/mp4)%end;
+%else %if &ext=doc %then %do;%str(application/msword)%end;
+%else %if &ext=dot %then %do;%str(application/msword)%end;
+%else %if &ext=opf %then %do;%str(application/oebps-package+xml)%end;
+%else %if &ext=ogx %then %do;%str(application/ogg)%end;
+%else %if &ext=xer %then %do;%str(application/patch-ops-error+xml)%end;
+%else %if &ext=pgp %then %do;%str(application/pgp-encrypted)%end;
+%else %if &ext=asc %then %do;%str(application/pgp-signature)%end;
+%else %if &ext=sig %then %do;%str(application/pgp-signature)%end;
+%else %if &ext=prf %then %do;%str(application/pics-rules)%end;
+%else %if &ext=p10 %then %do;%str(application/pkcs10)%end;
+%else %if &ext=p7m %then %do;%str(application/pkcs7-mime)%end;
+%else %if &ext=p7c %then %do;%str(application/pkcs7-mime)%end;
+%else %if &ext=p7s %then %do;%str(application/pkcs7-signature)%end;
+%else %if &ext=p8 %then %do;%str(application/pkcs8)%end;
+%else %if &ext=ac %then %do;%str(application/pkix-attr-cert)%end;
+%else %if &ext=cer %then %do;%str(application/pkix-cert)%end;
+%else %if &ext=crl %then %do;%str(application/pkix-crl)%end;
+%else %if &ext=pkipath %then %do;%str(application/pkix-pkipath)%end;
+%else %if &ext=pki %then %do;%str(application/pkixcmp)%end;
+%else %if &ext=cww %then %do;%str(application/prs.cww)%end;
+%else %if &ext=pskcxml %then %do;%str(application/pskc+xml)%end;
+%else %if &ext=rif %then %do;%str(application/reginfo+xml)%end;
+%else %if &ext=rnc %then %do;%str(application/relax-ng-compact-syntax)%end;
+%else %if &ext=rld %then %do;%str(application/resource-lists-diff+xml)%end;
+%else %if &ext=rl %then %do;%str(application/resource-lists+xml)%end;
+%else %if &ext=gbr %then %do;%str(application/rpki-ghostbusters)%end;
+%else %if &ext=mft %then %do;%str(application/rpki-manifest)%end;
+%else %if &ext=roa %then %do;%str(application/rpki-roa)%end;
+%else %if &ext=scq %then %do;%str(application/scvp-cv-request)%end;
+%else %if &ext=scs %then %do;%str(application/scvp-cv-response)%end;
+%else %if &ext=spq %then %do;%str(application/scvp-vp-request)%end;
+%else %if &ext=spp %then %do;%str(application/scvp-vp-response)%end;
+%else %if &ext=setpay %then %do;%str(application/set-payment-initiation)%end;
+%else %if &ext=setreg %then %do;%str(application/set-registration-initiation)%end;
+%else %if &ext=smi %then %do;%str(application/smil+xml)%end;
+%else %if &ext=rq %then %do;%str(application/sparql-query)%end;
+%else %if &ext=srx %then %do;%str(application/sparql-results+xml)%end;
+%else %if &ext=gram %then %do;%str(application/srgs)%end;
+%else %if &ext=grxml %then %do;%str(application/srgs+xml)%end;
+%else %if &ext=teicorpus %then %do;%str(application/tei+xml)%end;
+%else %if &ext=tfi %then %do;%str(application/thraud+xml)%end;
+%else %if &ext=tsd %then %do;%str(application/timestamped-data)%end;
+%else %if &ext=vxml %then %do;%str(application/voicexml+xml)%end;
+%else %if &ext=wgt %then %do;%str(application/widget)%end;
+%else %if &ext=hlp %then %do;%str(application/winhlp)%end;
+%else %if &ext=7z %then %do;%str(application/x-7z-compressed)%end;
+%else %if &ext=abw %then %do;%str(application/x-abiword)%end;
+%else %if &ext=ace %then %do;%str(application/x-ace-compressed)%end;
+%else %if &ext=dmg %then %do;%str(application/x-apple-diskimage)%end;
+%else %if &ext=aam %then %do;%str(application/x-authorware-map)%end;
+%else %if &ext=aas %then %do;%str(application/x-authorware-seg)%end;
+%else %if &ext=bcpio %then %do;%str(application/x-bcpio)%end;
+%else %if &ext=torrent %then %do;%str(application/x-bittorrent)%end;
+%else %if &ext=blb %then %do;%str(application/x-blorb)%end;
+%else %if &ext=blorb %then %do;%str(application/x-blorb)%end;
+%else %if &ext=bz %then %do;%str(application/x-bzip)%end;
+%else %if &ext=bz2 %then %do;%str(application/x-bzip2)%end;
+%else %if &ext=boz %then %do;%str(application/x-bzip2)%end;
+%else %if &ext=vcd %then %do;%str(application/x-cdlink)%end;
+%else %if &ext=cfs %then %do;%str(application/x-cfs-compressed)%end;
+%else %if &ext=chat %then %do;%str(application/x-chat)%end;
+%else %if &ext=pgn %then %do;%str(application/x-chess-pgn)%end;
+%else %if &ext=nsc %then %do;%str(application/x-conference)%end;
+%else %if &ext=cpio %then %do;%str(application/x-cpio)%end;
+%else %if &ext=csh %then %do;%str(application/x-csh)%end;
+%else %if &ext=deb %then %do;%str(application/x-debian-package)%end;
+%else %if &ext=udeb %then %do;%str(application/x-debian-package)%end;
+%else %if &ext=dgc %then %do;%str(application/x-dgc-compressed)%end;
+%else %if &ext=wad %then %do;%str(application/x-doom)%end;
+%else %if &ext=ncx %then %do;%str(application/x-dtbncx+xml)%end;
+%else %if &ext=dtb %then %do;%str(application/x-dtbook+xml)%end;
+%else %if &ext=res %then %do;%str(application/x-dtbresource+xml)%end;
+%else %if &ext=dvi %then %do;%str(application/x-dvi)%end;
+%else %if &ext=evy %then %do;%str(application/x-envoy)%end;
+%else %if &ext=eva %then %do;%str(application/x-eva)%end;
+%else %if &ext=bdf %then %do;%str(application/x-font-bdf)%end;
+%else %if &ext=gsf %then %do;%str(application/x-font-ghostscript)%end;
+%else %if &ext=psf %then %do;%str(application/x-font-linux-psf)%end;
+%else %if &ext=pcf %then %do;%str(application/x-font-pcf)%end;
+%else %if &ext=snf %then %do;%str(application/x-font-snf)%end;
+%else %if &ext=arc %then %do;%str(application/x-freearc)%end;
+%else %if &ext=spl %then %do;%str(application/x-futuresplash)%end;
+%else %if &ext=gca %then %do;%str(application/x-gca-compressed)%end;
+%else %if &ext=ulx %then %do;%str(application/x-glulx)%end;
+%else %if &ext=gnumeric %then %do;%str(application/x-gnumeric)%end;
+%else %if &ext=gramps %then %do;%str(application/x-gramps-xml)%end;
+%else %if &ext=gtar %then %do;%str(application/x-gtar)%end;
+%else %if &ext=hdf %then %do;%str(application/x-hdf)%end;
+%else %if &ext=install %then %do;%str(application/x-install-instructions)%end;
+%else %if &ext=iso %then %do;%str(application/x-iso9660-image)%end;
+%else %if &ext=jnlp %then %do;%str(application/x-java-jnlp-file)%end;
+%else %if &ext=latex %then %do;%str(application/x-latex)%end;
+%else %if &ext=lzh %then %do;%str(application/x-lzh-compressed)%end;
+%else %if &ext=lha %then %do;%str(application/x-lzh-compressed)%end;
+%else %if &ext=mie %then %do;%str(application/x-mie)%end;
+%else %if &ext=prc %then %do;%str(application/x-mobipocket-ebook)%end;
+%else %if &ext=mobi %then %do;%str(application/x-mobipocket-ebook)%end;
+%else %if &ext=application %then %do;%str(application/x-ms-application)%end;
+%else %if &ext=lnk %then %do;%str(application/x-ms-shortcut)%end;
+%else %if &ext=wmd %then %do;%str(application/x-ms-wmd)%end;
+%else %if &ext=wmz %then %do;%str(application/x-ms-wmz)%end;
+%else %if &ext=xbap %then %do;%str(application/x-ms-xbap)%end;
+%else %if &ext=mdb %then %do;%str(application/x-msaccess)%end;
+%else %if &ext=obd %then %do;%str(application/x-msbinder)%end;
+%else %if &ext=crd %then %do;%str(application/x-mscardfile)%end;
+%else %if &ext=clp %then %do;%str(application/x-msclip)%end;
+%else %if &ext=mny %then %do;%str(application/x-msmoney)%end;
+%else %if &ext=pub %then %do;%str(application/x-mspublisher)%end;
+%else %if &ext=scd %then %do;%str(application/x-msschedule)%end;
+%else %if &ext=trm %then %do;%str(application/x-msterminal)%end;
+%else %if &ext=wri %then %do;%str(application/x-mswrite)%end;
+%else %if &ext=nc %then %do;%str(application/x-netcdf)%end;
+%else %if &ext=cdf %then %do;%str(application/x-netcdf)%end;
+%else %if &ext=nzb %then %do;%str(application/x-nzb)%end;
+%else %if &ext=p12 %then %do;%str(application/x-pkcs12)%end;
+%else %if &ext=pfx %then %do;%str(application/x-pkcs12)%end;
+%else %if &ext=p7b %then %do;%str(application/x-pkcs7-certificates)%end;
+%else %if &ext=spc %then %do;%str(application/x-pkcs7-certificates)%end;
+%else %if &ext=p7r %then %do;%str(application/x-pkcs7-certreqresp)%end;
+%else %if &ext=rar %then %do;%str(application/x-rar-compressed)%end;
+%else %if &ext=ris %then %do;%str(application/x-research-info-systems)%end;
+%else %if &ext=sh %then %do;%str(application/x-sh)%end;
+%else %if &ext=shar %then %do;%str(application/x-shar)%end;
+%else %if &ext=swf %then %do;%str(application/x-shockwave-flash)%end;
+%else %if &ext=xap %then %do;%str(application/x-silverlight-app)%end;
+%else %if &ext=sql %then %do;%str(application/x-sql)%end;
+%else %if &ext=sit %then %do;%str(application/x-stuffit)%end;
+%else %if &ext=sitx %then %do;%str(application/x-stuffitx)%end;
+%else %if &ext=srt %then %do;%str(application/x-subrip)%end;
+%else %if &ext=sv4cpio %then %do;%str(application/x-sv4cpio)%end;
+%else %if &ext=sv4crc %then %do;%str(application/x-sv4crc)%end;
+%else %if &ext=t3 %then %do;%str(application/x-t3vm-image)%end;
+%else %if &ext=gam %then %do;%str(application/x-tads)%end;
+%else %if &ext=tar %then %do;%str(application/x-tar)%end;
+%else %if &ext=tcl %then %do;%str(application/x-tcl)%end;
+%else %if &ext=tex %then %do;%str(application/x-tex)%end;
+%else %if &ext=tfm %then %do;%str(application/x-tex-tfm)%end;
+%else %if &ext=texinfo %then %do;%str(application/x-texinfo)%end;
+%else %if &ext=texi %then %do;%str(application/x-texinfo)%end;
+%else %if &ext=obj %then %do;%str(application/x-tgif)%end;
+%else %if &ext=ustar %then %do;%str(application/x-ustar)%end;
+%else %if &ext=src %then %do;%str(application/x-wais-source)%end;
+%else %if &ext=der %then %do;%str(application/x-x509-ca-cert)%end;
+%else %if &ext=crt %then %do;%str(application/x-x509-ca-cert)%end;
+%else %if &ext=fig %then %do;%str(application/x-xfig)%end;
+%else %if &ext=xlf %then %do;%str(application/x-xliff+xml)%end;
+%else %if &ext=xpi %then %do;%str(application/x-xpinstall)%end;
+%else %if &ext=xz %then %do;%str(application/x-xz)%end;
+%else %if &ext=xdf %then %do;%str(application/xcap-diff+xml)%end;
+%else %if &ext=xht %then %do;%str(application/xhtml+xml)%end;
+%else %if &ext=xsl %then %do;%str(application/xml)%end;
+%else %if &ext=dtd %then %do;%str(application/xml-dtd)%end;
+%else %if &ext=xpl %then %do;%str(application/xproc+xml)%end;
+%else %if &ext=adp %then %do;%str(audio/adpcm)%end;
+%else %if &ext=au %then %do;%str(audio/basic)%end;
+%else %if &ext=snd %then %do;%str(audio/basic)%end;
+%else %if &ext=m4a %then %do;%str(audio/mp4)%end;
+%else %if &ext=mp4a %then %do;%str(audio/mp4)%end;
+%else %if &ext=s3m %then %do;%str(audio/s3m)%end;
+%else %if &ext=sil %then %do;%str(audio/silk)%end;
+%else %if &ext=uva %then %do;%str(audio/vnd.dece.audio)%end;
+%else %if &ext=uvva %then %do;%str(audio/vnd.dece.audio)%end;
+%else %if &ext=eol %then %do;%str(audio/vnd.digital-winds)%end;
+%else %if &ext=dra %then %do;%str(audio/vnd.dra)%end;
+%else %if &ext=dts %then %do;%str(audio/vnd.dts)%end;
+%else %if &ext=dtshd %then %do;%str(audio/vnd.dts.hd)%end;
+%else %if &ext=lvp %then %do;%str(audio/vnd.lucent.voice)%end;
+%else %if &ext=pya %then %do;%str(audio/vnd.ms-playready.media.pya)%end;
+%else %if &ext=ecelp4800 %then %do;%str(audio/vnd.nuera.ecelp4800)%end;
+%else %if &ext=ecelp7470 %then %do;%str(audio/vnd.nuera.ecelp7470)%end;
+%else %if &ext=ecelp9600 %then %do;%str(audio/vnd.nuera.ecelp9600)%end;
+%else %if &ext=rip %then %do;%str(audio/vnd.rip)%end;
+%else %if &ext=weba %then %do;%str(audio/webm)%end;
+%else %if &ext=mka %then %do;%str(audio/x-matroska)%end;
+%else %if &ext=m3u %then %do;%str(audio/x-mpegurl)%end;
+%else %if &ext=wax %then %do;%str(audio/x-ms-wax)%end;
+%else %if &ext=wma %then %do;%str(audio/x-ms-wma)%end;
+%else %if &ext=ra %then %do;%str(audio/x-pn-realaudio)%end;
+%else %if &ext=ram %then %do;%str(audio/x-pn-realaudio)%end;
+%else %if &ext=rmp %then %do;%str(audio/x-pn-realaudio-plugin)%end;
+%else %if &ext=xm %then %do;%str(audio/xm)%end;
+%else %if &ext=ttc %then %do;%str(font/collection)%end;
+%else %if &ext=g3 %then %do;%str(image/g3fax)%end;
+%else %if &ext=btif %then %do;%str(image/prs.btif)%end;
+%else %if &ext=svg %then %do;%str(image/svg+xml)%end;
+%else %if &ext=svgz %then %do;%str(image/svg+xml)%end;
+%else %if &ext=tif %then %do;%str(image/tiff)%end;
+%else %if &ext=psd %then %do;%str(image/vnd.adobe.photoshop)%end;
+%else %if &ext=djv %then %do;%str(image/vnd.djvu)%end;
+%else %if &ext=djvu %then %do;%str(image/vnd.djvu)%end;
+%else %if &ext=sub %then %do;%str(image/vnd.dvb.subtitle)%end;
+%else %if &ext=dwg %then %do;%str(image/vnd.dwg)%end;
+%else %if &ext=dxf %then %do;%str(image/vnd.dxf)%end;
+%else %if &ext=fbs %then %do;%str(image/vnd.fastbidsheet)%end;
+%else %if &ext=fpx %then %do;%str(image/vnd.fpx)%end;
+%else %if &ext=fst %then %do;%str(image/vnd.fst)%end;
+%else %if &ext=mmr %then %do;%str(image/vnd.fujixerox.edmics-mmr)%end;
+%else %if &ext=rlc %then %do;%str(image/vnd.fujixerox.edmics-rlc)%end;
+%else %if &ext=mdi %then %do;%str(image/vnd.ms-modi)%end;
+%else %if &ext=wdp %then %do;%str(image/vnd.ms-photo)%end;
+%else %if &ext=npx %then %do;%str(image/vnd.net-fpx)%end;
+%else %if &ext=wbmp %then %do;%str(image/vnd.wap.wbmp)%end;
+%else %if &ext=xif %then %do;%str(image/vnd.xiff)%end;
+%else %if &ext=ras %then %do;%str(image/x-cmu-raster)%end;
+%else %if &ext=ico %then %do;%str(image/x-icon)%end;
+%else %if &ext=sid %then %do;%str(image/x-mrsid-image)%end;
+%else %if &ext=pct %then %do;%str(image/x-pict)%end;
+%else %if &ext=pic %then %do;%str(image/x-pict)%end;
+%else %if &ext=pnm %then %do;%str(image/x-portable-anymap)%end;
+%else %if &ext=pbm %then %do;%str(image/x-portable-bitmap)%end;
+%else %if &ext=pgm %then %do;%str(image/x-portable-graymap)%end;
+%else %if &ext=ppm %then %do;%str(image/x-portable-pixmap)%end;
+%else %if &ext=xbm %then %do;%str(image/x-xbitmap)%end;
+%else %if &ext=xpm %then %do;%str(image/x-xpixmap)%end;
+%else %if &ext=xwd %then %do;%str(image/x-xwindowdump)%end;
+%else %if &ext=eml %then %do;%str(message/rfc822)%end;
+%else %if &ext=mime %then %do;%str(message/rfc822)%end;
+%else %if &ext=iges %then %do;%str(model/iges)%end;
+%else %if &ext=igs %then %do;%str(model/iges)%end;
+%else %if &ext=dae %then %do;%str(model/vnd.collada+xml)%end;
+%else %if &ext=dwf %then %do;%str(model/vnd.dwf)%end;
+%else %if &ext=gdl %then %do;%str(model/vnd.gdl)%end;
+%else %if &ext=gtw %then %do;%str(model/vnd.gtw)%end;
+%else %if &ext=vtu %then %do;%str(model/vnd.vtu)%end;
+%else %if &ext=vrml %then %do;%str(model/vrml)%end;
+%else %if &ext=wrl %then %do;%str(model/vrml)%end;
+%else %if &ext=x3db %then %do;%str(model/x3d+binary)%end;
+%else %if &ext=x3dbz %then %do;%str(model/x3d+binary)%end;
+%else %if &ext=x3dv %then %do;%str(model/x3d+vrml)%end;
+%else %if &ext=x3dvz %then %do;%str(model/x3d+vrml)%end;
+%else %if &ext=x3d %then %do;%str(model/x3d+xml)%end;
+%else %if &ext=x3dz %then %do;%str(model/x3d+xml)%end;
+%else %if &ext=appcache %then %do;%str(text/cache-manifest)%end;
+%else %if &ext=ics %then %do;%str(text/calendar)%end;
+%else %if &ext=ifb %then %do;%str(text/calendar)%end;
+%else %if &ext=htm %then %do;%str(text/html)%end;
+%else %if &ext=js %then %do;%str(text/javascript)%end;
+%else %if &ext=mjs %then %do;%str(text/javascript)%end;
+%else %if &ext=dsc %then %do;%str(text/prs.lines.tag)%end;
+%else %if &ext=rtx %then %do;%str(text/richtext)%end;
+%else %if &ext=sgm %then %do;%str(text/sgml)%end;
+%else %if &ext=tsv %then %do;%str(text/tab-separated-values)%end;
+%else %if &ext=ttl %then %do;%str(text/turtle)%end;
+%else %if &ext=curl %then %do;%str(text/vnd.curl)%end;
+%else %if &ext=dcurl %then %do;%str(text/vnd.curl.dcurl)%end;
+%else %if &ext=mcurl %then %do;%str(text/vnd.curl.mcurl)%end;
+%else %if &ext=scurl %then %do;%str(text/vnd.curl.scurl)%end;
+%else %if &ext=sub %then %do;%str(text/vnd.dvb.subtitle)%end;
+%else %if &ext=fly %then %do;%str(text/vnd.fly)%end;
+%else %if &ext=flx %then %do;%str(text/vnd.fmi.flexstor)%end;
+%else %if &ext=gv %then %do;%str(text/vnd.graphviz)%end;
+%else %if &ext=3dml %then %do;%str(text/vnd.in3d.3dml)%end;
+%else %if &ext=spot %then %do;%str(text/vnd.in3d.spot)%end;
+%else %if &ext=jad %then %do;%str(text/vnd.sun.j2me.app-descriptor)%end;
+%else %if &ext=wml %then %do;%str(text/vnd.wap.wml)%end;
+%else %if &ext=wmls %then %do;%str(text/vnd.wap.wmlscript)%end;
+%else %if &ext=s %then %do;%str(text/x-asm)%end;
+%else %if &ext=java %then %do;%str(text/x-java-source)%end;
+%else %if &ext=p %then %do;%str(text/x-pascal)%end;
+%else %if &ext=pas %then %do;%str(text/x-pascal)%end;
+%else %if &ext=etx %then %do;%str(text/x-setext)%end;
+%else %if &ext=uu %then %do;%str(text/x-uuencode)%end;
+%else %if &ext=vcs %then %do;%str(text/x-vcalendar)%end;
+%else %if &ext=vcf %then %do;%str(text/x-vcard)%end;
+%else %if &ext=3gp %then %do;%str(video/3gpp)%end;
+%else %if &ext=3g2 %then %do;%str(video/3gpp2)%end;
+%else %if &ext=jpgv %then %do;%str(video/jpeg)%end;
+%else %if &ext=jpgm %then %do;%str(video/jpm)%end;
+%else %if &ext=mjp2 %then %do;%str(video/mj2)%end;
+%else %if &ext=ogv %then %do;%str(video/ogg)%end;
+%else %if &ext=mov %then %do;%str(video/quicktime)%end;
+%else %if &ext=qt %then %do;%str(video/quicktime)%end;
+%else %if &ext=uvh %then %do;%str(video/vnd.dece.hd)%end;
+%else %if &ext=uvvh %then %do;%str(video/vnd.dece.hd)%end;
+%else %if &ext=uvm %then %do;%str(video/vnd.dece.mobile)%end;
+%else %if &ext=uvvm %then %do;%str(video/vnd.dece.mobile)%end;
+%else %if &ext=uvp %then %do;%str(video/vnd.dece.pd)%end;
+%else %if &ext=uvvp %then %do;%str(video/vnd.dece.pd)%end;
+%else %if &ext=uvs %then %do;%str(video/vnd.dece.sd)%end;
+%else %if &ext=uvvs %then %do;%str(video/vnd.dece.sd)%end;
+%else %if &ext=uvv %then %do;%str(video/vnd.dece.video)%end;
+%else %if &ext=uvvv %then %do;%str(video/vnd.dece.video)%end;
+%else %if &ext=dvb %then %do;%str(video/vnd.dvb.file)%end;
+%else %if &ext=fvt %then %do;%str(video/vnd.fvt)%end;
+%else %if &ext=m4u %then %do;%str(video/vnd.mpegurl)%end;
+%else %if &ext=mxu %then %do;%str(video/vnd.mpegurl)%end;
+%else %if &ext=pyv %then %do;%str(video/vnd.ms-playready.media.pyv)%end;
+%else %if &ext=uvu %then %do;%str(video/vnd.uvvu.mp4)%end;
+%else %if &ext=uvvu %then %do;%str(video/vnd.uvvu.mp4)%end;
+%else %if &ext=viv %then %do;%str(video/vnd.vivo)%end;
+%else %if &ext=asf %then %do;%str(video/x-ms-asf)%end;
+%else %if &ext=asx %then %do;%str(video/x-ms-asf)%end;
+%else %if &ext=vob %then %do;%str(video/x-ms-vob)%end;
+%else %if &ext=wm %then %do;%str(video/x-ms-wm)%end;
+%else %if &ext=wmv %then %do;%str(video/x-ms-wmv)%end;
+%else %if &ext=wmx %then %do;%str(video/x-ms-wmx)%end;
+%else %if &ext=wvx %then %do;%str(video/x-ms-wvx)%end;
+%else %if &ext=avi %then %do;%str(video/x-msvideo)%end;
+%else %if &ext=movie %then %do;%str(video/x-sgi-movie)%end;
+%else %if &ext=ice %then %do;%str(x-conference/x-cooltalk)%end;
+%else %if "&ext"="in" %then %do;%str(text/plain)%end;
+%else %do;%str(application/octet-stream)%end;
+
+%mend mf_mimetype;

base/mp_assert.sas

@@ -35,7 +35,7 @@
 )/*/STORE SOURCE*/;
 
   data ;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     test_comments="&sysmacroname: Test result of "!!symget('iftrue');
   %if %eval(%unquote(&iftrue)) %then %do;
@@ -52,5 +52,6 @@
   run;
   proc sql;
   drop table &ds;
+  quit;
 
 %mend mp_assert;

base/mp_assertcols.sas

@@ -108,7 +108,7 @@
   %end;
 
   data;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     if test_description='0'
     then test_description="Testing &inds for existence of &test of: &cols";

base/mp_assertcolvals.sas

@@ -143,7 +143,7 @@
   )
 
   data;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
     test_comments="&sysmacroname: &lib..&ds..&col has &result/&orig values "

base/mp_assertdsobs.sas

@@ -87,7 +87,7 @@
   %end;
 
   data &ds;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
     test_comments="&sysmacroname: Dataset &inds has &nobs observations.";

base/mp_assertscope.sas

@@ -73,10 +73,6 @@
   ignorelist=,
   outds=work.test_results
 )/*/STORE SOURCE*/;
-%local ds test_result test_comments del add mod ilist;
-%let ilist=%upcase(&sasjs_prefix._FUNCTIONS SYS_PROCHTTP_STATUS_CODE
-  SYS_PROCHTTP_STATUS_CODE SYS_PROCHTTP_STATUS_PHRASE &ignorelist);
-
 /**
   * this sets up the global vars, it will also enter STRICT mode.  If this
   * behaviour is not desired, simply initiate the following global macro
@@ -84,6 +80,10 @@
   */
 %mp_init()
 
+%local ds test_result test_comments del add mod ilist;
+%let ilist=%upcase(&sasjs_prefix._FUNCTIONS SYS_PROCHTTP_STATUS_CODE
+  SYS_PROCHTTP_STATUS_CODE SYS_PROCHTTP_STATUS_PHRASE &ignorelist);
+
 /* get current variables */
 %if &action=SNAPSHOT %then %do;
   proc sql;
@@ -92,6 +92,7 @@
     from dictionary.macros
     where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
     order by name,offset;
+  quit;
 %end;
 %else %if &action=COMPARE %then %do;
 
@@ -129,9 +130,8 @@
     %let test_comments=%str(Mod:(&mod) Add:(&add) Del:(&del));
   %end;
 
-
   data ;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     test_comments=symget('test_comments');
     test_result=symget('test_result');
@@ -142,6 +142,7 @@
   run;
   proc sql;
   drop table &ds;
+  quit;
 %end;
 
 %mend mp_assertscope;

base/mp_cntlout.sas

@@ -79,8 +79,8 @@ data &cntlout/nonote2err;
   end;
 
   /* create row marker. Data cannot be sorted without it! */
-  if first.fmtname then fmtrow=0;
+  if first.fmtname then fmtrow=1;
-  fmtrow+1;
+  else fmtrow+1;
 
 run;
 proc sort;

base/mp_dirlist.sas

@@ -85,7 +85,7 @@ data;run;
 data &out_ds(compress=no
     keep=file_or_folder filepath filename ext msg directory level
   );
-  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
+  length directory filepath $2000 fref fref2 $8 file_or_folder $6 filename $255
     ext $20 msg $200 foption $16;
   if _n_=1 then call missing(of _all_);
   retain level &level;

base/mp_ds2csv.sas

@@ -118,14 +118,22 @@ data _null_;
     header = cats(coalescec(varlabel(dsid,i),varnm),dlm);
   %end;
   %else %if &headerformat=SASJS %then %do;
-    if vartype(dsid,i)='C' then header=cats(varnm,':$char',varlen(dsid,i),'.');
+    vlen=varlen(dsid,i);
+    if vartype(dsid,i)='C' then header=cats(varnm,':$char',vlen,'.');
     else do;
       vfmt=coalescec(varfmt(dsid,i),'0');
       fmttype=mcf_getfmttype(vfmt);
       if fmttype='DATE' then header=cats(varnm,':date9.');
       else if fmttype='DATETIME' then header=cats(varnm,':E8601DT26.6');
       else if fmttype='TIME' then header=cats(varnm,':TIME12.');
-      else header=cats(varnm,':best.');
+      /**
+        * there is not much point importing a short length numeric like this,
+        * eg with best4., as the resulting variable will still be stored as
+        * length 8.  We need a length or format statement to ensure variable
+        * is created with the smaller length...
+        **/
+      else if vlen<8 then header=cats(varnm,':best',vlen,'.');
+      else header=cats(varnm,':best32.');
     end;
   %end;
   %else %do;
@@ -151,6 +159,7 @@ data _null_;
   set &ds end=last;
 %do i=1 %to &vcnt;
   %let var=%scan(&varlist,&i);
+  %local vlen&i;
   %if %mf_getvartype(&ds,&var)=C %then %do;
     %let dsv1=%mf_getuniquename(prefix=csvcol1_);
     %let dsv2=%mf_getuniquename(prefix=csvcol2_);

base/mp_filtercheck.sas

@@ -86,15 +86,14 @@
 /**
   * Sanitise the values based on valid value lists, then strip out
   * quotes, commas, periods and spaces.
-  * Only numeric values should remain
   */
 %local reason_cd nobs;
 %let nobs=0;
 data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
-  set &inds;
+  set &inds end=last;
-  length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000;
+  length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000;
   drop tmp;
 
   /* quick check to ensure column exists */
@@ -110,7 +109,8 @@ data &outds;
   end;
 
   /* need to open the dataset to get the column type */
-  dsid=open("&targetds","i");
+  retain dsid;
+  if _n_=1 then dsid=open("&targetds","i");
   if dsid>0 then do;
     vnum=varnum(dsid,VARIABLE_NM);
     if vnum<1 then do;
@@ -120,11 +120,19 @@ data &outds;
       call symputx('reason_cd',reason_cd,'l');
       call symputx('nobs',_n_,'l');
       output;
-      return;
+      goto endstep;
     end;
     /* now we can get the type */
     else vtype=vartype(dsid,vnum);
   end;
+  else do;
+      REASON_CD=cats("Could not open &targetds");
+      putlog REASON_CD= dsid=;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
+      output;
+      stop;
+  end;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
@@ -159,9 +167,8 @@ data &outds;
   end;
 
   /* special missing logic */
-  if vtype='N'
+  if vtype='N' & OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE') then do;
-    and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE')
+    if cats(upcase(raw_value)) in (
-    and cats(upcase(raw_value)) in (
       '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N'
       '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._'
     )
@@ -169,6 +176,32 @@ data &outds;
       /* valid numeric - exit data step loop */
       return;
     end;
+    else if subpad(upcase(raw_value),1,1) in (
+      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+    )
+    then do;
+      /* check if the raw_value contains a valid variable NAME */
+      vnum=varnum(dsid,subpad(raw_value,1,32));
+      if vnum>0 then do;
+        /* now we can get the type */
+        vtype2=vartype(dsid,vnum);
+        /* check type matches */
+        if vtype2=vtype then do;
+          /* valid target var - exit loop */
+          return;
+        end;
+        else do;
+          REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
+          putlog REASON_CD= dsid=;
+          call symputx('reason_cd',reason_cd,'l');
+          call symputx('nobs',_n_,'l');
+          output;
+          goto endstep;
+        end;
+      end;
+    end;
+  end;
 
   /* special logic */
   if OPERATOR_NM in ('IN','NOT IN','BETWEEN') then do;
@@ -189,6 +222,32 @@ data &outds;
     if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
       tmp=scan(raw_value1,i,',');
       if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
+        if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in (
+          'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+          'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+        )
+        then do;
+          /* check if the raw_value contains a valid variable NAME */
+          /* is not valid syntax for IN or NOT IN */
+          vnum=varnum(dsid,subpad(tmp,1,32));
+          if vnum>0 then do;
+            /* now we can get the type */
+            vtype2=vartype(dsid,vnum);
+            /* check type matches */
+            if vtype2=vtype then do;
+              /* valid target var - exit loop */
+              return;
+            end;
+            else do;
+              REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
+              putlog REASON_CD= dsid=;
+              call symputx('reason_cd',reason_cd,'l');
+              call symputx('nobs',_n_,'l');
+              output;
+              goto endstep;
+            end;
+          end;
+        end;
         REASON_CD='Non Numeric value provided';
         putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
         call symputx('reason_cd',reason_cd,'l');
@@ -213,14 +272,42 @@ data &outds;
 
   /* output records that contain values other than digits and spaces */
   if notdigit(compress(raw_value3,' '))>0 then do;
+    if vtype='C' and subpad(upcase(raw_value),1,1) in (
+      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+    )
+    then do;
+      /* check if the raw_value contains a valid variable NAME */
+      vnum=varnum(dsid,subpad(raw_value,1,32));
+      if vnum>0 then do;
+        /* now we can get the type */
+        vtype2=vartype(dsid,vnum);
+        /* check type matches */
+        if vtype2=vtype then do;
+          /* valid target var - exit loop */
+          return;
+        end;
+        else do;
+          REASON_CD=cats("Compared Char Type (",vtype2,") is not (",vtype,")");
+          putlog REASON_CD= dsid=;
+          call symputx('reason_cd',reason_cd,'l');
+          call symputx('nobs',_n_,'l');
+          output;
+          goto endstep;
+        end;
+      end;
+    end;
+
     putlog raw_value3= $hex32.;
     REASON_CD=cats('Invalid RAW_VALUE:',raw_value);
-    putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
+    putlog (_all_)(=);
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');
     output;
   end;
 
+  endstep:
+  if last then rc=close(dsid);
 run;
 
 

base/mp_getcols.sas

@@ -53,10 +53,11 @@ data &outds(keep=name type length varnum format label ddtype fmtname);
     else if formatd=0 then format=cats(fmtname,formatl,'.');
     else format=cats(fmtname,formatl,'.',formatd);
     type='N';
-    if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
+    if format=:'DATETIME' or format=:'E8601DT' or format=:'NLDATM'
+    then ddtype='DATETIME';
     else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
       or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
-      or format=:'MONYY'
+      or format=:'MONYY' or format=:'NLDATE'
     then ddtype='DATE';
     else if format=:'TIME' then ddtype='TIME';
     else ddtype='NUMERIC';

base/mp_hashdataset.sas

@@ -17,7 +17,7 @@
   @li mf_getattrn.sas
   @li mf_getuniquename.sas
   @li mf_getvarlist.sas
-  @li mp_md5.sas
+  @li mp_rowhash.sas
 
   <h4> Related Files </h4>
   @li mp_hashdataset.test.sas
@@ -46,8 +46,7 @@
 
 %local keyvar /* roll up the md5 */
   prevkeyvar /* retain prev record md5 */
-  lastvar /* last var in input ds */
+  lastvar /* last var in input ds */;
-  cvars nvars;
 
 %if not(%eval(%unquote(&iftrue))) %then %return;
 
@@ -79,10 +78,11 @@
     retain &prevkeyvar;
     if _n_=1 then &prevkeyvar=put(md5("&salt"),$hex32.);
     set &libds end=&lastvar;
-    /* hash should include previous row */
+    /* hash should include previous row - listed first so it is hashed first */
-    &keyvar=%mp_md5(
+    %mp_rowhash(
-      cvars=%mf_getvarlist(&libds,typefilter=C) &prevkeyvar,
+      md5_col=&keyvar
-      nvars=%mf_getvarlist(&libds,typefilter=N)
+      ,cvars=&prevkeyvar %mf_getvarlist(&libds,typefilter=C)
+      ,nvars=%mf_getvarlist(&libds,typefilter=N)
     );
     &prevkeyvar=&keyvar;
     if &lastvar then output;

base/mp_init.sas

@@ -37,12 +37,14 @@
 
 %global
   SASJS_PREFIX       /* the ONLY hard-coded global macro variable in SASjs    */
+  &prefix.PROCESSMODE &prefix._STPSRV_HEADER_LOC /* SASjs server specific vars*/
   &prefix._FUNCTIONS /* used in mcf_init() to track core function compilation */
   &prefix._INIT_NUM  /* initialisation time as numeric                        */
   &prefix._INIT_DTTM /* initialisation time in E8601DT26.6 format             */
   &prefix.WORK       /* avoid typing %sysfunc(pathname(work)) every time      */
+  &prefix.PROCESSMODE
+  &prefix._STPSRV_HEADER_LOC
 ;
-
 %let sasjs_prefix=&prefix;
 
 data _null_;
@@ -70,6 +72,7 @@ options
 %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
   noautocorrect           /* disallow misspelled procedure names            */
   dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
+                          /* turn off with dsoptions=nonote2err             */
 %end;
 ;
 

base/mp_loadformat.sas

@@ -9,6 +9,9 @@
   format, to prevent loss of data - UNLESS the input dataset contains a marker
   column, specifying that a particular row needs to be deleted (`delete_col=`).
 
+  Positions of formats are made using the FMTROW variable - this must be present
+  and unique (on TYPE / FMTNAME / FMTROW).
+
   This macro can also be used to identify which records would be (or were)
   considered new, modified or deleted (`loadtarget=`) by creating the following
   tables:
@@ -17,7 +20,7 @@
   @li work.outds_del
   @li work.outds_mod
 
-  For example usage, see mp_loadformat.test.sas
+  For example usage, see test (under Related Macros)
 
   @param [in] libcat The format catalog to be loaded
   @param [in] libds The staging table to load
@@ -34,12 +37,15 @@
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_existvar.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
+  @li mp_rowhash.sas
   @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
@@ -83,6 +89,16 @@
 %let libcat=%scan(&libcat,1,-);
 
 /* perform input validations */
+%mp_abort(
+  iftrue=(%mf_existds(&libds)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(&libds could not be found)
+)
+%mp_abort(
+  iftrue=(%mf_existvar(&libds,FMTROW)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(FMTROW not found in &libds)
+)
 %let err=0;
 %let msg=0;
 data _null_;
@@ -103,13 +119,6 @@ data _null_;
       stop;
     end;
   end;
-  else if name='LIBDS' then do;
-    if exist(value) le 0 then do;
-      call symputx('msg',"Unable to open staging table: "!!value);
-      call symputx('err',1);
-      stop;
-    end;
-  end;
   else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
   and missing(value) then do;
     call symputx('msg',"missing value in var: "!!name);
@@ -117,6 +126,14 @@ data _null_;
     stop;
   end;
 run;
+data _null_;
+  set &libds;
+  if missing(fmtrow) then do;
+    call symputx('msg',"missing fmtrow in format: "!!FMTNAME);
+    call symputx('err',1);
+    stop;
+  end;
+run;
 
 %mp_abort(
   iftrue=(&err ne 0)
@@ -124,6 +141,15 @@ run;
   ,msg=%str(&msg)
 )
 
+%local cnt;
+proc sql noprint;
+select count(distinct catx('|',type,fmtname,fmtrow)) into: cnt from &libds;
+%mp_abort(
+  iftrue=(&cnt ne %mf_nobs(&libds))
+  ,mac=&sysmacroname
+  ,msg=%str(Non-unique primary key on &libds)
+)
+
 /**
   * First, extract only relevant formats from the catalog
   */
@@ -148,14 +174,19 @@ select distinct
 %let nvars=FMTROW MIN MAX DEFAULT LENGTH FUZZ MULT NOEDIT;
 data &base_fmts/note2err;
   set &base_fmts;
-  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
+  length fmthash $32;
+  %mp_rowhash(
+    md5_col=fmthash
+    ,cvars=&cvars
+    ,nvars=&nvars
+  )
 run;
 
 /**
   * Ensure input table and base_formats have consistent lengths and types
   */
 data &inlibds/nonote2err;
-  length &delete_col $3 FMTROW 8 start end label $32767;
+  length &delete_col $3 FMTROW 8 start end label $32767 fmthash $32;
   if 0 then set &base_fmts;
   set &libds;
   by type fmtname notsorted;
@@ -177,13 +208,11 @@ data &inlibds/nonote2err;
     %mp_aligndecimal(end,width=16)
   end;
 
-  /* update row marker - retain new var as fmtrow may already be in libds */
+  %mp_rowhash(
-  if first.fmtname then row=1;
+    md5_col=fmthash
-  else row+1;
+    ,cvars=&cvars
-  drop row;
+    ,nvars=&nvars
-  fmtrow=row;
+  )
-
-  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
 run;
 
 /**

base/mp_md5.sas

@@ -28,11 +28,16 @@
   @li Global option:  `options dsoptions=nonote2err;`
   @li Data step option: `data YOURLIB.YOURDATASET /nonote2err;`
 
+  For very wide tables (hundreds or thousands of columns) consider using
+  `mp_rowhash.sas`, which builds the same type of row hash iteratively and
+  avoids creating a single, very long concatenated SAS expression.
+
   @param [in] cvars= () Space seperated list of character variables
   @param [in] nvars= () Space seperated list of numeric variables
 
   <h4> Related Programs </h4>
   @li mp_init.sas
+  @li mp_rowhash.sas
 
   @version 9.2
   @author Allan Bowe

base/mp_replace.sas

@@ -33,7 +33,8 @@
   LUA, you can also use this macro: mp_gsubfile.sas
 
   @param [in] infile The QUOTED path to the file on which to perform the
-    substitution
+    substitution.  Note that you can extract the pathname from a fileref using
+    the pathname function, eg: `"%sysfunc(pathname(fref))"`;
   @param [in] findvar= Macro variable NAME containing the string to search for
   @param [in] replacevar= Macro variable NAME containing the replacement string
   @param [out] outfile= (0) Optional QUOTED path to the adjusted output file (to
@@ -147,6 +148,6 @@ data _null_;
 run;
 
 /* END */
-%put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) seconds to run;
+/* %put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) secs to run; */
 
 %mend mp_replace;

base/mp_retainedkey.sas

@@ -148,7 +148,8 @@
 %mp_abort(iftrue=(&iserr=1),mac=mp_retainedkey,msg=%superq(msg))
 
 proc sql noprint;
-select sum(max(&retained_key),0) into: maxkey from &base_libds;
+select sum(max(&retained_key),0) format=32. into: maxkey trimmed
+  from &base_libds;
 
 /**
   * get base table RK and bus field values for lookup
@@ -217,19 +218,19 @@ quit;
   insert into &maxkeytable
     set keytable="&base_libds"
       ,keycolumn="&retained_key"
-      ,max_key=%eval(&maxkey+&newkey_cnt)
+      ,max_key=%sysevalf(&maxkey+&newkey_cnt)
       ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt;
   %end;
   %else %do;
   update &maxkeytable
-    set max_key=%eval(&maxkey+&newkey_cnt)
+    set max_key=%sysevalf(&maxkey+&newkey_cnt)
       ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt
     where keytable="&base_libds";
   %end;
   %mp_lockanytable(UNLOCK
     ,lib=%scan(&maxkeytable,1,.)
     ,ds=%scan(&maxkeytable,2,.)
-    ,ref=Updating maxkeyvalues with maxkey=%eval(&maxkey+&newkey_cnt)
+    ,ref=Updating maxkeyvalues with maxkey=%sysevalf(&maxkey+&newkey_cnt)
     ,ctl_ds=&locktable
   )
 %end;
@@ -237,7 +238,7 @@ quit;
 /* fill in the missing retained key values */
 %let tempvar=%mf_getuniquename();
 data &outds(drop=&tempvar);
-  retain &tempvar %eval(&maxkey+1);
+  retain &tempvar %sysevalf(&maxkey+1);
   set &tempds2;
   if &retained_key =. then &retained_key=&tempvar;
   &tempvar=&tempvar+1;

base/mp_rowhash.sas

@@ -0,0 +1,90 @@
+/**
+  @file
+  @brief Iterative row-level MD5 hash generator
+  @details Generates DATA step statements that compute a deterministic,
+    row-level MD5 hash using a Merkle-style construction.  Each variable is
+    hashed independently and the resulting raw 16-byte digests are combined
+    iteratively.  This avoids concatenating the per-variable hex digests into a
+    single SAS character expression, which overflows when datasets contain many
+    variables.
+
+    Temporary variables use five leading underscores.
+
+    This macro is called from inside a DATA step.  The caller is responsible
+    for declaring the output hash column if it does not already exist on the
+    input dataset.
+
+    Variables are hashed in the order supplied.  If a particular variable
+    needs to influence the hash first (for example the previous row hash in
+    `mp_hashdataset`, or business dates in Data Controller's bitemporal
+    loader) simply list it first in `cvars` or `nvars`.
+
+  @param [in] md5_col= Name of the output hash column.
+  @param [in] cvars= Space separated list of character variables to hash.
+  @param [in] nvars= Space separated list of numeric variables to hash.
+
+  @version 9.3M5
+  @author Allan Bowe
+**/
+
+%macro mp_rowhash(
+    md5_col=
+    ,cvars=
+    ,nvars=
+  );
+
+  /* DATA step temp variables use five leading underscores.  The names
+      are generated at macro invocation to avoid clashing with data columns. */
+  %local state digest pair numtext normal i chars nums;
+  %let state=%mf_getuniquename(prefix=_____state_);
+  %let digest=%mf_getuniquename(prefix=_____digest_);
+  %let pair=%mf_getuniquename(prefix=_____pair_);
+  %let numtext=%mf_getuniquename(prefix=_____numtext_);
+  %let normal=%mf_getuniquename(prefix=_____normal_);
+  %let i=%mf_getuniquename(prefix=_____i_);
+  %let chars=%mf_getuniquename(prefix=_____chars_);
+  %let nums=%mf_getuniquename(prefix=_____nums_);
+
+  length &state $16
+        &digest $16
+        &pair $32
+        &numtext $64
+        &normal &i 8;
+  if _n_=1 then call missing(&state,&digest,&pair,&numtext,&normal,&i);
+  drop &state &digest &pair &numtext &normal &i;
+
+  /* Versioned seed prevents confusion with other hashing schemes. */
+  &state = md5('DC HASH v2');
+
+  %if %length(&cvars)>0 %then %do;
+    array &chars{*} $ &cvars;
+    do &i = 1 to dim(&chars);
+      /* Leading blanks are retained. */
+      &digest = md5(trimn(&chars[&i]));
+      substr(&pair,  1, 16) = &state;
+      substr(&pair, 17, 16) = &digest;
+      &state = md5(&pair);
+    end;
+  %end;
+
+  %if %length(&nvars)>0 %then %do;
+    array &nums{*} &nvars;
+    do &i = 1 to dim(&nums);
+      /*
+        multiply-by-one for consistent cross-system precision.
+        Ignore null to protect SAS special missing values.
+        Cannot use IFN() as it evaluates both sides
+      */
+      if missing(&nums[&i]) then &normal = &nums[&i];
+      else &normal = &nums[&i] * 1;
+      &numtext = put(&normal, binary64.);
+      &digest = md5(trim(&numtext));
+      substr(&pair,  1, 16) = &state;
+      substr(&pair, 17, 16) = &digest;
+      &state = md5(&pair);
+    end;
+  %end;
+
+  &md5_col = put(&state, $hex32.);
+
+%mend mp_rowhash;

base/mp_stackdiffs.sas

@@ -197,6 +197,7 @@
   @li mp_coretable.sas
   @li mp_stackdiffs.test.sas
   @li mp_storediffs.sas
+  @li mp_stripdiffs.sas
 
   @todo The current approach assumes that a variable called KEY_HASH is not on
     the base table.  This part will need to be refactored (eg using

base/mp_storediffs.sas

@@ -64,6 +64,7 @@
   <h4> Related Macros </h4>
   @li mp_stackdiffs.sas
   @li mp_storediffs.test.sas
+  @li mp_stripdiffs.sas
 
   @version 9.2
   @author Allan Bowe
@@ -183,7 +184,7 @@ data &ds4;
 run;
 
 %if "&loadref"="0" %then %let loadref=%sysfunc(uuidgen());
-%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime());
+%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime(),8.6);
 %let libds=%upcase(&libds);
 
 /* join orig vals for modified & deleted */

base/mp_stripdiffs.sas

@@ -0,0 +1,256 @@
+/**
+  @file
+  @brief Generates a stage dataset to revert diffs tracked in an audit table
+  @details A big benefit of tracking data changes in an audit table is that
+    those changes can be subsequently reverted if necessary!
+
+    This macro prepares a staging dataset containing those differences - eg for:
+
+    @li deleted rows - these are re-inserted
+    @li changed rows - differences are reverted
+    @li added rows - marked with `_____DELETE__THIS__RECORD_____="YES"`
+
+    These changes are NOT applied to the base table - a staging dataset is
+    simply prepared for an ETL process to action.  In Data Controller, this
+    dataset is used directly as an input to the APPROVE process (so that the
+    reversion diffs can be reviewed prior to being applied).
+
+
+  @param [in] libds Base library.dataset (will not be modified).  The library
+    must be assigned.
+  @param [in] loadref Unique identifier for the version to be reverted.  This
+    change, plus ALL SUBSEQUENT CHANGES, will be reverted in the output table.
+  @param [in] difftable The dataset containing the diffs.  Definition available
+    in mddl_dc_difftable.sas
+  @param [in] filtervar= (0) If provided, the contents of this macro variable
+    will be applied as an additional filter against &libds
+  @param [out] outds= (work.mp_stripdiffs) Output table containing the diffs.
+    Has the same format as the base datset, plus a
+    `_____DELETE__THIS__RECORD_____` variable.
+  @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
+  @li mf_islibds.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_abort.sas
+  @li mp_ds2squeeze.sas
+
+  <h4> Related Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mp_stackdiffs.sas
+  @li mp_storediffs.sas
+  @li mp_stripdiffs.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mp_stripdiffs(libds
+  ,loadref
+  ,difftable
+  ,filtervar=0
+  ,outds=work.mp_stripdiffs
+  ,mdebug=0
+)/*/STORE SOURCE*/;
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+%let libds=%upcase(&libds);
+
+/* safety checks */
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc on entry.  Clean session required!)
+)
+%let libds=%upcase(&libds);
+%mp_abort(iftrue= (%mf_islibds(&libds)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid library.dataset reference - %superq(libds))
+)
+
+/* set up unique and temporary vars */
+%local ds1 ds2 ds3 ds4 ds5 fref1 filterstr;
+%let fref1=%mf_getuniquefileref();
+%if &filtervar ne 0 %then %let filterstr=%superq(&filtervar);
+%else %let filterstr=%str(1=1);
+
+/* get timestamp of the diff to be reverted */
+%local ts;
+proc sql noprint;
+select put(processed_dttm,datetime19.6) into: ts
+  from &difftable where load_ref="&loadref";
+%mp_abort(iftrue= (&sqlobs=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Load ref %superq(loadref) not found!)
+)
+
+/* extract diffs for this base table from this timestamp onwards */
+%let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_diffs));
+create table &ds1 (drop=libref dsn) as
+  select * from &difftable
+  where upcase(cats(libref))="%scan(&libds,1,.)"
+  and upcase(cats(dsn))="%scan(&libds,2,.)"
+  and processed_dttm ge "&ts"dt
+  order by processed_dttm desc, key_hash, is_pk;
+
+/* extract key values only */
+%let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_pks));
+%local keyhash processed;
+%let keyhash=%upcase(%mf_getuniquename(prefix=mpsdvar_keyhash));
+%let processed=%upcase(%mf_getuniquename(prefix=mpsdvar_processed));
+create table &ds2 as
+  select key_hash as &keyhash,
+    tgtvar_nm,
+    tgtvar_type,
+    coalescec(oldval_char,newval_char) as charval,
+    coalesce(oldval_num, newval_num) as numval,
+    processed_dttm as &processed
+  from &ds1
+  where is_pk=1
+  order by &keyhash, &processed;
+
+/* grab pk values */
+%local pk;
+select distinct upcase(tgtvar_nm) into: pk separated by ' ' from &ds2;
+
+%let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_keychar));
+proc transpose data=&ds2(where=(tgtvar_type='C'))
+    out=&ds3(drop=_name_);
+  by &keyhash &processed;
+  id TGTVAR_NM;
+  var charval;
+run;
+
+%let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_keynum));
+proc transpose data=&ds2(where=(tgtvar_type='N'))
+    out=&ds4(drop=_name_);
+  by &keyhash &processed;
+  id TGTVAR_NM;
+  var numval;
+run;
+/* shorten the lengths */
+%mp_ds2squeeze(&ds3,outds=&ds3)
+%mp_ds2squeeze(&ds4,outds=&ds4)
+
+/* now merge to get all key values and de-dup */
+%let ds5=%upcase(work.%mf_getuniquename(prefix=mpsd_merged));
+data &ds5;
+  length &keyhash $32 &processed 8;
+  merge &ds3 &ds4;
+  by &keyhash &processed;
+  if not missing(&keyhash);
+run;
+proc sort data=&ds5 nodupkey;
+  by &pk;
+run;
+
+/* join to base table for preliminary stage DS */
+proc sql;
+create table &outds as select "No " as _____DELETE__THIS__RECORD_____
+  %do x=1 %to %sysfunc(countw(&pk,%str( )));
+    ,a.%scan(&pk,&x,%str( ))
+  %end;
+  %local notpkcols;
+  %let notpkcols=%upcase(%mf_getvarlist(&libds));
+  %let notpkcols=%mf_wordsinstr1butnotstr2(str1=&notpkcols,str2=&pk);
+  %do x=1 %to %sysfunc(countw(&notpkcols,%str( )));
+    ,b.%scan(&notpkcols,&x,%str( ))
+  %end;
+  from &ds5 a
+  left join &libds (where=(&filterstr)) b
+  on 1=1
+%do x=1 %to %sysfunc(countw(&pk,%str( )));
+  and a.%scan(&pk,&x,%str( ))=b.%scan(&pk,&x,%str( ))
+%end;
+;
+
+/* create SAS code to apply to stage_ds */
+data _null_;
+  set &ds1;
+  file &fref1 lrecl=33000;
+  length charval $32767;
+  if _n_=1 then put 'proc sql noprint;';
+  by descending processed_dttm key_hash is_pk;
+  if move_type='M' then do;
+    if first.key_hash then do;
+      put "update &outds set " @@;
+    end;
+    if IS_PK=0 then do;
+      put "  " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+      if not last.is_pk then put ',';
+    end;
+    else do;
+      if first.is_pk then put "  where 1=1 " @@;
+      put "  and " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+  end;
+  else if move_type='A' then do;
+    if first.key_hash then do;
+      put "update &outds set _____DELETE__THIS__RECORD_____='Yes' where 1=1 "@@;
+    end;
+    /* gating if - as only need PK now */
+    if is_pk=1;
+    put '  AND ' tgtvar_nm '=' @@;
+    cnt=count(newval_char,'"');
+    charval=quote(trim(substr(newval_char,1,32765-cnt)));
+    if tgtvar_type='C' then put charval @@;
+    else put newval_num @@;
+  end;
+  else if move_type='D' then do;
+    if first.key_hash then do;
+      put "update &outds set _____DELETE__THIS__RECORD_____='No' " @@;
+    end;
+    if IS_PK=0 then do;
+      put "  ," tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+    else do;
+      if first.is_pk then put "  where 1=1 " @@;
+      put "  and " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+  end;
+  if last.key_hash then put ';';
+run;
+
+/* apply the modification statements */
+%inc &fref1/source2 lrecl=33000;
+
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &ds1, &ds2, &ds3, &ds4, &ds5;
+  file &fref1 clear;
+%end;
+%else %do;
+  data _null_;
+    infile &fref1;
+    input;
+    if _n_=1 then putlog "Contents of SQL adjustments";
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_stripdiffs;
+/** @endcond */

base/mp_validatecol.sas

@@ -1,6 +1,6 @@
 /**
   @file
-  @brief Used to validate variables in a dataset
+  @brief Used to validate values in a data step
   @details Useful when sanitising inputs, to ensure that they arrive with a
   certain pattern.
   Usage:
@@ -27,6 +27,7 @@
   @param [in] incol The column to be validated
   @param [in] rule The rule to apply.  Current rules:
     @li ISINT - checks if the variable is an integer
+    @li ISLIB - checks if the value is a valid libref (NOT whether it exists)
     @li ISNUM - checks if the variable is numeric
     @li LIBDS - matches LIBREF.DATASET format
     @li FORMAT - checks if the provided format is syntactically valid
@@ -57,7 +58,7 @@
 %end;
 %else %if &rule=ISNUM %then %do;
   /*
-    credit SØREN LASSEN
+    credit SOREN LASSEN
     https://sasmacro.blogspot.com/2009/06/welcome-isnum-macro.html
   */
   &tempcol=input(&incol,?? best32.);
@@ -65,6 +66,19 @@
   else &outcol=1;
   drop &tempcol;
 %end;
+%else %if &rule=ISLIB %then %do;
+  if _n_=1 then do;
+    retain &tempcol;
+    &tempcol=prxparse('/^[_a-z]\w{0,7}$/i');
+    if missing(&tempcol) then do;
+      putlog 'ERR' +(-1) "OR: Invalid expression for ISLIB";
+      stop;
+    end;
+    drop &tempcol;
+  end;
+  if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
+  else &outcol=0;
+%end;
 %else %if &rule=LIBDS %then %do;
   /* match libref.dataset */
   if _n_=1 then do;

ddl/mddl_sas_cntlout.sas

@@ -6,6 +6,22 @@
   (given various practical restrictions) are described here to enable
   consistency when dealing with format data.
 
+  The HLO variable may have a number of values, documented here due to the
+  256 char label description length limit:
+
+    F=Standard format/informat.
+    H=Range ending value is HIGH.
+    I=Numeric informat.
+    J=Justification for an informat.
+    L=Range starting value is LOW.
+    M=MultiLabel.
+    N=Format or informat has no ranges, including no OTHER= range.
+    O=Range is OTHER.
+    R=ROUND option is in effect.
+    S=Specifies that NOTSORTED is in effect.
+    U=Specifies that the UPCASE option for an informat be used.
+
+
 **/
 
 
@@ -13,9 +29,11 @@
 
   proc sql;
   create table &libds(
-    TYPE char(1) label='Type of format - either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
+    TYPE char(1) label=
+'Format Type: either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
     ,FMTNAME char(32)     label='Format name'
-    ,FMTROW num label='CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
+    ,FMTROW num label=
+'CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
     ,START char(32767)    label='Starting value for format'
     /*
       Keep lengths of START and END the same to avoid this err:
@@ -35,18 +53,8 @@
     ,NOEDIT num length=3  label='Is picture string noedit?'
     ,SEXCL char(1)        label='Start exclusion'
     ,EEXCL char(1)        label='End exclusion'
-    ,HLO char(13)         label='Additional information.
+    ,HLO char(13) label=
-F=Standard format/informat.
+'More info: https://core.sasjs.io/mddl__sas__cntlout_8sas_source.html'
-H=Range ending value is HIGH.
-I=Numeric informat.
-J=Justification for an informat.
-L=Range starting value is LOW.
-M=MultiLabel.
-N=Format or informat has no ranges, including no OTHER= range.
-O=Range is OTHER.
-R=ROUND option is in effect.
-S=Specifies that NOTSORTED is in effect.
-U=Specifies that the UPCASE option for an informat be used.'
     ,DECSEP char(1)       label='Decimal separator'
     ,DIG3SEP char(1)      label='Three-digit separator'
     ,DATATYPE char(8)     label='Date/time/datetime?'

fcmp/mcf_getfmttype.sas

@@ -80,8 +80,8 @@ function mcf_getfmttype(fmtnm $) $8;
     /* apply lookups */
     if cats(fmt) in ('DATETIME','B8601DN','B8601DN','B8601DT','B8601DT'
       ,'B8601DZ','B8601DZ','DATEAMPM','DTDATE','DTMONYY','DTWKDATX','DTYEAR'
-      ,'DTYYQC','E8601DN','E8601DN','E8601DT','E8601DT','E8601DZ','E8601DZ')
+      ,'DTYYQC','E8601DN','E8601DN','E8601DT','E8601DT','E8601DZ','E8601DZ'
-      then return('DATETIME');
+      ,'NLDATM') then return('DATETIME');
     else if fmt in ('DATE','YYMMDD','B8601DA','B8601DA','DAY','DDMMYY'
       ,'DDMMYYB','DDMMYYC','DDMMYYD','DDMMYYN','DDMMYYP','DDMMYYS','DDMMYYx'
       ,'DOWNAME','E8601DA','E8601DA','JULDAY','JULIAN','MMDDYY','MMDDYYB'
@@ -92,7 +92,7 @@ function mcf_getfmttype(fmtnm $) $8;
       ,'YYMMD','YYMMDDB','YYMMDDC','YYMMDDD','YYMMDDN','YYMMDDP','YYMMDDS'
       ,'YYMMDDx','YYMMN','YYMMP','YYMMS','YYMMx','YYMON','YYQ','YYQC','YYQD'
       ,'YYQN','YYQP','YYQR','YYQRC','YYQRD','YYQRN','YYQRP','YYQRS','YYQRx'
-      ,'YYQS','YYQx','YYQZ') then return('DATE');
+      ,'YYQS','YYQx','YYQZ','NLDATE') then return('DATE');
     else if fmt in ('TIME','B8601LZ','B8601LZ','B8601TM','B8601TM','B8601TZ'
       ,'B8601TZ','E8601LZ','E8601LZ','E8601TM','E8601TM','E8601TZ','E8601TZ'
       ,'HHMM','HOUR','MMSS','TIMEAMPM','TOD') then return('TIME');

meta/mm_assigndirectlib.sas

@@ -54,7 +54,8 @@
 
 %local cur_engine;
 %let cur_engine=%mf_getengine(&libref);
-%if &cur_engine ne META and &cur_engine ne %then %do;
+%if &cur_engine ne META and &cur_engine ne and %length(&open_passthrough)=0
+%then %do;
   %put NOTE:  &libref already has a direct (&cur_engine) libname connection;
   %return;
 %end;
@@ -210,9 +211,10 @@ run;
   %end;
 %end;
 %else %if &engine=ODBC %then %do;
-  &mD.%put NOTE: Retrieving ODBC connection details;
+  %&mD.put NOTE: Retrieving ODBC connection details;
   data _null_;
-    length connx_uri conprop_uri value datasource up_uri schema $256.;
+    length connx_uri conprop_uri value datasource up_uri schema domprop_uri
+      authdomain $256.;
     call missing (of _all_);
     /* get source connection ID */
     rc=metadata_getnasn("&liburi",'LibraryConnection',1,connx_uri);
@@ -227,6 +229,13 @@ run;
         rc2=-1;
       end;
     end;
+
+    /* get auth domain */
+    autrc=metadata_getnasn(connx_uri,"Domain",1,domprop_uri);
+    arc=metadata_getattr(domprop_uri,"Name",authdomain);
+    if not missing(authdomain) then authdomain=cats('AUTHDOMAIN=',authdomain);
+    call symputx('authdomain',authdomain,'l');
+
     /* get SCHEMA */
     rc6=metadata_getnasn("&liburi",'UsingPackages',1,up_uri);
     rc7=metadata_getattr(up_uri,'SchemaName',schema);
@@ -243,7 +252,7 @@ run;
       (INSERT_SQL=YES DATASRC=&sql_dsn. CONNECTION=global);
   %end;
   %else %do;
-    libname &libref ODBC DATASRC=&sql_dsn SCHEMA=&sql_schema;
+    libname &libref ODBC DATASRC=&sql_dsn SCHEMA=&sql_schema &authdomain;
   %end;
 %end;
 %else %if &engine=POSTGRES %then %do;
@@ -405,6 +414,55 @@ run;
 
   libname &libref SQLSVR datasrc=&path schema=&schema user="&user" pass="&pass";
 %end;
+%else %if &engine=SASIOSNF or &engine=SNOW %then %do;
+  %&mD.put NOTE: Retrieving SNOW connection details;
+  data _null_;
+    length connx_uri conprop_uri value server up_uri schema domprop_uri
+      authdomain database $256.;
+    call missing (of _all_);
+    /* get source connection ID */
+    rc=metadata_getnasn("&liburi",'LibraryConnection',1,connx_uri);
+    /* get connection properties */
+    i=0;
+    do until (rc2<0);
+      i+1;
+      rc2=metadata_getnasn(connx_uri,'Properties',i,conprop_uri);
+      rc3=metadata_getattr(conprop_uri,'Name',value);
+      if value='Connection.DBMS.Property.SERVER.Name.xmlKey.txt' then do;
+        rc4=metadata_getattr(conprop_uri,'DefaultValue',server);
+        rc2=-1;
+      end;
+    end;
+
+    /* get auth domain */
+    autrc=metadata_getnasn(connx_uri,"Domain",1,domprop_uri);
+    arc=metadata_getattr(domprop_uri,"Name",authdomain);
+    if not missing(authdomain) then authdomain=cats('AUTHDOMAIN=',authdomain);
+    call symputx('authdomain',authdomain,'l');
+
+    /* get SCHEMA */
+    rc6=metadata_getnasn("&liburi",'UsingPackages',1,up_uri);
+    rc7=metadata_getattr(up_uri,'SchemaName',schema);
+    &mD.put rc= connx_uri= rc2= conprop_uri= rc3= value= rc4= server=
+      rc6= up_uri= rc7= schema=;
+
+    /* get database value */
+    prop='Connection.DBMS.Property.DB.Name.xmlKey.txt';
+    rc=metadata_getprop("&liburi",prop,database,"");
+    if database^='' then database='database='!!quote(trim(database));
+    call symputx('database',database,'l');
+
+    call symputx('snow_schema',schema,'l');
+    call symputx('snow_server',server,'l');
+  run;
+
+  libname &libref SNOW SERVER="&snow_server" SCHEMA=&snow_schema &authdomain
+    &database;
+  %if %length(&open_passthrough)>0 %then %do;
+    proc sql;
+    connect using &libref as &open_passthrough;
+  %end;
+%end;
 %else %if &engine=TERADATA %then %do;
   %put NOTE: Obtaining &engine library details;
   data _null;

meta/mm_createstp.sas

@@ -67,6 +67,11 @@
     a conflict)
   @param [in] repo= ServerContext is tied to a repo, if you are not using the
     foundation repo then select a different one here
+  @param [in] LogicalServerType= (Sps) Server Type to use. Valid options:
+    @li Any - Uses the default server.
+    @li Sps - Stored Process Server, best choice for web app development. Runs
+      under a system account identity (eg sassrv).
+    @li Wks - Workspace Server.  Runs under the end user identity.
 
   @returns outds  dataset containing the following columns:
     - stpuri
@@ -106,6 +111,7 @@
     ,minify=NO
     ,frefin=mm_in
     ,frefout=mm_out
+    ,LogicalServerType=Sps
 )/*/STORE SOURCE*/;
 
 %local mD;
@@ -122,6 +128,17 @@
 
 %mp_dropmembers(%scan(&outds,2,.))
 
+/* check LogicalServerType validity */
+%mp_abort(
+  iftrue=(
+    &LogicalServerType ne Sps
+    and &LogicalServerType ne Wks
+    and &LogicalServerType ne Any
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for LogicalServerType (&LogicalServerType))
+)
+
 /**
   * check tree exists
   */
@@ -354,8 +371,9 @@ run;
     '  <TextStore IsHidden="0" Name="Stored Process" UsageVersion="0" '/
     '    TextRole="StoredProcessConfiguration" TextType="XML" '/
     '    StoredText="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&qu'@@
-    'ot;?&gt;&lt;StoredProcess&gt;&lt;ServerContext LogicalServerType=&quot;S'@@
+    'ot;?&gt;&lt;StoredProcess&gt;&lt;ServerContext LogicalServerType=&quot;'@@
-    'ps&quot; OtherAllowed=&quot;false&quot;/&gt;&lt;ResultCapabilities Packa'@@
+    "&LogicalServerType"@@
+    '&quot; OtherAllowed=&quot;false&quot;/&gt;&lt;ResultCapabilities Packa'@@
     'ge=&quot;' @@ "&package" @@ '&quot; Streaming=&quot;' @@ "&streaming" @@
     '&quot;/&gt;&lt;OutputParameters/&gt;&lt;/StoredProcess&gt;" />' /
     "  </Notes> "/

meta/mm_getdetails.sas

@@ -1,22 +1,25 @@
 /**
   @file mm_getdetails.sas
   @brief extracts metadata attributes and associations for a particular uri
-
   @param [in] uri the metadata object for which to return
     attributes / associations
+  @param [in] sortoptions= Enables sorting of the output datasets, for example,
+    `SORTSEQ=LINGUISTIC`
   @param [out] outattrs= (work.attributes)
     The dataset to create that contains the list of attributes
   @param [out] outassocs= (work.associations)
     The dataset to contain the list of associations
 
-  @version 9.2
+  <h4> Related Files </h4>
-  @author Allan Bowe
+  @li mm_getobjects.sas
+  @li mm_gettypes.sas
 
 **/
 
 %macro mm_getdetails(uri
   ,outattrs=work.attributes
   ,outassocs=work.associations
+  ,sortoptions=
 )/*/STORE SOURCE*/;
 
 data &outassocs;
@@ -41,7 +44,7 @@ data &outassocs;
     n1+1;
   end;
 run;
-proc sort;
+proc sort &sortoptions;
   by assoc name;
 run;
 
@@ -61,7 +64,7 @@ data &outattrs;
     n1+1;
   end;
 run;
-proc sort;
+proc sort &sortoptions;
   by type name;
 run;
 

meta/mm_getgroups.sas

@@ -11,10 +11,11 @@
 
   @param [in] user= the metadata user to return groups for.  Leave blank for all
     groups.
-  @param [in] repo= the metadata repository that contains the user/group
+  @param [in] repo= (foundation) the metadata repository that contains the
-    information
+    user/group information
-  @param [in] mDebug= set to 1 to show debug messages in the log
+  @param [in] mDebug= (0) set to 1 to show debug messages in the log
-  @param [out] outds= the dataset to create that contains the list of groups
+  @param [out] outds= (work.mm_getgroups) The dataset to create that contains
+    the list of groups
 
   @returns outds  dataset containing all groups in a column named "metagroup"
     - groupuri

package.json

@@ -7,7 +7,7 @@
     "Viya",
     "SASjs"
   ],
-  "author": "Allan Bowe <support@macropeople.com>",
+  "author": "Allan Bowe",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/sasjs/core.git"
@@ -26,13 +26,13 @@
   "homepage": "https://core.sasjs.io",
   "main": "index.js",
   "scripts": {
-    "build": "sasjs cbd -t server",
+    "build": "npx @sasjs/cli cbd -t server",
-    "docs": "sasjs doc -t docsonly && ./sasjs/utils/build.sh",
+    "docs": "npx @sasjs/cli doc -t docsonly && ./sasjs/utils/build.sh",
-    "test": "sasjs test -t server",
+    "test": "npx @sasjs/cli test -t server",
-    "lint": "sasjs lint",
+    "lint": "npx @sasjs/cli lint",
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
-  "devDependencies": {
+  "maintainers": [
-    "@sasjs/cli": "^4.4.1"
+    "https://sasapps.io"
-  }
+  ]
 }

sasjs/doxy/new_header.html

@@ -21,6 +21,7 @@
   <link href="$relpath^tabs.css" rel="stylesheet" type="text/css" />
   <script type="text/javascript" src="$relpath^jquery.js"></script>
   <script type="text/javascript" src="$relpath^dynsections.js"></script>
+  <script type="text/javascript" src="$relpath^cookie.js"></script>
   $treeview $search $mathjax
   <link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
   <link rel="shortcut icon" href="$relpath^favicon.ico" type="image/x-icon" />
@@ -39,6 +40,10 @@
       g.async = true; g.src = u + 'matomo.js'; s.parentNode.insertBefore(g, s);
     })();
   </script>
+  <noscript>
+    <p><img referrerpolicy="no-referrer-when-downgrade" src="https://analytics.4gl.io/matomo.php?idsite=6&amp;rec=1"
+        style="border:0;" alt="" /></p>
+  </noscript>
   <!-- End Matomo Code -->
 </head>
 
@@ -80,7 +85,3 @@
     </div>
     <!--END TITLEAREA-->
     <!-- end header part -->
-  </div>
-</body>
-
-</html>

sasjs/sasjsconfig.json

@@ -5,7 +5,10 @@
     "ddl",
     "fcmp",
     "lua",
+    "meta",
+    "metax",
     "server",
+    "viya",
     "xplatform",
     "tests/base",
     "tests/ddlonly",
@@ -22,10 +25,7 @@
   },
   "testConfig": {
     "initProgram": "tests/testinit.sas",
-    "termProgram": "tests/testterm.sas",
+    "termProgram": "tests/testterm.sas"
-    "macroVars": {
-      "mcTestAppLoc": "/Public/temp/macrocore"
-    }
   },
   "defaultTarget": "server",
   "targets": [
@@ -37,8 +37,11 @@
         "allowInsecureRequests": false
       },
       "appLoc": "/Public/app/macrocore",
+      "deployConfig": {
+        "deployServicePack": true,
+        "deployScripts": []
+      },
       "macroFolders": [
-        "viya",
         "tests/viyaonly"
       ],
       "contextName": "SAS Job Execution compute context"
@@ -52,8 +55,6 @@
       },
       "appLoc": "/Shared Data/temp/macrocore",
       "macroFolders": [
-        "meta",
-        "metax",
         "tests/sas9only"
       ],
       "programFolders": [],
@@ -67,7 +68,7 @@
     },
     {
       "name": "server",
-      "serverUrl": "https://sas9.4gl.io",
+      "serverUrl": "https://sas.4gl.io",
       "serverType": "SASJS",
       "httpsAgentOptions": {
         "allowInsecureRequests": false
@@ -78,7 +79,6 @@
         "deployScripts": []
       },
       "macroFolders": [
-        "server",
         "tests/serveronly"
       ]
     },

sasjs/utils/build.sh

@@ -1,22 +1,27 @@
 #!/bin/bash
 ####################################################################
-# PROJECT: Macro Core Docs Build                                   #
+# PROJECT: SASjs Core Docs Build
-# To execute, use the npm command (npm run docs)                   #
+# To execute, use the npm command (npm run docs)
+# Target repo will have github action to create sitemap
+# https://github.com/marketplace/actions/generate-sitemap
 ####################################################################
 
 # refresh github pages site
 rm -rf sasjsbuild/docsite
 git clone git@github.com:sasjs/core.github.io.git sasjsbuild/docsite
-rm -rf sasjsbuild/docsite/*
+rm -rf sasjsbuild/docsite/*.html
-mv sasjsbuild/docs/* sasjsbuild/docsite/
+rm -rf sasjsbuild/docsite/*.js
+rm -rf sasjsbuild/docsite/*.png
+rm -rf sasjsbuild/docsite/*.dot
+rm -rf sasjsbuild/docsite/*.css
+rm -rf sasjsbuild/docsite/*.svg
+rm -rf search
+cp -R sasjsbuild/docs/* sasjsbuild/docsite/
 cd sasjsbuild/docsite/
+git config user.name sasjs
 echo 'core.sasjs.io' > CNAME
 git add .
 git commit -m "build.sh build on $(date +%F:%H:%M:%S)"
 git push
-npx sitemap-generator-cli https://core.sasjs.io
-git add .
-git commit -m "adding sitemap"
-git push
 
 echo "check it out:  https://sasjs.github.io/core.github.io/files.html"

sasjs/utils/create_sas_package.sas

@@ -8,6 +8,12 @@
 
     Requires the server to have SSH keys.
 
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mp_gitadd.sas
+  @li mp_gitreleaseinfo.sas
+  @li mp_gitstatus.sas
+
   <h4> SAS Macros </h4>
   @li mp_gitadd.sas
   @li mp_gitreleaseinfo.sas
@@ -124,11 +130,15 @@ data members(compress=char);
   keep name name2 path;
 run;
 
+proc sort data=members;
+  by name name2;
+run;
+
 %let temp_options = %sysfunc(getoption(source)) %sysfunc(getoption(notes));
 options nosource nonotes;
 data _null_;
   set members;
-  by name notsorted;
+  by name;
 
   ord + first.name;
 

server/ms_triggerstp.sas

@@ -0,0 +1,231 @@
+/**
+  @file
+  @brief Triggers a SASjs Server STP using the /SASjsApi/stp/trigger endpoint
+  @details Triggers the STP and returns the sessionId
+
+  Example:
+
+      %ms_triggerstp(/some/stored/program
+        ,debug=131
+        ,outds=work.myresults
+      )
+
+  @param [in] pgm The full path to the Stored Program in SASjs Drive (_program
+    parameter)
+  @param [in] debug= (131) The value to supply to the _debug URL parameter
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] inputparams=(_null_) A dataset containing name/value pairs in the
+    following format:
+    |name:$32|value:$10000|
+    |---|---|
+    |stpmacname|some value|
+    |mustbevalidname|can be anything, oops, %abort!!|
+  @param [in] inputfiles= (_null_) A dataset containing fileref/name/filename in
+    the following format:
+    |fileref:$8|name:$32|filename:$256|
+    |---|---|--|
+    |someref|some_name|some_filename.xls|
+    |fref2|another_file|zyx_v2.csv|
+  @param [in] expiresaftermins= (15) The number of minutes to retain the session
+    folder after the session ends.
+
+  @param [out] outds= (work.ms_triggerstp) Set to the name of a dataset to
+  contain the sessionId. If this dataset already exists, and contains the
+  sessionId, it will be appended to.
+  Format:
+    |sessionId:$36|
+    |---|
+    |20241028074744-54132-1730101664824|
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+  @li mp_dropmembers.sas
+  @li mf_nobs.sas
+
+**/
+
+%macro ms_triggerstp(pgm
+    ,debug=131
+    ,inputparams=_null_
+    ,inputfiles=_null_
+    ,expiresAfterMins=15
+    ,outds=work.ms_triggerstp
+    ,mdebug=0
+  );
+  %local dbg mainref authref boundary libref triggered_sid;
+  %let mainref=%mf_getuniquefileref();
+  %let authref=%mf_getuniquefileref();
+  %let boundary=%mf_getuniquename();
+  %if &inputparams=0 %then %let inputparams=_null_;
+
+  %if &mdebug=1 %then %do;
+    %put &sysmacroname entry vars:;
+    %put _local_;
+  %end;
+  %else %let dbg=*;
+
+
+  %mp_abort(iftrue=("&pgm"="")
+    ,mac=&sysmacroname
+    ,msg=%str(Program not provided)
+  )
+  %mp_abort(iftrue=("&outds"="")
+    ,mac=&sysmacroname
+    ,msg=%str(Output dataset not provided)
+  )
+
+  /* avoid sending bom marker to API */
+  %local optval;
+  %let optval=%sysfunc(getoption(bomfile));
+  options nobomfile;
+
+  /* Add params to the content */
+  data _null_;
+    file &mainref termstr=crlf lrecl=32767 mod;
+    length line $1000 name $32 value $32767;
+    if _n_=1 then call missing(of _all_);
+    set &inputparams;
+    put "--&boundary";
+    line=cats('Content-Disposition: form-data; name="',name,'"');
+    put line;
+    put ;
+    put value;
+  run;
+
+  /* parse input file list */
+  %local webcount;
+  %let webcount=0;
+  data _null_;
+    set &inputfiles end=last;
+    length fileref $8 name $32 filename $256;
+    call symputx(cats('webref',_n_),fileref,'l');
+    call symputx(cats('webname',_n_),name,'l');
+    call symputx(cats('webfilename',_n_),filename,'l');
+    if last then do;
+      call symputx('webcount',_n_);
+      call missing(of _all_);
+    end;
+  run;
+
+  /* write out the input files to the content */
+  %local i;
+  %do i=1 %to &webcount;
+    data _null_;
+      file &mainref termstr=crlf lrecl=32767 mod;
+      infile &&webref&i lrecl=32767;
+      if _n_ = 1 then do;
+        length line $32767;
+        line=cats(
+          'Content-Disposition: form-data; name="'
+          ,"&&webname&i"
+          ,'"; filename="'
+          ,"&&webfilename&i"
+          ,'"'
+        );
+        put "--&boundary";
+        put line;
+        put "Content-Type: text/plain";
+        put ;
+      end;
+      input;
+      put _infile_; /* add the actual file to be sent */
+    run;
+  %end;
+
+  /* Add footer to the content */
+  data _null_;
+    file &mainref termstr=crlf mod;
+    put / "--&boundary--";
+  run;
+
+  data _null_;
+    file &authref lrecl=1000;
+    infile "&_sasjs_tokenfile" lrecl=1000;
+    input;
+    if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary";
+    put _infile_;
+  run;
+
+  %if &mdebug=1 %then %do;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** authref=&authref content *****";
+      infile &authref;
+      input;
+      put _infile_;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** mainref=&mainref content *****";
+      infile &mainref;
+      input;
+      put _infile_;
+    run;
+  %end;
+
+  %local resp_path outref;
+  %let resp_path=%sysfunc(pathname(work))/%mf_getuniquename();
+  %let outref=%mf_getuniquefileref();
+  filename &outref "&resp_path" lrecl=32767;
+
+  /* prepare request*/
+  proc http method='POST' headerin=&authref in=&mainref out=&outref
+    url="&_sasjs_apiserverurl/SASjsApi/stp/trigger?%trim(
+      )_program=&pgm%str(&)_debug=131%str(&)expiresAfterMins=&expiresaftermins";
+  %if &mdebug=1 %then %do;
+    debug level=2;
+  %end;
+  run;
+
+  %if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  or &mdebug=1
+  %then %do;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** outref=&outref content *****";
+      infile &outref;
+      input;
+      putlog _infile_;
+    run;
+  %end;
+  %mp_abort(
+    iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200
+        and &SYS_PROCHTTP_STATUS_CODE ne 201)
+    ,mac=&sysmacroname
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+
+  /* reset options */
+  options &optval;
+
+  %let libref=%mf_getuniquelibref();
+  libname &libref JSON fileref=&outref;
+  %let triggered_sid=%mf_getuniquename(prefix=triggered_sid_);
+
+  data work.&triggered_sid (keep=sessionid);
+    set &libref..root;
+
+    %if &mdebug=1 %then %do;
+      putlog (_all_)(=);
+    %end;
+  run;
+
+  %if %mf_nobs(work.&triggered_sid)>0 %then %do;
+    proc append base=&outds data=work.&triggered_sid;
+    run;
+  %end;
+
+  %if &mdebug=1 %then %do;
+    %put &sysmacroname exit vars:;
+    %put _local_;
+  %end;
+  %else %do;
+    /* clear refs */
+    filename &authref;
+    filename &mainref;
+    filename &outref;
+    libname &libref clear;
+    /* and remove temp dataset */
+    %mp_dropmembers(&triggered_sid,libref=work);
+  %end;
+
+%mend ms_triggerstp;

tests/base/mf_getfilesize.test.sas

@@ -14,7 +14,7 @@ data test;
 run;
 
 %mp_assertscope(SNAPSHOT)
-%put %mf_getfilesize(libds=work.test)
+%put %mf_getfilesize(libds=work.test);
 %mp_assertscope(COMPARE)
 
 %mp_assert(

tests/base/mf_getfmtlist.test.sas

@@ -5,9 +5,14 @@
   <h4> SAS Macros </h4>
   @li mf_getfmtlist.sas
   @li mp_assert.sas
+  @li mp_assertscope.sas
 
 **/
 
+%mp_assertscope(SNAPSHOT)
+%put %mf_getfmtlist(sashelp.prdsale);
+%mp_assertscope(COMPARE)
+
 %mp_assert(
   iftrue=(
     "%mf_getfmtlist(sashelp.prdsale)"="DOLLAR $CHAR W MONNAME"
@@ -23,11 +28,3 @@
   desc=Checking basic char,
   outds=work.test_results
 )
-
-%mp_assert(
-  iftrue=(
-    "%mf_getfmtlist(sashelp.demographics)"="BEST Z $CHAR COMMA PERCENTN"
-  ),
-  desc=Checking longer numeric,
-  outds=work.test_results
-)

tests/base/mf_getvalue.test.sas

@@ -0,0 +1,78 @@
+/**
+  @file
+  @brief Testing mf_getvalue macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getvalue.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+data work.test_data;
+  do i = 1 to 10;
+    output;
+  end;
+  stop;
+run;
+
+/* - Test 1 -
+  Get value from default first observation.
+  No filter.
+*/
+%mp_assertscope(SNAPSHOT)
+%let test_value=%mf_getvalue(work.test_data,i);
+%mp_assertscope(COMPARE,ignorelist=test_value)
+
+%mp_assert(
+  iftrue=(&test_value=1 and &syscc eq 0),
+  desc=Basic test fetching value from default first obs,
+  outds=work.test_results
+)
+
+/* - Test 2 -
+  Get value from 10th observation.
+  No filter.
+*/
+%let test_value=%mf_getvalue(work.test_data,i,fetchobs=10);
+%mp_assert(
+  iftrue=(&test_value=10 and &syscc eq 0),
+  desc=Test fetching value from specifically the 10th row,
+  outds=work.test_results
+)
+
+/* - Test 3 -
+  Get value from default first observation.
+  With filter.
+*/
+%let test_value=%mf_getvalue(work.test_data,i,filter=(i>4));
+%mp_assert(
+  iftrue=(&test_value=5 and &syscc eq 0),
+  desc=Test fetching value from default row of filtered data,
+  outds=work.test_results
+)
+
+/* - Test 4 -
+  Get value from specified observation.
+  With filter.
+*/
+%let test_value=%mf_getvalue(work.test_data,i,filter=(i>4),fetchobs=5);
+%mp_assert(
+  iftrue=(&test_value=9 and &syscc eq 0),
+  desc=Test fetching value from 5th row of filtered data,
+  outds=work.test_results
+)
+
+/* - Test 5 -
+  Get value from default observation.
+  Filter removes all rows. This simulates providing an empty dataset
+  or specifying an observation number beyond the set returned by the filter.
+*/
+%let test_value=%mf_getvalue(work.test_data,i,filter=(i>10));
+%mp_assert(
+  iftrue=(&test_value=%str() and &syscc eq 4),
+  desc=Test fetching value from 1st row of empty (filtered) data,
+  outds=work.test_results
+)
+%let syscc=0;
+

tests/base/mf_getvarlist.test.sas

@@ -15,7 +15,7 @@
 %let test5=%mf_getvarlist(sashelp.class,typefilter=C);
 
 data work.test_results;
-  length test_description $256 test_result $4 test_comments base result  $256;
+  length test_result $4 test_description $256 test_comments base result  $256;
   test_description="Basic test";
   base=symget('test1');
   result='Name Sex Age Height Weight';

tests/base/mf_mimetype.test.sas

@@ -0,0 +1,19 @@
+/**
+  @file
+  @brief Testing mf_mimetype macro
+
+  <h4> SAS Macros </h4>
+  @li mf_mimetype.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assertscope(SNAPSHOT)
+%let test_value=%mf_mimetype(CSV);
+%mp_assertscope(COMPARE,ignorelist=test_value)
+
+%mp_assert(
+  iftrue=("%mf_mimetype(XLS)"="application/vnd.ms-excel"),
+  desc=Checking correct value
+)
+

tests/base/mp_ds2csv.test.1.sas

@@ -0,0 +1,246 @@
+/**
+  @file
+  @brief Testing mp_ds2csv.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2csv.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+data work.somedata;
+  x=1;
+  y='  t"w"o';
+  z=.z;
+  label x='x factor';
+run;
+
+/**
+  * Test 1 - default CSV
+  */
+%mp_assertscope(SNAPSHOT)
+%mp_ds2csv(work.somedata,outfile="&sasjswork/test1.csv")
+%mp_assertscope(COMPARE)
+
+%let test1b=FAIL;
+data _null_;
+  infile "&sasjswork/test1.csv";
+  input;
+  list;
+  if _n_=1 then call symputx('test1a',_infile_);
+  else if _infile_=:'1,"  t""w""o",.Z' then call symputx('test1b','PASS');
+run;
+
+%mp_assert(
+  iftrue=("&test1a"="x factor, Y, Z"),
+  desc=Checking header row Test 1,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test1b"="PASS"),
+  desc=Checking data row Test 1,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - NAME header with fileref and semicolons
+  */
+filename test2 "&sasjswork/test2.csv";
+%mp_ds2csv(work.somedata,outref=test2,dlm=SEMICOLON,headerformat=NAME)
+
+%let test2b=FAIL;
+data _null_;
+  infile test2;
+  input;
+  list;
+  if _n_=1 then call symputx('test2a',_infile_);
+  else if _infile_=:'1;"  t""w""o";.Z' then call symputx('test2b','PASS');
+run;
+
+%mp_assert(
+  iftrue=("&test2a"="X; Y; Z"),
+  desc=Checking header row Test 2,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test2b"="PASS"),
+  desc=Checking data row Test 2,
+  outds=work.test_results
+)
+
+/**
+  * Test 3 - SASjs format
+  */
+filename test3 "&sasjswork/test3.csv";
+%mp_ds2csv(work.somedata,outref=test3,headerformat=SASJS)
+
+%let test3b=FAIL;
+data _null_;
+  infile test3;
+  input;
+  list;
+  if _n_=1 then call symputx('test3a',_infile_);
+  else if _infile_=:'1,"  t""w""o",.Z' then call symputx('test3b','PASS');
+run;
+
+%mp_assert(
+  iftrue=("&test3a"="X:best32. Y:$char7. Z:best32."),
+  desc=Checking header row Test 3,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test3b"="PASS"),
+  desc=Checking data row Test 3,
+  outds=work.test_results
+)
+
+/* test 4 - sasjs with compare */
+data work.baseball ;
+attrib
+Name                             length= $18 label="Player's Name"
+Team                             length= $14 label="Team at the End of 1986"
+nAtBat                           length= 8 label="Times at Bat in 1986"
+nHits                            length= 8 label="Hits in 1986"
+nHome                            length= 8 label="Home Runs in 1986"
+nRuns                            length= 8 label="Runs in 1986"
+nRBI                             length= 8 label="RBIs in 1986"
+nBB                              length= 8 label="Walks in 1986"
+YrMajor                          length= 8 label="Years in the Major Leagues"
+CrAtBat                          length= 8 label="Career Times at Bat"
+CrHits                           length= 8 label="Career Hits"
+CrHome                           length= 8 label="Career Home Runs"
+CrRuns                           length= 8 label="Career Runs"
+CrRbi                            length= 8 label="Career RBIs"
+CrBB                             length= 8 label="Career Walks"
+League                           length= $8 label="League at the End of 1986"
+Division                         length= $8 label="Division at the End of 1986"
+Position                         length= $8 label="Position(s) in 1986"
+nOuts                            length= 8 label="Put Outs in 1986"
+nAssts                           length= 8 label="Assists in 1986"
+nError                           length= 8 label="Errors in 1986"
+Salary                           length= 8 label="1987 Salary in $ Thousands"
+Div                              length= $16 label="League and Division"
+logSalary                        length= 8 label="Log Salary"
+;
+infile cards dsd;
+input
+  Name                             :$char.
+  Team                             :$char.
+  nAtBat
+  nHits
+  nHome
+  nRuns
+  nRBI
+  nBB
+  YrMajor
+  CrAtBat
+  CrHits
+  CrHome
+  CrRuns
+  CrRbi
+  CrBB
+  League                           :$char.
+  Division                         :$char.
+  Position                         :$char.
+  nOuts
+  nAssts
+  nError
+  Salary
+  Div                              :$char.
+  logSalary
+;
+missing a b c d e f g h i j k l m n o p q r s t u v w x y z _;
+/* fix precision issues */
+if not missing(logSalary) then logsalary=logSalary*1;
+datalines4;
+"Allanson, Andy",Cleveland,293,66,1,30,29,14,1,293,66,1,30,29,14,American,East,C,446,33,20,.,AE,.
+"Ashby, Alan",Houston,315,81,7,24,38,39,14,3449,835,69,321,414,375,National,West,C,632,43,10,475,NW,6.16331480403464
+"Davis, Alan",Seattle,479,130,18,66,72,76,3,1624,457,63,224,266,263,American,West,1B,880,82,14,480,AW,6.17378610390193
+"Dawson, Andre",Montreal,496,141,20,65,78,37,11,5628,1575,225,828,838,354,National,East,RF,200,11,3,500,NE,6.21460809842219
+"Galarraga, Andres",Montreal,321,87,10,39,42,30,2,396,101,12,48,46,33,National,East,1B,805,40,4,91.5,NE,4.51633897228147
+"Griffin, Alfredo",Oakland,594,169,4,74,51,35,11,4408,1133,19,501,336,194,American,West,SS,282,421,25,750,AW,6.62007320653035
+"Newman, Al",Montreal,185,37,1,23,8,21,2,214,42,1,30,9,24,National,East,2B,76,127,7,70,NE,4.24849524204936
+"Salazar, Argenis",Kansas City,298,73,0,24,24,7,3,509,108,0,41,37,12,American,West,SS,121,283,9,100,AW,4.60517018598809
+"Thomas, Andres",Atlanta,323,81,6,26,32,8,2,341,86,6,32,34,8,National,West,SS,143,290,19,75,NW,4.31748811353631
+"Thornton, Andre",Cleveland,401,92,17,49,66,65,13,5206,1332,253,784,890,866,American,East,DH,0,0,0,1100,AE,7.00306545878646
+"Trammell, Alan",Detroit,574,159,21,107,75,59,10,4631,1300,90,702,504,488,American,East,SS,238,445,22,517.143,AE,6.24831943200756
+"Trevino, Alex",Los Angeles,202,53,4,31,26,27,9,1876,467,15,192,186,161,National,West,C,304,45,11,512.5,NW,6.23930071101256
+"Van Slyke, Andy",St Louis,418,113,13,48,61,47,4,1512,392,41,205,204,203,National,East,RF,211,11,7,550,NE,6.30991827822651
+"Wiggins, Alan",Baltimore,239,60,0,30,11,22,6,1941,510,4,309,103,207,American,East,2B,121,151,6,700,AE,6.5510803350434
+"Almon, Bill",Pittsburgh,196,43,7,29,27,30,13,3231,825,36,376,290,238,National,East,UT,80,45,8,240,NE,5.48063892334199
+"Beane, Billy",Minneapolis,183,39,3,20,15,11,3,201,42,3,20,16,11,American,West,OF,118,0,0,.,AW,.
+"Bell, Buddy",Cincinnati,568,158,20,89,75,73,15,8068,2273,177,1045,993,732,National,West,3B,105,290,10,775,NW,6.65286302935334
+"Biancalana, Buddy",Kansas City,190,46,2,24,8,15,5,479,102,5,65,23,39,American,West,SS,102,177,16,175,AW,5.16478597392351
+"Bochte, Bruce",Oakland,407,104,6,57,43,65,12,5233,1478,100,643,658,653,American,West,1B,912,88,9,.,AW,.
+;;;;
+run;
+filename example temp lrecl=5000;
+%mp_ds2csv(work.baseball,outref=example,headerformat=SASJS)
+data _null_; infile example; input;put _infile_; if _n_>5 then stop;run;
+
+data _null_;
+  infile example;
+  input;
+  call symputx('stmnt',_infile_);
+  stop;
+run;
+data work.want;
+  infile example dsd firstobs=2;
+  input &stmnt;
+run;
+
+%mp_assert(
+  iftrue=(&syscc =0),
+  desc=Checking syscc prior to compare of work.baseball,
+  outds=work.test_results
+)
+
+proc compare base=want compare=work.baseball
+  method = absolute criterion = 0.0000000001   ;
+run;
+%mp_assert(
+  iftrue=(&sysinfo le 41),
+  desc=Checking compare of work.baseball,
+  outds=work.test_results
+)
+
+/* test 5 - sasjs with time/datetime/date */
+filename f2 temp;
+data work.test5;
+  do x=1 to 5;
+    y=x;
+    z=x;
+  end;
+  format x date9. y datetime19. z time.;
+run;
+%mp_ds2csv(work.test5,outref=f2,headerformat=SASJS)
+data _null_; infile example; input;put _infile_; if _n_>5 then stop;run;
+
+data _null_;
+  infile f2;
+  input;
+  putlog _infile_;
+  call symputx('stmnt2',_infile_);
+  stop;
+run;
+data work.want5;
+  infile f2 dsd firstobs=2;
+  input &stmnt2;
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking syscc prior to compare of test5,
+  outds=work.test_results
+)
+
+proc compare base=want5 compare=work.test5;
+run;
+%mp_assert(
+  iftrue=(&sysinfo le 41),
+  desc=Checking compare of work.test5,
+  outds=work.test_results
+)
+

tests/base/mp_ds2csv.test.2.sas

@@ -0,0 +1,43 @@
+/**
+  @file
+  @brief Testing mp_ds2csv.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2csv.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+data work.shortnum;
+  length a 3 b 4 c 8;
+  a=1;b=2;c=3;
+  output;
+  stop;
+run;
+
+/**
+  * Test 1 - default CSV
+  */
+
+%mp_ds2csv(work.shortnum,outfile="&sasjswork/test1.csv",headerformat=SASJS)
+
+%let test1b=FAIL;
+data _null_;
+  infile "&sasjswork/test1.csv";
+  input;
+  list;
+  if _n_=1 then call symputx('test1a',_infile_);
+  else if _infile_=:'1,2,3' then call symputx('test1b','PASS');
+run;
+
+%mp_assert(
+  iftrue=("&test1a"="A:best3. B:best4. C:best32."),
+  desc=Checking header row Test 1,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test1b"="PASS"),
+  desc=Checking data row Test 1,
+  outds=work.test_results
+)

tests/base/mp_ds2csv.test.sas

@@ -1,166 +0,0 @@
-/**
-  @file
-  @brief Testing mp_ds2csv.sas macro
-
-  <h4> SAS Macros </h4>
-  @li mp_ds2csv.sas
-  @li mp_assert.sas
-  @li mp_assertscope.sas
-
-**/
-
-data work.somedata;
-  x=1;
-  y='  t"w"o';
-  z=.z;
-  label x='x factor';
-run;
-
-/**
-  * Test 1 - default CSV
-  */
-%mp_assertscope(SNAPSHOT)
-%mp_ds2csv(work.somedata,outfile="&sasjswork/test1.csv")
-%mp_assertscope(COMPARE)
-
-%let test1b=FAIL;
-data _null_;
-  infile "&sasjswork/test1.csv";
-  input;
-  list;
-  if _n_=1 then call symputx('test1a',_infile_);
-  else if _infile_=:'1,"  t""w""o",.Z' then call symputx('test1b','PASS');
-run;
-
-%mp_assert(
-  iftrue=("&test1a"="x factor, Y, Z"),
-  desc=Checking header row Test 1,
-  outds=work.test_results
-)
-%mp_assert(
-  iftrue=("&test1b"="PASS"),
-  desc=Checking data row Test 1,
-  outds=work.test_results
-)
-
-/**
-  * Test 2 - NAME header with fileref and semicolons
-  */
-filename test2 "&sasjswork/test2.csv";
-%mp_ds2csv(work.somedata,outref=test2,dlm=SEMICOLON,headerformat=NAME)
-
-%let test2b=FAIL;
-data _null_;
-  infile test2;
-  input;
-  list;
-  if _n_=1 then call symputx('test2a',_infile_);
-  else if _infile_=:'1;"  t""w""o";.Z' then call symputx('test2b','PASS');
-run;
-
-%mp_assert(
-  iftrue=("&test2a"="X; Y; Z"),
-  desc=Checking header row Test 2,
-  outds=work.test_results
-)
-%mp_assert(
-  iftrue=("&test2b"="PASS"),
-  desc=Checking data row Test 2,
-  outds=work.test_results
-)
-
-/**
-  * Test 3 - SASjs format
-  */
-filename test3 "&sasjswork/test3.csv";
-%mp_ds2csv(work.somedata,outref=test3,headerformat=SASJS)
-
-%let test3b=FAIL;
-data _null_;
-  infile test3;
-  input;
-  list;
-  if _n_=1 then call symputx('test3a',_infile_);
-  else if _infile_=:'1,"  t""w""o",.Z' then call symputx('test3b','PASS');
-run;
-
-%mp_assert(
-  iftrue=("&test3a"="X:best. Y:$char7. Z:best."),
-  desc=Checking header row Test 3,
-  outds=work.test_results
-)
-%mp_assert(
-  iftrue=("&test3b"="PASS"),
-  desc=Checking data row Test 3,
-  outds=work.test_results
-)
-
-/* test 4 - sasjs with compare */
-filename example temp;
-%mp_ds2csv(sashelp.air,outref=example,headerformat=SASJS)
-data _null_; infile example; input;put _infile_; if _n_>5 then stop;run;
-
-data _null_;
-  infile example;
-  input;
-  call symputx('stmnt',_infile_);
-  stop;
-run;
-data work.want;
-  infile example dsd firstobs=2;
-  input &stmnt;
-run;
-
-%mp_assert(
-  iftrue=(&syscc =0),
-  desc=Checking syscc prior to compare of sashelp.air,
-  outds=work.test_results
-)
-
-proc compare base=want compare=sashelp.air;
-run;
-%mp_assert(
-  iftrue=(&sysinfo le 41),
-  desc=Checking compare of sashelp.air,
-  outds=work.test_results
-)
-
-/* test 5 - sasjs with time/datetime/date */
-filename f2 temp;
-data work.test5;
-  do x=1 to 5;
-    y=x;
-    z=x;
-  end;
-  format x date9. y datetime19. z time.;
-run;
-%mp_ds2csv(work.test5,outref=f2,headerformat=SASJS)
-data _null_; infile example; input;put _infile_; if _n_>5 then stop;run;
-
-data _null_;
-  infile f2;
-  input;
-  putlog _infile_;
-  call symputx('stmnt2',_infile_);
-  stop;
-run;
-data work.want5;
-  infile f2 dsd firstobs=2;
-  input &stmnt2;
-  putlog _infile_;
-run;
-
-%mp_assert(
-  iftrue=(&syscc=0),
-  desc=Checking syscc prior to compare of test5,
-  outds=work.test_results
-)
-
-proc compare base=want5 compare=work.test5;
-run;
-%mp_assert(
-  iftrue=(&sysinfo le 41),
-  desc=Checking compare of work.test5,
-  outds=work.test_results
-)
-

tests/base/mp_filtercheck.test.sas

@@ -53,7 +53,10 @@ AND,AND,1,age,=,.A
 AND,AND,1,height,<,.B
 AND,AND,1,age,IN,"(.a,.b,.)"
 AND,AND,1,age,IN,"(.A)"
-
+AND,AND,1,AGE,=,AGE
+AND,AND,1,AGE,<,Weight
+AND,AND,1,AGE,BETWEEN,"HEIGHT AND WEIGHT"
+AND,OR,2,Name,=,name
 ;;;;
 run;
 
@@ -204,3 +207,26 @@ run;
   outds=work.test_results
 )
 %let syscc=0;
+
+
+/* invalid IN value (cannot use var names) */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,AGE,NOT IN,"(height, age)"
+;;;;
+run;
+
+%mp_filtercheck(work.inds,
+  targetds=work.class,
+  outds=work.badrecords,
+  abort=NO
+)
+%let syscc=0;
+%mp_assertdsobs(work.badrecords,
+  desc=Invalid IN syntax,
+  test=HASOBS,
+  outds=work.test_results
+)

tests/base/mp_gitreleaseinfo.test.sas

@@ -24,7 +24,7 @@ data _null_;
 run;
 
 %mp_assert(
-  iftrue=(&author=sasjsbot),
+  iftrue=("&author"="github-actions[bot]"),
   desc=release info extracted successfully,
   outds=work.test_results
 )

tests/base/mp_loadformat.test.1.sas

@@ -189,8 +189,10 @@ data work.stagedata3;
     if last.fmtname then do;
       output; /* 6 new records */
       x=_n_;
-      x+1;start=cats("mod",x);end=start;label='newlabel1';output;
+      x+1;start=cats("mod",x);end=start;label='newlabel1';fmtrow=fmtrow+1;
-      x+1;start=cats("mod",x);end=start;label='newlabel2';output;
+      output;
+      x+1;start=cats("mod",x);end=start;label='newlabel2';fmtrow=fmtrow+2;
+      output;
     end;
     else if fmtrow le 3 then do; /* 9 more changed values */
       start= cats("mod",_n_);

tests/base/mp_loadformat.test.2.sas

@@ -58,6 +58,9 @@ proc format library=&cat1;
   value agemlb (multilabel)
     19-120='Adults'
     1-18='Children'
+    0-1='Preschool'
+    1-2='Preschool'
+    2-3='Preschool'
     1-4='Preschool';
   value agemlc (multilabel notsorted)
     19-120='Adults'
@@ -67,16 +70,19 @@ run;
 
 %mp_cntlout(libcat=&cat1,cntlout=work.cntlout1)
 %mp_assertdsobs(work.cntlout1,
-  desc=Has 16 records,
+  desc=Has 19 records,
-  test=EQUALS 16
+  test=EQUALS 19
 )
 
 data work.stagedata3;
   set work.cntlout1;
   if fmtname='AGEMLA' and label ne 'Preschool' then deleteme='Yes';
   if fmtname='AGEMLB' and label = 'Preschool' then label='Kids';
-  if fmtname='GENDERML' and label='Farmale' then output;
+  if fmtname='GENDERML' and label='Farmale' then do;
     output;
+    fmtrow=101; output;
+  end;
+  else output;
 run;
 
 
@@ -113,14 +119,17 @@ run;
 
 %let check1=0;
 %let check2=0;
+%let check3=0;
 data test;
   set work.cntlout2;
   where fmtname='GENDERML';
+  putlog fmtrow= label=;
   if _n_=4 and label='Farmale' then call symputx('check1',1);
-  if _n_=5 and label='Farmale' then call symputx('check2',1);
+  if _n_=5 and label ne 'Farmale' then call symputx('check2',1);
+  if _n_=8 and label = 'Farmale' then call symputx('check3',1);
 run;
 %mp_assert(
-  iftrue=(&check1=1 and &check2=1),
+  iftrue=(&check1=1 and &check2=1 and &check3=1),
   desc=Ensuring Farmale values retain their order,
   outds=work.test_results
 )

tests/base/mp_retainedkey.test.sas

@@ -5,6 +5,7 @@
   <h4> SAS Macros </h4>
   @li mp_assert.sas
   @li mp_assertcolvals.sas
+  @li mp_assertscope.sas
   @li mp_retainedkey.sas
 
 **/
@@ -114,3 +115,53 @@ run;
   desc=Checking append records created,
   outds=work.test_results
 )
+
+/** Test 3 - large numeric RK values (avoid exponential notation issue) **/
+data work.targetds3;
+  set sashelp.class;
+  rk_col=_n_+209100000000;
+run;
+
+data work.appendtable3;
+  set sashelp.class;
+  if mod(_n_,2)=0 then name=cats('New',_n_);
+  if _n_<7;
+run;
+
+%mp_assertscope(SNAPSHOT)
+
+%mp_retainedkey(
+  base_lib=X
+  ,base_dsn=targetds3
+  ,append_lib=X
+  ,append_dsn=APPENDTABLE3
+  ,retained_key=rk_col
+  ,business_key= name
+  ,check_uniqueness=NO
+  ,maxkeytable=0
+  ,locktable=0
+  ,outds=work.APPEND3
+  ,filter_str=
+)
+
+%mp_assertscope(COMPARE,
+  desc=Checking macro scope leakage in mp_retainedkey (test 3)
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking errors in test 3 (large numeric RK),
+  outds=work.test_results
+)
+
+data work.check3;
+  do val=209100000001,209100000003,209100000005,
+    209100000020,209100000021,209100000022;
+    output;
+  end;
+run;
+%mp_assertcolvals(work.append3.rk_col,
+  checkvals=work.check3.val,
+  desc=All large numeric values have a match,
+  test=ALLVALS
+)

tests/base/mp_rowhash.test.sas

@@ -0,0 +1,195 @@
+/**
+  @file
+  @brief Testing mp_rowhash.sas macro
+  @details The mp_rowhash macro generates DATA step statements that compute a
+    deterministic row-level MD5 hash.  These tests exercise the versioned seed,
+    character and numeric hashing rules, variable ordering, sensitivity to
+    whitespace, and reproducibility.
+
+  <h4> SAS Macros </h4>
+  @li mp_rowhash.sas
+  @li mp_assert.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
+
+**/
+
+%mp_assertscope(SNAPSHOT)
+
+/* test 1 - versioned seed only (no input variables) */
+data _null_;
+  length actual expected $32;
+  actual='';
+  %mp_rowhash(md5_col=actual)
+  expected=put(md5('DC HASH v2'),$hex32.);
+  call symputx('t1_actual',actual);
+  call symputx('t1_expected',expected);
+run;
+%mp_assertscope(COMPARE,ignorelist=T1_ACTUAL T1_EXPECTED)
+%mp_assert(
+  iftrue=("&t1_actual"="&t1_expected"),
+  desc=Versioned seed is returned when no variables are supplied,
+  outds=work.test_results
+)
+
+/* test 2 - single character variable reference */
+data _null_;
+  length c1 $1 actual expected $32;
+  length state digest $16 pair $32;
+  c1='A';
+  actual='';
+  %mp_rowhash(md5_col=actual, cvars=c1)
+  state=md5('DC HASH v2');
+  digest=md5(trimn(c1));
+  pair=state||digest;
+  expected=put(md5(pair),$hex32.);
+  call symputx('t2_actual',actual);
+  call symputx('t2_expected',expected);
+run;
+%mp_assert(
+  iftrue=("&t2_actual"="&t2_expected"),
+  desc=Single character variable hash matches reference construction,
+  outds=work.test_results
+)
+
+/* test 3 - single numeric variable reference */
+data _null_;
+  length actual expected $32;
+  length state digest $16 numtext $64 pair $32;
+  n1=42;
+  actual='';
+  %mp_rowhash(md5_col=actual, nvars=n1)
+  state=md5('DC HASH v2');
+  /* multiply-by-one matches the internal normalisation */
+  normal=n1*1;
+  numtext=put(normal,binary64.);
+  digest=md5(trim(numtext));
+  pair=state||digest;
+  expected=put(md5(pair),$hex32.);
+  call symputx('t3_actual',actual);
+  call symputx('t3_expected',expected);
+run;
+%mp_assert(
+  iftrue=("&t3_actual"="&t3_expected"),
+  desc=Single numeric variable hash matches reference construction,
+  outds=work.test_results
+)
+
+/* test 4 - order of cvars affects the resulting hash */
+data _null_;
+  length c1 c2 $1 h1 h2 $32;
+  c1='A';
+  c2='B';
+  h1='';
+  %mp_rowhash(md5_col=h1, cvars=c1 c2)
+  h2='';
+  %mp_rowhash(md5_col=h2, cvars=c2 c1)
+  call symputx('t4_h1',h1);
+  call symputx('t4_h2',h2);
+run;
+%mp_assert(
+  iftrue=("&t4_h1" ne "&t4_h2"),
+  desc=Order of cvars changes the resulting hash,
+  outds=work.test_results
+)
+
+/* test 5 - order of nvars affects the resulting hash */
+data _null_;
+  length h1 h2 $32;
+  n1=1;
+  n2=2;
+  h1='';
+  %mp_rowhash(md5_col=h1, nvars=n1 n2)
+  h2='';
+  %mp_rowhash(md5_col=h2, nvars=n2 n1)
+  call symputx('t5_h1',h1);
+  call symputx('t5_h2',h2);
+run;
+%mp_assert(
+  iftrue=("&t5_h1" ne "&t5_h2"),
+  desc=Order of nvars changes the resulting hash,
+  outds=work.test_results
+)
+
+/* test 6 - identical rows produce identical hashes */
+data work.test6;
+  length c $5 n 8;
+  c='ABC';
+  n=42;
+  output;
+  output;
+run;
+
+data work.test6_hashed;
+  set work.test6;
+  length hash $32;
+  %mp_rowhash(md5_col=hash, cvars=c, nvars=n)
+run;
+
+proc sql noprint;
+  select count(distinct hash) into: t6_distinct trimmed
+    from work.test6_hashed;
+quit;
+
+%mp_assert(
+  iftrue=(&t6_distinct=1),
+  desc=Identical rows produce identical hashes,
+  outds=work.test_results
+)
+
+/* test 7 - different rows produce different hashes */
+data work.test7;
+  length c $5 n 8;
+  c='ABC';
+  n=42;
+  output;
+  c='ABD';
+  n=42;
+  output;
+run;
+
+data work.test7_hashed;
+  set work.test7;
+  length hash $32;
+  %mp_rowhash(md5_col=hash, cvars=c, nvars=n)
+run;
+
+proc sql noprint;
+  select count(distinct hash) into: t7_distinct trimmed
+    from work.test7_hashed;
+quit;
+
+%mp_assert(
+  iftrue=(&t7_distinct=2),
+  desc=Different rows produce different hashes,
+  outds=work.test_results
+)
+
+/* test 8 - leading blanks in character variables are retained */
+data work.test8;
+  length c $5;
+  c=' f';
+  output;
+  c='f';
+  output;
+run;
+
+data work.test8_hashed;
+  set work.test8;
+  length hash $32;
+  %mp_rowhash(md5_col=hash, cvars=c)
+run;
+
+proc sql noprint;
+  select count(distinct hash) into: t8_distinct trimmed
+    from work.test8_hashed;
+quit;
+
+%mp_assert(
+  iftrue=(&t8_distinct=2),
+  desc=Leading blanks are retained in character hashes,
+  outds=work.test_results
+)
+
+
+

tests/base/mp_stripdiffs.test.sas

@@ -0,0 +1,106 @@
+/**
+  @file
+  @brief Testing mp_stripdiffs.sas macro
+  @details
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mp_ds2md.sas
+  @li mp_stripdiffs.sas
+
+**/
+
+/* make an adjustable base dataset */
+/* use a composite key also (name weight) */
+libname libby (work);
+data libby.class;
+  set sashelp.class;
+run;
+
+/* first, store some diffs */
+data work.orig work.deleted work.changed work.appended;
+  set libby.class;
+  if _n_=1 then do;
+    call symputx('delname',name);
+    output work.orig work.deleted;
+  end;
+  else if _n_=2 then do;
+    output work.orig;
+    call symputx('modname',name);
+    call symputx('modval',age);
+    age=99;
+    output work.changed;
+  end;
+  else do;
+    name='Newbie';
+    output work.appended;
+    stop;
+  end;
+run;
+%mp_storediffs(libby.class,work.orig,NAME WEIGHT
+  ,delds=work.deleted
+  ,modds=work.changed
+  ,appds=work.appended
+  ,outds=work.audit
+  ,loadref=UPLOAD1
+  ,mdebug=0
+)
+%mp_ds2md(work.audit)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking preparation case,
+  outds=work.test_results
+)
+
+/* apply the changes */
+proc sql;
+delete from libby.class where name in ("&delname","&modname");
+proc append base=libby.class data=work.appended;
+proc append base=libby.class data=work.changed;
+run;
+
+/* now, prepare the revert dataset */
+%mp_assertscope(SNAPSHOT)
+%mp_stripdiffs(libby.class
+  ,UPLOAD1
+  ,work.audit
+  ,outds=work.mp_stripdiffs
+  ,mdebug=1
+)
+%mp_ds2md(work.mp_stripdiffs)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking error condition,
+  outds=work.test_results
+)
+
+%let delpass=0;
+%let modpass=0;
+%let addpass=0;
+data _null_;
+  set work.mp_stripdiffs;
+  if upcase(_____DELETE__THIS__RECORD_____)='NO' and name="&delname"
+  then call symputx('delpass',1);
+  if name="&modname" and age=&modval then call symputx('modpass',1);
+  if upcase(_____DELETE__THIS__RECORD_____)='YES' and name="Newbie"
+  then call symputx('addpass',1);
+run;
+
+%mp_assert(
+  iftrue=(&delpass=1),
+  desc=Ensuring deleted record is back in the dataset,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&modpass=1),
+  desc=Ensuring modified record now has old value,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&addpass=1),
+  desc=Ensuring added record is now marked for deletion,
+  outds=work.test_results
+)

tests/base/mp_validatecol.test.sas

@@ -130,3 +130,29 @@ run;
   test=EQUALS 6,
   outds=work.test_results
 )
+
+/**
+  * Test 5 - ISLIB
+  */
+data test5;
+  infile datalines4 dsd;
+  input;
+  inf=_infile_;
+  %mp_validatecol(inf,ISLIB,islib)
+  if islib=1;
+datalines4;
+some
+!lib
+%abort
+definite
+2fail
+nineletrs
+.failalso
+_valid
+;;;;
+run;
+%mp_assertdsobs(work.test5,
+  desc=Testing ISLIB,
+  test=EQUALS 3,
+  outds=work.test_results
+)

tests/sas9only/mm_assigndirectlib.test.sas

@@ -0,0 +1,30 @@
+/**
+  @file
+  @brief Testing mm_assigndirectlib macro
+  @details  A valid library must first be configured in metadata.
+  To test success, also define a table for which we can test the existence.
+  This is a unit test - not part of the full test run, as it would be a
+  lot of overhead to create an external DB and metadata setup each time.
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mm_assigndirectlib.sas
+
+**/
+
+%let runtest=0;
+%let libref=XXX;
+%let ds=XXXX;
+
+
+%mp_assertscope(SNAPSHOT)
+%mm_assigndirectlib(&libref)
+%mp_assertscope(COMPARE)
+
+
+%mp_assert(
+  iftrue=(&runtest=1 and %mf_existds(&libref..&ds)),
+  desc=Check if &libref..&ds exists
+)

tests/serveronly/ms_adduser2group.test.sas

@@ -32,6 +32,7 @@ run;
 %ms_adduser2group(uid=1,gid=&groupid,mdebug=&sasjs_mdebug,outds=test1)
 %mp_assertscope(COMPARE
   ,ignorelist=MCLIB2_JADP1LEN MCLIB2_JADP2LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
+    MC2_JADP1LEN MC2_JADP2LEN MC2_JADPNUM MC2_JADVLEN
 )
 
 /* check the user is in the output list */

tests/serveronly/ms_creategroup.test.sas

@@ -16,7 +16,8 @@
 %mp_assertscope(SNAPSHOT)
 %ms_creategroup(&group, desc=The description,mdebug=&sasjs_mdebug,outds=test1)
 %mp_assertscope(COMPARE
-  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN
+  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN MC0_JADP1LEN
+    MC0_JADPNUM MC0_JADVLEN
 )
 
 %let id=0;

tests/serveronly/ms_createuser.test.sas

@@ -16,7 +16,8 @@
 %mp_assertscope(SNAPSHOT)
 %ms_createuser(&user,passwrd,outds=test1,mdebug=&sasjs_mdebug)
 %mp_assertscope(COMPARE
-  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN
+  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN MC0_JADP1LEN
+    MC0_JADPNUM MC0_JADVLEN
 )
 
 %let id=0;

tests/serveronly/ms_getgroups.test.sas

@@ -21,7 +21,8 @@
 %mp_assertscope(SNAPSHOT)
 %ms_getgroups(outds=work.test1,mdebug=&sasjs_mdebug)
 %mp_assertscope(COMPARE
-  ,ignorelist=MCLIB2_JADP1LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
+  ,ignorelist=MCLIB2_JADP1LEN MCLIB2_JADPNUM MCLIB2_JADVLEN MC2_JADP1LEN
+    MC2_JADPNUM MC2_JADVLEN
 )
 
 /* check the group was created */

tests/serveronly/ms_getusers.test.sas

@@ -15,7 +15,8 @@
 %mp_assertscope(SNAPSHOT)
 %ms_getusers(outds=work.test1,mdebug=&sasjs_mdebug)
 %mp_assertscope(COMPARE
-  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN
+  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN MC0_JADP1LEN
+    MC0_JADPNUM MC0_JADVLEN
 )
 
 %mp_assertdsobs(work.test1,test=ATLEAST 1)

tests/serveronly/ms_triggerstp.test.sas

@@ -0,0 +1,91 @@
+/**
+  @file
+  @brief Testing ms_triggerstp.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li ms_createfile.sas
+  @li ms_triggerstp.sas
+  @li mf_existds.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertcols.sas
+  @li mf_getvartype.sas
+  @li ms_deletefile.sas
+
+**/
+
+/* first, create multiple STPs to run */
+filename stpcode1 temp;
+data _null_;
+  file stpcode1;
+  put '%put hello world;';
+  put '%put _all_;';
+  put 'data _null_; file _webout1; put "triggerstp test 1";run;';
+run;
+filename stpcode2 temp;
+data _null_;
+  file stpcode2;
+  put '%put Lorem Ipsum;';
+  put '%put _all_;';
+  put 'data _null_; file _webout2; put "triggerstp test 2";run;';
+run;
+options mprint;
+%let fname1=%mf_getuniquename();
+%let fname2=%mf_getuniquename();
+
+%ms_createfile(/sasjs/tests/&fname1..sas
+  ,inref=stpcode1
+  ,mdebug=1
+)
+%ms_createfile(/sasjs/tests/&fname2..sas
+  ,inref=stpcode2
+)
+
+%mp_assertscope(SNAPSHOT)
+  %ms_triggerstp(/sasjs/tests/&fname1
+    ,debug=131
+    ,outds=work.mySessions
+  )
+  %ms_triggerstp(/sasjs/tests/&fname2
+    ,outds=work.mySessions
+  )
+%mp_assertscope(COMPARE,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM
+  MCLIB0_JADVLEN MC0_JADP1LEN MC0_JADPNUM MC0_JADVLEN
+)
+
+%mp_assert(iftrue=%str(%mf_existds(work.mySessions)=1)
+  ,desc=Testing output exists
+  ,outds=work.test_results
+)
+
+%mp_assertdsobs(work.mySessions,
+  test=EQUALS 2,
+  desc=Testing observations,
+  outds=work.test_results
+)
+%mp_assertcols(work.mySessions,
+  cols=sessionid,
+  test=ALL,
+  desc=Testing column exists,
+  outds=work.test_results
+)
+
+data _null_;
+  retain contentCheck 1;
+  set work.mySessions end=last;
+  if missing(sessionID) then contentCheck = 0;
+  if last then do;
+    call symputx("contentCheck",contentCheck,"l");
+  end;
+run;
+%let typeCheck = %mf_getvartype(work.mySessions,sessionid);
+
+%mp_assert(iftrue=%str(&typeCheck = C and &contentCheck = 1)
+          ,desc=Testing type and content of output
+          ,outds=work.test_results
+)
+
+%ms_deletefile(/sasjs/tests/&fname1..sas)
+%ms_deletefile(/sasjs/tests/&fname2..sas)

tests/testinit.sas

@@ -10,7 +10,9 @@
 **/
 
 /* location in metadata or SAS Drive for temporary files */
-%let mcTestAppLoc=/tmp/tests/sasjs/core/%mf_uid();
+%let mcTestAppLoc=/Users/&sysuserid/testresults/sasjs_core/%mf_uid();
+
+%let mcTestContext=SAS Job Execution compute context;
 
 /* set defaults */
 %mp_init()
@@ -23,6 +25,7 @@ options lrecl=80;
 
 %macro loglevel();
   %if "&_debug"="2477" or "&_debug"="fields,log,trace" or "&_debug"="131"
+  or "&_debug"="128"
   %then %do;
     %put debug mode activated;
     options mprint mprintnest;

tests/viyaonly/mfv_existsashdat.test.sas

@@ -0,0 +1,74 @@
+/**
+  @file
+  @brief Testing mfv_existsashdat macro function
+
+  <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mfv_existsashdat.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+options mprint;
+
+/* ------------------------------------------------------------------------ */
+/* Setup: start a CAS session and stage a sashdat file in the Public caslib */
+/* ------------------------------------------------------------------------ */
+cas mysess;
+caslib _all_ assign;
+
+%let testcaslib = Public;   /* change this if Public isn't available */
+proc cas;
+  table.caslibInfo result=r / ;
+  found = 0;
+  do row over r.CASLibInfo;
+    if upcase(row.Name) = upcase("&testcaslib") then found = 1;
+  end;
+  if found = 0 then do;
+    print "ERROR: caslib &testcaslib not available";
+    exit;
+  end;
+quit;
+%put NOTE: Using testcaslib=&testcaslib;
+
+%let tab1=T%mf_uid();
+
+proc casutil;
+  load data=sashelp.baseball outcaslib="&testcaslib" casout="&tab1" replace;
+  save casdata="&tab1" incaslib="&testcaslib"
+    casout="&tab1..sashdat" outcaslib="&testcaslib" replace;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+quit;
+
+
+/* ------------------------------------------------------------------------ */
+%put TEST 1 - returns 1 when the sashdat file exists in the caslib;
+/* ------------------------------------------------------------------------ */
+%mp_assert(
+  iftrue=(%mfv_existsashdat(&testcaslib..&tab1)=1),
+  desc=Test 1 - Check returns 1 for a sashdat that exists
+)
+
+/* ------------------------------------------------------------------------ */
+%put TEST 2 - returns 0 when the file does not exist in the caslib;
+/* ------------------------------------------------------------------------ */
+%mp_assertscope(SNAPSHOT)
+%mp_assert(
+  iftrue=(%mfv_existsashdat(&testcaslib..DOESNOTEXIST_%mf_uid())=0),
+  desc=Check returns 0 for a sashdat that does not exist
+)
+%mp_assertscope(COMPARE,
+  desc=Check mfv_existsashdat does not leak macro variables into GLOBAL scope
+)
+
+/* ------------------------------------------------------------------------ */
+/* Teardown                                                                 */
+/* ------------------------------------------------------------------------ */
+proc casutil;
+  deletesource casdata="&tab1..sashdat" incaslib="&testcaslib" quiet;
+quit;
+
+cas mysess terminate;
+
+%let syscc=0;

tests/viyaonly/mfv_getcaslib.test.sas

@@ -0,0 +1,69 @@
+/**
+  @file
+  @brief Testing mfv_getcaslib macro function
+
+  <h4> SAS Macros </h4>
+  @li mfv_getcaslib.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+options mprint;
+
+/* ------------------------------------------------------------------------ */
+/* Setup: start a CAS session and assign caslibs                            */
+/* ------------------------------------------------------------------------ */
+cas mysess;
+caslib _all_ assign;
+
+%let testcaslib=Public;
+
+libname castest cas caslib=&testcaslib;
+
+/* ------------------------------------------------------------------------ */
+%put TEST 1 - returns the caslib name for a valid CAS libref;
+/* ------------------------------------------------------------------------ */
+%mp_assert(
+  iftrue=(%mfv_getcaslib(castest)=%upcase(&testcaslib)),
+  desc=Check correct caslib name returned for a valid CAS libref
+)
+
+
+/* ------------------------------------------------------------------------ */
+%put TEST 2 - returns empty for a non-CAS libref (WORK);
+/* ------------------------------------------------------------------------ */
+%mp_assert(
+  iftrue=(%mfv_getcaslib(WORK)=),
+  desc=Check empty string returned for a non-CAS libref
+)
+
+
+/* ------------------------------------------------------------------------ */
+%put TEST 3 - returns empty for a libref that does not exist;
+/* ------------------------------------------------------------------------ */
+%mp_assert(
+  iftrue=(%mfv_getcaslib(DOESNOTEXIST)=),
+  desc=Check empty string returned for a non-existent libref
+)
+
+
+/* ------------------------------------------------------------------------ */
+%put TEST 5 - no scope leakage into global macro variables;
+/* ------------------------------------------------------------------------ */
+%mp_assertscope(SNAPSHOT)
+
+%let _rc=%mfv_getcaslib(castest);
+
+%mp_assertscope(COMPARE,
+  desc=Check mfv_getcaslib does not leak macro variables into GLOBAL scope,
+  ignorelist=_RC
+)
+
+
+/* ------------------------------------------------------------------------ */
+/* Teardown                                                                  */
+/* ------------------------------------------------------------------------ */
+cas mysess terminate;
+
+

tests/viyaonly/mfv_getfolderpath.test.sas

@@ -0,0 +1,31 @@
+/**
+  @file
+  @brief Testing mfv_getfolderpath macro function
+
+  <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mfv_getfolderpath.sas
+  @li mfv_getpathuri.sas
+  @li mp_assert.sas
+  @li mv_createfolder.sas
+
+**/
+
+options mprint sgen;
+
+%let folder=%mf_uid();
+/* create a folder */
+%mv_createfolder(path=&mcTestAppLoc/&folder)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=no errs on folder creation
+)
+
+%let uri=%mfv_getpathuri(&mcTestAppLoc/&folder);
+%put %mfv_getfolderpath(&uri);
+
+%mp_assert(
+  iftrue=("%mfv_getfolderpath(&uri)"="&mcTestAppLoc/&folder"),
+  desc=Check if correct folder was returned
+)
+

tests/viyaonly/mfv_getpathuri.test.sas

@@ -0,0 +1,35 @@
+/**
+  @file
+  @brief Testing mfv_getpathuri macro function
+
+  <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mfv_getpathuri.sas
+  @li mp_assert.sas
+  @li mv_createfile.sas
+
+**/
+
+options mprint sgen;
+
+%let file=%mf_uid();
+
+/* create a file */
+filename somefile temp;
+data _null_;
+  file somefile;
+  put 'hello testings';
+run;
+%let path=&mcTestAppLoc/temp;
+%mv_createfile(path=&path, name=&file..txt,inref=somefile)
+
+
+%mp_assert(
+  iftrue=(%mfv_existfile(&path/&file..txt)=1),
+  desc=Check if created file exists
+)
+
+%mp_assert(
+  iftrue=(%length(%mfv_getpathuri(&path/&file..txt))>0),
+  desc=Check that a URI was returned
+)

tests/viyaonly/mv_castabload.test.sas

@@ -0,0 +1,152 @@
+/**
+  @file
+  @brief Testing mv_castabload macro
+
+  <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mv_castabload.sas
+
+**/
+
+options mprint;
+
+/* -------------------------------------------------------------------- */
+/* Setup: start a CAS session and stage a source file in the caslib     */
+/* -------------------------------------------------------------------- */
+cas mysess;
+caslib _all_ assign;
+
+%let testcaslib=Public;
+
+proc cas;
+  table.caslibInfo result=r / ;
+  found=0;
+  do row over r.CASLibInfo;
+    if upcase(row.Name)=upcase("&testcaslib") then found=1;
+  end;
+  if found=0 then do;
+    print "ERROR: caslib &testcaslib not available";
+    exit;
+  end;
+quit;
+%put NOTE: Using testcaslib=&testcaslib;
+
+%let tab1=T%mf_uid();
+
+/* Save a sashdat source file then drop the in-memory copy so the first
+    mv_castabload call has something to load                             */
+proc casutil;
+  load data=sashelp.baseball
+    outcaslib="&testcaslib" casout="&tab1" replace;
+  save casdata="&tab1" incaslib="&testcaslib"
+    casout="&tab1..sashdat" outcaslib="&testcaslib" replace;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+quit;
+
+libname mylib cas caslib="&testcaslib";
+
+
+/* -------------------------------------------------------------------- */
+%put TEST 1 - load a table that is not in memory;
+/* -------------------------------------------------------------------- */
+
+/* Confirm table is absent before the call */
+%let _tabexists=0;
+proc cas;
+  table.tableExists result=r /
+    caslib="&testcaslib" name="&tab1";
+  if r.exists > 0 then call symputx('_tabexists','1');
+quit;
+
+%mp_assert(
+  iftrue=(&_tabexists=0),
+  desc=Check table is not in memory before mv_castabload
+)
+
+%mv_castabload(lib=mylib, table=&tab1, mdebug=1)
+
+%let _tabexists=0;
+proc cas;
+  table.tableExists result=r /
+    caslib="&testcaslib" name="&tab1";
+  if r.exists > 0 then call symputx('_tabexists','1');
+quit;
+
+%mp_assert(
+  iftrue=(&_tabexists=1),
+  desc=Check table is in memory after mv_castabload
+)
+
+
+/* -------------------------------------------------------------------- */
+%put TEST 2 - reload fetches a fresh copy and discards in-memory changes;
+/* -------------------------------------------------------------------- */
+
+/* Append a sentinel row to the in-memory table */
+data work.extra;
+  set mylib.&tab1;
+  name='TESTROW';
+  output;
+  stop;
+run;
+proc casutil;
+  load data=work.extra casout="&tab1"
+    outcaslib="&testcaslib" append;
+quit;
+
+%let _modified=0;
+proc sql noprint;
+  select count(*) into :_modified
+  from mylib.&tab1
+  where name='TESTROW';
+quit;
+
+%mp_assert(
+  iftrue=(&_modified=1),
+  desc=Check sentinel row is present in memory before reload
+)
+
+/* Drop the table and reload - source file does not have the sentinel   */
+proc casutil;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+quit;
+
+%mp_assertscope(SNAPSHOT)
+
+%mv_castabload(lib=mylib, table=&tab1, mdebug=1)
+
+%mp_assertscope(COMPARE,
+  desc=Check mv_castabload does not leak macro variables into GLOBAL scope
+)
+
+%let _after=0;
+proc sql noprint;
+  select count(*) into :_after
+  from mylib.&tab1
+  where name='TESTROW';
+quit;
+
+%mp_assert(
+  iftrue=(&_after=0),
+  desc=Check sentinel row is absent after reload from source
+)
+
+
+/* -------------------------------------------------------------------- */
+/* Teardown                                                              */
+/* -------------------------------------------------------------------- */
+libname mylib clear;
+
+proc casutil;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  deletesource casdata="&tab1..sashdat"
+    incaslib="&testcaslib" quiet;
+quit;
+
+cas mysess terminate;
+
+%let syscc=0;

tests/viyaonly/mv_castabsave.test.sas

@@ -0,0 +1,158 @@
+/**
+  @file
+  @brief Testing mv_castabsave macro
+
+  <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mv_castabsave.sas
+
+**/
+
+options mprint;
+
+/* -------------------------------------------------------------------- */
+/* Setup: start a CAS session and load a table that has a tracked       */
+/*        source file so mv_castabsave can discover it via the REST API */
+/* -------------------------------------------------------------------- */
+cas mysess;
+caslib _all_ assign;
+
+%let testcaslib=Public;
+
+proc cas;
+  table.caslibInfo result=r / ;
+  found=0;
+  do row over r.CASLibInfo;
+    if upcase(row.Name)=upcase("&testcaslib") then found=1;
+  end;
+  if found=0 then do;
+    print "ERROR: caslib &testcaslib not available";
+    exit;
+  end;
+quit;
+%put NOTE: Using testcaslib=&testcaslib;
+
+%let tab1=T%mf_uid();
+
+/* Load sashelp.class into CAS, save as sashdat, reload from that file
+    so the table has a tracked source path (needed for REST discovery) */
+proc casutil;
+  load data=sashelp.class
+    outcaslib="&testcaslib" casout="&tab1" replace;
+  save casdata="&tab1" incaslib="&testcaslib"
+    casout="&tab1..sashdat" outcaslib="&testcaslib" replace;
+  /* Drop any existing global-scope version before promoting */
+  /* runs twice (with quiet) as first would drop local scope if exists */
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+
+  load casdata="&tab1..sashdat" incaslib="&testcaslib"
+    casout="&tab1" outcaslib="&testcaslib" promote;
+quit;
+
+libname mylib cas caslib="&testcaslib";
+
+
+/* -------------------------------------------------------------------- */
+%put TEST 1 - save in-memory table back to disk + no scope leakage;
+/* -------------------------------------------------------------------- */
+
+/* Source file is removed so that the reload proves mv_castabsave
+    created the file from scratch, not that a prior version existed   */
+proc casutil;
+  deletesource casdata="&tab1..sashdat"
+    incaslib="&testcaslib" quiet;
+quit;
+
+/* Insert a sentinel row - it must survive the full save/drop/reload   */
+data work.appendme;
+  set mylib.&tab1;
+  name='TESTROW';
+  output;
+  stop;
+proc casutil;
+  load data=work.appendme casout="&tab1" outcaslib="&testcaslib" append;
+quit;
+
+%mp_assertscope(SNAPSHOT)
+
+%mv_castabsave(lib=mylib, table=&tab1, mdebug=1)
+
+%mp_assertscope(COMPARE,
+  desc=Check mv_castabsave does not leak macro variables into GLOBAL scope,
+  ignorelist=MC0_JADP1LEN MC0_JADP2LEN MC0_JADP3LEN MC0_JADPNUM MC0_JADVLEN
+)
+
+proc casutil;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  load casdata="&tab1..sashdat" incaslib="&testcaslib"
+    casout="&tab1" outcaslib="&testcaslib" promote;
+quit;
+
+%let _rowcount=0;
+proc sql noprint;
+  select count(*) into :_rowcount
+  from mylib.&tab1
+  where name='TESTROW';
+quit;
+
+%mp_assert(
+  iftrue=(&_rowcount=1),
+  desc=Check inserted row survives mv_castabsave round-trip to disk
+)
+
+
+/* -------------------------------------------------------------------- */
+%put TEST 2 - save overwrites an existing source file;
+/* -------------------------------------------------------------------- */
+
+/* Source file already exists from the TEST 1 save - append a new row  */
+data work.appendme;
+  set mylib.&tab1;
+  name='TESTROW2';
+  output;
+  stop;
+proc casutil;
+  load data=work.appendme casout="&tab1"
+    outcaslib="&testcaslib" append;
+quit;
+
+%mv_castabsave(lib=mylib, table=&tab1, mdebug=1)
+
+proc casutil;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  load casdata="&tab1..sashdat" incaslib="&testcaslib"
+    casout="&tab1" outcaslib="&testcaslib" promote;
+quit;
+
+%let _rowcount=0;
+proc sql noprint;
+  select count(*) into :_rowcount
+  from mylib.&tab1
+  where name='TESTROW2';
+quit;
+
+%mp_assert(
+  iftrue=(&_rowcount=1),
+  desc=Check inserted row survives save over an existing source file
+)
+
+
+/* -------------------------------------------------------------------- */
+/* Teardown                                                              */
+/* -------------------------------------------------------------------- */
+libname mylib clear;
+
+proc casutil;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  deletesource casdata="&tab1..sashdat"
+    incaslib="&testcaslib" quiet;
+quit;
+
+cas mysess terminate;
+
+%let syscc=0;

tests/viyaonly/mv_createfile.test.sas

@@ -6,6 +6,7 @@
   @li mf_uid.sas
   @li mfv_existfile.sas
   @li mp_assert.sas
+  @li mp_assertscope.sas
   @li mv_createfile.sas
 
 
@@ -21,22 +22,90 @@ data _null_;
   file somefile;
   put 'hello testings';
 run;
-%mv_createfile(path=&mcTestAppLoc/temp, name=&file..txt,inref=somefile)
+%mp_assertscope(SNAPSHOT)
+%mv_createfile(path=&mcTestAppLoc, name=&file..txt,inref=somefile,mdebug=1)
+%mp_assertscope(COMPARE
+  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADP2LEN MCLIB0_JADPNUM
+    MCLIB0_JADVLEN MCLIB2_JADP1LEN
+    SASJSPROCESSMODE SASJS_STPSRV_HEADER_LOC
+    MCLIB2_JADP2LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
+)
 
 %mp_assert(
-  iftrue=(%mfv_existfile(&mcTestAppLoc/temp/&file..txt)=1),
+  iftrue=(%mfv_existfile(&mcTestAppLoc/&file..txt)=1),
   desc=Check if created file exists
 )
 
-%put TEST 2 - dataset upload ;
+%put TEST 2 - html file;
+filename f2 temp;
+data _null_;
+  file f2;
+  put '<html><body><p>Hello world</p></body></html>';
+run;
+%mv_createfile(path=&mcTestAppLoc, name=test.html,inref=f2,mdebug=1)
+
+%mp_assert(
+  iftrue=(%mfv_existfile(&mcTestAppLoc/test.html)=1),
+  desc=Check if created file exists
+)
+
+%put TEST 3 - dataset upload ;
 data temp;
 x=1;
 run;
 filename ds "%sysfunc(pathname(work))/temp.sas7bdat";
 
-%mv_createfile(path=&mcTestAppLoc/temp, name=&file..sas7bdat,inref=ds)
+%mv_createfile(path=&mcTestAppLoc, name=&file..sas7bdat,inref=ds,mdebug=1)
 
 %mp_assert(
-  iftrue=(%mfv_existfile(&mcTestAppLoc/temp/&file..sas7bdat)=1),
+  iftrue=(%mfv_existfile(&mcTestAppLoc/&file..sas7bdat)=1),
   desc=Check if created dataset exists
 )
+
+%put TEST 4 - create a .sas file;
+filename f4 temp;
+data _null_;
+  file f4;
+  put '%put hello FromSASStudioBailey; ';
+run;
+%mv_createfile(path=&mcTestAppLoc, name=test4.sas,inref=f4,mdebug=1)
+
+%mp_assert(
+  iftrue=(%mfv_existfile(&mcTestAppLoc/test4.sas)=1),
+  desc=Check if created sas program exists
+)
+
+
+
+%put TEST 5 - reading from files service and writing back;
+filename sendfrom filesrvc folderpath="&mcTestAppLoc" filename='test4.sas';
+
+OPTIONS MERROR SYMBOLGEN MLOGIC MPRINT;
+
+%mv_createfile(path=&mcTestAppLoc,name=test5.sas,inref=sendfrom,mdebug=1) ;
+
+%put TEST 6 - try the find and replace;
+filename f6 temp;
+data _null_;
+  file f6;
+  put '//Hello world!';
+  put 'let var=/some/path/name;';
+run;
+%let in=/some/path/name;
+%let out=/final/destination;
+%mv_createfile(path=&mcTestAppLoc, name=test6.js,inref=f6,mdebug=1,swap=in out)
+
+filename getback filesrvc folderpath="&mcTestAppLoc" filename='test6.js';
+
+%let test6=0;
+data _null_;
+  infile getback;
+  input;
+  if _infile_="let var=&out;" then call symputx('test6',1);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(&test6=1),
+  desc=Check if find & replace worked
+)

tests/viyaonly/mv_createfolder.test.sas

@@ -5,6 +5,7 @@
   <h4> SAS Macros </h4>
   @li mf_uid.sas
   @li mp_assert.sas
+  @li mp_assertscope.sas
   @li mv_createfolder.sas
   @li mv_deleteviyafolder.sas
   @li mv_getfoldermembers.sas
@@ -15,7 +16,11 @@
 %let folder=%mf_uid();
 
 /* create a folder */
+%mp_assertscope(SNAPSHOT)
 %mv_createfolder(path=&mcTestAppLoc/temp/&folder/&folder)
+%mp_assertscope(COMPARE, ignorelist=MC0_JADP1LEN MC0_JADP2LEN MC0_JADPNUM
+  MC0_JADVLEN MC2_JADP1LEN MC2_JADP2LEN MC2_JADPNUM MC2_JADVLEN
+)
 
 %mv_getfoldermembers(root=&mcTestAppLoc/temp/&folder, outds=work.folders)
 
@@ -30,3 +35,41 @@ run;
   iftrue=(&test=1),
   desc=Check if temp folder can be successfully created
 )
+
+/* create a folder without output dataset as part of the original macro */
+%mv_createfolder(path=&mcTestAppLoc/temp/&folder/f2
+  ,outds=folders2,mdebug=&sasjs_mdebug
+)
+
+%let test=0;
+data _null_;
+  set work.folders2;
+  putlog (_all_)(=);
+  if not missing(self_uri) then call symputx('test2',1);
+run;
+
+%mp_assert(
+  iftrue=(&test2=1),
+  desc=Check if outds param works
+)
+
+/* create a folder with full stops */
+%let newfolder=%mf_uid().2.1;
+%mv_createfolder(path=&mcTestAppLoc/temp/&newfolder
+  ,outds=work.folders3
+  ,mdebug=&sasjs_mdebug
+)
+
+%mv_getfoldermembers(root=&mcTestAppLoc/temp, outds=work.folders3)
+
+%let test3=0;
+data _null_;
+  set work.folders3;
+  putlog (_all_)(=);
+  if name="&newfolder" then call symputx('test3',1);
+run;
+
+%mp_assert(
+  iftrue=(&test3=1),
+  desc=Check if folder with full stops can be successfully created
+)

tests/viyaonly/mv_createwebservice.test.sas

@@ -32,7 +32,7 @@ run;
 )
 %put TEST1: checking web service code;
 data work.test_results;
-  length test_description $256 test_result $4 test_comments $256;
+  length test_result $4 test_description $256 test_comments $256;
   if _n_=1 then call missing (of _all_);
   infile compare end=eof;
   input;

tests/viyaonly/mv_getviyafileextparms.test.sas

@@ -0,0 +1,96 @@
+/**
+  @file
+  @brief Testing mv_getviyafileextparms macro
+
+  <h4> SAS Macros </h4>
+  @li mf_isblank.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mv_getviyafileextparms.sas
+
+**/
+
+options mprint;
+
+%let mvarIgnoreList =
+  MC0_JADP1LEN MC0_JADP2LEN MC0_JADP3LEN MC0_JADPNUM MC0_JADVLEN
+  SASJSPROCESSMODE SASJS_STPSRV_HEADER_LOC;
+
+%put TEST 1 - Test with common extension, requesting only typeDefName parameter;
+%mp_assertscope(SNAPSHOT)
+%mv_getviyafileextparms(ext=txt, typeDefNameVar=viyaTypeDefName)
+%mp_assertscope(COMPARE
+  ,ignorelist=&mvarIgnoreList viyaTypeDefName
+)
+
+%mp_assert(
+  iftrue=(not %mf_isBlank(&viyaTypeDefName)),
+  desc=Check the requested macro variable viyaTypeDefName is not blank.
+)
+
+%put TEST 2 - Test with common extension, requesting only properties parameter;
+%mp_assertscope(SNAPSHOT)
+%mv_getviyafileextparms(ext=html, propertiesVar=viyaProperties)
+%mp_assertscope(COMPARE
+  ,ignorelist=&mvarIgnoreList viyaProperties
+)
+
+%mp_assert(
+  iftrue=(not %mf_isBlank(%superq(viyaProperties))),
+  desc=Check the requested macro variable viyaProperties is not blank.
+)
+
+%put TEST 3 - Test with common extension, requesting only mediaType parameter;
+%mp_assertscope(SNAPSHOT)
+%mv_getviyafileextparms(ext=mp3, mediaTypeVar=viyaMediaType)
+%mp_assertscope(COMPARE
+  ,ignorelist=&mvarIgnoreList viyaMediaType
+)
+
+%mp_assert(
+  iftrue=(not %mf_isBlank(&viyaMediaType)),
+  desc=Check the requested macro variable viyaMediaType is not blank.
+)
+
+%put TEST 4 - Test with common extension, requesting all parameters;
+%mp_assertscope(SNAPSHOT)
+%mv_getviyafileextparms(
+  ext=css,
+  typeDefNameVar=cssViyaTypeDefName,
+  propertiesVar=cssViyaProperties,
+  mediaTypeVar=cssViyaMediaType
+  )
+%mp_assertscope(COMPARE
+  ,ignorelist=
+    &mvarIgnoreList cssViyaTypeDefName cssViyaProperties  cssViyaMediaType
+)
+
+%mp_assert(
+  iftrue=(not ( %mf_isBlank(&cssViyaTypeDefName) or
+                %mf_isBlank(%superq(cssViyaProperties)) or
+                %mf_isBlank(&cssViyaMediaType) ) ),
+  desc=Check a full set of requested macro variables are not blank.
+)
+
+
+%put TEST 5 - Test with invalid extension - requested parameters will be blank;
+%mp_assertscope(SNAPSHOT)
+%mv_getviyafileextparms(
+  ext=xxxINVALIDxxx,
+  typeDefNameVar=invalidTypeDefName,
+  propertiesVar=invalidProperties,
+  mediaTypeVar=invalidMediaType
+)
+%mp_assertscope(COMPARE
+  ,ignorelist=
+    &mvarIgnoreList invalidTypeDefName invalidProperties invalidMediaType
+)
+
+%mp_assert(
+  iftrue=(
+    %mf_isBlank(&invalidTypeDefName) and
+    %mf_isBlank(%superq(invalidProperties)) and
+    %mf_isBlank(&invalidMediaType)
+    ),
+  desc=Check the requested macro variables are all blank.
+)

tests/viyaonly/mv_jobflow.test.1.sas

@@ -33,19 +33,19 @@ data _null_;
   ;
 run;
 
-%mv_createjob(path=/Public/temp,name=demo1,code=testprog)
+%mv_createjob(path=&mcTestAppLoc,name=demo1,code=testprog)
-%mv_createjob(path=/Public/temp,name=demo2,code=testprog)
+%mv_createjob(path=&mcTestAppLoc,name=demo2,code=testprog)
 
 data work.inputjobs;
-  _contextName='SAS Job Execution compute context';
+  _contextName="&mcTestContext";
   do flow_id=1 to 2;
     do i=1 to 4;
-      _program='/Public/temp/demo1';
+      _program="&mcTestAppLoc/demo1";
       macrovar1=10*i;
       macrovar2=4*i;
       output;
       i+1;
-      _program='/Public/temp/demo2';
+      _program="&mcTestAppLoc/demo2";
       macrovar1=40*i;
       macrovar2=44*i;
       output;

tests/viyaonly/mv_jobflow.test.2.sas

@@ -29,19 +29,19 @@ data _null_;
   ;
 run;
 
-%mv_createjob(path=/Public/temp,name=demo1,code=testprog)
+%mv_createjob(path=&mcTestAppLoc,name=demo1,code=testprog)
-%mv_createjob(path=/Public/temp,name=demo2,code=testprog)
+%mv_createjob(path=&mcTestAppLoc,name=demo2,code=testprog)
 
 data work.inputjobs;
-  _contextName='SAS Job Execution compute context';
+  _contextName="&mcTestContext";
   do flow_id=1 to 2;
     do i=1 to 4;
-      _program='/Public/temp/demo1';
+      _program="&mcTestAppLoc/demo1";
       macrovar1=10*i;
       macrovar2=4*i;
       output;
       i+1;
-      _program='/Public/temp/demo2';
+      _program="&mcTestAppLoc/demo2";
       macrovar1=40*i;
       macrovar2=44*i;
       output;

tests/x-platform/mx_append2pgm.test.sas

@@ -0,0 +1,192 @@
+/**
+  @file
+  @brief Testing mx_append2pgm.sas macro
+
+  Be sure to run <code>%let mcTestAppLoc=/Public/temp/macrocore;</code> when
+  running in Studio
+
+  <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
+  @li mf_uid.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li ms_createfile.sas
+  @li mv_createfile.sas
+  @li mm_createstp.sas
+  @li mx_append2pgm.sas
+  @li mx_getcode.sas
+
+**/
+
+/**
+  * Test 1 - Append content to an existing program and verify combined output
+  * Also checking for scope leakage
+  */
+
+/* create a unique name for the program */
+%let item=test_%mf_uid();
+
+/* create the initial program with some code */
+filename initpgm temp;
+data _null_;
+  file initpgm;
+  put '%put ORIGINAL LINE;';
+run;
+
+%macro setup_pgm();
+  %let platform=%mf_getplatform();
+  %if &platform=SASJS %then %do;
+    %ms_createfile(&mcTestAppLoc/temp/&item..sas, inref=initpgm)
+  %end;
+  %else %if &platform=SASVIYA %then %do;
+    %mv_createfile(path=&mcTestAppLoc/temp, name=&item..sas, inref=initpgm)
+  %end;
+  %else %do;
+    %let work=%sysfunc(pathname(work));
+    data _null_;
+      file "&work/&item..sas";
+      infile initpgm;
+      input;
+      put _infile_;
+    run;
+    %mm_createstp(stpname=&item
+      ,filename=&item..sas
+      ,directory=&work
+      ,tree=&mcTestAppLoc/temp
+      ,stptype=2
+      ,minify=NO
+    )
+  %end;
+%mend setup_pgm;
+%setup_pgm()
+
+/* create the content to append */
+filename toappnd temp;
+data _null_;
+  file toappnd;
+  put '%put APPENDED LINE;';
+run;
+
+/* run the macro under test with scope checks */
+%mp_assertscope(SNAPSHOT)
+%mx_append2pgm(&mcTestAppLoc/temp/&item, inref=toappnd)
+%mp_assertscope(COMPARE,
+  desc=Test 1: mx_append2pgm does not leak scope,
+  outds=work.test_results,
+  ignorelist=MC2_JADP1LEN MC2_JADP2LEN MC2_JADPNUM MC2_JADVLEN MC2_JADP3LEN
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 1: No errors after appending content to program,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - Verify the appended content is present
+  * Fetch the modified program and check both original and appended lines exist
+  */
+
+%let test2_orig=0;
+%let test2_appd=0;
+
+%macro verify_test2();
+  %let platform=%mf_getplatform();
+  %if &platform=SASVIYA %then %do;
+    filename verifrf filesrvc folderpath="&mcTestAppLoc/temp";
+    data _null_;
+      infile verifrf("&item..sas") lrecl=32000;
+      input;
+      if index(_infile_,'ORIGINAL LINE') then call symputx('test2_orig','1');
+      if index(_infile_,'APPENDED LINE') then call symputx('test2_appd','1');
+    run;
+    filename verifrf clear;
+  %end;
+  %else %do;
+    %mx_getcode(&mcTestAppLoc/temp/&item, outref=verifrf)
+    data _null_;
+      infile verifrf lrecl=32000;
+      input;
+      if index(_infile_,'ORIGINAL LINE') then call symputx('test2_orig','1');
+      if index(_infile_,'APPENDED LINE') then call symputx('test2_appd','1');
+    run;
+  %end;
+%mend verify_test2;
+%verify_test2()
+
+%mp_assert(
+  iftrue=(&test2_orig=1),
+  desc=Test 2a: Original content is preserved after append,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&test2_appd=1),
+  desc=Test 2b: Appended content is present in modified program,
+  outds=work.test_results
+)
+
+/**
+  * Test 3 - Append multiple times to ensure repeated appends work
+  */
+filename toappd2 temp;
+data _null_;
+  file toappd2;
+  put '%put SECOND APPEND;';
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mx_append2pgm(&mcTestAppLoc/temp/&item, inref=toappd2)
+%mp_assertscope(COMPARE,
+  desc=Test 3: mx_append2pgm does not leak scope on second call,
+  outds=work.test_results
+)
+
+/* verify all three pieces of content exist */
+%let test3_orig=0;
+%let test3_appd=0;
+%let test3_app2=0;
+
+%macro verify_test3();
+  %let platform=%mf_getplatform();
+  %if &platform=SASVIYA %then %do;
+    filename verifr2 filesrvc folderpath="&mcTestAppLoc/temp";
+    data _null_;
+      infile verifr2("&item..sas") lrecl=32000;
+      input;
+      if index(_infile_,'ORIGINAL LINE') then call symputx('test3_orig','1');
+      if index(_infile_,'APPENDED LINE') then call symputx('test3_appd','1');
+      if index(_infile_,'SECOND APPEND') then call symputx('test3_app2','1');
+    run;
+    filename verifr2 clear;
+  %end;
+  %else %do;
+    %mx_getcode(&mcTestAppLoc/temp/&item, outref=verifr2)
+    data _null_;
+      infile verifr2 lrecl=32000;
+      input;
+      if index(_infile_,'ORIGINAL LINE') then call symputx('test3_orig','1');
+      if index(_infile_,'APPENDED LINE') then call symputx('test3_appd','1');
+      if index(_infile_,'SECOND APPEND') then call symputx('test3_app2','1');
+    run;
+  %end;
+%mend verify_test3;
+%verify_test3()
+
+%mp_assert(
+  iftrue=(&test3_orig=1),
+  desc=Test 3a: Original content still present after second append,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&test3_appd=1),
+  desc=Test 3b: First appended content still present after second append,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&test3_app2=1),
+  desc=Test 3c: Second appended content is present,
+  outds=work.test_results
+)

tests/x-platform/mx_createjob.test.sas

@@ -0,0 +1,307 @@
+/**
+  @file
+  @brief Testing mx_createjob.sas macro
+
+  Be sure to run <code>%let mcTestAppLoc=/Public/temp/macrocore;</code> when
+  running in Studio
+
+  <h4> SAS Macros </h4>
+  @li mx_createjob.sas
+  @li mp_assert.sas
+  @li mf_getuniquefileref.sas
+  @li mp_assertscope.sas
+
+**/
+
+/**
+  * Test 1 - Basic job creation with default parameters
+  * Also checking for scope leakage
+  */
+filename ft15f001 temp;
+parmcards4;
+  data example1;
+    set sashelp.class;
+  run;
+  %put Job executed successfully;
+;;;;
+%mp_assertscope(SNAPSHOT)
+%mx_createjob(path=&mcTestAppLoc/jobs,name=testjob1,replace=YES)
+%mp_assertscope(COMPARE,ignorelist=MC0_JADP1LEN MC0_JADP2LEN MC0_JADPNUM
+  MC0_JADVLEN MC2_JADP1LEN MC2_JADP2LEN MC2_JADPNUM MC2_JADVLEN MC4_JADP1LEN
+  MC4_JADP2LEN MC4_JADPNUM MC4_JADVLEN
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 1: No errors after basic job creation,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - Job creation with custom description
+  */
+filename ft15f001 temp;
+parmcards4;
+  data example2;
+    set sashelp.cars;
+  run;
+;;;;
+%mx_createjob(
+  path=&mcTestAppLoc/jobs,
+  name=testjob2,
+  desc=Custom job description for testing,
+  replace=YES
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 2: Job created with custom description,
+  outds=work.test_results
+)
+
+/**
+  * Test 3 - Job creation with precode
+  */
+filename precode1 temp;
+data _null_;
+  file precode1;
+  put '%let testvar=PreCodeValue;';
+  put '%put &=testvar;';
+run;
+
+filename ft15f001 temp;
+parmcards4;
+  data example3;
+    set sashelp.class;
+    precode_var="&testvar";
+  run;
+;;;;
+%mx_createjob(
+  path=&mcTestAppLoc/jobs,
+  name=testjob3,
+  precode=precode1,
+  replace=YES
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 3: Job created with precode parameter,
+  outds=work.test_results
+)
+
+filename precode1 clear;
+
+/**
+  * Test 4 - Job creation with multiple code filerefs
+  */
+%let code1=%mf_getuniquefileref();
+%let code2=%mf_getuniquefileref();
+
+filename &code1 temp;
+data _null_;
+  file &code1;
+  put 'data work.part1;';
+  put '  set sashelp.class(obs=5);';
+  put 'run;';
+run;
+
+filename &code2 temp;
+data _null_;
+  file &code2;
+  put 'data work.part2;';
+  put '  set sashelp.class(firstobs=6);';
+  put 'run;';
+run;
+
+%mx_createjob(
+  path=&mcTestAppLoc/jobs,
+  name=testjob4,
+  code=&code1 &code2,
+  replace=YES
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 4: Job created with multiple code filerefs,
+  outds=work.test_results
+)
+
+filename &code1 clear;
+filename &code2 clear;
+
+/**
+  * Test 5 - Job creation with both precode and multiple code files
+  */
+%let pre1=%mf_getuniquefileref();
+%let pre2=%mf_getuniquefileref();
+%let main1=%mf_getuniquefileref();
+
+filename &pre1 temp;
+data _null_;
+  file &pre1;
+  put '%let globalvar1=Value1;';
+run;
+
+filename &pre2 temp;
+data _null_;
+  file &pre2;
+  put '%let globalvar2=Value2;';
+run;
+
+filename &main1 temp;
+data _null_;
+  file &main1;
+  put 'data work.combined;';
+  put '  var1="&globalvar1";';
+  put '  var2="&globalvar2";';
+  put '  output;';
+  put 'run;';
+run;
+
+%mx_createjob(
+  path=&mcTestAppLoc/jobs,
+  name=testjob5,
+  precode=&pre1 &pre2,
+  code=&main1,
+  desc=Job with multiple precode and code files,
+  replace=YES
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 5: Job created with multiple precode and code files,
+  outds=work.test_results
+)
+
+filename &pre1 clear;
+filename &pre2 clear;
+filename &main1 clear;
+
+/**
+  * Test 6 - Job creation with special characters in code
+  */
+filename ft15f001 temp;
+parmcards4;
+  data example6;
+    length text $200;
+    text='Special chars: & % $ # @ !';
+    output;
+    text="Quotes: 'single' and ""double""";
+    output;
+  run;
+  %put Test with special characters;
+;;;;
+%mx_createjob(
+  path=&mcTestAppLoc/jobs,
+  name=testjob6,
+  desc=Job with special characters in code,
+  replace=YES
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 6: Job created with special characters in code,
+  outds=work.test_results
+)
+
+/**
+  * Test 7 - Job creation with macro code
+  */
+filename ft15f001 temp;
+parmcards4;
+  %macro testmacro();
+    data example7;
+      set sashelp.class;
+      where age > 12;
+    run;
+  %mend testmacro;
+
+  %testmacro()
+;;;;
+%mx_createjob(
+  path=&mcTestAppLoc/jobs,
+  name=testjob7,
+  desc=Job containing macro definitions,
+  replace=YES
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 7: Job created with macro code,
+  outds=work.test_results
+)
+
+/**
+  * Test 8 - Job creation with empty code (edge case)
+  */
+filename ft15f001 temp;
+data _null_;
+  file ft15f001;
+  put '/* Empty job for testing */';
+run;
+
+%mx_createjob(
+  path=&mcTestAppLoc/jobs,
+  name=testjob8,
+  desc=Job with minimal code,
+  replace=YES
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 8: Job created with minimal code,
+  outds=work.test_results
+)
+
+/**
+  * Test 9 - Job creation with long code block
+  */
+filename ft15f001 temp;
+data _null_;
+  file ft15f001;
+  put 'data work.longtest;';
+  do i=1 to 50;
+    put '  var' i +(-1) '=' i ';';
+  end;
+  put '  output;';
+  put 'run;';
+run;
+
+%mx_createjob(
+  path=&mcTestAppLoc/jobs,
+  name=testjob9,
+  desc=Job with many variables,
+  replace=YES
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 9: Job created with long code block,
+  outds=work.test_results
+)
+
+/**
+  * Test 10 - Replace existing job (replace=YES)
+  */
+filename ft15f001 temp;
+parmcards4;
+  data example10_v1;
+    set sashelp.class;
+  run;
+;;;;
+%mx_createjob(path=&mcTestAppLoc/jobs,name=testjob10,replace=YES)
+
+/* Now replace it */
+filename ft15f001 temp;
+parmcards4;
+  data example10_v2;
+    set sashelp.cars;
+  run;
+;;;;
+%mx_createjob(path=&mcTestAppLoc/jobs,name=testjob10,replace=YES)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 10: Job replaced successfully with replace=YES,
+  outds=work.test_results
+)

tests/x-platform/mx_getgroups.test.sas

@@ -0,0 +1,41 @@
+/**
+  @file
+  @brief Testing mx_getgroups.test.sas macro
+
+  Be sure to run <code>%let mcTestAppLoc=/Public/temp/macrocore;</code> when
+  running in Studio
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mf_getuser.sas
+  @li mp_assert.sas
+  @li mx_getgroups.sas
+
+**/
+
+
+%mx_getgroups(outds=work.test1)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.test1)>0),
+  desc=groups were found,
+  outds=work.test_results
+)
+%mp_assertcols(work.test1,
+  cols=groupuri groupname groupdesc,
+  test=ALL,
+  desc=check all columns exist
+)
+
+%mx_getgroups(outds=work.test2,user=%mf_getuser())
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.test2)>0),
+  desc=groups for current user were found,
+  outds=work.test_results
+)
+%mp_assertcols(work.test2,
+  cols=groupuri groupname groupdesc,
+  test=ALL,
+  desc=check all columns exist
+)

tests/x-platform/mx_testservice.test.sas

@@ -3,10 +3,10 @@
   @brief Testing mx_testservice.sas macro
 
   Be sure to run <code>%let mcTestAppLoc=/Public/temp/macrocore;</code> when
-  runnin in Studio
+  running in Studio
 
   <h4> SAS Macros </h4>
-  @li mp_createwebservice.sas
+  @li mx_createwebservice.sas
   @li mx_testservice.sas
   @li mp_assert.sas
 

viya/mfv_existfolder.sas

@@ -33,16 +33,23 @@
     msg=Cannot enter mfv_existfolder.sas with syscc=&syscc
   )
 
-  %local fref rc;
+  %local fref rc var;
   %let fref=%mf_getuniquefileref();
 
   %if %sysfunc(filename(fref,,filesrvc,folderPath="&path"))=0 %then %do;
     1
+    %let var=_FILESRVC_&fref._URI;
     %let rc=%sysfunc(filename(fref));
+    %symdel &var;
   %end;
   %else %do;
     0
     %let syscc=0;
   %end;
 
+  %mf_abort(
+    iftrue=(&syscc ne 0),
+    msg=Cannot leave mfv_existfolder.sas with syscc=&syscc
+  )
+
 %mend mfv_existfolder;

viya/mfv_existsashdat.sas

@@ -6,12 +6,12 @@
       %if %mfv_existsashdat(libds=casuser.sometable) %then %put  yes it does!;
 
   The function uses `dosubl()` to run the `table.fileinfo` action, for the
-  specified library, filtering for `*.sashdat` tables.  The results are stored
+  specified library, filtering for `*.sashdat` tables.
-  in a WORK table (&outprefix._&lib). If that table already exists, it is
-  queried instead, to avoid the dosubl() performance hit.
 
-  To force a rescan, just use a new `&outprefix` value, or delete the table(s)
+  Results are cached in a WORK table (&outprefix._&lib).  If that table
-  before running the function.
+  already exists it is queried directly to avoid the dosubl() overhead.
+  To force a rescan, use a new `&outprefix` value or delete the cache
+  table before calling.
 
   @param [in] libds library.dataset
   @param [out] outprefix= (work.mfv_existsashdat)
@@ -24,13 +24,12 @@
   @author Mathieu Blauw
 **/
 
-%macro mfv_existsashdat(libds,outprefix=work.mfv_existsashdat
+%macro mfv_existsashdat(libds,outprefix=work.mfv_existsashdat);
-);
 %local rc dsid name lib ds;
 %let lib=%upcase(%scan(&libds,1,'.'));
 %let ds=%upcase(%scan(&libds,-1,'.'));
 
-/* if table does not exist, create it */
+/* if cache table does not exist, build it */
 %if %sysfunc(exist(&outprefix._&lib)) ne 1 %then %do;
   %let rc=%sysfunc(dosubl(%nrstr(
     /* Read in table list (once per &lib per session) */
@@ -41,7 +40,7 @@
     quit;
     /* Only keep name, without file extension */
     data &outprefix._&lib;
-      set &outprefix._&lib(where=(Name like '%.sashdat') keep=Name);
+      set &outprefix._&lib(where=(upcase(Name) like '%.SASHDAT') keep=Name);
       Name=upcase(scan(Name,1,'.'));
     run;
   )));

viya/mfv_getcaslib.sas

@@ -0,0 +1,37 @@
+/**
+  @file mfv_getcaslib.sas
+  @brief Returns the CAS caslib name for a given SAS libref
+  @details Pure macro function.  Reads sashelp.vlibnam and returns
+    the sysvalue where sysname='Caslib' for the given libref.  This
+    is useful when the caslib name and libref name may differ.
+
+    Usage:
+
+        %put %mfv_getcaslib(lib=PUBLIC);
+
+  @param [in] lib SAS libref for which to return the CAS caslib name
+
+  @return Returns the CAS caslib name, or empty string if not found
+
+**/
+
+%macro mfv_getcaslib(lib);
+
+%local dsid rc result;
+
+%let dsid=%sysfunc(open(sashelp.vlibnam(
+  where=(libname="%upcase(&lib)" and sysname="Caslib")
+)));
+
+%if &dsid %then %do;
+  %let rc=%sysfunc(fetch(&dsid));
+  %if &rc=0 %then
+    %let result=%sysfunc(
+      getvarc(&dsid,%sysfunc(varnum(&dsid,SYSVALUE)))
+    );
+  %let rc=%sysfunc(close(&dsid));
+%end;
+
+&result
+
+%mend mfv_getcaslib;

viya/mfv_getfolderpath.sas

@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Returns the path of a folder from the URI
+  @details Makes use of the SYSMSG() ER8OR response, which resolves the uri,
+  seemingly without entering an er8or state.
+
+  Usage:
+
+      %mv_createfolder(path=/public/demo)
+      %let uri=%mfv_getpathuri(/public/demo);
+      %put %mfv_getfolderpath(&uri);
+
+  Notice above the new path has an uppercase P - the correct path.
+
+  @param [in] uri The uri of the folder -eg /folders/folders/xxxx)
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+
+  <h4> Related Macros </h4>
+  @li mfv_getpathuri.sas
+
+  @version 4
+  @author [Allan Bowe](https://www.linkedin.com/in/allanbowe/)
+**/
+%macro mfv_getfolderpath(uri
+)/*/STORE SOURCE*/;
+
+  %local fref rc path msg var /* var used to avoid delete timing issue */;
+  %let fref=%mf_getuniquefileref();
+  %if %quote(%substr(%str(&uri),1,17)) ne %quote(/folders/folders/)
+  %then %do;
+    %put &sysmacroname: Invalid URI: &uri;
+  %end;
+  %else %if %sysfunc(filename(fref,,filesrvc,folderuri="&uri" ))=0
+  %then %do;
+    %let var=_FILESRVC_&fref._URI;
+    %local fid ;
+    %let fid= %sysfunc(fopen(&fref,I));
+    %let msg=%quote(%sysfunc(sysmsg()));
+
+    %unquote(%scan(&msg,2,%str(,.)))
+
+    %let rc=%sysfunc(fclose(&fid));
+    %let rc=%sysfunc(filename(fref));
+    %symdel &var;
+  %end;
+  %else %do;
+    %put &sysmacroname: Not Found: &uri;
+    %let syscc=0;
+  %end;
+
+%mend mfv_getfolderpath ;