Merge pull request #342 from sasjs/issue341

Full Format Deletion

.all-contributorsrc

@@ -135,6 +135,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "henrik-forsell",
+      "name": "Henrik Forsell",
+      "avatar_url": "https://avatars.githubusercontent.com/u/109935936?v=4",
+      "profile": "https://github.com/henrik-forsell",
+      "contributions": [
+        "doc"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.git-hooks/pre-commit

@@ -2,7 +2,7 @@
 
 # Ensure lint is passing
 LINT=`sasjs lint`
-if [[ "$LINT" != "✔ All matched files use @sasjs/lint code style!" ]]; then
+if [[ "$LINT" != *"✔ All matched files use @sasjs/lint code style!" ]]; then
   echo "$LINT"
   echo "To commit in spite of these warnings, use the -n parameter."
   exit 1

.github/FUNDING.yml

@@ -1,3 +1,3 @@
 # These are supported funding model platforms
 
-github: [sasjs]
+custom: https://getalby.com/p/sasjs

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,3 +15,4 @@ What code changes have been made to achieve the intent.
 - [ ] Code is formatted correctly (`sasjs lint`).
 - [ ] Any new functionality has been unit tested.
 - [ ] All unit tests are passing (`sasjs test`).
+- [ ] `all.sas` has been regenerated (`python3 build.py`)

.github/vpn/config.ovpn

@@ -0,0 +1,25 @@
+# Client
+client
+tls-client
+dev tun
+# this will connect with whatever proto DNS tells us (https://community.openvpn.net/openvpn/ticket/934)
+proto tcp
+remote vpn.4gl.io 7494
+resolv-retry infinite
+cipher AES-256-CBC
+auth SHA256
+script-security 2
+keepalive 10 120
+remote-cert-tls server
+
+# Keys
+ca ca.crt
+cert user.crt
+key user.key
+tls-auth tls.key 1
+
+# Security
+nobind
+persist-key
+persist-tun
+verb 3

.github/workflows/main.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v2
+        uses: cycjimmy/semantic-release-action@v3
         env:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/run-tests.yml

@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [lts/fermium]
+        node-version: [lts/hydrogen]
 
     steps:
       - uses: actions/checkout@v2
@@ -21,32 +21,52 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
 
+      - name: Write VPN Files
+        run: |
+          echo "$CA_CRT" > .github/vpn/ca.crt
+          echo "$USER_CRT" > .github/vpn/user.crt
+          echo "$USER_KEY" > .github/vpn/user.key
+          echo "$TLS_KEY" > .github/vpn/tls.key
+        shell: bash
+        env:
+          CA_CRT: ${{ secrets.CA_CRT}}
+          USER_CRT: ${{ secrets.USER_CRT }}
+          USER_KEY: ${{ secrets.USER_KEY }}
+          TLS_KEY: ${{ secrets.TLS_KEY }}
+
+      - name: Install Open VPN
+        run: |
+          sudo apt install apt-transport-https
+          sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
+          sudo apt-key add openvpn-repo-pkg-key.pub
+          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
+          sudo apt update
+          sudo apt install openvpn3=17~betaUb22042+jammy
+      - name: Start Open VPN 3
+        run: openvpn3 session-start --config .github/vpn/config.ovpn
       - name: Install Doxygen
         run: sudo apt-get install doxygen
 
       - name: Install dependencies
         run: npm ci
 
-      - name: Check code style
-        run: npm run lint
+      - name: Check code style (aborts if errors found)
+        run: npx @sasjs/cli lint
 
       - name: Add client
-        run: echo "CLIENT=${{secrets.CLIENT}}"> .env.viya
-
-      - name: Add secret
-        run: echo "SECRET=${{secrets.SECRET}}" >> .env.viya
+        run: echo "CLIENT=${{secrets.SAS9_4GL_IO_CLIENT}}"> .env.server
 
       - name: Add access token
-        run: echo "ACCESS_TOKEN=${{secrets.ACCESS_TOKEN}}" >> .env.viya
+        run: echo "ACCESS_TOKEN=${{secrets.SAS9_4GL_IO_ACCESS_TOKEN}}" >> .env.server
 
       - name: Add refresh token
-        run: echo "REFRESH_TOKEN=${{secrets.REFRESH_TOKEN}}" >> .env.viya
+        run: echo "REFRESH_TOKEN=${{secrets.SAS9_4GL_IO_REFRESH_TOKEN}}" >> .env.server
 
-      - name: Build Project
-        run: npm run build
+      - name: Build & Deploy Project to SAS server
+        run: npx @sasjs/cli cbd -t server
 
-      - name: Run SASjs  tests
-        run: npm run test
+      - name: Run all tests
+        run: npx @sasjs/cli test -t server
         env:
           CI: true
           CLIENT: ${{secrets.CLIENT}}

.sasjslint

@@ -4,6 +4,7 @@
   "hasDoxygenHeader": true,
   "hasMacroNameInMend": true,
   "hasMacroParentheses": true,
+  "lineEndings": "lf",
   "noGremlins": true,
   "noNestedMacros": false,
   "noSpacesInFileNames": true,

README.md

@@ -246,7 +246,7 @@ The following repositories are also worth checking out:
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-12-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-13-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -254,22 +254,25 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
 <!-- prettier-ignore-start -->
 <!-- markdownlint-disable -->
 <table>
-  <tr>
-    <td align="center"><a href="https://github.com/allanbowe"><img src="https://avatars.githubusercontent.com/u/4420615?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Allan Bowe</b></sub></a><br /><a href="#business-allanbowe" title="Business development">💼</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Code">💻</a> <a href="#content-allanbowe" title="Content">🖋</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Documentation">📖</a> <a href="#infra-allanbowe" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#maintenance-allanbowe" title="Maintenance">🚧</a> <a href="#mentoring-allanbowe" title="Mentoring">🧑‍🏫</a> <a href="#question-allanbowe" title="Answering Questions">💬</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3Aallanbowe" title="Reviewed Pull Requests">👀</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Tests">⚠️</a></td>
-    <td align="center"><a href="https://github.com/rafgag"><img src="https://avatars.githubusercontent.com/u/69139928?v=4?s=100" width="100px;" alt=""/><br /><sub><b>rafgag</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rafgag" title="Code">💻</a></td>
-    <td align="center"><a href="https://github.com/tmoody"><img src="https://avatars.githubusercontent.com/u/79837106?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Trevor Moody</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=tmoody" title="Code">💻</a></td>
-    <td align="center"><a href="https://krishna-acondy.io/"><img src="https://avatars.githubusercontent.com/u/2980428?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Krishna Acondy</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=krishna-acondy" title="Code">💻</a> <a href="#infra-krishna-acondy" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#blog-krishna-acondy" title="Blogposts">📝</a> <a href="#content-krishna-acondy" title="Content">🖋</a> <a href="#ideas-krishna-acondy" title="Ideas, Planning, & Feedback">🤔</a> <a href="#video-krishna-acondy" title="Videos">📹</a></td>
-    <td align="center"><a href="https://github.com/saadjutt01"><img src="https://avatars.githubusercontent.com/u/8914650?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Muhammad Saad </b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=saadjutt01" title="Code">💻</a> <a href="#ideas-saadjutt01" title="Ideas, Planning, & Feedback">🤔</a></td>
-    <td align="center"><a href="https://www.erudicat.com/"><img src="https://avatars.githubusercontent.com/u/25773492?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Yury Shkoda</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=YuryShkoda" title="Code">💻</a> <a href="#infra-YuryShkoda" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#video-YuryShkoda" title="Videos">📹</a></td>
-    <td align="center"><a href="https://github.com/medjedovicm"><img src="https://avatars.githubusercontent.com/u/18329105?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Mihajlo Medjedovic</b></sub></a><br /><a href="#infra-medjedovicm" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
-  </tr>
-  <tr>
-    <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
-    <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
-    <td align="center"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
-    <td align="center"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
-    <td align="center"><a href="https://bandism.net/"><img src="https://avatars.githubusercontent.com/u/22633385?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Ikko Ashimine</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=eltociear" title="Code">💻</a></td>
-  </tr>
+  <tbody>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/allanbowe"><img src="https://avatars.githubusercontent.com/u/4420615?v=4?s=100" width="100px;" alt="Allan Bowe"/><br /><sub><b>Allan Bowe</b></sub></a><br /><a href="#business-allanbowe" title="Business development">💼</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Code">💻</a> <a href="#content-allanbowe" title="Content">🖋</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Documentation">📖</a> <a href="#infra-allanbowe" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#maintenance-allanbowe" title="Maintenance">🚧</a> <a href="#mentoring-allanbowe" title="Mentoring">🧑‍🏫</a> <a href="#question-allanbowe" title="Answering Questions">💬</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3Aallanbowe" title="Reviewed Pull Requests">👀</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Tests">⚠️</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/rafgag"><img src="https://avatars.githubusercontent.com/u/69139928?v=4?s=100" width="100px;" alt="rafgag"/><br /><sub><b>rafgag</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rafgag" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/tmoody"><img src="https://avatars.githubusercontent.com/u/79837106?v=4?s=100" width="100px;" alt="Trevor Moody"/><br /><sub><b>Trevor Moody</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=tmoody" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://krishna-acondy.io/"><img src="https://avatars.githubusercontent.com/u/2980428?v=4?s=100" width="100px;" alt="Krishna Acondy"/><br /><sub><b>Krishna Acondy</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=krishna-acondy" title="Code">💻</a> <a href="#infra-krishna-acondy" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#blog-krishna-acondy" title="Blogposts">📝</a> <a href="#content-krishna-acondy" title="Content">🖋</a> <a href="#ideas-krishna-acondy" title="Ideas, Planning, & Feedback">🤔</a> <a href="#video-krishna-acondy" title="Videos">📹</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/saadjutt01"><img src="https://avatars.githubusercontent.com/u/8914650?v=4?s=100" width="100px;" alt="Muhammad Saad "/><br /><sub><b>Muhammad Saad </b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=saadjutt01" title="Code">💻</a> <a href="#ideas-saadjutt01" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://www.erudicat.com/"><img src="https://avatars.githubusercontent.com/u/25773492?v=4?s=100" width="100px;" alt="Yury Shkoda"/><br /><sub><b>Yury Shkoda</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=YuryShkoda" title="Code">💻</a> <a href="#infra-YuryShkoda" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#video-YuryShkoda" title="Videos">📹</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/medjedovicm"><img src="https://avatars.githubusercontent.com/u/18329105?v=4?s=100" width="100px;" alt="Mihajlo Medjedovic"/><br /><sub><b>Mihajlo Medjedovic</b></sub></a><br /><a href="#infra-medjedovicm" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt="kkchandok"/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt="Vladislav Parhomchik"/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt="Vignesh T."/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt="Bart Jablonski"/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://bandism.net/"><img src="https://avatars.githubusercontent.com/u/22633385?v=4?s=100" width="100px;" alt="Ikko Ashimine"/><br /><sub><b>Ikko Ashimine</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=eltociear" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/henrik-forsell"><img src="https://avatars.githubusercontent.com/u/109935936?v=4?s=100" width="100px;" alt="Henrik Forsell"/><br /><sub><b>Henrik Forsell</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=henrik-forsell" title="Documentation">📖</a></td>
+    </tr>
+  </tbody>
 </table>
 
 <!-- markdownlint-restore -->

all.sas

@@ -312,13 +312,17 @@ https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionex
   %local dsid rc;
   %let dsid=%sysfunc(open(&libds,is));
 
-  %if &dsid=0 or %length(&var)=0 %then %do;
+  %if &dsid=0 %then %do;
     %put %sysfunc(sysmsg());
-      0
+    0
+  %end;
+  %else %if %length(&var)=0 %then %do;
+    0
+    %let rc=%sysfunc(close(&dsid));
   %end;
   %else %do;
-      %sysfunc(varnum(&dsid,&var))
-      %let rc=%sysfunc(close(&dsid));
+    %sysfunc(varnum(&dsid,&var))
+    %let rc=%sysfunc(close(&dsid));
   %end;
 
 %mend mf_existvar;
@@ -691,10 +695,9 @@ or %index(&pgm,/tests/testteardown)
 
   returns:
 
-  > DOLLAR $CHAR W MONNAME
-  > $CHAR BEST DOLLAR
-  > BEST Z $CHAR COMMA PERCENTN
-
+      DOLLAR $CHAR W MONNAME
+      $CHAR BEST DOLLAR
+      BEST Z $CHAR COMMA PERCENTN
 
   @param [in] libds Two part library.dataset reference.
 
@@ -2654,6 +2657,101 @@ and %superq(SYSPROCESSNAME) ne %str(Compute Server)
 %mend mp_abort;
 
 /** @endcond */
+/**
+  @file
+  @brief Apply leading blanks to align numbers vertically in a char variable
+  @details This is particularly useful when storing numbers (as character) that
+  need to be sorted.
+
+  It works by splitting the number left and right of the decimal place, and
+  aligning it accordingly.  A temporary variable is created as part of this
+  process (which is automatically dropped)
+
+  The macro can be used only in data step, eg as follows:
+
+      data _null_;
+        length myvar $50;
+        do i=1 to 1000 by 50;
+          if mod(i,2)=0 then j=ranuni(0)*i*100;
+          else j=i*100;
+
+          %mp_aligndecimal(myvar,width=7)
+
+          leading_spaces=length(myvar)-length(cats(myvar));
+          putlog +leading_spaces myvar;
+        end;
+      run;
+
+  The generated code will look something like this:
+
+      length aligndp4e49996 $7;
+      if index(myvar,'.') then do;
+        aligndp4e49996=cats(scan(myvar,1,'.'));
+        aligndp4e49996=right(aligndp4e49996);
+        myvar=aligndp4e49996!!'.'!!cats(scan(myvar,2,'.'));
+      end;
+      else do;
+        aligndp4e49996=myvar;
+        aligndp4e49996=right(aligndp4e49996);
+        myvar=aligndp4e49996;
+      end;
+      drop aligndp4e49996;
+
+  Results (myvar variable):
+
+                0.7683559324
+              122.8232796
+            99419.50552
+            42938.5143414
+              763.3799189
+            15170.606073
+            15083.285773
+            85443.198707
+          2022999.2251
+            12038.658867
+          1350582.6734
+            52777.258221
+            11723.347628
+            33101.268376
+          6181622.8603
+          7390614.0669
+            73384.537893
+          1788362.1016
+          2774586.2219
+          7998580.8415
+
+
+  @param var The (data step, character) variable to modify
+  @param width= (8) The number of characters BEFORE the decimal point
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+
+  <h4> Related Programs </h4>
+  @li mp_aligndecimal.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_aligndecimal(var,width=8);
+
+  %local tmpvar;
+  %let tmpvar=%mf_getuniquename(prefix=aligndp);
+  length &tmpvar $&width;
+  if index(&var,'.') then do;
+    &tmpvar=cats(scan(&var,1,'.'));
+    &tmpvar=right(&tmpvar);
+    &var=&tmpvar!!'.'!!cats(scan(&var,2,'.'));
+  end;
+  else do;
+    &tmpvar=cats(&var);
+    &tmpvar=right(&tmpvar);
+    &var=&tmpvar;
+  end;
+  drop &tmpvar;
+
+%mend mp_aligndecimal;
 /**
   @file
   @brief Append (concatenate) two or more files.
@@ -2914,7 +3012,7 @@ run;
   results.  If it does not exist, it will be created, with the following format:
   |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
   |---|---|---|
-  |User Provided description|PASS|Column &inds contained ALL columns|
+  |User Provided description|PASS|Dataset &inds contained ALL columns|
 
   @version 9.2
   @author Allan Bowe
@@ -2988,7 +3086,7 @@ run;
   results.  If it does not exist, it will be created, with the following format:
   |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
   |---|---|---|
-  |User Provided description|PASS|Column &inds contained ALL columns|
+  |User Provided description|PASS|Dataset &inds contained ALL columns|
 
 
   <h4> Related Macros </h4>
@@ -3133,6 +3231,7 @@ run;
   @param [in] test= (ALLVALS) The test to apply.  Valid values are:
     @li ALLVALS - Test is a PASS if ALL values have a match in checkvals
     @li ANYVAL - Test is a PASS if at least 1 value has a match in checkvals
+    @li NOVAL - Test is a PASS if there are NO matches in checkvals
   @param [out] outds= (work.test_results) The output dataset to contain the
   results.  If it does not exist, it will be created, with the following format:
   |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
@@ -3188,7 +3287,7 @@ run;
 
   %let test=%upcase(&test);
 
-  %if &test ne ALLVALS and &test ne ANYVAL %then %do;
+  %if &test ne ALLVALS and &test ne ANYVAL and &test ne NOVAL %then %do;
     %mp_abort(
       mac=&sysmacroname,
       msg=%str(Invalid test - &test)
@@ -3199,12 +3298,12 @@ run;
   %let result=-1;
   %let orig=-1;
   proc sql noprint;
-  select count(*) into: result
+  select count(*) into: result trimmed
     from &lib..&ds
     where &col not in (
       select &ccol from &clib..&cds
     );
-  select count(*) into: orig from &lib..&ds;
+  select count(*) into: orig trimmed from &lib..&ds;
   quit;
 
   %local notfound tmp1 tmp2;
@@ -3236,7 +3335,7 @@ run;
     length test_description $256 test_result $4 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
-    test_comments="&sysmacroname: &lib..&ds..&col has &result values "
+    test_comments="&sysmacroname: &lib..&ds..&col has &result/&orig values "
       !!"not in &clib..&cds..&ccol.. First 10 vals:"!!symget('notfound');
   %if &test=ANYVAL %then %do;
     if &result < &orig then test_result='PASS';
@@ -3244,6 +3343,9 @@ run;
   %else %if &test=ALLVALS %then %do;
     if &result=0 then test_result='PASS';
   %end;
+  %else %if &test=NOVAL %then %do;
+    if &result=&orig then test_result='PASS';
+  %end;
   %else %do;
     test_comments="&sysmacroname: Unsatisfied test condition - &test";
   %end;
@@ -4016,6 +4118,7 @@ run;
   <h4> SAS Macros </h4>
   @li mddl_sas_cntlout.sas
   @li mf_getuniquename.sas
+  @li mp_aligndecimal.sas
 
   <h4> Related Macros </h4>
   @li mf_getvarformat.sas
@@ -4048,25 +4151,33 @@ run;
 %end;
 
 proc format lib=&libcat cntlout=&cntlds;
-%if "&fmtlist" ne "0" %then %do;
+%if "&fmtlist" ne "0" and "&fmtlist" ne "" %then %do;
   select
-  %do i=1 %to %sysfunc(countw(&fmtlist));
+  %do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
     %scan(&fmtlist,&i,%str( ))
   %end;
   ;
 %end;
 run;
 
-data &cntlout;
+data &cntlout/nonote2err;
   if 0 then set &ddlds;
   set &cntlds;
-  if type="N" then do;
-    start=cats(start);
-    end=cats(end);
+  by type fmtname notsorted;
+
+  /* align the numeric values to avoid overlapping ranges */
+  if type in ("I","N") then do;
+    %mp_aligndecimal(start,width=16)
+    %mp_aligndecimal(end,width=16)
   end;
+
+  /* create row marker. Data cannot be sorted without it! */
+  if first.fmtname then fmtrow=0;
+  fmtrow+1;
+
 run;
 proc sort;
-  by fmtname start;
+  by type fmtname fmtrow;
 run;
 
 proc sql;
@@ -5910,6 +6021,7 @@ run;
   options ps=max lrecl=max;
   data _null_;
     infile &outref;
+    if _n_=1 then putlog "# &libds" /;
     input;
     putlog _infile_;
   run;
@@ -7781,10 +7893,15 @@ run;
   Formats are taken from the library / dataset reference and / or a static
   format list.
 
+  Note - the source for this information is the dictionary.formats table. This
+  cannot show formats that are not already declared in the FMTSEARCH path.
+
   Example usage:
 
       %mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable)
 
+      %mp_getformats(fmtlist=FORMAT1 $FORMAT2 @INFMT3,outsummary=work.table2)
+
   @param [in] lib= (0) The libref for which to return formats.
   @todo Enable exporting of formats for an entire library
   @param [in] ds= (0) The dataset from which to obtain format definitions
@@ -7823,7 +7940,9 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 
 
   <h4> Related Macros </h4>
+  @li mf_getfmtlist.sas
   @li mp_applyformats.sas
+  @li mp_cntlout.sas
   @li mp_getformats.test.sas
 
   @version 9.2
@@ -7840,7 +7959,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 
 %local i fmt allfmts tempds fmtcnt;
 
-%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( )));
+%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
   /* ensure format list contains format _name_ only */
   %let fmt=%scan(&fmtlist,&i,%str( ));
   %let fmt=%mf_getfmtname(&fmt);
@@ -7864,8 +7983,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 proc sql;
 create table &outsummary as
   select * from dictionary.formats
-  where fmtname in (%mf_getquotedstr(&allfmts,quote=D))
-    and fmttype='F';
+  where fmtname in (%mf_getquotedstr(&allfmts,quote=D));
 
 %if "&outdetail" ne "0" %then %do;
   /* ensure base table always exists */
@@ -7889,6 +8007,10 @@ create table &outsummary as
     data &tempds;
       if 0 then set &outdetail;
       set &tempds;
+      /* set fmtrow (position of record within the format) */
+      by type fmtname notsorted;
+      if first.fmtname then fmtrow=1;
+      else fmtrow+1;
     run;
     proc append base=&outdetail data=&tempds ;
     run;
@@ -10015,10 +10137,10 @@ select distinct lowcase(memname)
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
-  @li mddl_sas_cntlout.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
+  @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
   @li mp_storediffs.sas
@@ -10026,7 +10148,8 @@ select distinct lowcase(memname)
   <h4> Related Macros </h4>
   @li mddl_dc_difftable.sas
   @li mddl_dc_locktable.sas
-  @li mp_loadformat.test.sas
+  @li mp_loadformat.test.1.sas
+  @li mp_loadformat.test.2.sas
   @li mp_lockanytable.sas
   @li mp_stackdiffs.sas
 
@@ -10048,7 +10171,7 @@ select distinct lowcase(memname)
 );
 /* set up local macro variables and temporary tables (with a prefix) */
 %local err msg prefix dslist i var fmtlist ibufsize;
-%let dslist=base_fmts template inlibds ds1 stagedata storediffs;
+%let dslist=base_fmts template inlibds ds1 stagedata storediffs del1 del2;
 %if &outds_add=0 %then %let dslist=&dslist outds_add;
 %if &outds_del=0 %then %let dslist=&dslist outds_del;
 %if &outds_mod=0 %then %let dslist=&dslist outds_mod;
@@ -10059,13 +10182,6 @@ select distinct lowcase(memname)
   %let &var=%upcase(&prefix._&var);
 %end;
 
-/*
-format values can be up to 32767 wide.  SQL joins on such a wide column can
-cause buffer issues.  Update ibufsize and reset at the end.
-*/
-%let ibufsize=%sysfunc(getoption(ibufsize));
-options ibufsize=32767 ;
-
 /* in DC, format catalogs maybe specified in the libds with a -FC extension */
 %let libcat=%scan(&libcat,1,-);
 
@@ -10115,29 +10231,62 @@ run;
   * First, extract only relevant formats from the catalog
   */
 proc sql noprint;
-select distinct upcase(fmtname) into: fmtlist separated by ' ' from &libds;
+select distinct
+  case
+    when type='N' then upcase(fmtname)
+    when type='C' then cats('$',upcase(fmtname))
+    when type='I' then cats('@',upcase(fmtname))
+    when type='J' then cats('@$',upcase(fmtname))
+    else "&sysmacroname:UNHANDLED"
+  end
+  into: fmtlist separated by ' '
+  from &libds;
 
 %mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
 
+/* get a hash of the row */
+%local cvars nvars;
+%let cvars=TYPE FMTNAME START END LABEL PREFIX FILL SEXCL EEXCL HLO DECSEP
+  DIG3SEP DATATYPE LANGUAGE;
+%let nvars=FMTROW MIN MAX DEFAULT LENGTH FUZZ MULT NOEDIT;
+data &base_fmts/note2err;
+  set &base_fmts;
+  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
+run;
 
 /**
   * Ensure input table and base_formats have consistent lengths and types
   */
-%mddl_sas_cntlout(libds=&template)
-data &inlibds;
-  length &delete_col $3;
-  if 0 then set &template;
+data &inlibds/nonote2err;
+  length &delete_col $3 FMTROW 8 start end label $32767;
+  if 0 then set &base_fmts;
   set &libds;
+  by type fmtname notsorted;
   if &delete_col='' then &delete_col='No';
   fmtname=upcase(fmtname);
+  type=upcase(type);
   if missing(type) then do;
-    if substr(fmtname,1,1)='$' then type='C';
-    else type='N';
+    if substr(fmtname,1,1)='@' then do;
+      if substr(fmtname,2,1)='$' then type='J';
+      else type='I';
+    end;
+    else do;
+      if substr(fmtname,1,1)='$' then type='C';
+      else type='N';
+    end;
   end;
-  if type='N' then do;
-    start=cats(start);
-    end=cats(end);
+  if type in ('N','I') then do;
+    %mp_aligndecimal(start,width=16)
+    %mp_aligndecimal(end,width=16)
   end;
+
+  /* update row marker - retain new var as fmtrow may already be in libds */
+  if first.fmtname then row=1;
+  else row+1;
+  drop row;
+  fmtrow=row;
+
+  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
 run;
 
 /**
@@ -10148,23 +10297,10 @@ create table &outds_add(drop=&delete_col) as
   select a.*
   from &inlibds a
   left join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where b.fmtname is null
     and upcase(a.&delete_col) ne "YES"
-  order by fmtname, start;;
-
-/**
-  * Identify deleted records
-  */
-create table &outds_del(drop=&delete_col) as
-  select a.*
-  from &inlibds a
-  inner join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
-  where upcase(a.&delete_col)="YES"
-  order by fmtname, start;
+  order by type, fmtname, fmtrow;
 
 /**
   * Identify modified records
@@ -10173,12 +10309,40 @@ create table &outds_mod (drop=&delete_col) as
   select a.*
   from &inlibds a
   inner join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where upcase(a.&delete_col) ne "YES"
-  order by fmtname, start;
+    and a.fmthash ne b.fmthash
+  order by type, fmtname, fmtrow;
+
+/**
+  * Identify deleted records
+  */
+create table &outds_del(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
+  where upcase(a.&delete_col)="YES"
+  order by type, fmtname, fmtrow;
+
+/**
+  * Identify fully deleted formats (where every record is removed)
+  * These require to be explicitly deleted in proc format
+  * del1 - identify _partial_ deletes
+  * del2 - exclude these, and also formats that come with _additions_
+  */
+create table &del1 as
+  select a.*
+  from &base_fmts a
+  left join &outds_del b
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
+  where b.fmtrow is null;
+
+create table &del2 as
+  select * from &outds_del
+  where cats(type,fmtname) not in (select cats(type,fmtname) from &outds_add)
+    and cats(type,fmtname) not in (select cats(type,fmtname) from &del1);
 
-options ibufsize=&ibufsize;
 
 %mp_abort(
   iftrue=(&syscc ne 0)
@@ -10187,19 +10351,21 @@ options ibufsize=&ibufsize;
 )
 
 %if &loadtarget=YES %then %do;
+  /* new records plus base records that are not deleted or modified */
   data &ds1;
     merge &base_fmts(in=base)
       &outds_mod(in=mod)
       &outds_add(in=add)
       &outds_del(in=del);
     if not del and not mod;
-    by fmtname start;
+    by type fmtname fmtrow;
   run;
+  /* add back the modified records */
   data &stagedata;
     set &ds1 &outds_mod;
   run;
   proc sort;
-    by fmtname start;
+    by type fmtname fmtrow;
   run;
 %end;
 /* mp abort needs to run outside of conditional blocks */
@@ -10209,7 +10375,7 @@ options ibufsize=&ibufsize;
   ,msg=%str(SYSCC=&syscc prior to actual load)
 )
 %if &loadtarget=YES %then %do;
-  %if %mf_nobs(&stagedata)=0 %then %do;
+  %if %mf_nobs(&stagedata)=0 and %mf_nobs(&del2)=0 %then %do;
     %put There are no changes to load in &libcat!;
     %return;
   %end;
@@ -10225,6 +10391,22 @@ options ibufsize=&ibufsize;
   /* do the actual load */
   proc format lib=&libcat cntlin=&stagedata;
   run;
+  /* apply any full deletes */
+  %if %mf_nobs(&del2)>0 %then %do;
+    %local delfmtlist;
+    proc sql noprint;
+    select distinct case when type='N' then cats(fmtname,'.FORMAT')
+        when type='C' then cats(fmtname,'.FORMATC')
+        when type='J' then cats(fmtname,'.INFMTC')
+        when type='I' then cats(fmtname,'.INFMT')
+        else cats(fmtname,'.BADENTRY!!!') end
+      into: delfmtlist
+      separated by ' '
+      from &del2;
+    proc catalog catalog=&libcat;
+      delete &delfmtlist;
+    quit;
+  %end;
   %if &locklibds ne 0 %then %do;
     /* unlock the table */
     %mp_lockanytable(UNLOCK
@@ -10247,7 +10429,7 @@ options ibufsize=&ibufsize;
 
     %mp_storediffs(&libcat-FC
       ,&base_fmts
-      ,FMTNAME START
+      ,TYPE FMTNAME FMTROW
       ,delds=&outds_del
       ,modds=&outds_mod
       ,appds=&outds_add
@@ -10451,7 +10633,7 @@ run;
         data _null_;
           putlog 'NOTE-' / 'NOTE-';
           putlog "NOTE- &sysmacroname: table locked, waiting "@;
-          putlog "%sysfunc(sleep(&loop_inc)) seconds.. ";
+          putlog "%sysfunc(sleep(&loop_secs)) seconds.. ";
           putlog "NOTE- (iteration &x of &loops)";
           putlog 'NOTE-' / 'NOTE-';
         run;
@@ -10478,32 +10660,39 @@ run;
   %end;
 %end;
 %else %if &ACTION=UNLOCK %then %do;
-  %local status;
+  %local status cnt;
+  %let cnt=0;
   proc sql noprint;
-  select LOCK_STATUS_CD into: status from &ctl_ds
-    where LOCK_LIB ="&lib" and LOCK_DS="&ds";
-  quit;
-  %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
-  %if &status=LOCKED %then %do;
-    data _null_;
-      putlog "&sysmacroname: unlocking &lib..&ds:";
-    run;
-    proc sql;
-    update &ctl_ds
-      set LOCK_STATUS_CD='UNLOCKED'
-        , LOCK_END_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
-        , LOCK_USER_NM="&user"
-        , LOCK_PID="&sysjobid"
-        , LOCK_REF="&ref"
-      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
-    quit;
-  %end;
-  %else %if &status=UNLOCKED %then %do;
-    %put %str(WAR)NING: &lib..&ds is already unlocked!;
+  select count(*) into: cnt from &ctl_ds where LOCK_LIB ="&lib" & LOCK_DS="&ds";
+  %if &cnt=0 %then %do;
+    %put %str(WAR)NING: &lib..&ds was not previously locked in &ctl_ds!;
   %end;
   %else %do;
-    %put NOTE: Unrecognised STATUS_CD (&status) in &ctl_ds;
-    %let abortme=1;
+    select LOCK_STATUS_CD into: status from &ctl_ds
+      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+    quit;
+    %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
+    %if &status=LOCKED %then %do;
+      data _null_;
+        putlog "&sysmacroname: unlocking &lib..&ds:";
+      run;
+      proc sql;
+      update &ctl_ds
+        set LOCK_STATUS_CD='UNLOCKED'
+          , LOCK_END_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
+          , LOCK_USER_NM="&user"
+          , LOCK_PID="&sysjobid"
+          , LOCK_REF="&ref"
+        where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+      quit;
+    %end;
+    %else %if &status=UNLOCKED %then %do;
+      %put %str(WAR)NING: &lib..&ds is already unlocked!;
+    %end;
+    %else %do;
+      %put NOTE: Unrecognised STATUS_CD (&status) in &ctl_ds;
+      %let abortme=1;
+    %end;
   %end;
 %end;
 %else %do;
@@ -12668,9 +12857,9 @@ run;
 
 %if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do;
   /* this is a format catalog - cannot query cols directly */
-  %let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH"
-    ,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO"
-    ,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
+  %let vlist="TYPE","FMTNAME","FMTROW","START","END","LABEL","MIN","MAX"
+    ,"DEFAULT","LENGTH","FUZZ","PREFIX","MULT","FILL","NOEDIT","SEXCL"
+    ,"EEXCL","HLO","DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
 %end;
 %else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
 
@@ -12686,7 +12875,7 @@ data &ds4;
   if upcase(&inds_auto)="&ds2" then tgtvar_type='N';
   else if upcase(&inds_auto)="&ds3" then tgtvar_type='C';
   else do;
-    putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
+    putlog 'ERR' +(-1) "OR: unidentified vartype input!" &inds_auto;
     call symputx('syscc',98);
   end;
 
@@ -12695,7 +12884,7 @@ data &ds4;
   else if &inds_keep="&modds" then move_type='M';
   else if &inds_keep="&origds" then move_type='O';
   else do;
-    putlog "%str(ERR)OR: unidentified movetype input!" &inds_keep;
+    putlog 'ERR' +(-1) "OR: unidentified movetype input!" &inds_keep;
     call symputx('syscc',99);
   end;
   tgtvar_nm=upcase(tgtvar_nm);
@@ -13938,24 +14127,27 @@ ods package close;
 %mend mddl_dc_maxkeytable;/**
   @file
   @brief The CNTLOUT table generated by proc format
-  @details This table will actually change format depending on the data values,
-  therefore the max possible lengths are described here to enable consistency
-  when dealing with format data.
+  @details The actual CNTLOUT table may have varying variable lengths,
+  depending on the data values, therefore the max possible lengths
+  (given various practical restrictions) are described here to enable
+  consistency when dealing with format data.
 
 **/
 
 
 %macro mddl_sas_cntlout(libds=WORK.CNTLOUT);
 
-proc sql;
-create table &libds(
-    FMTNAME char(32)      label='Format name'
+  proc sql;
+  create table &libds(
+    TYPE char(1) label='Type of format - either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
+    ,FMTNAME char(32)     label='Format name'
+    ,FMTROW num label='CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
+    ,START char(32767)    label='Starting value for format'
     /*
-      to accommodate larger START values, mp_loadformat.sas will need the
-      SQL dependency removed (proc sql needs to accommodate 3 index values in
-      a 32767 ibufsize limit)
+      Keep lengths of START and END the same to avoid this err:
+      "Start is greater than end:  -<."
+      Similar usage note: https://support.sas.com/kb/69/330.html
     */
-    ,START char(10000)    label='Starting value for format'
     ,END char(32767)      label='Ending value for format'
     ,LABEL char(32767)    label='Format value label'
     ,MIN num length=3     label='Minimum length'
@@ -13967,15 +14159,26 @@ create table &libds(
     ,MULT num             label='Multiplier'
     ,FILL char(1)         label='Fill character'
     ,NOEDIT num length=3  label='Is picture string noedit?'
-    ,TYPE char(1)         label='Type of format'
     ,SEXCL char(1)        label='Start exclusion'
     ,EEXCL char(1)        label='End exclusion'
-    ,HLO char(13)         label='Additional information'
+    ,HLO char(13)         label='Additional information. M=MultiLabel'
     ,DECSEP char(1)       label='Decimal separator'
     ,DIG3SEP char(1)      label='Three-digit separator'
     ,DATATYPE char(8)     label='Date/time/datetime?'
     ,LANGUAGE char(8)     label='Language for date strings'
-);
+  );
+
+  %local lib;
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create
+      pk_cntlout=(type fmtname fmtrow)
+      /nomiss unique;
+  quit;
 
 %mend mddl_sas_cntlout;
 /**
@@ -14061,7 +14264,8 @@ run;
 filename __us2grp temp;
 
 proc metadata in= "<UpdateMetadata><Reposid>$METAREPOSITORY</Reposid><Metadata>
-    <Person Id='&uuri'><IdentityGroups><IdentityGroup ObjRef='&guri' />
+    <Person Id='%nrstr(&uuri)'>
+    <IdentityGroups><IdentityGroup ObjRef='%nrstr(&guri)' />
     </IdentityGroups></Person></Metadata>
     <NS>SAS</NS><Flags>268435456</Flags></UpdateMetadata>"
   out=__us2grp verbose;
@@ -14078,7 +14282,8 @@ run;
 
 filename __us2grp clear;
 
-%mend mm_adduser2group;/**
+%mend mm_adduser2group;
+/**
   @file
   @brief Assigns library directly using details from metadata
   @details Queries metadata to get the libname definition then allocates the
@@ -16541,9 +16746,11 @@ data _null_;
   put ' ';
   put '%mend mm_webout; ';
 /* WEBOUT END */
-  put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO);';
+  put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO';
+  put '    ,maxobs=MAX';
+  put ');';
   put '  %mm_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt,missing=&missing';
-  put '    ,showmeta=&showmeta';
+  put '    ,showmeta=&showmeta,maxobs=&maxobs';
   put '  )';
   put '%mend;';
 run;
@@ -22106,8 +22313,9 @@ options &optval;
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
-  @li mf_getuniquelibref.sas
+  @li mf_getuniquename.sas
   @li mp_abort.sas
+  @li mp_chop.sas
 
 **/
 
@@ -22220,7 +22428,10 @@ run;
   run;
 %end;
 
-filename &outref temp lrecl=32767;
+%local resp_path;
+%let resp_path=%sysfunc(pathname(work))/%mf_getuniquename();
+filename &outref "&resp_path" lrecl=32767;
+
 /* prepare request*/
 proc http method='POST' headerin=&authref in=&mainref out=&outref
   url="&_sasjs_apiserverurl.&_sasjs_apipath?_program=&pgm%str(&)_debug=131";
@@ -22228,6 +22439,7 @@ proc http method='POST' headerin=&authref in=&mainref out=&outref
   debug level=2;
 %end;
 run;
+
 %if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
 or &mdebug=1
 %then %do;
@@ -22243,11 +22455,22 @@ or &mdebug=1
 options &optval;
 
 %if &outlogds ne _null_ or &mdebug=1 %then %do;
-  %local dumplib;
-  %let dumplib=%mf_getuniquelibref();
-  libname &dumplib json fileref=&outref;
+  %local matchstr chopout;
+  %let matchstr=SASJS_LOGS_SEPARATOR_163ee17b6ff24f028928972d80a26784;
+  %let chopout=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop);
+
+  %mp_chop("&resp_path"
+    ,matchvar=matchstr
+    ,keep=LAST
+    ,matchpoint=END
+    ,outfile="&chopout"
+    ,mdebug=&mdebug
+  )
+
   data &outlogds;
-    set &dumplib..log;
+    infile "&chopout" lrecl=2000;
+    length line $2000;
+    line=_infile_;
   %if &mdebug=1 %then %do;
     putlog line=;
   %end;
@@ -22374,50 +22597,38 @@ run;
 )
 
 
-/* SASjs services have the _webout embedded in wrapper JSON */
-/* Files can also be very large - so use a dedicated macro to chop it out */
-%local matchstr1 matchstr2 ;
-%let matchstr1={"status":"success","_webout":{;
-%let matchstr2=},"log":[{;
-%let chopout1=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop1);
-%let chopout2=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop2);
+/* chop out JSON section */
+%local matchstr chopout;
+%let matchstr=SASJS_LOGS_SEPARATOR_163ee17b6ff24f028928972d80a26784;
+%let chopout=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop);
 
 %mp_chop("%sysfunc(pathname(&fref1,F))"
-  ,matchvar=matchstr1
-  ,keep=LAST
-  ,matchpoint=END
-  ,offset=-1
-  ,outfile="&chopout1"
-  ,mdebug=&mdebug
-)
-
-%mp_chop("&chopout1"
-  ,matchvar=matchstr2
+  ,matchvar=matchstr
   ,keep=FIRST
   ,matchpoint=START
-  ,offset=1
-  ,outfile="&chopout2"
+  ,offset=-1
+  ,outfile="&chopout"
   ,mdebug=&mdebug
 )
 
 %if &outlib ne 0 %then %do;
-  libname &outlib json "&chopout2";
+  libname &outlib json "&chopout";
 %end;
 %if &outref ne 0 %then %do;
-  filename &outref "&chopout2";
+  filename &outref "&chopout";
 %end;
 
 %if &mdebug=0 %then %do;
   filename &webref clear;
   filename &fref1 clear;
-  filename &fref2 clear;
 %end;
 %else %do;
   %put &sysmacroname exit vars:;
   %put _local_;
 %end;
 
-%mend ms_testservice;/**
+%mend ms_testservice;
+/**
   @file
   @brief Send data to/from sasjs/server
   @details This macro should be added to the start of each web service,

base/mf_existvar.sas

@@ -25,13 +25,17 @@
   %local dsid rc;
   %let dsid=%sysfunc(open(&libds,is));
 
-  %if &dsid=0 or %length(&var)=0 %then %do;
+  %if &dsid=0 %then %do;
     %put %sysfunc(sysmsg());
-      0
+    0
+  %end;
+  %else %if %length(&var)=0 %then %do;
+    0
+    %let rc=%sysfunc(close(&dsid));
   %end;
   %else %do;
-      %sysfunc(varnum(&dsid,&var))
-      %let rc=%sysfunc(close(&dsid));
+    %sysfunc(varnum(&dsid,&var))
+    %let rc=%sysfunc(close(&dsid));
   %end;
 
 %mend mf_existvar;

base/mf_getfmtlist.sas

@@ -10,10 +10,9 @@
 
   returns:
 
-  > DOLLAR $CHAR W MONNAME
-  > $CHAR BEST DOLLAR
-  > BEST Z $CHAR COMMA PERCENTN
-
+      DOLLAR $CHAR W MONNAME
+      $CHAR BEST DOLLAR
+      BEST Z $CHAR COMMA PERCENTN
 
   @param [in] libds Two part library.dataset reference.
 

base/mp_aligndecimal.sas

@@ -0,0 +1,95 @@
+/**
+  @file
+  @brief Apply leading blanks to align numbers vertically in a char variable
+  @details This is particularly useful when storing numbers (as character) that
+  need to be sorted.
+
+  It works by splitting the number left and right of the decimal place, and
+  aligning it accordingly.  A temporary variable is created as part of this
+  process (which is automatically dropped)
+
+  The macro can be used only in data step, eg as follows:
+
+      data _null_;
+        length myvar $50;
+        do i=1 to 1000 by 50;
+          if mod(i,2)=0 then j=ranuni(0)*i*100;
+          else j=i*100;
+
+          %mp_aligndecimal(myvar,width=7)
+
+          leading_spaces=length(myvar)-length(cats(myvar));
+          putlog +leading_spaces myvar;
+        end;
+      run;
+
+  The generated code will look something like this:
+
+      length aligndp4e49996 $7;
+      if index(myvar,'.') then do;
+        aligndp4e49996=cats(scan(myvar,1,'.'));
+        aligndp4e49996=right(aligndp4e49996);
+        myvar=aligndp4e49996!!'.'!!cats(scan(myvar,2,'.'));
+      end;
+      else do;
+        aligndp4e49996=myvar;
+        aligndp4e49996=right(aligndp4e49996);
+        myvar=aligndp4e49996;
+      end;
+      drop aligndp4e49996;
+
+  Results (myvar variable):
+
+                0.7683559324
+              122.8232796
+            99419.50552
+            42938.5143414
+              763.3799189
+            15170.606073
+            15083.285773
+            85443.198707
+          2022999.2251
+            12038.658867
+          1350582.6734
+            52777.258221
+            11723.347628
+            33101.268376
+          6181622.8603
+          7390614.0669
+            73384.537893
+          1788362.1016
+          2774586.2219
+          7998580.8415
+
+
+  @param var The (data step, character) variable to modify
+  @param width= (8) The number of characters BEFORE the decimal point
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+
+  <h4> Related Programs </h4>
+  @li mp_aligndecimal.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_aligndecimal(var,width=8);
+
+  %local tmpvar;
+  %let tmpvar=%mf_getuniquename(prefix=aligndp);
+  length &tmpvar $&width;
+  if index(&var,'.') then do;
+    &tmpvar=cats(scan(&var,1,'.'));
+    &tmpvar=right(&tmpvar);
+    &var=&tmpvar!!'.'!!cats(scan(&var,2,'.'));
+  end;
+  else do;
+    &tmpvar=cats(&var);
+    &tmpvar=right(&tmpvar);
+    &var=&tmpvar;
+  end;
+  drop &tmpvar;
+
+%mend mp_aligndecimal;

base/mp_assert.sas

@@ -22,7 +22,7 @@
   results.  If it does not exist, it will be created, with the following format:
   |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
   |---|---|---|
-  |User Provided description|PASS|Column &inds contained ALL columns|
+  |User Provided description|PASS|Dataset &inds contained ALL columns|
 
   @version 9.2
   @author Allan Bowe

base/mp_assertcols.sas

@@ -41,7 +41,7 @@
   results.  If it does not exist, it will be created, with the following format:
   |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
   |---|---|---|
-  |User Provided description|PASS|Column &inds contained ALL columns|
+  |User Provided description|PASS|Dataset &inds contained ALL columns|
 
 
   <h4> Related Macros </h4>

base/mp_assertcolvals.sas

@@ -42,6 +42,7 @@
   @param [in] test= (ALLVALS) The test to apply.  Valid values are:
     @li ALLVALS - Test is a PASS if ALL values have a match in checkvals
     @li ANYVAL - Test is a PASS if at least 1 value has a match in checkvals
+    @li NOVAL - Test is a PASS if there are NO matches in checkvals
   @param [out] outds= (work.test_results) The output dataset to contain the
   results.  If it does not exist, it will be created, with the following format:
   |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
@@ -97,7 +98,7 @@
 
   %let test=%upcase(&test);
 
-  %if &test ne ALLVALS and &test ne ANYVAL %then %do;
+  %if &test ne ALLVALS and &test ne ANYVAL and &test ne NOVAL %then %do;
     %mp_abort(
       mac=&sysmacroname,
       msg=%str(Invalid test - &test)
@@ -108,12 +109,12 @@
   %let result=-1;
   %let orig=-1;
   proc sql noprint;
-  select count(*) into: result
+  select count(*) into: result trimmed
     from &lib..&ds
     where &col not in (
       select &ccol from &clib..&cds
     );
-  select count(*) into: orig from &lib..&ds;
+  select count(*) into: orig trimmed from &lib..&ds;
   quit;
 
   %local notfound tmp1 tmp2;
@@ -145,7 +146,7 @@
     length test_description $256 test_result $4 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
-    test_comments="&sysmacroname: &lib..&ds..&col has &result values "
+    test_comments="&sysmacroname: &lib..&ds..&col has &result/&orig values "
       !!"not in &clib..&cds..&ccol.. First 10 vals:"!!symget('notfound');
   %if &test=ANYVAL %then %do;
     if &result < &orig then test_result='PASS';
@@ -153,6 +154,9 @@
   %else %if &test=ALLVALS %then %do;
     if &result=0 then test_result='PASS';
   %end;
+  %else %if &test=NOVAL %then %do;
+    if &result=&orig then test_result='PASS';
+  %end;
   %else %do;
     test_comments="&sysmacroname: Unsatisfied test condition - &test";
   %end;

base/mp_cntlout.sas

@@ -25,6 +25,7 @@
   <h4> SAS Macros </h4>
   @li mddl_sas_cntlout.sas
   @li mf_getuniquename.sas
+  @li mp_aligndecimal.sas
 
   <h4> Related Macros </h4>
   @li mf_getvarformat.sas
@@ -57,25 +58,33 @@
 %end;
 
 proc format lib=&libcat cntlout=&cntlds;
-%if "&fmtlist" ne "0" %then %do;
+%if "&fmtlist" ne "0" and "&fmtlist" ne "" %then %do;
   select
-  %do i=1 %to %sysfunc(countw(&fmtlist));
+  %do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
     %scan(&fmtlist,&i,%str( ))
   %end;
   ;
 %end;
 run;
 
-data &cntlout;
+data &cntlout/nonote2err;
   if 0 then set &ddlds;
   set &cntlds;
-  if type="N" then do;
-    start=cats(start);
-    end=cats(end);
+  by type fmtname notsorted;
+
+  /* align the numeric values to avoid overlapping ranges */
+  if type in ("I","N") then do;
+    %mp_aligndecimal(start,width=16)
+    %mp_aligndecimal(end,width=16)
   end;
+
+  /* create row marker. Data cannot be sorted without it! */
+  if first.fmtname then fmtrow=0;
+  fmtrow+1;
+
 run;
 proc sort;
-  by fmtname start;
+  by type fmtname fmtrow;
 run;
 
 proc sql;

base/mp_ds2md.sas

@@ -95,6 +95,7 @@ run;
   options ps=max lrecl=max;
   data _null_;
     infile &outref;
+    if _n_=1 then putlog "# &libds" /;
     input;
     putlog _infile_;
   run;

base/mp_getformats.sas

@@ -7,10 +7,15 @@
   Formats are taken from the library / dataset reference and / or a static
   format list.
 
+  Note - the source for this information is the dictionary.formats table. This
+  cannot show formats that are not already declared in the FMTSEARCH path.
+
   Example usage:
 
       %mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable)
 
+      %mp_getformats(fmtlist=FORMAT1 $FORMAT2 @INFMT3,outsummary=work.table2)
+
   @param [in] lib= (0) The libref for which to return formats.
   @todo Enable exporting of formats for an entire library
   @param [in] ds= (0) The dataset from which to obtain format definitions
@@ -49,7 +54,9 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 
 
   <h4> Related Macros </h4>
+  @li mf_getfmtlist.sas
   @li mp_applyformats.sas
+  @li mp_cntlout.sas
   @li mp_getformats.test.sas
 
   @version 9.2
@@ -66,7 +73,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 
 %local i fmt allfmts tempds fmtcnt;
 
-%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( )));
+%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
   /* ensure format list contains format _name_ only */
   %let fmt=%scan(&fmtlist,&i,%str( ));
   %let fmt=%mf_getfmtname(&fmt);
@@ -90,8 +97,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 proc sql;
 create table &outsummary as
   select * from dictionary.formats
-  where fmtname in (%mf_getquotedstr(&allfmts,quote=D))
-    and fmttype='F';
+  where fmtname in (%mf_getquotedstr(&allfmts,quote=D));
 
 %if "&outdetail" ne "0" %then %do;
   /* ensure base table always exists */
@@ -115,6 +121,10 @@ create table &outsummary as
     data &tempds;
       if 0 then set &outdetail;
       set &tempds;
+      /* set fmtrow (position of record within the format) */
+      by type fmtname notsorted;
+      if first.fmtname then fmtrow=1;
+      else fmtrow+1;
     run;
     proc append base=&outdetail data=&tempds ;
     run;

base/mp_loadformat.sas

@@ -34,10 +34,10 @@
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
-  @li mddl_sas_cntlout.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
+  @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
   @li mp_storediffs.sas
@@ -45,7 +45,8 @@
   <h4> Related Macros </h4>
   @li mddl_dc_difftable.sas
   @li mddl_dc_locktable.sas
-  @li mp_loadformat.test.sas
+  @li mp_loadformat.test.1.sas
+  @li mp_loadformat.test.2.sas
   @li mp_lockanytable.sas
   @li mp_stackdiffs.sas
 
@@ -67,7 +68,7 @@
 );
 /* set up local macro variables and temporary tables (with a prefix) */
 %local err msg prefix dslist i var fmtlist ibufsize;
-%let dslist=base_fmts template inlibds ds1 stagedata storediffs;
+%let dslist=base_fmts template inlibds ds1 stagedata storediffs del1 del2;
 %if &outds_add=0 %then %let dslist=&dslist outds_add;
 %if &outds_del=0 %then %let dslist=&dslist outds_del;
 %if &outds_mod=0 %then %let dslist=&dslist outds_mod;
@@ -78,13 +79,6 @@
   %let &var=%upcase(&prefix._&var);
 %end;
 
-/*
-format values can be up to 32767 wide.  SQL joins on such a wide column can
-cause buffer issues.  Update ibufsize and reset at the end.
-*/
-%let ibufsize=%sysfunc(getoption(ibufsize));
-options ibufsize=32767 ;
-
 /* in DC, format catalogs maybe specified in the libds with a -FC extension */
 %let libcat=%scan(&libcat,1,-);
 
@@ -134,29 +128,62 @@ run;
   * First, extract only relevant formats from the catalog
   */
 proc sql noprint;
-select distinct upcase(fmtname) into: fmtlist separated by ' ' from &libds;
+select distinct
+  case
+    when type='N' then upcase(fmtname)
+    when type='C' then cats('$',upcase(fmtname))
+    when type='I' then cats('@',upcase(fmtname))
+    when type='J' then cats('@$',upcase(fmtname))
+    else "&sysmacroname:UNHANDLED"
+  end
+  into: fmtlist separated by ' '
+  from &libds;
 
 %mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
 
+/* get a hash of the row */
+%local cvars nvars;
+%let cvars=TYPE FMTNAME START END LABEL PREFIX FILL SEXCL EEXCL HLO DECSEP
+  DIG3SEP DATATYPE LANGUAGE;
+%let nvars=FMTROW MIN MAX DEFAULT LENGTH FUZZ MULT NOEDIT;
+data &base_fmts/note2err;
+  set &base_fmts;
+  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
+run;
 
 /**
   * Ensure input table and base_formats have consistent lengths and types
   */
-%mddl_sas_cntlout(libds=&template)
-data &inlibds;
-  length &delete_col $3;
-  if 0 then set &template;
+data &inlibds/nonote2err;
+  length &delete_col $3 FMTROW 8 start end label $32767;
+  if 0 then set &base_fmts;
   set &libds;
+  by type fmtname notsorted;
   if &delete_col='' then &delete_col='No';
   fmtname=upcase(fmtname);
+  type=upcase(type);
   if missing(type) then do;
-    if substr(fmtname,1,1)='$' then type='C';
-    else type='N';
+    if substr(fmtname,1,1)='@' then do;
+      if substr(fmtname,2,1)='$' then type='J';
+      else type='I';
+    end;
+    else do;
+      if substr(fmtname,1,1)='$' then type='C';
+      else type='N';
+    end;
   end;
-  if type='N' then do;
-    start=cats(start);
-    end=cats(end);
+  if type in ('N','I') then do;
+    %mp_aligndecimal(start,width=16)
+    %mp_aligndecimal(end,width=16)
   end;
+
+  /* update row marker - retain new var as fmtrow may already be in libds */
+  if first.fmtname then row=1;
+  else row+1;
+  drop row;
+  fmtrow=row;
+
+  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
 run;
 
 /**
@@ -167,23 +194,10 @@ create table &outds_add(drop=&delete_col) as
   select a.*
   from &inlibds a
   left join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where b.fmtname is null
     and upcase(a.&delete_col) ne "YES"
-  order by fmtname, start;;
-
-/**
-  * Identify deleted records
-  */
-create table &outds_del(drop=&delete_col) as
-  select a.*
-  from &inlibds a
-  inner join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
-  where upcase(a.&delete_col)="YES"
-  order by fmtname, start;
+  order by type, fmtname, fmtrow;
 
 /**
   * Identify modified records
@@ -192,12 +206,40 @@ create table &outds_mod (drop=&delete_col) as
   select a.*
   from &inlibds a
   inner join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where upcase(a.&delete_col) ne "YES"
-  order by fmtname, start;
+    and a.fmthash ne b.fmthash
+  order by type, fmtname, fmtrow;
+
+/**
+  * Identify deleted records
+  */
+create table &outds_del(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
+  where upcase(a.&delete_col)="YES"
+  order by type, fmtname, fmtrow;
+
+/**
+  * Identify fully deleted formats (where every record is removed)
+  * These require to be explicitly deleted in proc format
+  * del1 - identify _partial_ deletes
+  * del2 - exclude these, and also formats that come with _additions_
+  */
+create table &del1 as
+  select a.*
+  from &base_fmts a
+  left join &outds_del b
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
+  where b.fmtrow is null;
+
+create table &del2 as
+  select * from &outds_del
+  where cats(type,fmtname) not in (select cats(type,fmtname) from &outds_add)
+    and cats(type,fmtname) not in (select cats(type,fmtname) from &del1);
 
-options ibufsize=&ibufsize;
 
 %mp_abort(
   iftrue=(&syscc ne 0)
@@ -206,19 +248,21 @@ options ibufsize=&ibufsize;
 )
 
 %if &loadtarget=YES %then %do;
+  /* new records plus base records that are not deleted or modified */
   data &ds1;
     merge &base_fmts(in=base)
       &outds_mod(in=mod)
       &outds_add(in=add)
       &outds_del(in=del);
     if not del and not mod;
-    by fmtname start;
+    by type fmtname fmtrow;
   run;
+  /* add back the modified records */
   data &stagedata;
     set &ds1 &outds_mod;
   run;
   proc sort;
-    by fmtname start;
+    by type fmtname fmtrow;
   run;
 %end;
 /* mp abort needs to run outside of conditional blocks */
@@ -228,7 +272,7 @@ options ibufsize=&ibufsize;
   ,msg=%str(SYSCC=&syscc prior to actual load)
 )
 %if &loadtarget=YES %then %do;
-  %if %mf_nobs(&stagedata)=0 %then %do;
+  %if %mf_nobs(&stagedata)=0 and %mf_nobs(&del2)=0 %then %do;
     %put There are no changes to load in &libcat!;
     %return;
   %end;
@@ -244,6 +288,22 @@ options ibufsize=&ibufsize;
   /* do the actual load */
   proc format lib=&libcat cntlin=&stagedata;
   run;
+  /* apply any full deletes */
+  %if %mf_nobs(&del2)>0 %then %do;
+    %local delfmtlist;
+    proc sql noprint;
+    select distinct case when type='N' then cats(fmtname,'.FORMAT')
+        when type='C' then cats(fmtname,'.FORMATC')
+        when type='J' then cats(fmtname,'.INFMTC')
+        when type='I' then cats(fmtname,'.INFMT')
+        else cats(fmtname,'.BADENTRY!!!') end
+      into: delfmtlist
+      separated by ' '
+      from &del2;
+    proc catalog catalog=&libcat;
+      delete &delfmtlist;
+    quit;
+  %end;
   %if &locklibds ne 0 %then %do;
     /* unlock the table */
     %mp_lockanytable(UNLOCK
@@ -266,7 +326,7 @@ options ibufsize=&ibufsize;
 
     %mp_storediffs(&libcat-FC
       ,&base_fmts
-      ,FMTNAME START
+      ,TYPE FMTNAME FMTROW
       ,delds=&outds_del
       ,modds=&outds_mod
       ,appds=&outds_add

base/mp_lockanytable.sas

@@ -167,7 +167,7 @@ run;
         data _null_;
           putlog 'NOTE-' / 'NOTE-';
           putlog "NOTE- &sysmacroname: table locked, waiting "@;
-          putlog "%sysfunc(sleep(&loop_inc)) seconds.. ";
+          putlog "%sysfunc(sleep(&loop_secs)) seconds.. ";
           putlog "NOTE- (iteration &x of &loops)";
           putlog 'NOTE-' / 'NOTE-';
         run;
@@ -194,32 +194,39 @@ run;
   %end;
 %end;
 %else %if &ACTION=UNLOCK %then %do;
-  %local status;
+  %local status cnt;
+  %let cnt=0;
   proc sql noprint;
-  select LOCK_STATUS_CD into: status from &ctl_ds
-    where LOCK_LIB ="&lib" and LOCK_DS="&ds";
-  quit;
-  %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
-  %if &status=LOCKED %then %do;
-    data _null_;
-      putlog "&sysmacroname: unlocking &lib..&ds:";
-    run;
-    proc sql;
-    update &ctl_ds
-      set LOCK_STATUS_CD='UNLOCKED'
-        , LOCK_END_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
-        , LOCK_USER_NM="&user"
-        , LOCK_PID="&sysjobid"
-        , LOCK_REF="&ref"
-      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
-    quit;
-  %end;
-  %else %if &status=UNLOCKED %then %do;
-    %put %str(WAR)NING: &lib..&ds is already unlocked!;
+  select count(*) into: cnt from &ctl_ds where LOCK_LIB ="&lib" & LOCK_DS="&ds";
+  %if &cnt=0 %then %do;
+    %put %str(WAR)NING: &lib..&ds was not previously locked in &ctl_ds!;
   %end;
   %else %do;
-    %put NOTE: Unrecognised STATUS_CD (&status) in &ctl_ds;
-    %let abortme=1;
+    select LOCK_STATUS_CD into: status from &ctl_ds
+      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+    quit;
+    %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
+    %if &status=LOCKED %then %do;
+      data _null_;
+        putlog "&sysmacroname: unlocking &lib..&ds:";
+      run;
+      proc sql;
+      update &ctl_ds
+        set LOCK_STATUS_CD='UNLOCKED'
+          , LOCK_END_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
+          , LOCK_USER_NM="&user"
+          , LOCK_PID="&sysjobid"
+          , LOCK_REF="&ref"
+        where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+      quit;
+    %end;
+    %else %if &status=UNLOCKED %then %do;
+      %put %str(WAR)NING: &lib..&ds is already unlocked!;
+    %end;
+    %else %do;
+      %put NOTE: Unrecognised STATUS_CD (&status) in &ctl_ds;
+      %let abortme=1;
+    %end;
   %end;
 %end;
 %else %do;

base/mp_storediffs.sas

@@ -147,9 +147,9 @@ run;
 
 %if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do;
   /* this is a format catalog - cannot query cols directly */
-  %let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH"
-    ,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO"
-    ,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
+  %let vlist="TYPE","FMTNAME","FMTROW","START","END","LABEL","MIN","MAX"
+    ,"DEFAULT","LENGTH","FUZZ","PREFIX","MULT","FILL","NOEDIT","SEXCL"
+    ,"EEXCL","HLO","DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
 %end;
 %else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
 
@@ -165,7 +165,7 @@ data &ds4;
   if upcase(&inds_auto)="&ds2" then tgtvar_type='N';
   else if upcase(&inds_auto)="&ds3" then tgtvar_type='C';
   else do;
-    putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
+    putlog 'ERR' +(-1) "OR: unidentified vartype input!" &inds_auto;
     call symputx('syscc',98);
   end;
 
@@ -174,7 +174,7 @@ data &ds4;
   else if &inds_keep="&modds" then move_type='M';
   else if &inds_keep="&origds" then move_type='O';
   else do;
-    putlog "%str(ERR)OR: unidentified movetype input!" &inds_keep;
+    putlog 'ERR' +(-1) "OR: unidentified movetype input!" &inds_keep;
     call symputx('syscc',99);
   end;
   tgtvar_nm=upcase(tgtvar_nm);

ddl/mddl_sas_cntlout.sas

@@ -1,24 +1,27 @@
 /**
   @file
   @brief The CNTLOUT table generated by proc format
-  @details This table will actually change format depending on the data values,
-  therefore the max possible lengths are described here to enable consistency
-  when dealing with format data.
+  @details The actual CNTLOUT table may have varying variable lengths,
+  depending on the data values, therefore the max possible lengths
+  (given various practical restrictions) are described here to enable
+  consistency when dealing with format data.
 
 **/
 
 
 %macro mddl_sas_cntlout(libds=WORK.CNTLOUT);
 
-proc sql;
-create table &libds(
-    FMTNAME char(32)      label='Format name'
+  proc sql;
+  create table &libds(
+    TYPE char(1) label='Type of format - either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
+    ,FMTNAME char(32)     label='Format name'
+    ,FMTROW num label='CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
+    ,START char(32767)    label='Starting value for format'
     /*
-      to accommodate larger START values, mp_loadformat.sas will need the
-      SQL dependency removed (proc sql needs to accommodate 3 index values in
-      a 32767 ibufsize limit)
+      Keep lengths of START and END the same to avoid this err:
+      "Start is greater than end:  -<."
+      Similar usage note: https://support.sas.com/kb/69/330.html
     */
-    ,START char(10000)    label='Starting value for format'
     ,END char(32767)      label='Ending value for format'
     ,LABEL char(32767)    label='Format value label'
     ,MIN num length=3     label='Minimum length'
@@ -30,14 +33,25 @@ create table &libds(
     ,MULT num             label='Multiplier'
     ,FILL char(1)         label='Fill character'
     ,NOEDIT num length=3  label='Is picture string noedit?'
-    ,TYPE char(1)         label='Type of format'
     ,SEXCL char(1)        label='Start exclusion'
     ,EEXCL char(1)        label='End exclusion'
-    ,HLO char(13)         label='Additional information'
+    ,HLO char(13)         label='Additional information. M=MultiLabel'
     ,DECSEP char(1)       label='Decimal separator'
     ,DIG3SEP char(1)      label='Three-digit separator'
     ,DATATYPE char(8)     label='Date/time/datetime?'
     ,LANGUAGE char(8)     label='Language for date strings'
-);
+  );
+
+  %local lib;
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create
+      pk_cntlout=(type fmtname fmtrow)
+      /nomiss unique;
+  quit;
 
 %mend mddl_sas_cntlout;

meta/mm_adduser2group.sas

@@ -81,7 +81,8 @@ run;
 filename __us2grp temp;
 
 proc metadata in= "<UpdateMetadata><Reposid>$METAREPOSITORY</Reposid><Metadata>
-    <Person Id='&uuri'><IdentityGroups><IdentityGroup ObjRef='&guri' />
+    <Person Id='%nrstr(&uuri)'>
+    <IdentityGroups><IdentityGroup ObjRef='%nrstr(&guri)' />
     </IdentityGroups></Person></Metadata>
     <NS>SAS</NS><Flags>268435456</Flags></UpdateMetadata>"
   out=__us2grp verbose;
@@ -98,4 +99,4 @@ run;
 
 filename __us2grp clear;
 
-%mend mm_adduser2group;
+%mend mm_adduser2group;

meta/mm_createwebservice.sas

@@ -646,9 +646,11 @@ data _null_;
   put ' ';
   put '%mend mm_webout; ';
 /* WEBOUT END */
-  put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO);';
+  put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO';
+  put '    ,maxobs=MAX';
+  put ');';
   put '  %mm_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt,missing=&missing';
-  put '    ,showmeta=&showmeta';
+  put '    ,showmeta=&showmeta,maxobs=&maxobs';
   put '  )';
   put '%mend;';
 run;

package.json

@@ -33,6 +33,6 @@
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "3.24.0"
+    "@sasjs/cli": "^4.4.1"
   }
-}
+}

server/ms_runstp.sas

@@ -39,8 +39,9 @@
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
-  @li mf_getuniquelibref.sas
+  @li mf_getuniquename.sas
   @li mp_abort.sas
+  @li mp_chop.sas
 
 **/
 
@@ -153,7 +154,10 @@ run;
   run;
 %end;
 
-filename &outref temp lrecl=32767;
+%local resp_path;
+%let resp_path=%sysfunc(pathname(work))/%mf_getuniquename();
+filename &outref "&resp_path" lrecl=32767;
+
 /* prepare request*/
 proc http method='POST' headerin=&authref in=&mainref out=&outref
   url="&_sasjs_apiserverurl.&_sasjs_apipath?_program=&pgm%str(&)_debug=131";
@@ -161,6 +165,7 @@ proc http method='POST' headerin=&authref in=&mainref out=&outref
   debug level=2;
 %end;
 run;
+
 %if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
 or &mdebug=1
 %then %do;
@@ -176,11 +181,22 @@ or &mdebug=1
 options &optval;
 
 %if &outlogds ne _null_ or &mdebug=1 %then %do;
-  %local dumplib;
-  %let dumplib=%mf_getuniquelibref();
-  libname &dumplib json fileref=&outref;
+  %local matchstr chopout;
+  %let matchstr=SASJS_LOGS_SEPARATOR_163ee17b6ff24f028928972d80a26784;
+  %let chopout=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop);
+
+  %mp_chop("&resp_path"
+    ,matchvar=matchstr
+    ,keep=LAST
+    ,matchpoint=END
+    ,outfile="&chopout"
+    ,mdebug=&mdebug
+  )
+
   data &outlogds;
-    set &dumplib..log;
+    infile "&chopout" lrecl=2000;
+    length line $2000;
+    line=_infile_;
   %if &mdebug=1 %then %do;
     putlog line=;
   %end;

server/ms_testservice.sas

@@ -108,47 +108,34 @@ run;
 )
 
 
-/* SASjs services have the _webout embedded in wrapper JSON */
-/* Files can also be very large - so use a dedicated macro to chop it out */
-%local matchstr1 matchstr2 ;
-%let matchstr1={"status":"success","_webout":{;
-%let matchstr2=},"log":[{;
-%let chopout1=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop1);
-%let chopout2=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop2);
+/* chop out JSON section */
+%local matchstr chopout;
+%let matchstr=SASJS_LOGS_SEPARATOR_163ee17b6ff24f028928972d80a26784;
+%let chopout=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop);
 
 %mp_chop("%sysfunc(pathname(&fref1,F))"
-  ,matchvar=matchstr1
-  ,keep=LAST
-  ,matchpoint=END
-  ,offset=-1
-  ,outfile="&chopout1"
-  ,mdebug=&mdebug
-)
-
-%mp_chop("&chopout1"
-  ,matchvar=matchstr2
+  ,matchvar=matchstr
   ,keep=FIRST
   ,matchpoint=START
-  ,offset=1
-  ,outfile="&chopout2"
+  ,offset=-1
+  ,outfile="&chopout"
   ,mdebug=&mdebug
 )
 
 %if &outlib ne 0 %then %do;
-  libname &outlib json "&chopout2";
+  libname &outlib json "&chopout";
 %end;
 %if &outref ne 0 %then %do;
-  filename &outref "&chopout2";
+  filename &outref "&chopout";
 %end;
 
 %if &mdebug=0 %then %do;
   filename &webref clear;
   filename &fref1 clear;
-  filename &fref2 clear;
 %end;
 %else %do;
   %put &sysmacroname exit vars:;
   %put _local_;
 %end;
 
-%mend ms_testservice;
+%mend ms_testservice;

tests/base/mf_existvar.test.sas

@@ -17,4 +17,24 @@
 %mp_assert(
   iftrue=(%mf_existvar(sashelp.class,isjustanumber)=0),
   desc=Checking non existing var does not exist
+)
+
+data work.lockcheck;
+  a=1;
+  output;
+  stop;
+run;
+
+%mp_assert(
+  iftrue=(%mf_existvar(work.lockcheck,)=0),
+  desc=Checking non-provided var does not exist
+)
+
+proc sql;
+update work.lockcheck set a=2;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking the lock was released,
+  outds=work.test_results
 )

tests/base/mp_aligndecimal.test.sas

@@ -0,0 +1,46 @@
+/**
+  @file
+  @brief Testing mp_aligndecimal macro
+  @details Creates an aligned variable and checks the number of leading blanks
+
+  <h4> SAS Macros </h4>
+  @li mp_aligndecimal.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertscope.sas
+
+**/
+
+
+
+/* target values */
+data work.checkds;
+  do checkval='   0.56',' 123.45',' 123.4 ','   1.2 ','   0';
+    output;
+  end;
+run;
+
+/* raw values */
+data work.rawds;
+  set work.checkds;
+  tgtvar=cats(checkval);
+  drop checkval;
+run;
+%mp_assertcolvals(work.rawds.tgtvar,
+  checkvals=work.checkds.checkval,
+  desc=No values match (ready to align),
+  test=NOVAL
+)
+
+/* aligned values */
+%mp_assertscope(SNAPSHOT)
+data work.finalds;
+  set work.rawds;
+  %mp_aligndecimal(tgtvar,width=4)
+run;
+%mp_assertscope(COMPARE)
+
+%mp_assertcolvals(work.finalds.tgtvar,
+  checkvals=work.checkds.checkval,
+  desc=All values match (aligned),
+  test=ALLVALS
+)

tests/base/mp_getcols.test.sas

@@ -7,23 +7,35 @@
   @li mp_assertcols.sas
   @li mp_assertcolvals.sas
   @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
 
 **/
 
 
-/* valid filter */
-%mp_getcols(sashelp.airline,outds=work.info)
+/* make some data */
+proc sql;
+create table work.src(
+  SOME_DATETIME float format=datetime19.,
+  SOME_CHAR char(16),
+  SOME_NUM num,
+  SOME_TIME num format=time8.,
+  SOME_DATE num format=date9.
+);
 
+/* run macro, checking for scope leakage */
+%mp_assertscope(SNAPSHOT)
+%mp_getcols(work.src,outds=work.info)
+%mp_assertscope(COMPARE)
 
 %mp_assertdsobs(work.info,
-  desc=Has 3 records,
-  test=EQUALS 3,
+  desc=Has 5 records,
+  test=EQUALS 5,
   outds=work.test_results
 )
 
 data work.check;
   length val $10;
-  do val='NUMERIC','DATE','CHARACTER';
+  do val='NUMERIC','DATE','CHARACTER','DATETIME','TIME';
     output;
   end;
 run;

tests/base/mp_loadformat.test.1.sas

@@ -0,0 +1,231 @@
+/**
+  @file
+  @brief Testing mp_loadformat.sas macro
+  @details first test regular formats, then informats
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mp_aligndecimal.sas
+  @li mp_cntlout.sas
+  @li mp_loadformat.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+/* prep format catalog */
+libname perm (work);
+
+%mddl_dc_difftable(libds=perm.audit)
+
+/* set up regular formats */
+data work.loadfmts;
+  /* matching start / end lengths (to baseds) are important */
+  length fmtname $32 start end $10000;
+  eexcl='Y';
+  type='N';
+  do i=1 to 10;
+    fmtname=cats('SASJS_',put(i,z4.),'X');
+    do j=1 to 20;
+      start=cats(j);
+      end=cats(j+1);
+      %mp_aligndecimal(start,width=16)
+      %mp_aligndecimal(end,width=16)
+      label= cats('Numeric Format ',start);
+      output;
+    end;
+  end;
+run;
+proc format cntlin=work.loadfmts library=perm.testcat;
+run;
+
+/*
+  use actual format data as test baseline, as proc format adds attributes eg
+  min/max etc
+*/
+%mp_cntlout(libcat=perm.testcat,cntlout=work.loadfmts2)
+
+/* make some test data */
+data work.stagedata;
+  set work.loadfmts2 end=lastobs;
+  by type fmtname;
+
+  if lastobs then do;
+    output;
+    fmtname='NEWFMT'!!cats(_n_,'x'); /* 1 new record */
+    start=cats(_n_);
+    end=cats(_n_+1);
+    %mp_aligndecimal(start,width=16)
+    %mp_aligndecimal(end,width=16)
+    label='newval'!!cats(_N_,'X');
+    output;
+    stop;
+  end;
+  else if last.fmtname then deleteme='Yes'; /* 9 deletions */
+  else if first.fmtname then label='modified '!!cats(_n_); /* 10 changes */
+
+  output;
+run;
+
+/* load the above */
+%mp_assertscope(SNAPSHOT)
+%mp_loadformat(perm.testcat
+  ,work.stagedata
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_test1
+  ,outds_del=del_test1
+  ,outds_mod=mod_test1
+  ,mdebug=1
+)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(%mf_nobs(del_test1)=9),
+  desc=Test 1 - delete obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(add_test1)=1),
+  desc=Test 1 - add obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(mod_test1)=10),
+  desc=Test 1 - mod obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(perm.audit)=440),
+  desc=Test 1 - audit table updated,
+  outds=work.test_results
+)
+data work.difftest;
+  set perm.audit;
+  where is_diff=1;
+run;
+%mp_assert(
+  iftrue=(%mf_nobs(work.difftest)>0),
+  desc=Test 1 - diffs were found,
+  outds=work.test_results
+)
+
+/* set up a mix of formats */
+data work.loadfmts3;
+  length fmtname $32 start end $10000;
+  eexcl='Y';
+  type='J';
+  do i=1 to 3;
+    fmtname=cats('SASJS_CI_',i,'X');
+    do j=1 to 4;
+      start=cats(j);
+      end=start;
+      label= cats('Char INFORMAT ',start);
+      output;
+    end;
+  end;
+  type='I';
+  do i=1 to 3;
+    fmtname=cats('SASJS_NI_',i,'X');
+    do j=1 to 4;
+      start=cats(j);
+      end=cats(j+1);
+      %mp_aligndecimal(start,width=16)
+      %mp_aligndecimal(end,width=16)
+      label= cats(ranuni(0));
+      output;
+    end;
+  end;
+  type='N';
+  do i=1 to 3;
+    fmtname=cats('SASJS_NF_',i,'X');
+    do j=1 to 4;
+      start=cats(j);
+      end=cats(j+1);
+      %mp_aligndecimal(start,width=16)
+      %mp_aligndecimal(end,width=16)
+      label= cats('Numeric Format ',start);
+      output;
+    end;
+  end;
+  type='C';
+  do i=1 to 3;
+    fmtname=cats('SASJS_CF_',i,'X');
+    do j=1 to 4;
+      start=cats(j);
+      end=start;
+      label= cats('Char Format ',start);
+      output;
+    end;
+  end;
+  drop i j;
+run;
+proc format cntlin=work.loadfmts3 library=perm.testcat3;
+run;
+%mp_cntlout(libcat=perm.testcat3,cntlout=work.loadfmts4)
+
+/* make some test data */
+data work.stagedata3;
+  set work.loadfmts4;
+  where type in ('I','J');
+  by type fmtname notsorted;
+  if type='I' then do;
+    if last.fmtname then do;
+      deleteme='Yes'; /* 3 deletions */
+      output;
+    end;
+    else if fmtrow le 3 then do; /* 9 changed values */
+      z=ranuni(0)*1000000;
+      start=cats(z);
+      end=cats(z+1);
+      %mp_aligndecimal(start,width=16)
+      %mp_aligndecimal(end,width=16)
+      output;
+    end;
+  end;
+  else do;
+    if last.fmtname then do;
+      output; /* 6 new records */
+      x=_n_;
+      x+1;start=cats("mod",x);end=start;label='newlabel1';output;
+      x+1;start=cats("mod",x);end=start;label='newlabel2';output;
+    end;
+    else if fmtrow le 3 then do; /* 9 more changed values */
+      start= cats("mod",_n_);
+      end=start;
+      label= "mod "||cats(ranuni(0)*100);
+      output;
+    end;
+  end;
+run;
+
+%mp_loadformat(perm.testcat3
+  ,work.stagedata3
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_test2
+  ,outds_del=del_test2
+  ,outds_mod=mod_test2
+  ,mdebug=1
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(del_test2)=3),
+  desc=Test 2 - delete obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(mod_test2)=18),
+  desc=Test 2 - mod obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(add_test2)=6),
+  desc=Test 2 - add obs,
+  outds=work.test_results
+)
+

tests/base/mp_loadformat.test.2.sas

@@ -0,0 +1,234 @@
+/**
+  @file
+  @brief Testing mp_loadformat.sas macro for multilabel formats
+  @details Multilabel records can be complete duplicates!!  Also, the order is
+  important.
+
+  The provided formats create a table as follows:
+
+
+|TYPE:$1.|FMTNAME:$32.|START:$10000.|END:$10000.|LABEL:$32767.|MIN:best.|MAX:best.|DEFAULT:best.|LENGTH:best.|FUZZ:best.|PREFIX:$2.|MULT:best.|FILL:$1.|NOEDIT:best.|SEXCL:$1.|EEXCL:$1.|HLO:$13.|DECSEP:$1.|DIG3SEP:$1.|DATATYPE:$8.|LANGUAGE:$8.|
+|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
+|`C `|`GENDERML `|` `|` `|`Total people `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`1 `|`1 `|`Male `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`1 `|`1 `|`Total people `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`2 `|`2 `|`Female `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`2 `|`2 `|`Female `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`2 `|`2 `|`Thormale `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`2 `|`2 `|`Total people `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`N `|`AGEMLA `|`1 `|`4 `|`Preschool `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLA `|`1 `|`18 `|`Children `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLA `|`19 `|`120 `|`Adults `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLB `|`1 `|`4 `|`Preschool `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLB `|`1 `|`18 `|`Children `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLB `|`19 `|`120 `|`Adults `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLC `|`1 `|`18 `|`Children `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLC `|`1 `|`4 `|`Preschool `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLC `|`19 `|`120 `|`Adults `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_cntlout.sas
+  @li mp_loadformat.sas
+  @li mp_assert.sas
+  @li mp_assertdsobs.sas
+  @li mp_getformats.sas
+  @li mp_ds2md.sas
+
+
+**/
+
+/* prep format catalog */
+libname perm (work);
+
+/* create some multilabel formats */
+%let cat1=perm.test1;
+proc format library=&cat1;
+  value $genderml (multilabel notsorted)
+    '1'='Male'
+    '2'='Female'
+    '2'='Female'
+    '2'='Farmale'
+    '1','2',' '='Total people';
+  value agemla (multilabel)
+    1-4='Preschool'
+    1-18='Children'
+    19-120='Adults';
+  value agemlb (multilabel)
+    19-120='Adults'
+    1-18='Children'
+    1-4='Preschool';
+  value agemlc (multilabel notsorted)
+    19-120='Adults'
+    1-18='Children'
+    1-4='Preschool';
+run;
+
+%mp_cntlout(libcat=&cat1,cntlout=work.cntlout1)
+%mp_assertdsobs(work.cntlout1,
+  desc=Has 16 records,
+  test=EQUALS 16
+)
+
+data work.stagedata3;
+  set work.cntlout1;
+  if fmtname='AGEMLA' and label ne 'Preschool' then deleteme='Yes';
+  if fmtname='AGEMLB' and label = 'Preschool' then label='Kids';
+  if fmtname='GENDERML' and label='Farmale' then output;
+  output;
+run;
+
+
+%mp_loadformat(&cat1
+  ,work.stagedata3
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_test1
+  ,outds_del=del_test1
+  ,outds_mod=mod_test1
+  ,mdebug=1
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(del_test1)=2),
+  desc=Test 1 - deleted obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(mod_test1)=4),
+  desc=Test 1 - mod obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(add_test1)=1),
+  desc=Test 1 - add obs,
+  outds=work.test_results
+)
+
+/* now check the order of the notsorted format */
+%mp_cntlout(libcat=&cat1,cntlout=work.cntlout2)
+
+%let check1=0;
+%let check2=0;
+data test;
+  set work.cntlout2;
+  where fmtname='GENDERML';
+  if _n_=4 and label='Farmale' then call symputx('check1',1);
+  if _n_=5 and label='Farmale' then call symputx('check2',1);
+run;
+%mp_assert(
+  iftrue=(&check1=1 and &check2=1),
+  desc=Ensuring Farmale values retain their order,
+  outds=work.test_results
+)
+
+/**
+  * completely delete a format and make sure it is removed
+  */
+
+/* first, make sure these three formats exist */
+options insert=(fmtsearch=(&cat1));
+%mp_getformats(fmtlist=AGEMLA AGEMLB AGEMLC $GENDERML,outsummary=work.fmtdels)
+
+%let fmtlist=NONE;
+proc sql;
+select distinct cats(fmtname) into: fmtlist separated by ' ' from work.fmtdels;
+
+%mp_assert(
+  iftrue=(%mf_nobs(fmtdels)=4),
+  desc=Deletion test 1 - ensure formats exist for deletion (&fmtlist found),
+  outds=work.test_results
+)
+
+/* deltest1 - deleting every record */
+%mp_cntlout(libcat=&cat1,cntlout=work.cntloutdel1)
+data work.stagedatadel1;
+  set work.cntloutdel1;
+  if fmtname='AGEMLA';
+  deleteme='Yes';
+run;
+%mp_loadformat(&cat1
+  ,work.stagedatadel1
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_testdel1
+  ,outds_del=del_testdel1
+  ,outds_mod=mod_testdel1
+  ,mdebug=1
+)
+%mp_getformats(fmtlist=AGEMLA,outsummary=work.fmtdel1)
+%mp_assert(
+  iftrue=(%mf_nobs(fmtdel1)=0),
+  desc=Deletion test 1 - ensure AGEMLA format was fully deleted,
+  outds=work.test_results
+)
+
+/* deltest2 - deleting every record except 1 */
+data work.stagedatadel2;
+  set work.cntloutdel1;
+  if fmtname='AGEMLB';
+  x+1;
+  if x>1 then deleteme='Yes';
+run;
+%mp_loadformat(&cat1
+  ,work.stagedatadel2
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_testdel2
+  ,outds_del=del_testdel2
+  ,outds_mod=mod_testdel2
+  ,mdebug=1
+)
+%mp_getformats(fmtlist=AGEMLB,outsummary=work.fmtdel2)
+%mp_assert(
+  iftrue=(%mf_nobs(fmtdel2)=1),
+  desc=Deletion test 2 - ensure AGEMLB format was not fully deleted,
+  outds=work.test_results
+)
+
+
+/* deltest3 - deleting every record, and adding a new one */
+data work.stagedatadel3;
+  set work.cntloutdel1;
+  if fmtname='GENDERML';
+  deleteme='Yes';
+run;
+data work.stagedatadel3;
+  set work.stagedatadel3 end=last;
+  output;
+  if last then do;
+    deleteme='No';
+    /* must be a new fmtrow (key value) if adding new row in same load! */
+    fmtrow=1000;
+    start='Mail';
+    end='Mail';
+    output;
+  end;
+run;
+
+%mp_loadformat(&cat1
+  ,work.stagedatadel3
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_testdel2
+  ,outds_del=del_testdel2
+  ,outds_mod=mod_testdel2
+  ,mdebug=1
+)
+%mp_getformats(fmtlist=$GENDERML,outsummary=work.fmtdel3)
+%mp_assert(
+  iftrue=(%mf_nobs(fmtdel3)=1),
+  desc=Deletion test 3 - ensure GENDERML format was not fully deleted,
+  outds=work.test_results
+)
+
+%mp_ds2md(work.fmtdel3)

tests/base/mp_loadformat.test.sas

@@ -1,93 +0,0 @@
-/**
-  @file
-  @brief Testing mp_loadformat.sas macro
-
-  <h4> SAS Macros </h4>
-  @li mddl_dc_difftable.sas
-  @li mp_loadformat.sas
-  @li mp_assert.sas
-  @li mp_assertscope.sas
-
-**/
-
-/* prep format catalog */
-libname perm (work);
-
-%mddl_dc_difftable(libds=perm.audit)
-
-data work.loadfmts;
-  length fmtname $32;
-  eexcl='Y';
-  type='N';
-  do i=1 to 100;
-    fmtname=cats('SASJS_',i,'X');
-    do j=1 to 100;
-      start=cats(j);
-      end=cats(j+1);
-      label= cats('Dummy ',start);
-      output;
-    end;
-  end;
-run;
-proc format cntlin=work.loadfmts library=perm.testcat;
-run;
-
-/* make some test data */
-data work.stagedata;
-  set work.loadfmts;
-  type='N';
-  eexcl='Y';
-  if _n_<150 then deleteme='Yes';
-  else if _n_<250 then label='mod'!!cats(_n_);
-  else if _n_<350 then do;
-    start=cats(_n_);
-    end=cats(_n_+1);
-    label='newval'!!cats(_N_);
-  end;
-  else stop;
-run;
-
-/* load the above */
-%mp_assertscope(SNAPSHOT)
-%mp_loadformat(perm.testcat
-  ,work.stagedata
-  ,loadtarget=YES
-  ,auditlibds=perm.audit
-  ,locklibds=0
-  ,delete_col=deleteme
-  ,outds_add=add_test1
-  ,outds_del=del_test1
-  ,outds_mod=mod_test1
-  ,mdebug=1
-)
-%mp_assertscope(COMPARE)
-
-%mp_assert(
-  iftrue=(%mf_nobs(del_test1)=149),
-  desc=Test 1 - delete obs,
-  outds=work.test_results
-)
-%mp_assert(
-  iftrue=(%mf_nobs(add_test1)=100),
-  desc=Test 1 - add obs,
-  outds=work.test_results
-)
-%mp_assert(
-  iftrue=(%mf_nobs(mod_test1)=100),
-  desc=Test 1 - mod obs,
-  outds=work.test_results
-)
-%mp_assert(
-  iftrue=(%mf_nobs(perm.audit)=7329),
-  desc=Test 1 - audit table updated,
-  outds=work.test_results
-)
-data work.difftest;
-  set perm.audit;
-  where is_diff=1;
-run;
-%mp_assert(
-  iftrue=(%mf_nobs(work.difftest)>0),
-  desc=Test 1 - diffs were found,
-  outds=work.test_results
-)

tests/base/mp_lockanytable.test.sas

@@ -4,8 +4,10 @@
 
   <h4> SAS Macros </h4>
   @li mp_lockanytable.sas
+  @li mp_assert.sas
   @li mp_assertcols.sas
   @li mp_assertcolvals.sas
+  @li mp_assertscope.sas
   @li mp_coretable.sas
 
 **/
@@ -61,3 +63,18 @@ run;
   desc=Ref is captured in unlock,
   test=ANYVAL
 )
+
+/* attempt unlock of a table that was never locked */
+
+%mp_lockanytable(UNLOCK,lib=no,ds=doesnotexist,ref=bye, ctl_ds=work.controller)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Ability to unlock a table that was never locked,
+  outds=work.test_results
+)
+
+/* test for macro variable scope leakage */
+%mp_assertscope(SNAPSHOT)
+%mp_lockanytable(LOCK,lib=tmp,ds=testscope,ref=This Ref, ctl_ds=work.controller)
+%mp_assertscope(COMPARE)

tests/serveronly/ms_adduser2group.test.sas

@@ -31,7 +31,7 @@ run;
 %mp_assertscope(SNAPSHOT)
 %ms_adduser2group(uid=1,gid=&groupid,mdebug=&sasjs_mdebug,outds=test1)
 %mp_assertscope(COMPARE
-  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN
+  ,ignorelist=MCLIB2_JADP1LEN MCLIB2_JADP2LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
 )
 
 /* check the user is in the output list */

tests/serveronly/ms_deletefile.test.sas

@@ -53,7 +53,7 @@ data _null_;
 run;
 
 %mp_assert(
-  iftrue=("&test2"="%str(Err)or: File doesn't exist."),
+  iftrue=("&test2"="File doesn't exist."),
   desc=Make sure the file was deleted,
   outds=work.test_results
 )

tests/serveronly/ms_getgroups.test.sas

@@ -21,7 +21,7 @@
 %mp_assertscope(SNAPSHOT)
 %ms_getgroups(outds=work.test1,mdebug=&sasjs_mdebug)
 %mp_assertscope(COMPARE
-  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN
+  ,ignorelist=MCLIB2_JADP1LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
 )
 
 /* check the group was created */

tests/serveronly/ms_runstp.test.sas

@@ -17,6 +17,7 @@ data _null_;
   file stpcode;
   put '%put hello world;';
   put '%put _all_;';
+  put 'data _null_; file _webout; put "runstptest";run;';
 run;
 
 options mprint;
@@ -34,25 +35,29 @@ options mprint;
 )
 %mp_assertscope(COMPARE)
 
-libname webeen json fileref=weboot;
-
+%let test1=0;
+%let test2=0;
 data _null_;
   infile weboot;
   input;
-  putlog _infile_;
+  if _n_=1 then call symputx('test1',_infile_);
+  if _n_=3 then do;
+    call symputx('test2',substr(_infile_,1,30));
+    putlog "SASJS_LOGS_SEPARATOR_xxx"; /* this marker affects the CLI parser */
+  end;
+  else putlog _infile_;
 run;
 
-%let test1=0;
-data work.log;
-  set webeen.log;
-  put (_all_)(=);
-  if _n_>10 then call symputx('test1',1);
-run;
+
 
 %mp_assert(
-  iftrue=("&test1"="1"),
-  desc=Checking log was returned,
+  iftrue=("&test1"="runstptest"),
+  desc=Checking webout was created,
   outds=work.test_results
 )
 
-
+%mp_assert(
+  iftrue=("&test2"="SASJS_LOGS_SEPARATOR_163ee17b6"),
+  desc=Checking debug was enabled,
+  outds=work.test_results
+)