Merge pull request #393 from sasjs/issue1403

feat: adding mp_replace option to mv_createfile

.all-contributorsrc

@@ -144,8 +144,27 @@
       "contributions": [
         "doc"
       ]
+    },
+    {
+      "login": "rudvfaden",
+      "name": "Rud Faden",
+      "avatar_url": "https://avatars.githubusercontent.com/u/2445577?v=4",
+      "profile": "http://rudvfaden.github.io/",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "andyjessen",
+      "name": "andyjessen",
+      "avatar_url": "https://avatars.githubusercontent.com/u/62343929?v=4",
+      "profile": "https://github.com/andyjessen",
+      "contributions": [
+        "doc"
+      ]
     }
   ],
   "contributorsPerLine": 7,
-  "skipCi": true
+  "skipCi": true,
+  "commitType": "docs"
 }

.git-hooks/pre-commit

@@ -2,7 +2,7 @@
 
 # Ensure lint is passing
 LINT=`sasjs lint`
-if [[ "$LINT" != "✔ All matched files use @sasjs/lint code style!" ]]; then
+if [[ "$LINT" != *"All matched files use @sasjs/lint code style!" ]]; then
   echo "$LINT"
   echo "To commit in spite of these warnings, use the -n parameter."
   exit 1

.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

.github/CONTRIBUTING.md

@@ -27,5 +27,6 @@ To contribute:
 1.  Create your feature branch (`git checkout -b myfeature`)
 2.  Make your change
 3.  Update the `all.sas` file (`python3 build.py`)
-4.  Push and make a PR
+4.  Commit using a [Conventional Commit](https://www.conventionalcommits.org)
+5.  Push and make a PR
 

.github/FUNDING.yml

@@ -1,3 +1,3 @@
 # These are supported funding model platforms
 
-github: [sasjs]
+custom: https://getalby.com/p/sasjs

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,4 +15,4 @@ What code changes have been made to achieve the intent.
 - [ ] Code is formatted correctly (`sasjs lint`).
 - [ ] Any new functionality has been unit tested.
 - [ ] All unit tests are passing (`sasjs test`).
-- [ ] `all.sas` has been regenerated (`python3 build.py`)
+- [ ] The PR desc or underlying commits follow the [Conventional Commit](https://www.conventionalcommits.org) standard

.github/vpn/config.ovpn

@@ -3,10 +3,12 @@ client
 tls-client
 dev tun
 # this will connect with whatever proto DNS tells us (https://community.openvpn.net/openvpn/ticket/934)
-proto tcp
-remote vpn.4gl.io 7494
+proto udp
+remote vpn.4gl.io 7194
 resolv-retry infinite
-cipher AES-256-CBC
+# this will fallback from udp6 to udp4 as well
+connect-timeout 5
+data-ciphers AES-256-CBC:AES-256-GCM
 auth SHA256
 script-security 2
 keepalive 10 120

.github/workflows/main.yml

@@ -13,16 +13,54 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-      - name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v3
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          npm ci
+
+      - name: Check code style (aborts if errors found)
+        run: npx @sasjs/cli lint
+
+      - name: Write VPN Files
+        run: |
+          echo "$CA_CRT" > .github/vpn/ca.crt
+          echo "$USER_CRT" > .github/vpn/user.crt
+          echo "$USER_KEY" > .github/vpn/user.key
+          echo "$TLS_KEY" > .github/vpn/tls.key
+        shell: bash
         env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+          CA_CRT: ${{ secrets.CA_CRT}}
+          USER_CRT: ${{ secrets.USER_CRT }}
+          USER_KEY: ${{ secrets.USER_KEY }}
+          TLS_KEY: ${{ secrets.TLS_KEY }}
+
+      - name: Install Open VPN
+        run: |
+          #sudo apt install apt-transport-https
+          #sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
+          #sudo apt-key add openvpn-repo-pkg-key.pub
+          #sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
+          #sudo apt update
+          #sudo apt install openvpn3=17~betaUb22042+jammy
+      - name: Start Open VPN 3
+        run: |
+          # openvpn3 session-start --config .github/vpn/config.ovpn
+
+      - name: Add credentials
+        run: |
+          echo "CLIENT=${{secrets.SAS9_4GL_IO_CLIENT}}"> .env.server
+          echo "ACCESS_TOKEN=${{secrets.SAS9_4GL_IO_ACCESS_TOKEN}}" >> .env.server
+          echo "REFRESH_TOKEN=${{secrets.SAS9_4GL_IO_REFRESH_TOKEN}}" >> .env.server
+
+      - name: Semantic Release
+        uses: cycjimmy/semantic-release-action@v4
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
       - name: SAS Packages Release
         run: |
-          npx @sasjs/cli compile job -s sasjs/utils/create_sas_package.sas -o sasjsbuild
-          # this part depends on https://github.com/sasjs/server/issues/307
-          # sasjs run sasjsbuild/jobs/utils/create_sas_package.sas -t sas9
-
-
+          npx @sasjs/cli compile job -s sasjs/utils/create_sas_package.sas -o sasjsbuild -t server
+          # need long duration token per https://github.com/sasjs/server/issues/307
+          # npx @sasjs/cli run sasjsbuild/jobs/utils/create_sas_package.sas -t server

.github/workflows/notmain.yml

@@ -0,0 +1,32 @@
+# This workflow will run tests using node and then publish a package to GitHub Packages when a release is created
+# For more information see: https://help.github.com/actions/language-and-framework-guides/publishing-nodejs-packages
+
+name: SASjs Core - Update all.sas
+
+on:
+  push:
+    branches-ignore:
+      - main
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          npm ci
+          npm i -g @sasjs/cli@latest
+
+      - name: Ensure all.sas is always up to date
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          python3 build.py
+          git add all.sas
+          git commit -m "chore: updating all.sas" --allow-empty
+          git push
+
+

.github/workflows/run-tests.yml

@@ -8,11 +8,11 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     strategy:
       matrix:
-        node-version: [lts/fermium]
+        node-version: [lts/hydrogen]
 
     steps:
       - uses: actions/checkout@v2
@@ -34,6 +34,10 @@ jobs:
           USER_KEY: ${{ secrets.USER_KEY }}
           TLS_KEY: ${{ secrets.TLS_KEY }}
 
+      - name: Chmod VPN files
+        run: |
+          chmod 600 .github/vpn/ca.crt .github/vpn/user.crt .github/vpn/user.key .github/vpn/tls.key
+
       - name: Install Open VPN
         run: |
           sudo apt install apt-transport-https
@@ -42,8 +46,13 @@ jobs:
           sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
           sudo apt update
           sudo apt install openvpn3=17~betaUb22042+jammy
+
       - name: Start Open VPN 3
         run: openvpn3 session-start --config .github/vpn/config.ovpn
+
+      - name: Fetch SASJS server
+        run: curl ${{ secrets.SASJS_SERVER_URL }}/SASjsApi/info
+
       - name: Install Doxygen
         run: sudo apt-get install doxygen
 
@@ -51,7 +60,7 @@ jobs:
         run: npm ci
 
       - name: Check code style (aborts if errors found)
-        run: npx sasjs lint
+        run: npx @sasjs/cli lint
 
       - name: Add client
         run: echo "CLIENT=${{secrets.SAS9_4GL_IO_CLIENT}}"> .env.server
@@ -59,11 +68,14 @@ jobs:
       - name: Add access token
         run: echo "ACCESS_TOKEN=${{secrets.SAS9_4GL_IO_ACCESS_TOKEN}}" >> .env.server
 
+      - name: Add refresh token
+        run: echo "REFRESH_TOKEN=${{secrets.SAS9_4GL_IO_REFRESH_TOKEN}}" >> .env.server
+
       - name: Build & Deploy Project to SAS server
-        run: npx sasjs cbd -t server
+        run: npx @sasjs/cli cbd -t server
 
       - name: Run all tests
-        run: npx sasjs test -t server
+        run: npx @sasjs/cli test -t server
         env:
           CI: true
           CLIENT: ${{secrets.CLIENT}}

.gitignore

@@ -13,3 +13,4 @@ mc_*
 *.env*
 
 ~
+

README.md

@@ -147,16 +147,17 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 - macro names must be lowercase
 - one macro per file
 - prefixes:
-  - _mcf_ for macro compiled functions (proc fcmp)
-  - _mddl_ for macros containing DDL (Data Definition Language)
-  - _mf_ for macro functions (can be used in open code).
-  - _ml_ for macros that are used to compile LUA modules
-  - _mm_ for metadata macros (interface with the metadata server).
-  - _mmx_ for macros that use metadata and are XCMD enabled (working on both windows and unix)
-  - _mp_ for macro procedures (which generate sas code)
-  - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
-  - _mv_ for macro procedures that will only work in Viya
-  - _mx_ for macros that work on Viya, SAS 9 EBI and SASjs Server
+  - _mcf__: macro compiled functions (proc fcmp)
+  - _mddl__: macros containing DDL (Data Definition Language)
+  - _mf__: macro functions (can be used in open code).
+  - _mfv__: macro functions that work only in Viya
+  - _ml__: macros that are used to compile LUA modules
+  - _mm__: metadata macros (interface with the metadata server).
+  - _mmx__: macros that use metadata and are XCMD enabled (working on both windows and unix)
+  - _mp__: macro procedures (which generate sas code)
+  - _ms__: macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
+  - _mv__: macro procedures that will only work in Viya
+  - _mx__: macros that work on Viya, SAS 9 EBI and SASjs Server
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
@@ -212,7 +213,8 @@ When contributing to this library, it is therefore important to ensure that all
 
 ## General Notes
 
-- All macros should be compatible with SAS versions from support level B and above (so currently 9.2 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully (eg `%if %sysevalf(&sysver<9.3) %then %return`).
+- All macros should be compatible with SAS versions from support level B and above (so currently 9.3 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully.
+- It's [best to avoid](https://git.datacontroller.io/dc/dc/issues/50) special / non-ASCII characters for compatibility with the widest variety of SAS installations.
 
 ## Breaking Changes
 
@@ -235,18 +237,20 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 The following repositories are also worth checking out:
 
-* [xieliaing/SAS](https://github.com/xieliaing/SAS)
-* [SASJedi/sas-macros](https://github.com/SASJedi/sas-macros)
 * [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
+* [Criptic/sas_snippets](https://github.com/Criptic/sas_snippets)
 * [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)
 * [KatjaGlassConsulting/SMILE-SmartSASMacros](https://github.com/KatjaGlassConsulting/SMILE-SmartSASMacros)
+* [paul-canals/toolbox](https://github.com/paul-canals/toolbox)
 * [rogerjdeangelis](https://github.com/rogerjdeangelis)
+* [SASJedi/sas-macros](https://github.com/SASJedi/sas-macros)
 * [scottbass/sas](https://github.com/scottbass/SAS)
+* [xieliaing/SAS](https://github.com/xieliaing/SAS)
 * [yabwon/sas_packages](https://github.com/yabwon/SAS_PACKAGES)
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-13-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-15-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -271,6 +275,10 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt="Bart Jablonski"/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://bandism.net/"><img src="https://avatars.githubusercontent.com/u/22633385?v=4?s=100" width="100px;" alt="Ikko Ashimine"/><br /><sub><b>Ikko Ashimine</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=eltociear" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/henrik-forsell"><img src="https://avatars.githubusercontent.com/u/109935936?v=4?s=100" width="100px;" alt="Henrik Forsell"/><br /><sub><b>Henrik Forsell</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=henrik-forsell" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://rudvfaden.github.io/"><img src="https://avatars.githubusercontent.com/u/2445577?v=4?s=100" width="100px;" alt="Rud Faden"/><br /><sub><b>Rud Faden</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rudvfaden" title="Code">💻</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/andyjessen"><img src="https://avatars.githubusercontent.com/u/62343929?v=4?s=100" width="100px;" alt="andyjessen"/><br /><sub><b>andyjessen</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=andyjessen" title="Documentation">📖</a></td>
     </tr>
   </tbody>
 </table>

base/mf_abort.sas

@@ -3,6 +3,11 @@
   @brief Abort, ungracefully
   @details Will abort with a straightforward %abort if the condition is true.
 
+  @param [in] mac= (mf_abort.sas) Name of calling macro (is printed to the log)
+  @param [in] msg= ( ) Additional string to print to the log
+  @param [in] iftrue= (%str(1=1)) Conditional logic under which to perform the
+    abort
+
   <h4> Related Macros </h4>
   @li mp_abort.sas
 

base/mf_deletefile.sas

@@ -10,7 +10,7 @@
       %mf_deletefile(&sasjswork/myfile.txt)
 
 
-  @param filepath Full path to the target file
+  @param [in] file Full path to the target file
 
   @returns The return code from the fdelete() invocation
 

base/mf_existds.sas

@@ -10,7 +10,7 @@
     expected results (depending on whether you 'expect' the result to be
     case insensitive in this context!)
 
-  @param libds library.dataset
+  @param [in] libds library.dataset
   @return output returns 1 or 0
 
   <h4> Related Macros </h4>

base/mf_existfileref.sas

@@ -4,7 +4,7 @@
   @details You can probably do without this macro as it is just a one liner.
   Mainly it is here as a convenient way to remember the syntax!
 
-  @param fref the fileref to detect
+  @param [in] fref the fileref to detect
 
   @return output Returns 1 if found and 0 if not found.  Note - it is possible
   that the fileref is found, but the file does not (yet) exist. If you need

base/mf_existfunction.sas

@@ -16,7 +16,7 @@
 https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionexists-macro
   ).
 
-  @param [in] name (positional) - function name
+  @param [in] name function name
 
   @author Allan Bowe
 **/

base/mf_existvarlist.sas

@@ -6,8 +6,8 @@
 
       %put %mf_existVarList(sashelp.class, age sex name dummyvar);
 
-  @param libds 2 part dataset or view reference
-  @param varlist space separated variable names
+  @param [in] libds 2 part dataset or view reference
+  @param [in] varlist space separated variable names
 
   @version 9.2
   @author Allan Bowe

base/mf_getapploc.sas

@@ -31,6 +31,7 @@
       %put %mf_getapploc(/some/location/jobs/extract/somejob/);
       %put %mf_getapploc(/some/location/tests/jobs/somejob/);
 
+  @param [in] pgm The _program value from which to extract the appLoc
 
   @author Allan Bowe
 **/

base/mf_getattrc.sas

@@ -6,8 +6,8 @@
       %put Dataset label = %mf_getattrc(sashelp.class,LABEL);
       %put Member Type = %mf_getattrc(sashelp.class,MTYPE);
 
-  @param libds library.dataset
-  @param attr full list in [documentation](
+  @param [in] libds library.dataset
+  @param [in] attr full list in [documentation](
     https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000147794.htm)
   @return output returns result of the attrc value supplied, or -1 and log
     message if err.

base/mf_getattrn.sas

@@ -6,8 +6,8 @@
       %put Number of observations=%mf_getattrn(sashelp.class,NLOBS);
       %put Number of variables = %mf_getattrn(sashelp.class,NVARS);
 
-  @param libds library.dataset
-  @param attr Common values are NLOBS and NVARS, full list in [documentation](
+  @param [in] libds library.dataset
+  @param [in] attr Common values are NLOBS and NVARS, full list in [documentation](
   http://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000212040.htm)
   @return output returns result of the attrn value supplied, or -1 and log
     message if err.

base/mf_getfmtlist.sas

@@ -10,10 +10,9 @@
 
   returns:
 
-  > DOLLAR $CHAR W MONNAME
-  > $CHAR BEST DOLLAR
-  > BEST Z $CHAR COMMA PERCENTN
-
+      DOLLAR $CHAR W MONNAME
+      $CHAR BEST DOLLAR
+      BEST Z $CHAR COMMA PERCENTN
 
   @param [in] libds Two part library.dataset reference.
 

base/mf_getkeyvalue.sas

@@ -7,8 +7,9 @@
       %put %mf_getkeyvalue(someindex)
 
 
-  @param key Provide a key on which to perform the lookup
-  @param libds= define the target table which holds the parameters
+  @param [in] key Provide a key on which to perform the lookup
+  @param [in] libds= (work.mp_setkeyvalue) The library.dataset which holds the
+    parameters
 
   @version 9.2
   @author Allan Bowe

base/mf_getplatform.sas

@@ -12,7 +12,7 @@
   @li SASJS
   @li BASESAS
 
-  @param switch the param for which to return a platform specific variable
+  @param [in] switch the param for which to return a platform specific variable
 
   <h4> SAS Macros </h4>
   @li mf_mval.sas

base/mf_getschema.sas

@@ -8,7 +8,7 @@
   returns:
   > dbo
 
-  @param libref Library reference (also accepts a 2 level libds ref).
+  @param [in] libref Library reference (also accepts a 2 level libds ref).
 
   @return output returns the library schema for the FIRST library encountered
 

base/mf_getuniquelibref.sas

@@ -19,7 +19,7 @@
   @param [in] prefix= (mclib) first part of the returned libref. As librefs can
     be as long as 8 characters, a maximum length of 7 characters is premitted
     for this prefix.
-  @param [in] maxtries= Deprecated parameter. Remains here to ensure a
+  @param [in] maxtries= (1000) Deprecated parameter. Remains here to ensure a
     non-breaking change.  Will be removed in v5.
 
   @version 9.2

base/mf_getuniquename.sas

@@ -10,7 +10,7 @@
 
 > MCc59c750610321d4c8bf75faadbcd22
 
-  @param prefix= set a prefix for the new name
+  @param [in] prefix= (MC) Sets a prefix for the new name
 
   @version 9.3
   @author Allan Bowe

base/mf_getuser.sas

@@ -13,8 +13,6 @@
         %let user= %mf_getUser();
         %put &user;
 
-  @param type - do not use, may be deprecated in a future release
-
   @return SYSUSERID (if workspace server)
   @return _METAPERSON (if stored process server)
   @return SYS_COMPUTE_SESSION_OWNER (if Viya compute session)

base/mf_getvalue.sas

@@ -13,9 +13,9 @@
   <h4> Related Macros </h4>
   @li mp_setkeyvalue.sas
 
-  @param libds dataset to query
-  @param variable the variable which contains the value to return.
-  @param filter contents of where clause
+  @param [in] libds dataset to query
+  @param [in] variable the variable which contains the value to return.
+  @param [in] filter= (1) contents of where clause
 
   @version 9.2
   @author Allan Bowe

base/mf_getvarformat.sas

@@ -25,7 +25,8 @@
 
   @param [in] libds Two part dataset (or view) reference.
   @param [in] var Variable name for which a format should be returned
-  @param [in] force=(0) Set to 1 to supply a default if the variable has no format
+  @param [in] force= (0) Set to 1 to supply a default if the variable has no
+    format
   @returns outputs format
 
   @author Allan Bowe

base/mf_getvarlen.sas

@@ -18,8 +18,8 @@
       8
       NOTE: Variable renegade does not exist in test
 
-  @param libds Two part dataset (or view) reference.
-  @param var Variable name for which a length should be returned
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] var Variable name for which a length should be returned
   @returns outputs length
 
   @author Allan Bowe

base/mf_getvarnum.sas

@@ -21,8 +21,8 @@ returns:
 
   > NOTE: Variable renegade does not exist in test
 
-  @param libds Two part dataset (or view) reference.
-  @param var Variable name for which a position should be returned
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] var Variable name for which a position should be returned
 
   @author Allan Bowe
   @version 9.2

base/mf_getvartype.sas

@@ -13,8 +13,8 @@ Usage:
 
 
 
-  @param libds Two part dataset (or view) reference.
-  @param var the variable name to be checked
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] var the variable name to be checked
   @return output returns C or N depending on variable type.  If variable
     does not exist then a blank is returned and a note is written to the log.
 

base/mf_getxengine.sas

@@ -10,7 +10,7 @@
   returns:
   > TEMP
 
-  @param fref The fileref to check
+  @param [in] fref The fileref to check
 
   @returns The XENGINE value in sashelp.vextfl or 0 if not found.
 

base/mf_increment.sas

@@ -11,8 +11,8 @@
       %put Now we have run %mf_increment(cnt) lines;
       %put There are %mf_increment(cnt) lines in total;
 
-  @param [in] MACRO_NAME the name of the macro variable to increment
-  @param [in] ITER= The amount to add or subtract to the macro
+  @param [in] macro_name The name of the macro variable to increment
+  @param [in] incr= (1) The amount to add or subtract to the macro
 
   <h4> Related Files </h4>
   @li mf_increment.test.sas

base/mf_isblank.sas

@@ -7,12 +7,12 @@
 
   Usage:
 
-      %put mf_isblank(&var);
+      %put %mf_isblank(&var);
 
   inspiration:
   https://support.sas.com/resources/papers/proceedings09/022-2009.pdf
 
-  @param param VALUE to be checked
+  @param [in] Param VALUE to be checked
 
   @return output returns 1 (if blank) else 0
 

base/mf_isdir.sas

@@ -9,7 +9,7 @@
   With thanks and full credit to Andrea Defronzo -
   https://www.linkedin.com/in/andrea-defronzo-b1a47460/
 
-  @param path full path of the file/directory to be checked
+  @param [in] path Full path of the file/directory to be checked
 
   @return output returns 1 if path is a directory, 0 if it is not
 

base/mf_loc.sas

@@ -6,6 +6,10 @@
 
       %put %mf_loc(POF); %*location of PlatformObjectFramework tools;
 
+  @param [in] loc The item to locate, eg:
+    @li PLAATFORMOBJECTFRAMEWORK (or POF)
+    @li VIYACONFG
+
   @version 9.2
   @author Allan Bowe
 **/
@@ -15,7 +19,8 @@
 %local root;
 
 %if &loc=POF or &loc=PLATFORMOBJECTFRAMEWORK %then %do;
-  %let root=%substr(%sysget(SASROOT),1,%index(%sysget(SASROOT),SASFoundation)-2);
+  %let root=%sysget(SASROOT);
+  %let root=%substr(&root,1,%index(&root,SASFoundation)-2);
   %let root=&root/SASPlatformObjectFramework/&sysver;
   %put Batch tools located at: &root;
   &root

base/mf_mimetype.sas

@@ -0,0 +1,436 @@
+/**
+  @file
+  @brief Returns a mime type from a file extension
+  @details Provide a file extension and get a mime type.
+    The mappings were derived from this source:
+    https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
+
+
+
+  @param [in] ext The file extension (unquoted)
+  @return output The mime type
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_mimetype(
+    ext
+)/*/STORE SOURCE*/ /minoperator mindelimiter=' ';
+
+%let ext=%lowcase(&ext);
+
+%if &ext in (sas txt text conf def list log)
+%then %do;%str(text/plain)%end;
+%else %if &ext=xlsx %then %do;
+%str(application/vnd.openxmlformats-officedocument.spreadsheetml.sheet)%end;
+%else %if &ext in (xls xlm xla xlc xlt xlw)
+%then %do;%str(application/vnd.ms-excel)%end;
+%else %if &ext=xlsm
+%then %do;%str(application/vnd.ms-excel.sheet.macroenabled.12)%end;
+%else %if &ext=xlsb
+%then %do;%str(application/vnd.ms-excel.sheet.binary.macroenabled.12)%end;
+%else %if &ext in (css csv html n3 sgml vcard)
+%then %do;%str(text/&ext)%end;
+%else %if &ext in (avif bmp cgm gif ief jxl ktx png sgi tiff webp)
+%then %do;%str(image/&ext)%end;
+%else %if &ext in (exi gxf ipfix json mbox mp21 mxf oda oxps pdf rtf sdp wasm
+  xml yang zip)
+%then %do;%str(application/&ext)%end;
+%else %if &ext in (jpeg jpg jpe) %then %do;%str(image/jpeg)%end;
+%else %if &ext in (mp4 mp4v mpg4) %then %do;%str(video/mp4)%end;
+%else %if &ext in (otf ttf woff woff2) %then %do;%str(font/&ext)%end;
+%else %if &ext in (mpeg mpg mpe m1v m2v)
+%then %do;%str(video/mpeg)%end;
+%else %if &ext in (h261 h263 h264 jpm mj2 webm)
+%then %do;%str(video/&ext)%end;
+%else %if &ext in (f4v fli flv m4v mng smv)
+%then %do;%str(video/x-&ext)%end;
+%else %if &ext in (3ds cmx pcx rgb tga)
+%then %do;%str(image/x-&ext)%end;
+%else %if &ext in (asm nfo opml sfv)
+%then %do;%str(text/x-&ext)%end;
+%else %if &ext in (aac caf flac wav)
+%then %do;%str(audio/x-&ext)%end;
+%else %if &ext in (ts m2t m2ts mts)
+%then %do;%str(video/mp2t)%end;
+%else %if &ext in (pfa pfb pfm afm)
+%then %do;%str(application/x-font-type1)%end;
+%else %if &ext in (oga ogg spx opus)
+%then %do;%str(audio/ogg)%end;
+%else %if &ext in (mid midi kar rmi)
+%then %do;%str(audio/midi)%end;
+%else %if &ext in (onetoc onetoc2 onetmp onepkg)
+%then %do;%str(application/onenote)%end;
+%else %if &ext in (mxml xhvml xvml xvm)
+%then %do;%str(application/xv+xml)%end;
+%else %if &ext in (f for f77 f90)
+%then %do;%str(text/x-fortran)%end;
+%else %if &ext in (wmf wmz emf emz)
+%then %do;%str(application/x-msmetafile)%end;
+%else %if &ext in (exe dll com bat msi)
+%then %do;%str(application/x-msdownload)%end;
+%else %if &ext in (bin dms lrf mar so dist distz pkg bpk dump elc deploy)
+%then %do;%str(application/octet-stream)%end;
+%else %if &ext in (atom atomcat atomsvc ccxml davmount emma gml gpx inkml mads
+  mathml metalink mets mods omdoc pls rdf rsd rss sbml shf smil sru ssdl ssml
+  tei wsdl wspolicy xaml xenc xhtml xop xslt xspf yin)
+%then %do;%str(application/&ext+xml)%end;
+%else %if &ext in (dir dcr dxr cst cct cxt w3d fgd swa)
+%then %do;%str(application/x-director)%end;
+%else %if &ext in (z1 z2 z3 z4 z5 z6 z7 z8)
+%then %do;%str(application/x-zmachine)%end;
+%else %if &ext in (c cc cxx cpp h hh dic)
+%then %do;%str(text/x-c)%end;
+%else %if &ext in (mpga mp2 mp2a mp3 m2a m3a)
+%then %do;%str(audio/mpeg)%end;
+%else %if &ext in (t tr roff man me ms)
+%then %do;%str(text/troff)%end;
+%else %if &ext in (cbr cba cbt cbz cb7)
+%then %do;%str(application/x-cbr)%end;
+%else %if &ext in (fh fhc fh4 fh5 fh7)
+%then %do;%str(image/x-freehand)%end;
+%else %if &ext in (aab x32 u32 vox)
+%then %do;%str(application/x-authorware-bin)%end;
+%else %if &ext in (uvi uvvi uvg uvvg)
+%then %do;%str(image/vnd.dece.graphic)%end;
+%else %if &ext in (cdx cif cmdf cml csml xyz)
+%then %do;%str(chemical/x-&ext)%end;
+%else %if &ext in (aif aiff aifc) %then %do;%str(audio/x-aiff)%end;
+%else %if &ext in (ma nb mb) %then %do;%str(application/mathematica)%end;
+%else %if &ext in (mvb m13 m14) %then %do;%str(application/x-msmediaview)%end;
+%else %if &ext in (msh mesh silo) %then %do;%str(model/mesh)%end;
+%else %if &ext in (uri uris urls) %then %do;%str(text/uri-list)%end;
+%else %if &ext in (mkv mk3d mks) %then %do;%str(video/x-matroska)%end;
+%else %if &ext=ez %then %do;%str(application/andrew-inset)%end;
+%else %if &ext=aw %then %do;%str(application/applixware)%end;
+%else %if &ext=cdmia %then %do;%str(application/cdmi-capability)%end;
+%else %if &ext=cdmic %then %do;%str(application/cdmi-container)%end;
+%else %if &ext=cdmid %then %do;%str(application/cdmi-domain)%end;
+%else %if &ext=cdmio %then %do;%str(application/cdmi-object)%end;
+%else %if &ext=cdmiq %then %do;%str(application/cdmi-queue)%end;
+%else %if &ext=cu %then %do;%str(application/cu-seeme)%end;
+%else %if &ext=dssc %then %do;%str(application/dssc+der)%end;
+%else %if &ext=xdssc %then %do;%str(application/dssc+xml)%end;
+%else %if &ext=ecma %then %do;%str(application/ecmascript)%end;
+%else %if &ext=epub %then %do;%str(application/epub+zip)%end;
+%else %if &ext=pfr %then %do;%str(application/font-tdpfr)%end;
+%else %if &ext=stk %then %do;%str(application/hyperstudio)%end;
+%else %if &ext=ink %then %do;%str(application/inkml+xml)%end;
+%else %if &ext=jar %then %do;%str(application/java-archive)%end;
+%else %if &ext=ser %then %do;%str(application/java-serialized-object)%end;
+%else %if &ext=class %then %do;%str(application/java-vm)%end;
+%else %if &ext=jsonml %then %do;%str(application/jsonml+json)%end;
+%else %if &ext=lostxml %then %do;%str(application/lost+xml)%end;
+%else %if &ext=hqx %then %do;%str(application/mac-binhex40)%end;
+%else %if &ext=cpt %then %do;%str(application/mac-compactpro)%end;
+%else %if &ext=mrc %then %do;%str(application/marc)%end;
+%else %if &ext=mrcx %then %do;%str(application/marcxml+xml)%end;
+%else %if &ext=mscml %then %do;%str(application/mediaservercontrol+xml)%end;
+%else %if &ext=meta4 %then %do;%str(application/metalink4+xml)%end;
+%else %if &ext=m21 %then %do;%str(application/mp21)%end;
+%else %if &ext=mp4s %then %do;%str(application/mp4)%end;
+%else %if &ext=doc %then %do;%str(application/msword)%end;
+%else %if &ext=dot %then %do;%str(application/msword)%end;
+%else %if &ext=opf %then %do;%str(application/oebps-package+xml)%end;
+%else %if &ext=ogx %then %do;%str(application/ogg)%end;
+%else %if &ext=xer %then %do;%str(application/patch-ops-error+xml)%end;
+%else %if &ext=pgp %then %do;%str(application/pgp-encrypted)%end;
+%else %if &ext=asc %then %do;%str(application/pgp-signature)%end;
+%else %if &ext=sig %then %do;%str(application/pgp-signature)%end;
+%else %if &ext=prf %then %do;%str(application/pics-rules)%end;
+%else %if &ext=p10 %then %do;%str(application/pkcs10)%end;
+%else %if &ext=p7m %then %do;%str(application/pkcs7-mime)%end;
+%else %if &ext=p7c %then %do;%str(application/pkcs7-mime)%end;
+%else %if &ext=p7s %then %do;%str(application/pkcs7-signature)%end;
+%else %if &ext=p8 %then %do;%str(application/pkcs8)%end;
+%else %if &ext=ac %then %do;%str(application/pkix-attr-cert)%end;
+%else %if &ext=cer %then %do;%str(application/pkix-cert)%end;
+%else %if &ext=crl %then %do;%str(application/pkix-crl)%end;
+%else %if &ext=pkipath %then %do;%str(application/pkix-pkipath)%end;
+%else %if &ext=pki %then %do;%str(application/pkixcmp)%end;
+%else %if &ext=cww %then %do;%str(application/prs.cww)%end;
+%else %if &ext=pskcxml %then %do;%str(application/pskc+xml)%end;
+%else %if &ext=rif %then %do;%str(application/reginfo+xml)%end;
+%else %if &ext=rnc %then %do;%str(application/relax-ng-compact-syntax)%end;
+%else %if &ext=rld %then %do;%str(application/resource-lists-diff+xml)%end;
+%else %if &ext=rl %then %do;%str(application/resource-lists+xml)%end;
+%else %if &ext=gbr %then %do;%str(application/rpki-ghostbusters)%end;
+%else %if &ext=mft %then %do;%str(application/rpki-manifest)%end;
+%else %if &ext=roa %then %do;%str(application/rpki-roa)%end;
+%else %if &ext=scq %then %do;%str(application/scvp-cv-request)%end;
+%else %if &ext=scs %then %do;%str(application/scvp-cv-response)%end;
+%else %if &ext=spq %then %do;%str(application/scvp-vp-request)%end;
+%else %if &ext=spp %then %do;%str(application/scvp-vp-response)%end;
+%else %if &ext=setpay %then %do;%str(application/set-payment-initiation)%end;
+%else %if &ext=setreg %then %do;%str(application/set-registration-initiation)%end;
+%else %if &ext=smi %then %do;%str(application/smil+xml)%end;
+%else %if &ext=rq %then %do;%str(application/sparql-query)%end;
+%else %if &ext=srx %then %do;%str(application/sparql-results+xml)%end;
+%else %if &ext=gram %then %do;%str(application/srgs)%end;
+%else %if &ext=grxml %then %do;%str(application/srgs+xml)%end;
+%else %if &ext=teicorpus %then %do;%str(application/tei+xml)%end;
+%else %if &ext=tfi %then %do;%str(application/thraud+xml)%end;
+%else %if &ext=tsd %then %do;%str(application/timestamped-data)%end;
+%else %if &ext=vxml %then %do;%str(application/voicexml+xml)%end;
+%else %if &ext=wgt %then %do;%str(application/widget)%end;
+%else %if &ext=hlp %then %do;%str(application/winhlp)%end;
+%else %if &ext=7z %then %do;%str(application/x-7z-compressed)%end;
+%else %if &ext=abw %then %do;%str(application/x-abiword)%end;
+%else %if &ext=ace %then %do;%str(application/x-ace-compressed)%end;
+%else %if &ext=dmg %then %do;%str(application/x-apple-diskimage)%end;
+%else %if &ext=aam %then %do;%str(application/x-authorware-map)%end;
+%else %if &ext=aas %then %do;%str(application/x-authorware-seg)%end;
+%else %if &ext=bcpio %then %do;%str(application/x-bcpio)%end;
+%else %if &ext=torrent %then %do;%str(application/x-bittorrent)%end;
+%else %if &ext=blb %then %do;%str(application/x-blorb)%end;
+%else %if &ext=blorb %then %do;%str(application/x-blorb)%end;
+%else %if &ext=bz %then %do;%str(application/x-bzip)%end;
+%else %if &ext=bz2 %then %do;%str(application/x-bzip2)%end;
+%else %if &ext=boz %then %do;%str(application/x-bzip2)%end;
+%else %if &ext=vcd %then %do;%str(application/x-cdlink)%end;
+%else %if &ext=cfs %then %do;%str(application/x-cfs-compressed)%end;
+%else %if &ext=chat %then %do;%str(application/x-chat)%end;
+%else %if &ext=pgn %then %do;%str(application/x-chess-pgn)%end;
+%else %if &ext=nsc %then %do;%str(application/x-conference)%end;
+%else %if &ext=cpio %then %do;%str(application/x-cpio)%end;
+%else %if &ext=csh %then %do;%str(application/x-csh)%end;
+%else %if &ext=deb %then %do;%str(application/x-debian-package)%end;
+%else %if &ext=udeb %then %do;%str(application/x-debian-package)%end;
+%else %if &ext=dgc %then %do;%str(application/x-dgc-compressed)%end;
+%else %if &ext=wad %then %do;%str(application/x-doom)%end;
+%else %if &ext=ncx %then %do;%str(application/x-dtbncx+xml)%end;
+%else %if &ext=dtb %then %do;%str(application/x-dtbook+xml)%end;
+%else %if &ext=res %then %do;%str(application/x-dtbresource+xml)%end;
+%else %if &ext=dvi %then %do;%str(application/x-dvi)%end;
+%else %if &ext=evy %then %do;%str(application/x-envoy)%end;
+%else %if &ext=eva %then %do;%str(application/x-eva)%end;
+%else %if &ext=bdf %then %do;%str(application/x-font-bdf)%end;
+%else %if &ext=gsf %then %do;%str(application/x-font-ghostscript)%end;
+%else %if &ext=psf %then %do;%str(application/x-font-linux-psf)%end;
+%else %if &ext=pcf %then %do;%str(application/x-font-pcf)%end;
+%else %if &ext=snf %then %do;%str(application/x-font-snf)%end;
+%else %if &ext=arc %then %do;%str(application/x-freearc)%end;
+%else %if &ext=spl %then %do;%str(application/x-futuresplash)%end;
+%else %if &ext=gca %then %do;%str(application/x-gca-compressed)%end;
+%else %if &ext=ulx %then %do;%str(application/x-glulx)%end;
+%else %if &ext=gnumeric %then %do;%str(application/x-gnumeric)%end;
+%else %if &ext=gramps %then %do;%str(application/x-gramps-xml)%end;
+%else %if &ext=gtar %then %do;%str(application/x-gtar)%end;
+%else %if &ext=hdf %then %do;%str(application/x-hdf)%end;
+%else %if &ext=install %then %do;%str(application/x-install-instructions)%end;
+%else %if &ext=iso %then %do;%str(application/x-iso9660-image)%end;
+%else %if &ext=jnlp %then %do;%str(application/x-java-jnlp-file)%end;
+%else %if &ext=latex %then %do;%str(application/x-latex)%end;
+%else %if &ext=lzh %then %do;%str(application/x-lzh-compressed)%end;
+%else %if &ext=lha %then %do;%str(application/x-lzh-compressed)%end;
+%else %if &ext=mie %then %do;%str(application/x-mie)%end;
+%else %if &ext=prc %then %do;%str(application/x-mobipocket-ebook)%end;
+%else %if &ext=mobi %then %do;%str(application/x-mobipocket-ebook)%end;
+%else %if &ext=application %then %do;%str(application/x-ms-application)%end;
+%else %if &ext=lnk %then %do;%str(application/x-ms-shortcut)%end;
+%else %if &ext=wmd %then %do;%str(application/x-ms-wmd)%end;
+%else %if &ext=wmz %then %do;%str(application/x-ms-wmz)%end;
+%else %if &ext=xbap %then %do;%str(application/x-ms-xbap)%end;
+%else %if &ext=mdb %then %do;%str(application/x-msaccess)%end;
+%else %if &ext=obd %then %do;%str(application/x-msbinder)%end;
+%else %if &ext=crd %then %do;%str(application/x-mscardfile)%end;
+%else %if &ext=clp %then %do;%str(application/x-msclip)%end;
+%else %if &ext=mny %then %do;%str(application/x-msmoney)%end;
+%else %if &ext=pub %then %do;%str(application/x-mspublisher)%end;
+%else %if &ext=scd %then %do;%str(application/x-msschedule)%end;
+%else %if &ext=trm %then %do;%str(application/x-msterminal)%end;
+%else %if &ext=wri %then %do;%str(application/x-mswrite)%end;
+%else %if &ext=nc %then %do;%str(application/x-netcdf)%end;
+%else %if &ext=cdf %then %do;%str(application/x-netcdf)%end;
+%else %if &ext=nzb %then %do;%str(application/x-nzb)%end;
+%else %if &ext=p12 %then %do;%str(application/x-pkcs12)%end;
+%else %if &ext=pfx %then %do;%str(application/x-pkcs12)%end;
+%else %if &ext=p7b %then %do;%str(application/x-pkcs7-certificates)%end;
+%else %if &ext=spc %then %do;%str(application/x-pkcs7-certificates)%end;
+%else %if &ext=p7r %then %do;%str(application/x-pkcs7-certreqresp)%end;
+%else %if &ext=rar %then %do;%str(application/x-rar-compressed)%end;
+%else %if &ext=ris %then %do;%str(application/x-research-info-systems)%end;
+%else %if &ext=sh %then %do;%str(application/x-sh)%end;
+%else %if &ext=shar %then %do;%str(application/x-shar)%end;
+%else %if &ext=swf %then %do;%str(application/x-shockwave-flash)%end;
+%else %if &ext=xap %then %do;%str(application/x-silverlight-app)%end;
+%else %if &ext=sql %then %do;%str(application/x-sql)%end;
+%else %if &ext=sit %then %do;%str(application/x-stuffit)%end;
+%else %if &ext=sitx %then %do;%str(application/x-stuffitx)%end;
+%else %if &ext=srt %then %do;%str(application/x-subrip)%end;
+%else %if &ext=sv4cpio %then %do;%str(application/x-sv4cpio)%end;
+%else %if &ext=sv4crc %then %do;%str(application/x-sv4crc)%end;
+%else %if &ext=t3 %then %do;%str(application/x-t3vm-image)%end;
+%else %if &ext=gam %then %do;%str(application/x-tads)%end;
+%else %if &ext=tar %then %do;%str(application/x-tar)%end;
+%else %if &ext=tcl %then %do;%str(application/x-tcl)%end;
+%else %if &ext=tex %then %do;%str(application/x-tex)%end;
+%else %if &ext=tfm %then %do;%str(application/x-tex-tfm)%end;
+%else %if &ext=texinfo %then %do;%str(application/x-texinfo)%end;
+%else %if &ext=texi %then %do;%str(application/x-texinfo)%end;
+%else %if &ext=obj %then %do;%str(application/x-tgif)%end;
+%else %if &ext=ustar %then %do;%str(application/x-ustar)%end;
+%else %if &ext=src %then %do;%str(application/x-wais-source)%end;
+%else %if &ext=der %then %do;%str(application/x-x509-ca-cert)%end;
+%else %if &ext=crt %then %do;%str(application/x-x509-ca-cert)%end;
+%else %if &ext=fig %then %do;%str(application/x-xfig)%end;
+%else %if &ext=xlf %then %do;%str(application/x-xliff+xml)%end;
+%else %if &ext=xpi %then %do;%str(application/x-xpinstall)%end;
+%else %if &ext=xz %then %do;%str(application/x-xz)%end;
+%else %if &ext=xdf %then %do;%str(application/xcap-diff+xml)%end;
+%else %if &ext=xht %then %do;%str(application/xhtml+xml)%end;
+%else %if &ext=xsl %then %do;%str(application/xml)%end;
+%else %if &ext=dtd %then %do;%str(application/xml-dtd)%end;
+%else %if &ext=xpl %then %do;%str(application/xproc+xml)%end;
+%else %if &ext=adp %then %do;%str(audio/adpcm)%end;
+%else %if &ext=au %then %do;%str(audio/basic)%end;
+%else %if &ext=snd %then %do;%str(audio/basic)%end;
+%else %if &ext=m4a %then %do;%str(audio/mp4)%end;
+%else %if &ext=mp4a %then %do;%str(audio/mp4)%end;
+%else %if &ext=s3m %then %do;%str(audio/s3m)%end;
+%else %if &ext=sil %then %do;%str(audio/silk)%end;
+%else %if &ext=uva %then %do;%str(audio/vnd.dece.audio)%end;
+%else %if &ext=uvva %then %do;%str(audio/vnd.dece.audio)%end;
+%else %if &ext=eol %then %do;%str(audio/vnd.digital-winds)%end;
+%else %if &ext=dra %then %do;%str(audio/vnd.dra)%end;
+%else %if &ext=dts %then %do;%str(audio/vnd.dts)%end;
+%else %if &ext=dtshd %then %do;%str(audio/vnd.dts.hd)%end;
+%else %if &ext=lvp %then %do;%str(audio/vnd.lucent.voice)%end;
+%else %if &ext=pya %then %do;%str(audio/vnd.ms-playready.media.pya)%end;
+%else %if &ext=ecelp4800 %then %do;%str(audio/vnd.nuera.ecelp4800)%end;
+%else %if &ext=ecelp7470 %then %do;%str(audio/vnd.nuera.ecelp7470)%end;
+%else %if &ext=ecelp9600 %then %do;%str(audio/vnd.nuera.ecelp9600)%end;
+%else %if &ext=rip %then %do;%str(audio/vnd.rip)%end;
+%else %if &ext=weba %then %do;%str(audio/webm)%end;
+%else %if &ext=mka %then %do;%str(audio/x-matroska)%end;
+%else %if &ext=m3u %then %do;%str(audio/x-mpegurl)%end;
+%else %if &ext=wax %then %do;%str(audio/x-ms-wax)%end;
+%else %if &ext=wma %then %do;%str(audio/x-ms-wma)%end;
+%else %if &ext=ra %then %do;%str(audio/x-pn-realaudio)%end;
+%else %if &ext=ram %then %do;%str(audio/x-pn-realaudio)%end;
+%else %if &ext=rmp %then %do;%str(audio/x-pn-realaudio-plugin)%end;
+%else %if &ext=xm %then %do;%str(audio/xm)%end;
+%else %if &ext=ttc %then %do;%str(font/collection)%end;
+%else %if &ext=g3 %then %do;%str(image/g3fax)%end;
+%else %if &ext=btif %then %do;%str(image/prs.btif)%end;
+%else %if &ext=svg %then %do;%str(image/svg+xml)%end;
+%else %if &ext=svgz %then %do;%str(image/svg+xml)%end;
+%else %if &ext=tif %then %do;%str(image/tiff)%end;
+%else %if &ext=psd %then %do;%str(image/vnd.adobe.photoshop)%end;
+%else %if &ext=djv %then %do;%str(image/vnd.djvu)%end;
+%else %if &ext=djvu %then %do;%str(image/vnd.djvu)%end;
+%else %if &ext=sub %then %do;%str(image/vnd.dvb.subtitle)%end;
+%else %if &ext=dwg %then %do;%str(image/vnd.dwg)%end;
+%else %if &ext=dxf %then %do;%str(image/vnd.dxf)%end;
+%else %if &ext=fbs %then %do;%str(image/vnd.fastbidsheet)%end;
+%else %if &ext=fpx %then %do;%str(image/vnd.fpx)%end;
+%else %if &ext=fst %then %do;%str(image/vnd.fst)%end;
+%else %if &ext=mmr %then %do;%str(image/vnd.fujixerox.edmics-mmr)%end;
+%else %if &ext=rlc %then %do;%str(image/vnd.fujixerox.edmics-rlc)%end;
+%else %if &ext=mdi %then %do;%str(image/vnd.ms-modi)%end;
+%else %if &ext=wdp %then %do;%str(image/vnd.ms-photo)%end;
+%else %if &ext=npx %then %do;%str(image/vnd.net-fpx)%end;
+%else %if &ext=wbmp %then %do;%str(image/vnd.wap.wbmp)%end;
+%else %if &ext=xif %then %do;%str(image/vnd.xiff)%end;
+%else %if &ext=ras %then %do;%str(image/x-cmu-raster)%end;
+%else %if &ext=ico %then %do;%str(image/x-icon)%end;
+%else %if &ext=sid %then %do;%str(image/x-mrsid-image)%end;
+%else %if &ext=pct %then %do;%str(image/x-pict)%end;
+%else %if &ext=pic %then %do;%str(image/x-pict)%end;
+%else %if &ext=pnm %then %do;%str(image/x-portable-anymap)%end;
+%else %if &ext=pbm %then %do;%str(image/x-portable-bitmap)%end;
+%else %if &ext=pgm %then %do;%str(image/x-portable-graymap)%end;
+%else %if &ext=ppm %then %do;%str(image/x-portable-pixmap)%end;
+%else %if &ext=xbm %then %do;%str(image/x-xbitmap)%end;
+%else %if &ext=xpm %then %do;%str(image/x-xpixmap)%end;
+%else %if &ext=xwd %then %do;%str(image/x-xwindowdump)%end;
+%else %if &ext=eml %then %do;%str(message/rfc822)%end;
+%else %if &ext=mime %then %do;%str(message/rfc822)%end;
+%else %if &ext=iges %then %do;%str(model/iges)%end;
+%else %if &ext=igs %then %do;%str(model/iges)%end;
+%else %if &ext=dae %then %do;%str(model/vnd.collada+xml)%end;
+%else %if &ext=dwf %then %do;%str(model/vnd.dwf)%end;
+%else %if &ext=gdl %then %do;%str(model/vnd.gdl)%end;
+%else %if &ext=gtw %then %do;%str(model/vnd.gtw)%end;
+%else %if &ext=vtu %then %do;%str(model/vnd.vtu)%end;
+%else %if &ext=vrml %then %do;%str(model/vrml)%end;
+%else %if &ext=wrl %then %do;%str(model/vrml)%end;
+%else %if &ext=x3db %then %do;%str(model/x3d+binary)%end;
+%else %if &ext=x3dbz %then %do;%str(model/x3d+binary)%end;
+%else %if &ext=x3dv %then %do;%str(model/x3d+vrml)%end;
+%else %if &ext=x3dvz %then %do;%str(model/x3d+vrml)%end;
+%else %if &ext=x3d %then %do;%str(model/x3d+xml)%end;
+%else %if &ext=x3dz %then %do;%str(model/x3d+xml)%end;
+%else %if &ext=appcache %then %do;%str(text/cache-manifest)%end;
+%else %if &ext=ics %then %do;%str(text/calendar)%end;
+%else %if &ext=ifb %then %do;%str(text/calendar)%end;
+%else %if &ext=htm %then %do;%str(text/html)%end;
+%else %if &ext=js %then %do;%str(text/javascript)%end;
+%else %if &ext=mjs %then %do;%str(text/javascript)%end;
+%else %if &ext=dsc %then %do;%str(text/prs.lines.tag)%end;
+%else %if &ext=rtx %then %do;%str(text/richtext)%end;
+%else %if &ext=sgm %then %do;%str(text/sgml)%end;
+%else %if &ext=tsv %then %do;%str(text/tab-separated-values)%end;
+%else %if &ext=ttl %then %do;%str(text/turtle)%end;
+%else %if &ext=curl %then %do;%str(text/vnd.curl)%end;
+%else %if &ext=dcurl %then %do;%str(text/vnd.curl.dcurl)%end;
+%else %if &ext=mcurl %then %do;%str(text/vnd.curl.mcurl)%end;
+%else %if &ext=scurl %then %do;%str(text/vnd.curl.scurl)%end;
+%else %if &ext=sub %then %do;%str(text/vnd.dvb.subtitle)%end;
+%else %if &ext=fly %then %do;%str(text/vnd.fly)%end;
+%else %if &ext=flx %then %do;%str(text/vnd.fmi.flexstor)%end;
+%else %if &ext=gv %then %do;%str(text/vnd.graphviz)%end;
+%else %if &ext=3dml %then %do;%str(text/vnd.in3d.3dml)%end;
+%else %if &ext=spot %then %do;%str(text/vnd.in3d.spot)%end;
+%else %if &ext=jad %then %do;%str(text/vnd.sun.j2me.app-descriptor)%end;
+%else %if &ext=wml %then %do;%str(text/vnd.wap.wml)%end;
+%else %if &ext=wmls %then %do;%str(text/vnd.wap.wmlscript)%end;
+%else %if &ext=s %then %do;%str(text/x-asm)%end;
+%else %if &ext=java %then %do;%str(text/x-java-source)%end;
+%else %if &ext=p %then %do;%str(text/x-pascal)%end;
+%else %if &ext=pas %then %do;%str(text/x-pascal)%end;
+%else %if &ext=etx %then %do;%str(text/x-setext)%end;
+%else %if &ext=uu %then %do;%str(text/x-uuencode)%end;
+%else %if &ext=vcs %then %do;%str(text/x-vcalendar)%end;
+%else %if &ext=vcf %then %do;%str(text/x-vcard)%end;
+%else %if &ext=3gp %then %do;%str(video/3gpp)%end;
+%else %if &ext=3g2 %then %do;%str(video/3gpp2)%end;
+%else %if &ext=jpgv %then %do;%str(video/jpeg)%end;
+%else %if &ext=jpgm %then %do;%str(video/jpm)%end;
+%else %if &ext=mjp2 %then %do;%str(video/mj2)%end;
+%else %if &ext=ogv %then %do;%str(video/ogg)%end;
+%else %if &ext=mov %then %do;%str(video/quicktime)%end;
+%else %if &ext=qt %then %do;%str(video/quicktime)%end;
+%else %if &ext=uvh %then %do;%str(video/vnd.dece.hd)%end;
+%else %if &ext=uvvh %then %do;%str(video/vnd.dece.hd)%end;
+%else %if &ext=uvm %then %do;%str(video/vnd.dece.mobile)%end;
+%else %if &ext=uvvm %then %do;%str(video/vnd.dece.mobile)%end;
+%else %if &ext=uvp %then %do;%str(video/vnd.dece.pd)%end;
+%else %if &ext=uvvp %then %do;%str(video/vnd.dece.pd)%end;
+%else %if &ext=uvs %then %do;%str(video/vnd.dece.sd)%end;
+%else %if &ext=uvvs %then %do;%str(video/vnd.dece.sd)%end;
+%else %if &ext=uvv %then %do;%str(video/vnd.dece.video)%end;
+%else %if &ext=uvvv %then %do;%str(video/vnd.dece.video)%end;
+%else %if &ext=dvb %then %do;%str(video/vnd.dvb.file)%end;
+%else %if &ext=fvt %then %do;%str(video/vnd.fvt)%end;
+%else %if &ext=m4u %then %do;%str(video/vnd.mpegurl)%end;
+%else %if &ext=mxu %then %do;%str(video/vnd.mpegurl)%end;
+%else %if &ext=pyv %then %do;%str(video/vnd.ms-playready.media.pyv)%end;
+%else %if &ext=uvu %then %do;%str(video/vnd.uvvu.mp4)%end;
+%else %if &ext=uvvu %then %do;%str(video/vnd.uvvu.mp4)%end;
+%else %if &ext=viv %then %do;%str(video/vnd.vivo)%end;
+%else %if &ext=asf %then %do;%str(video/x-ms-asf)%end;
+%else %if &ext=asx %then %do;%str(video/x-ms-asf)%end;
+%else %if &ext=vob %then %do;%str(video/x-ms-vob)%end;
+%else %if &ext=wm %then %do;%str(video/x-ms-wm)%end;
+%else %if &ext=wmv %then %do;%str(video/x-ms-wmv)%end;
+%else %if &ext=wmx %then %do;%str(video/x-ms-wmx)%end;
+%else %if &ext=wvx %then %do;%str(video/x-ms-wvx)%end;
+%else %if &ext=avi %then %do;%str(video/x-msvideo)%end;
+%else %if &ext=movie %then %do;%str(video/x-sgi-movie)%end;
+%else %if &ext=ice %then %do;%str(x-conference/x-cooltalk)%end;
+%else %if "&ext"="in" %then %do;%str(text/plain)%end;
+%else %do;%str(application/octet-stream)%end;
+
+%mend mf_mimetype;

base/mf_mkdir.sas

@@ -7,7 +7,7 @@ Usage:
     %mf_mkdir(/some/path/name)
 
 
-  @param dir relative or absolute pathname.  Unquoted.
+  @param [in] dir Relative or absolute pathname.  Unquoted.
   @version 9.2
 
 **/

base/mf_mval.sas

@@ -8,6 +8,8 @@
 
       %if %mf_mval(maynotexist)=itdid %then %do;
 
+  @param [in] var The macro variable NAME to return the (possible) value for
+
   @version 9.2
   @author Allan Bowe
 **/

base/mf_nobs.sas

@@ -9,7 +9,7 @@
   <h4> SAS Macros </h4>
   @li mf_getattrn.sas
 
-  @param libds library.dataset
+  @param [in] libds library.dataset
 
   @return output returns result of the attrn value supplied, or log message
     if err.

base/mf_trimstr.sas

@@ -11,8 +11,8 @@
   <h4> SAS Macros </h4>
 
 
-  @param basestr The string to be modified
-  @param trimstr The string to be removed from the end of `basestr`, if it
+  @param [in] basestr The string to be modified
+  @param [in] trimstr The string to be removed from the end of `basestr`, if it
     exists
 
   @return output returns result with the value of `trimstr` removed from the end

base/mf_wordsinstr1andstr2.sas

@@ -13,8 +13,8 @@
   returns:
   > blah blaaah brah
 
-  @param str1= string containing words to extract
-  @param str2= used to compare with the extract string
+  @param [in] str1= () string containing words to extract
+  @param [in] str2= () used to compare with the extract string
 
   @warning CASE SENSITIVE!
 

base/mf_wordsinstr1butnotstr2.sas

@@ -16,8 +16,8 @@
   returns:
   > sss bram boo
 
-  @param [in] str1= string containing words to extract
-  @param [in] str2= used to compare with the extract string
+  @param [in] str1= () String containing words to extract
+  @param [in] str2= () Used to compare with the extract string
 
   @version 9.2
   @author Allan Bowe

base/mf_writefile.sas

@@ -21,8 +21,8 @@
   @param [in] mode= (O) Available options are A or O as follows:
     @li A APPEND mode, writes new records after the current end of the file.
     @li O OUTPUT mode, writes new records from the beginning of the file.
-  @param [in] l1= First line
-  @param [in] l2= Second line (etc through to l10)
+  @param [in] l1= () First line
+  @param [in] l2= () Second line (etc through to l10)
 
   <h4> Related Macros </h4>
   @li mf_writefile.test.sas

base/mp_abort.sas

@@ -27,14 +27,15 @@
     currently investigating approaches to deal with this.
 
 
-  @param mac= (mp_abort.sas) To contain the name of the calling macro. Do not
-    use &sysmacroname as this will always resolve to MP_ABORT.
-  @param msg= message to be returned
-  @param iftrue= (1=1) Supply a condition for which the macro should be executed
-  @param errds= (work.mp_abort_errds) There is no clean way to end a process
-    within a %include called within a macro.  Furthermore, there is no way to
-    test if a macro is called within a %include.  To handle this particular
-    scenario, the %include should be switched for the mp_include.sas macro.
+  @param [in] mac= (mp_abort.sas) To contain the name of the calling macro. Do
+    not use &sysmacroname as this will always resolve to MP_ABORT.
+  @param [out] msg= message to be returned
+  @param [in] iftrue= (1=1) Condition under which the macro should be executed
+  @param [in] errds= (work.mp_abort_errds) There is no clean way to end a
+    process within a %include called within a macro.  Furthermore, there is no
+    way to test if a macro is called within a %include.  To handle this
+    particular scenario, the %include should be switched for the mp_include.sas
+    macro.
     This provides an indicator that we are running a macro within a \%include
     (`_SYSINCLUDEFILEDEVICE`) and allows us to provide a dataset with the abort
     values (msg, mac).
@@ -45,8 +46,8 @@
     @li msg (the message)
     @li mac (the mac param)
 
-  @param mode= (REGULAR) If mode=INCLUDE then the &errds dataset is checked for
-    an abort status.
+  @param [in] mode= (REGULAR) If mode=INCLUDE then the &errds dataset is checked
+    for an abort status.
     Valid values:
     @li REGULAR (default)
     @li INCLUDE

base/mp_aligndecimal.sas

@@ -36,6 +36,7 @@
         aligndp4e49996=right(aligndp4e49996);
         myvar=aligndp4e49996;
       end;
+      drop aligndp4e49996;
 
   Results (myvar variable):
 
@@ -61,8 +62,8 @@
           7998580.8415
 
 
-  @param var The (data step) variable to create
-  @param width= (8) The number of characters BEFORE the decimal point
+  @param [in] var The (data step, character) variable to modify
+  @param [in] width= (8) The number of characters BEFORE the decimal point
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas

base/mp_appendfile.sas

@@ -17,8 +17,8 @@
         %mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3)
 
 
-  @param [in] baseref= Fileref of the base file (should exist)
-  @param [in] appendrefs= One or more filerefs to be appended to the base
+  @param [in] baseref= (0) Fileref of the base file (should exist)
+  @param [in] appendrefs= (0) One or more filerefs to be appended to the base
     fileref.  Space separated.
 
   @version 9.2

base/mp_applyformats.sas

@@ -94,7 +94,7 @@ data &outds;
   if libref(lib) ne 0 then do;
     msg=catx(' ','libref',lib,'is not assigned!');
     %if &errds=0 %then %do;
-      putlog "%str(ERR)OR: " msg;
+      putlog 'ERR' +(-1) "OR: " msg;
     %end;
     output;
     return;
@@ -102,7 +102,7 @@ data &outds;
   if exist(cats(lib,'.',ds)) ne 1 then do;
     msg=catx(' ','libds',lib,'.',ds,'does not exist!');
     %if &errds=0 %then %do;
-      putlog "%str(ERR)OR: " msg;
+      putlog 'ERR' +(-1) "OR: " msg;
     %end;
     output;
     return;
@@ -111,7 +111,7 @@ data &outds;
   if is_fmt=0 then do;
     msg=catx(' ','format',fmt,'on libds',lib,'.',ds,'.',var,'is not valid!');
     %if &errds=0 %then %do;
-      putlog "%str(ERR)OR: " msg;
+      putlog 'ERR' +(-1) "OR: " msg;
     %end;
     output;
     return;
@@ -123,7 +123,7 @@ data &outds;
     if dsid=0 then do;
       msg=catx(' ','libds',lib,'.',ds,' could not be opened!');
       %if &errds=0 %then %do;
-        putlog "%str(ERR)OR: " msg;
+        putlog 'ERR' +(-1) "OR: " msg;
       %end;
       output;
       return;
@@ -131,7 +131,7 @@ data &outds;
     if varnum(dsid,var)<1 then do;
       msg=catx(' ','Variable',lib,'.',ds,'.',var,' was not found!');
       %if &errds=0 %then %do;
-        putlog "%str(ERR)OR: " msg;
+        putlog 'ERR' +(-1) "OR: " msg;
       %end;
       output;
     end;

base/mp_assertcols.sas

@@ -31,8 +31,8 @@
 
 
   @param [in] inds The input library.dataset to test for values
-  @param [in] cols= The list of columns to check for
-  @param [in] desc= (Testing observations) The user provided test description
+  @param [in] cols= (0) The list of columns to check for
+  @param [in] desc= (0) The user provided test description
   @param [in] test= (ALL) The test to apply.  Valid values are:
     @li ALL - Test is a PASS if ALL columns exist in &inds
     @li ANY - Test is a PASS if ANY of the columns exist in &inds

base/mp_assertcolvals.sas

@@ -36,7 +36,7 @@
 
 
   @param [in] indscol The input library.dataset.column to test for values
-  @param [in] checkvals= A library.dataset.column value containing a UNIQUE
+  @param [in] checkvals= (0) A library.dataset.column value containing a UNIQUE
     list of values to be compared against the source (indscol).
   @param [in] desc= (Testing observations) The user provided test description
   @param [in] test= (ALLVALS) The test to apply.  Valid values are:

base/mp_base64copy.sas

@@ -28,8 +28,8 @@
           put _infile_;
         run;
 
-  @param [in] inref= Fileref of the input file (should exist)
-  @param [out] outref= Output fileref. If it does not exist, it is created.
+  @param [in] inref= (0) Fileref of the input file (should exist)
+  @param [out] outref= (0) Output fileref. If it does not exist, it is created.
   @param [in] action= (ENCODE) The action to take. Valid values:
     @li ENCODE - Convert the file to base64 format
     @li DECODE - Decode the file from base64 format

base/mp_binarycopy.sas

@@ -22,16 +22,17 @@
       %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
 
 
-  @param [in] inloc quoted "path/and/filename.ext" of the file to be copied
-  @param [out] outloc quoted "path/and/filename.ext" of the file to be created
-  @param [in] inref (____in) If provided, this fileref will take precedence over
+  @param [in] inloc= () quoted "path/and/filename.ext" of the file to be copied
+  @param [out] outloc= () quoted "path/and/filename.ext" of the file to create
+  @param [in] inref= (____in) If provided, this fileref takes precedence over
     the `inloc` parameter
-  @param [out] outref (____in) If provided, this fileref will take precedence
+  @param [out] outref= (____in) If provided, this fileref takes precedence
     over the `outloc` parameter.  It must already exist!
-  @param [in] mode (CREATE) Valid values:
+  @param [in] mode= (CREATE) Valid values:
     @li CREATE - Create the file (even if it already exists)
     @li APPEND - Append to the file (don't overwrite)
-  @param iftrue= (1=1) Supply a condition for which the macro should be executed
+  @param [in] iftrue= (1=1)
+    Supply a condition for which the macro should be executed
 
   @returns nothing
 

base/mp_chop.sas

@@ -44,7 +44,7 @@
   For more examples, see mp_chop.test.sas
 
   @param [in] infile The QUOTED path to the file on which to perform the chop
-  @param [in] matchvar= Macro variable NAME containing the string to split by
+  @param [in] matchvar= () Macro variable NAME containing the string to split by
   @param [in] matchpoint= (START) Valid values:
     @li START - chop at the beginning of the string in `matchvar`.
     @li END - chop at the end of the string in `matchvar`.
@@ -54,7 +54,8 @@
     @li FIRST - keep the section of the file before the chop
     @li LAST - keep the section of the file after the chop
   @param [in] mdebug= (0) Set to 1 to provide macro debugging
-  @param outfile= (0) Optional QUOTED path to the adjusted output file (avoids
+  @param [out] outfile= (0)
+    Optional QUOTED path to the adjusted output file (avoids
     overwriting the first file).
 
   <h4> SAS Macros </h4>
@@ -185,7 +186,7 @@ run;
     infile &outfile lrecl=32767;
     input;
     list;
-    if _n_>50 then stop;
+    if _n_>200 then stop;
   run;
 %end;
 /* END */

base/mp_cleancsv.sas

@@ -12,11 +12,12 @@
       fileref mycsv "/path/your/csv";
       %mp_cleancsv(in=mycsv,out=/path/new.csv)
 
-  @param in= (NOTPROVIDED) Provide path or fileref to input csv. If a period is
+  @param [in] in= (NOTPROVIDED)
+    Provide path or fileref to input csv. If a period is
     found, it is assumed to be a file.
-  @param out= (NOTPROVIDED) Output path or fileref to output csv.  If a period
-    is found, it is assumed to be a file.
-  @param qchar= ('22'x) Quote char - hex code 22 is the double quote.
+  @param [in] out= (NOTPROVIDED) Output path or fileref to output csv.
+    If a period is found, it is assumed to be a file.
+  @param [in] qchar= ('22'x) Quote char - hex code 22 is the double quote.
 
   @version 9.2
   @author Allan Bowe

base/mp_cntlout.sas

@@ -25,10 +25,10 @@
   <h4> SAS Macros </h4>
   @li mddl_sas_cntlout.sas
   @li mf_getuniquename.sas
+  @li mp_aligndecimal.sas
 
   <h4> Related Macros </h4>
   @li mf_getvarformat.sas
-  @li mp_aligndecimal.sas
   @li mp_getformats.sas
   @li mp_loadformat.sas
   @li mp_ds2fmtds.sas
@@ -58,25 +58,33 @@
 %end;
 
 proc format lib=&libcat cntlout=&cntlds;
-%if "&fmtlist" ne "0" %then %do;
+%if "&fmtlist" ne "0" and "&fmtlist" ne "" %then %do;
   select
-  %do i=1 %to %sysfunc(countw(&fmtlist));
+  %do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
     %scan(&fmtlist,&i,%str( ))
   %end;
   ;
 %end;
 run;
 
-data &cntlout;
+data &cntlout/nonote2err;
   if 0 then set &ddlds;
   set &cntlds;
-  if type in ("I","N") then do; /* numeric (in)format */
+  by type fmtname notsorted;
+
+  /* align the numeric values to avoid overlapping ranges */
+  if type in ("I","N") then do;
     %mp_aligndecimal(start,width=16)
     %mp_aligndecimal(end,width=16)
   end;
+
+  /* create row marker. Data cannot be sorted without it! */
+  if first.fmtname then fmtrow=1;
+  else fmtrow+1;
+
 run;
 proc sort;
-  by type fmtname start;
+  by type fmtname fmtrow;
 run;
 
 proc sql;

base/mp_copyfolder.sas

@@ -18,7 +18,7 @@
 
   @param [in] source Unquoted path to the folder to copy from.
   @param [out] target Unquoted path to the folder to copy to.
-  @param [in] copymax=(MAX) Set to a positive integer to indicate the level of
+  @param [in] copymax= (MAX) Set to a positive integer to indicate the level of
     subdirectory copy recursion - eg 3, to go `./3/levels/deep`.  For unlimited
     recursion, set to MAX.
 
@@ -69,7 +69,7 @@
       rc2=filename(fref2,filepath2,'disk','recfm=n');
       if fcopy(fref1,fref2) ne 0 then do;
         msg=sysmsg();
-        putlog "%str(ERR)OR: Unable to copy " filepath " to " filepath2;
+        putlog 'ERR' +(-1) "OR: Unable to copy " filepath " to " filepath2;
         putlog msg=;
       end;
     end;

base/mp_createconstraints.sas

@@ -18,11 +18,11 @@
       %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
       %mp_createconstraints(inds=work.constraints,outds=created,execute=YES)
 
-  @param inds= (work.mp_getconstraints) The input table containing the
+  @param [in] inds= (work.mp_getconstraints) The input table containing the
     constraint info
-  @param outds= (work.mp_createconstraints) A table containing the create
+  @param [out] outds= (work.mp_createconstraints) A table containing the create
     statements (create_statement column)
-  @param execute= (NO) To actually create, use YES.
+  @param [in] execute= (NO) To actually create, use YES.
 
   <h4> Related Files </h4>
   @li mp_getconstraints.sas

base/mp_createwebservice.sas

@@ -1,6 +1,6 @@
 /**
   @file mp_createwebservice.sas
-  @brief Create a web service in SAS 9, Viya or SASjs Server
+  @brief Create a web service in SAS 9, Viya or SASjs Server (legacy)
   @details This is actually a wrapper for mx_createwebservice.sas, remaining
   for legacy purposes.  For new apps, use mx_createwebservice.sas.
 

base/mp_csv2ds.sas

@@ -19,11 +19,12 @@
       %mp_csv2ds(inref=mycsv,outds=myds,baseds=sashelp.class)
 
 
-  @param inref= fileref to the CSV
-  @param outds= output ds (lib.ds format)
-  @param view= Set to YES or NO to determine whether the output should be
-    a view or not.  Default is NO (not a view).
-  @param baseds= Template dataset on which to create the input statement.
+  @param [in] inref= (0) Fileref to the CSV
+  @param [out] outds= (0) Output ds (lib.ds format)
+  @param [in] view= (NO) Set to YES or NO to determine whether the output
+    should be a view or not.  Default is NO (not a view).
+  @param [in] baseds= (0)
+    Template dataset on which to create the input statement.
     Is used to determine types, lengths, and any informats.
 
   @version 9.2

base/mp_deleteconstraints.sas

@@ -17,9 +17,11 @@
       %mp_getconstraints(lib=work,ds=example,outds=work.constraints)
       %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
 
-  @param inds= The input table containing the constraint info
-  @param outds= a table containing the drop statements (drop_statement column)
-  @param execute= `YES|NO` - default is NO. To actually drop, use YES.
+  @param [in] inds= (mp_getconstraints)
+    The input table containing constraint info
+  @param [out] outds= (mp_deleteconstraints)
+    Table containing the drop statements (drop_statement column)
+  @param [in] execute= (NO) `YES|NO` - default is NO. To actually drop, use YES.
 
 
   @version 9.2

base/mp_deletefolder.sas

@@ -15,7 +15,7 @@
 
       %mp_deletefolder(&rootdir)
 
-  @param path Unquoted path to the folder to delete.
+  @param [in] folder Unquoted path to the folder to delete.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas

base/mp_dictionary.sas

@@ -29,7 +29,7 @@
 
   ![](https://user-images.githubusercontent.com/4420615/188278365-2987db97-0594-4a39-ac81-dbacdef5cdc8.png)
 
-  @param lib= (WORK) The libref in which to create the views
+  @param [in] lib= (WORK) The libref in which to create the views
 
   <h4> Related Files </h4>
   @li mp_dictionary.test.sas

base/mp_dirlist.sas

@@ -85,7 +85,7 @@ data;run;
 data &out_ds(compress=no
     keep=file_or_folder filepath filename ext msg directory level
   );
-  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
+  length directory filepath $2000 fref fref2 $8 file_or_folder $6 filename $255
     ext $20 msg $200 foption $16;
   if _n_=1 then call missing(of _all_);
   retain level &level;

base/mp_distinctfmtvalues.sas

@@ -7,11 +7,11 @@
 
         %mp_distinctfmtvalues(libds=sashelp.class,var=age,outvar=age,outds=test)
 
-  @param libds input dataset
-  @param var variable to get distinct values for
-  @param outvar variable to create.  Default:  `formatted_value`
-  @param outds dataset to create.  Default:  work.mp_distinctfmtvalues
-  @param varlen length of variable to create (default 200)
+  @param [in] libds= () input dataset
+  @param [in] var= (0) variable to get distinct values for
+  @param [out] outvar= (formatteed_value) variable to create.
+  @param [out] outds= (work.mp_distinctfmtvalues) dataset to create.
+  @param [in] varlen= (2000) length of variable to create
 
   @version 9.2
   @author Allan Bowe

base/mp_ds2csv.sas

@@ -118,13 +118,21 @@ data _null_;
     header = cats(coalescec(varlabel(dsid,i),varnm),dlm);
   %end;
   %else %if &headerformat=SASJS %then %do;
-    if vartype(dsid,i)='C' then header=cats(varnm,':$char',varlen(dsid,i),'.');
+    vlen=varlen(dsid,i);
+    if vartype(dsid,i)='C' then header=cats(varnm,':$char',vlen,'.');
     else do;
       vfmt=coalescec(varfmt(dsid,i),'0');
       fmttype=mcf_getfmttype(vfmt);
       if fmttype='DATE' then header=cats(varnm,':date9.');
       else if fmttype='DATETIME' then header=cats(varnm,':E8601DT26.6');
       else if fmttype='TIME' then header=cats(varnm,':TIME12.');
+      /**
+        * there is not much point importing a short length numeric like this,
+        * eg with best4., as the resulting variable will still be stored as
+        * length 8.  We need a length or format statement to ensure variable
+        * is creatd with the smaller length...
+        **/
+      else if vlen<8 then header=cats(varnm,':best',vlen,'.');
       else header=cats(varnm,':best.');
     end;
   %end;
@@ -151,6 +159,7 @@ data _null_;
   set &ds end=last;
 %do i=1 %to &vcnt;
   %let var=%scan(&varlist,&i);
+  %local vlen&i;
   %if %mf_getvartype(&ds,&var)=C %then %do;
     %let dsv1=%mf_getuniquename(prefix=csvcol1_);
     %let dsv2=%mf_getuniquename(prefix=csvcol2_);

base/mp_ds2ddl.sas

@@ -1,8 +1,20 @@
 /**
   @file
-  @brief A wrapper for mp_getddl.sas
-  @details In the next release, this will be the main version.
+  @brief Fetches DDL for a specific table
+  @details Uses mp_getddl under the hood
 
+  @param [in] libds library.dataset to create ddl for
+  @param [in] fref= (getddl) the fileref to which to _append_ the DDL.  If it
+    does not exist, it will be created.
+  @param [in] flavour= (SAS) The type of DDL to create. Options:
+    @li SAS
+    @li TSQL
+
+  @param [in]showlog= (NO) Set to YES to show the DDL in the log
+  @param [in] schema= () Choose a preferred schema name (default is to use
+    actual schema, else libref)
+  @param [in] applydttm= (NO) For non SAS DDL, choose if columns are created with
+    native datetime2 format or regular decimal type
 
   <h4> SAS Macros </h4>
   @li mp_getddl.sas

base/mp_ds2md.sas

@@ -95,6 +95,7 @@ run;
   options ps=max lrecl=max;
   data _null_;
     infile &outref;
+    if _n_=1 then putlog "# &libds" /;
     input;
     putlog _infile_;
   run;

base/mp_dsmeta.sas

@@ -49,8 +49,8 @@
   https://support.sas.com/resources/papers/proceedings14/1549-2014.pdf) by
   [Louise Hadden](https://www.linkedin.com/in/louisehadden/).
 
-  @param libds The library.dataset to export the metadata for
-  @param outds= (work.dsmeta) The output table to contain the metadata
+  @param [in] libds The library.dataset to export the metadata for
+  @param [out] outds= (work.dsmeta) The output table to contain the metadata
 
   <h4> Related Files </h4>
   @li mp_dsmeta.test.sas

base/mp_filtercheck.sas

@@ -37,10 +37,12 @@
   @param [in] targetds= The target dataset against which to verify VARIABLE_NM.
     This must be available (ie, the library must be assigned).
   @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
-  @param [out] outds= The output table, which is a copy of the &inds. table
-    plus a REASON_CD column, containing only bad records.  If bad records found,
-    the SYSCC value will be set to 1008 (general data problem).  Downstream
-    processes should check this table (and return code) before continuing.
+  @param [out] outds= (work.badrecords) The output table, which is a copy of the
+    &inds. table plus a REASON_CD column, containing only bad records.
+    If bad records are found, the SYSCC value will be set to 1008
+    (a general data problem).
+    Downstream processes should check this table (and return code) before
+    continuing.
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas
@@ -84,15 +86,14 @@
 /**
   * Sanitise the values based on valid value lists, then strip out
   * quotes, commas, periods and spaces.
-  * Only numeric values should remain
   */
 %local reason_cd nobs;
 %let nobs=0;
 data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
-  set &inds;
-  length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000;
+  set &inds end=last;
+  length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000;
   drop tmp;
 
   /* quick check to ensure column exists */
@@ -108,7 +109,8 @@ data &outds;
   end;
 
   /* need to open the dataset to get the column type */
-  dsid=open("&targetds","i");
+  retain dsid;
+  if _n_=1 then dsid=open("&targetds","i");
   if dsid>0 then do;
     vnum=varnum(dsid,VARIABLE_NM);
     if vnum<1 then do;
@@ -118,11 +120,19 @@ data &outds;
       call symputx('reason_cd',reason_cd,'l');
       call symputx('nobs',_n_,'l');
       output;
-      return;
+      goto endstep;
     end;
     /* now we can get the type */
     else vtype=vartype(dsid,vnum);
   end;
+  else do;
+      REASON_CD=cats("Could not open &targetds");
+      putlog REASON_CD= dsid=;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
+      output;
+      stop;
+  end;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
@@ -157,15 +167,40 @@ data &outds;
   end;
 
   /* special missing logic */
-  if vtype='N'
-    and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE')
-    and cats(upcase(raw_value)) in (
+  if vtype='N' & OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE') then do;
+    if cats(upcase(raw_value)) in (
       '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N'
       '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._'
     )
-  then do;
-    /* valid numeric - exit data step loop */
-    return;
+    then do;
+      /* valid numeric - exit data step loop */
+      return;
+    end;
+    else if subpad(upcase(raw_value),1,1) in (
+      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+    )
+    then do;
+      /* check if the raw_value contains a valid variable NAME */
+      vnum=varnum(dsid,subpad(raw_value,1,32));
+      if vnum>0 then do;
+        /* now we can get the type */
+        vtype2=vartype(dsid,vnum);
+        /* check type matches */
+        if vtype2=vtype then do;
+          /* valid target var - exit loop */
+          return;
+        end;
+        else do;
+          REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
+          putlog REASON_CD= dsid=;
+          call symputx('reason_cd',reason_cd,'l');
+          call symputx('nobs',_n_,'l');
+          output;
+          goto endstep;
+        end;
+      end;
+    end;
   end;
 
   /* special logic */
@@ -187,6 +222,32 @@ data &outds;
     if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
       tmp=scan(raw_value1,i,',');
       if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
+        if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in (
+          'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+          'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+        )
+        then do;
+          /* check if the raw_value contains a valid variable NAME */
+          /* is not valid syntax for IN or NOT IN */
+          vnum=varnum(dsid,subpad(tmp,1,32));
+          if vnum>0 then do;
+            /* now we can get the type */
+            vtype2=vartype(dsid,vnum);
+            /* check type matches */
+            if vtype2=vtype then do;
+              /* valid target var - exit loop */
+              return;
+            end;
+            else do;
+              REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
+              putlog REASON_CD= dsid=;
+              call symputx('reason_cd',reason_cd,'l');
+              call symputx('nobs',_n_,'l');
+              output;
+              goto endstep;
+            end;
+          end;
+        end;
         REASON_CD='Non Numeric value provided';
         putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
         call symputx('reason_cd',reason_cd,'l');
@@ -211,14 +272,42 @@ data &outds;
 
   /* output records that contain values other than digits and spaces */
   if notdigit(compress(raw_value3,' '))>0 then do;
+    if vtype='C' and subpad(upcase(raw_value),1,1) in (
+      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+    )
+    then do;
+      /* check if the raw_value contains a valid variable NAME */
+      vnum=varnum(dsid,subpad(raw_value,1,32));
+      if vnum>0 then do;
+        /* now we can get the type */
+        vtype2=vartype(dsid,vnum);
+        /* check type matches */
+        if vtype2=vtype then do;
+          /* valid target var - exit loop */
+          return;
+        end;
+        else do;
+          REASON_CD=cats("Compared Char Type (",vtype2,") is not (",vtype,")");
+          putlog REASON_CD= dsid=;
+          call symputx('reason_cd',reason_cd,'l');
+          call symputx('nobs',_n_,'l');
+          output;
+          goto endstep;
+        end;
+      end;
+    end;
+
     putlog raw_value3= $hex32.;
     REASON_CD=cats('Invalid RAW_VALUE:',raw_value);
-    putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
+    putlog (_all_)(=);
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');
     output;
   end;
 
+  endstep:
+  if last then rc=close(dsid);
 run;
 
 

base/mp_filtergenerate.sas

@@ -51,8 +51,8 @@
   > )
 
   @param [in] inds The input table with query values
-  @param [out] outref= The output fileref to contain the filter clause.  Will
-    be created (or replaced).
+  @param [out] outref= (filter) The output fileref to contain the filter clause.
+    Will be created (or replaced).
 
   <h4> Related Macros </h4>
   @li mp_filtercheck.sas

base/mp_filterstore.sas

@@ -44,9 +44,10 @@
     mp_coretable.sas as follows:  `mp_coretable(LOCKTABLE)`.
   @param [in] maxkeytable= (0) Optional permanent reference table used for
     retained key tracking.  Described in mp_retainedkey.sas.
-  @param [in] mdebug= set to 1 to enable DEBUG messages
-  @param [out] outresult= The result table with the FILTER_RK
-  @param [out] outquery= The original query, taken as extract after table load
+  @param [in] mdebug= (1) set to 1 to enable DEBUG messages
+  @param [out] outresult= (work.result) The result table with the FILTER_RK
+  @param [out] outquery= (work.query) The original query, taken as extract
+    after table load
 
 
   <h4> SAS Macros </h4>

base/mp_getcols.sas

@@ -12,14 +12,15 @@
 
       %mp_getcols(sashelp.airline,outds=work.myds)
 
-  @param ds The dataset from which to obtain column metadata
-  @param outds= (work.cols) The output dataset to create. Sample data:
+  @param [in] ds The dataset from which to obtain column metadata
+  @param [out] outds= (work.cols) The output dataset to create. Sample data:
 |NAME:$32.|LENGTH:best.|VARNUM:best.|LABEL:$256.|FMTNAME:$32.|FORMAT:$49.|TYPE:$1.|DDTYPE:$9.|
 |---|---|---|---|---|---|---|---|
 |`AIR `|`8 `|`2 `|`international airline travel (thousands) `|` `|`8. `|`N `|`NUMERIC `|
 |`DATE `|`8 `|`1 `|`DATE `|`MONYY `|`MONYY. `|`N `|`DATE `|
 |`REGION `|`3 `|`3 `|`REGION `|` `|`$3. `|`C `|`CHARACTER `|
 
+
   <h4> Related Macros </h4>
   @li mf_getvarlist.sas
   @li mm_getcols.sas
@@ -52,10 +53,11 @@ data &outds(keep=name type length varnum format label ddtype fmtname);
     else if formatd=0 then format=cats(fmtname,formatl,'.');
     else format=cats(fmtname,formatl,'.',formatd);
     type='N';
-    if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
+    if format=:'DATETIME' or format=:'E8601DT' or format=:'NLDATM'
+    then ddtype='DATETIME';
     else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
       or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
-      or format=:'MONYY'
+      or format=:'MONYY' or format=:'NLDATE'
     then ddtype='DATE';
     else if format=:'TIME' then ddtype='TIME';
     else ddtype='NUMERIC';

base/mp_getdbml.sas

@@ -23,10 +23,10 @@
   @li mf_getquotedstr.sas
   @li mp_getconstraints.sas
 
-  @param liblist= Space seperated list of librefs to take as
-    input (Default=SASHELP)
-  @param outref= Fileref to contain the DBML (Default=getdbml)
-  @param showlog= set to YES to show the DBML in the log (Default is NO)
+  @param [in] liblist= (SASHELP) Space seperated list of librefs to take as
+    input
+  @param [out] outref= (getdbml) Fileref to contain the DBML
+  @param [in] showlog= (NO) set to YES to show the DBML in the log
 
   @version 9.3
   @author Allan Bowe

base/mp_getddl.sas

@@ -22,16 +22,21 @@
   @li mf_getvarcount.sas
   @li mp_getconstraints.sas
 
-  @param lib libref of the library to create DDL for.  Should be assigned.
-  @param ds dataset to create ddl for (optional)
-  @param fref= the fileref to which to _append_ the DDL.  If it does not exist,
-    it will be created.
-  @param flavour= The type of DDL to create (default=SAS). Supported=TSQL
-  @param showlog= Set to YES to show the DDL in the log
-  @param schema= Choose a preferred schema name (default is to use actual schema
-    ,else libref)
-  @param applydttm= for non SAS DDL, choose if columns are created with native
-    datetime2 format or regular decimal type
+  @param [in] libref Libref of the library to create DDL for. Should already
+    be assigned.
+  @param [in] ds dataset to create ddl for (optional)
+  @param [in] fref= (getddl) the fileref to which to _append_ the DDL.  If it
+    does not exist, it will be created.
+  @param [in] flavour= (SAS) The type of DDL to create. Options:
+    @li SAS
+    @li TSQL
+
+  @param [in]showlog= (NO) Set to YES to show the DDL in the log
+  @param [in] schema= () Choose a preferred schema name (default is to use
+    actual schema, else libref)
+  @param [in] applydttm= (NO) For non SAS DDL, choose if columns are created
+    with native datetime2 format or regular decimal type
+
   @version 9.3
   @author Allan Bowe
 **/

base/mp_getformats.sas

@@ -7,10 +7,15 @@
   Formats are taken from the library / dataset reference and / or a static
   format list.
 
+  Note - the source for this information is the dictionary.formats table. This
+  cannot show formats that are not already declared in the FMTSEARCH path.
+
   Example usage:
 
       %mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable)
 
+      %mp_getformats(fmtlist=FORMAT1 $FORMAT2 @INFMT3,outsummary=work.table2)
+
   @param [in] lib= (0) The libref for which to return formats.
   @todo Enable exporting of formats for an entire library
   @param [in] ds= (0) The dataset from which to obtain format definitions
@@ -49,7 +54,9 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 
 
   <h4> Related Macros </h4>
+  @li mf_getfmtlist.sas
   @li mp_applyformats.sas
+  @li mp_cntlout.sas
   @li mp_getformats.test.sas
 
   @version 9.2
@@ -66,7 +73,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 
 %local i fmt allfmts tempds fmtcnt;
 
-%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( )));
+%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
   /* ensure format list contains format _name_ only */
   %let fmt=%scan(&fmtlist,&i,%str( ));
   %let fmt=%mf_getfmtname(&fmt);
@@ -90,8 +97,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 proc sql;
 create table &outsummary as
   select * from dictionary.formats
-  where fmtname in (%mf_getquotedstr(&allfmts,quote=D))
-    and fmttype='F';
+  where fmtname in (%mf_getquotedstr(&allfmts,quote=D));
 
 %if "&outdetail" ne "0" %then %do;
   /* ensure base table always exists */
@@ -115,6 +121,10 @@ create table &outsummary as
     data &tempds;
       if 0 then set &outdetail;
       set &tempds;
+      /* set fmtrow (position of record within the format) */
+      by type fmtname notsorted;
+      if first.fmtname then fmtrow=1;
+      else fmtrow+1;
     run;
     proc append base=&outdetail data=&tempds ;
     run;

base/mp_gsubfile.sas

@@ -22,15 +22,15 @@
         list;
       run;
 
-  @param file= (0) The file to perform the substitution on
-  @param patternvar= A macro variable containing the Lua
+  @param [in] file= (0) The file to perform the substitution on
+  @param [in] patternvar= A macro variable containing the Lua
     [pattern](https://www.lua.org/pil/20.2.html) to search for.  Due to the use
     of special (magic) characters in Lua patterns, it is safer to pass the NAME
     of the macro variable containing the string, rather than the value itself.
-  @param replacevar= The name of the macro variable containing the replacement
-    _string_.
-  @param outfile= (0) The file to write the output to. If zero, then the file
-    is overwritten in-place.
+  @param [in] replacevar= ()
+    The name of the macro variable containing the replacement _string_.
+  @param [out] outfile= (0) The file to write the output to.
+    If zero, then the file is overwritten in-place.
 
   <h4> SAS Macros </h4>
   @li ml_gsubfile.sas

base/mp_guesspk.sas

@@ -18,7 +18,7 @@
       %mp_guesspk(sashelp.class,outds=classpks)
 
   @param [in] baseds The dataset to analyse
-  @param [out] outds= The output dataset to contain the possible PKs
+  @param [out] outds= (mp_guesspk) Output dataset to contain the possible PKs
   @param [in] max_guesses= (3) The total number of possible primary keys to
     generate. A table may have multiple (unlikely) PKs, so no need to list them
     all.

base/mp_hashdataset.sas

@@ -24,7 +24,7 @@
   @li mp_hashdirectory.sas
 
   @param [in] libds dataset to hash
-  @param [in] salt= Provide a salt (could be, for instance, the dataset name)
+  @param [in] salt= () Provide a salt (could be, for instance, the dataset name)
   @param [in] iftrue= (1=1) A condition under which the macro should be executed
   @param [out] outds= (work._data_) The output dataset to create. This
   will contain one column (hashkey) with one observation (a $hex32.

base/mp_init.sas

@@ -70,6 +70,7 @@ options
 %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
   noautocorrect           /* disallow misspelled procedure names            */
   dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
+                          /* turn off with dsoptions=nonote2err             */
 %end;
 ;
 

base/mp_lib2cards.sas

@@ -24,19 +24,19 @@
   @li mf_trimstr.sas
   @li mp_ds2cards.sas
 
-  @param [in] lib= Library in which to convert all datasets
-  @param [out] outloc= Location in which to store output.  Defaults to WORK
-    library. No quotes.
-  @param [out] outfile= Optional output file NAME - if provided, then will create
-  a single output file instead of one file per input table.
-  @param [in] maxobs= limit output to the first <code>maxobs</code> observations
+  @param [in] lib= () Library in which to convert all datasets
+  @param [out] outloc= (%sysfunc(pathname(work))) Location in which to store
+    output. No quotes.
+  @param [out] outfile= (0) Optional output file NAME - if provided, then
+    will create a single output file instead of one file per input table.
+  @param [in] maxobs= (max) limit output to the first <code>maxobs</code> rows
 
   @version 9.2
   @author Allan Bowe
 **/
 
 %macro mp_lib2cards(lib=
-    ,outloc=%sysfunc(pathname(work)) /* without trailing slash */
+    ,outloc=%sysfunc(pathname(work))
     ,maxobs=max
     ,random_sample=NO
     ,outfile=0

base/mp_loadformat.sas

@@ -9,6 +9,9 @@
   format, to prevent loss of data - UNLESS the input dataset contains a marker
   column, specifying that a particular row needs to be deleted (`delete_col=`).
 
+  Positions of formats are made using the FMTROW variable - this must be present
+  and unique (on TYPE / FMTNAME / FMTROW).
+
   This macro can also be used to identify which records would be (or were)
   considered new, modified or deleted (`loadtarget=`) by creating the following
   tables:
@@ -17,7 +20,7 @@
   @li work.outds_del
   @li work.outds_mod
 
-  For example usage, see mp_loadformat.test.sas
+  For example usage, see test (under Related Macros)
 
   @param [in] libcat The format catalog to be loaded
   @param [in] libds The staging table to load
@@ -34,19 +37,22 @@
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
-  @li mddl_sas_cntlout.sas
+  @li mf_existds.sas
+  @li mf_existvar.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
+  @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
+  @li mp_md5.sas
   @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
   @li mddl_dc_difftable.sas
   @li mddl_dc_locktable.sas
-  @li mp_aligndecimal.sas
-  @li mp_loadformat.test.sas
+  @li mp_loadformat.test.1.sas
+  @li mp_loadformat.test.2.sas
   @li mp_lockanytable.sas
   @li mp_stackdiffs.sas
 
@@ -68,7 +74,7 @@
 );
 /* set up local macro variables and temporary tables (with a prefix) */
 %local err msg prefix dslist i var fmtlist ibufsize;
-%let dslist=base_fmts template inlibds ds1 stagedata storediffs;
+%let dslist=base_fmts template inlibds ds1 stagedata storediffs del1 del2;
 %if &outds_add=0 %then %let dslist=&dslist outds_add;
 %if &outds_del=0 %then %let dslist=&dslist outds_del;
 %if &outds_mod=0 %then %let dslist=&dslist outds_mod;
@@ -79,17 +85,20 @@
   %let &var=%upcase(&prefix._&var);
 %end;
 
-/*
-format values can be up to 32767 wide.  SQL joins on such a wide column can
-cause buffer issues.  Update ibufsize and reset at the end.
-*/
-%let ibufsize=%sysfunc(getoption(ibufsize));
-options ibufsize=32767 ;
-
 /* in DC, format catalogs maybe specified in the libds with a -FC extension */
 %let libcat=%scan(&libcat,1,-);
 
 /* perform input validations */
+%mp_abort(
+  iftrue=(%mf_existds(&libds)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(&libds could not be found)
+)
+%mp_abort(
+  iftrue=(%mf_existvar(&libds,FMTROW)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(FMTROW not found in &libds)
+)
 %let err=0;
 %let msg=0;
 data _null_;
@@ -110,13 +119,6 @@ data _null_;
       stop;
     end;
   end;
-  else if name='LIBDS' then do;
-    if exist(value) le 0 then do;
-      call symputx('msg',"Unable to open staging table: "!!value);
-      call symputx('err',1);
-      stop;
-    end;
-  end;
   else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
   and missing(value) then do;
     call symputx('msg',"missing value in var: "!!name);
@@ -124,6 +126,14 @@ data _null_;
     stop;
   end;
 run;
+data _null_;
+  set &libds;
+  if missing(fmtrow) then do;
+    call symputx('msg',"missing fmtrow in format: "!!FMTNAME);
+    call symputx('err',1);
+    stop;
+  end;
+run;
 
 %mp_abort(
   iftrue=(&err ne 0)
@@ -131,6 +141,15 @@ run;
   ,msg=%str(&msg)
 )
 
+%local cnt;
+proc sql noprint;
+select count(distinct catx('|',type,fmtname,fmtrow)) into: cnt from &libds;
+%mp_abort(
+  iftrue=(&cnt ne %mf_nobs(&libds))
+  ,mac=&sysmacroname
+  ,msg=%str(Non-unique primary key on &libds)
+)
+
 /**
   * First, extract only relevant formats from the catalog
   */
@@ -148,16 +167,24 @@ select distinct
 
 %mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
 
+/* get a hash of the row */
+%local cvars nvars;
+%let cvars=TYPE FMTNAME START END LABEL PREFIX FILL SEXCL EEXCL HLO DECSEP
+  DIG3SEP DATATYPE LANGUAGE;
+%let nvars=FMTROW MIN MAX DEFAULT LENGTH FUZZ MULT NOEDIT;
+data &base_fmts/note2err;
+  set &base_fmts;
+  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
+run;
 
 /**
   * Ensure input table and base_formats have consistent lengths and types
   */
-%mddl_sas_cntlout(libds=&template)
-data &inlibds;
-  length &delete_col $3;
-  if 0 then set &template;
-  length start end $10000;
+data &inlibds/nonote2err;
+  length &delete_col $3 FMTROW 8 start end label $32767;
+  if 0 then set &base_fmts;
   set &libds;
+  by type fmtname notsorted;
   if &delete_col='' then &delete_col='No';
   fmtname=upcase(fmtname);
   type=upcase(type);
@@ -175,6 +202,8 @@ data &inlibds;
     %mp_aligndecimal(start,width=16)
     %mp_aligndecimal(end,width=16)
   end;
+
+  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
 run;
 
 /**
@@ -185,25 +214,10 @@ create table &outds_add(drop=&delete_col) as
   select a.*
   from &inlibds a
   left join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
-    and a.type=b.type
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where b.fmtname is null
     and upcase(a.&delete_col) ne "YES"
-  order by type, fmtname, start;
-
-/**
-  * Identify deleted records
-  */
-create table &outds_del(drop=&delete_col) as
-  select a.*
-  from &inlibds a
-  inner join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
-    and a.type=b.type
-  where upcase(a.&delete_col)="YES"
-  order by type, fmtname, start;
+  order by type, fmtname, fmtrow;
 
 /**
   * Identify modified records
@@ -212,13 +226,40 @@ create table &outds_mod (drop=&delete_col) as
   select a.*
   from &inlibds a
   inner join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
-    and a.type=b.type
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where upcase(a.&delete_col) ne "YES"
-  order by type, fmtname, start;
+    and a.fmthash ne b.fmthash
+  order by type, fmtname, fmtrow;
+
+/**
+  * Identify deleted records
+  */
+create table &outds_del(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
+  where upcase(a.&delete_col)="YES"
+  order by type, fmtname, fmtrow;
+
+/**
+  * Identify fully deleted formats (where every record is removed)
+  * These require to be explicitly deleted in proc format
+  * del1 - identify _partial_ deletes
+  * del2 - exclude these, and also formats that come with _additions_
+  */
+create table &del1 as
+  select a.*
+  from &base_fmts a
+  left join &outds_del b
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
+  where b.fmtrow is null;
+
+create table &del2 as
+  select * from &outds_del
+  where cats(type,fmtname) not in (select cats(type,fmtname) from &outds_add)
+    and cats(type,fmtname) not in (select cats(type,fmtname) from &del1);
 
-options ibufsize=&ibufsize;
 
 %mp_abort(
   iftrue=(&syscc ne 0)
@@ -227,19 +268,21 @@ options ibufsize=&ibufsize;
 )
 
 %if &loadtarget=YES %then %do;
+  /* new records plus base records that are not deleted or modified */
   data &ds1;
     merge &base_fmts(in=base)
       &outds_mod(in=mod)
       &outds_add(in=add)
       &outds_del(in=del);
     if not del and not mod;
-    by type fmtname start;
+    by type fmtname fmtrow;
   run;
+  /* add back the modified records */
   data &stagedata;
     set &ds1 &outds_mod;
   run;
   proc sort;
-    by type fmtname start;
+    by type fmtname fmtrow;
   run;
 %end;
 /* mp abort needs to run outside of conditional blocks */
@@ -249,7 +292,7 @@ options ibufsize=&ibufsize;
   ,msg=%str(SYSCC=&syscc prior to actual load)
 )
 %if &loadtarget=YES %then %do;
-  %if %mf_nobs(&stagedata)=0 %then %do;
+  %if %mf_nobs(&stagedata)=0 and %mf_nobs(&del2)=0 %then %do;
     %put There are no changes to load in &libcat!;
     %return;
   %end;
@@ -265,6 +308,22 @@ options ibufsize=&ibufsize;
   /* do the actual load */
   proc format lib=&libcat cntlin=&stagedata;
   run;
+  /* apply any full deletes */
+  %if %mf_nobs(&del2)>0 %then %do;
+    %local delfmtlist;
+    proc sql noprint;
+    select distinct case when type='N' then cats(fmtname,'.FORMAT')
+        when type='C' then cats(fmtname,'.FORMATC')
+        when type='J' then cats(fmtname,'.INFMTC')
+        when type='I' then cats(fmtname,'.INFMT')
+        else cats(fmtname,'.BADENTRY!!!') end
+      into: delfmtlist
+      separated by ' '
+      from &del2;
+    proc catalog catalog=&libcat;
+      delete &delfmtlist;
+    quit;
+  %end;
   %if &locklibds ne 0 %then %do;
     /* unlock the table */
     %mp_lockanytable(UNLOCK
@@ -287,7 +346,7 @@ options ibufsize=&ibufsize;
 
     %mp_storediffs(&libcat-FC
       ,&base_fmts
-      ,TYPE FMTNAME START
+      ,TYPE FMTNAME FMTROW
       ,delds=&outds_del
       ,modds=&outds_mod
       ,appds=&outds_add

base/mp_lockanytable.sas

@@ -194,35 +194,39 @@ run;
   %end;
 %end;
 %else %if &ACTION=UNLOCK %then %do;
-  %local status;
+  %local status cnt;
+  %let cnt=0;
   proc sql noprint;
-  select LOCK_STATUS_CD into: status from &ctl_ds
-    where LOCK_LIB ="&lib" and LOCK_DS="&ds";
-  quit;
-  %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
-  %if &sqlobs=0 %then %do;
-    %put %str(WAR)NING: &lib..&ds has never been locked!;
-  %end;
-  %else %if &status=LOCKED %then %do;
-    data _null_;
-      putlog "&sysmacroname: unlocking &lib..&ds:";
-    run;
-    proc sql;
-    update &ctl_ds
-      set LOCK_STATUS_CD='UNLOCKED'
-        , LOCK_END_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
-        , LOCK_USER_NM="&user"
-        , LOCK_PID="&sysjobid"
-        , LOCK_REF="&ref"
-      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
-    quit;
-  %end;
-  %else %if &status=UNLOCKED %then %do;
-    %put %str(WAR)NING: &lib..&ds is already unlocked!;
+  select count(*) into: cnt from &ctl_ds where LOCK_LIB ="&lib" & LOCK_DS="&ds";
+  %if &cnt=0 %then %do;
+    %put %str(WAR)NING: &lib..&ds was not previously locked in &ctl_ds!;
   %end;
   %else %do;
-    %put NOTE: Unrecognised STATUS_CD (&status) in &ctl_ds;
-    %let abortme=1;
+    select LOCK_STATUS_CD into: status from &ctl_ds
+      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+    quit;
+    %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
+    %if &status=LOCKED %then %do;
+      data _null_;
+        putlog "&sysmacroname: unlocking &lib..&ds:";
+      run;
+      proc sql;
+      update &ctl_ds
+        set LOCK_STATUS_CD='UNLOCKED'
+          , LOCK_END_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
+          , LOCK_USER_NM="&user"
+          , LOCK_PID="&sysjobid"
+          , LOCK_REF="&ref"
+        where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+      quit;
+    %end;
+    %else %if &status=UNLOCKED %then %do;
+      %put %str(WAR)NING: &lib..&ds is already unlocked!;
+    %end;
+    %else %do;
+      %put NOTE: Unrecognised STATUS_CD (&status) in &ctl_ds;
+      %let abortme=1;
+    %end;
   %end;
 %end;
 %else %do;

base/mp_md5.sas

@@ -2,8 +2,7 @@
   @file
   @brief Generates an md5 expression for hashing a set of variables
   @details This is the same algorithm used to hash records in
-  [Data Controller for SAS](https://datacontroller.io) (free for up
-  to 5 users).
+  [Data Controller for SAS](https://datacontroller.io).
 
   It is not designed to be efficient - it is designed to be effective,
   given the range of edge cases (large floating points, special missing
@@ -29,8 +28,8 @@
   @li Global option:  `options dsoptions=nonote2err;`
   @li Data step option: `data YOURLIB.YOURDATASET /nonote2err;`
 
-  @param cvars= Space seperated list of character variables
-  @param nvars= Space seperated list of numeric variables
+  @param [in] cvars= () Space seperated list of character variables
+  @param [in] nvars= () Space seperated list of numeric variables
 
   <h4> Related Programs </h4>
   @li mp_init.sas

base/mp_perflog.sas

@@ -11,9 +11,9 @@
       %mp_perflog(finished)
 
 
-  @param label Provide label to go into the control dataset
-  @param libds= Provide a dataset in which to store performance stats.  Default
-              name is <code>work.mp_perflog</code>;
+  @param [in] label Provide label to go into the control dataset
+  @param [in] libds= (work.mp_perflog) Provide a dataset in which to store
+    performance stats.  Default name is <code>work.mp_perflog</code>;
 
   @version 9.2
   @author Allan Bowe

base/mp_prevobs.sas

@@ -34,12 +34,12 @@
   Credit is made to `data _null_` for authoring this very helpful paper:
   https://www.lexjansen.com/pharmasug/2008/cc/CC08.pdf
 
-  @param action Either FETCH a current or previous record, or INITialise.
-  @param record The relative (to current) position of the previous observation
+  @param [in] action Either FETCH a current or previous record, or INITialise.
+  @param [in] record The relative (to current) position of the previous row
     to return.
-  @param history= The number of records to retain in the hash table. Default=5
-  @param prefix= the prefix to give to the variables used to store the hash name
-    and index. Default=mp_prevobs
+  @param [in] history= (5) The number of records to retain in the hash table.
+  @param [in] prefix= (mp_prevobs) The prefix to give to the variables used to
+    store the hash name and index.
 
   @version 9.2
   @author Allan Bowe

base/mp_recursivejoin.sas

@@ -23,12 +23,13 @@
         ,childvar=c
         )
 
-  @param base_ds= base table containing hierarchy (not modified)
-  @param outds= the output dataset to create with the generated hierarchy
-  @param matchval= the ultimate parent from which to filter
-  @param parentvar= name of the parent variable
-  @param childvar= name of the child variable (should be same type as parent)
-  @param mdebug= set to 1 to prevent temp tables being dropped
+  @param [in] base_ds= base table containing hierarchy (not modified)
+  @param [out] outds= the output dataset to create with the generated hierarchy
+  @param [in] matchval= the ultimate parent from which to filter
+  @param [in] parentvar= name of the parent variable
+  @param [in] childvar= () name of the child variable (should be same type as
+    parent)
+  @param [in] mdebug= set to 1 to prevent temp tables being dropped
 
 
   @returns outds contains the following variables:

base/mp_replace.sas

@@ -32,10 +32,12 @@
   Note - if you are running a version of SAS that will allow the io package in
   LUA, you can also use this macro: mp_gsubfile.sas
 
-  @param infile The QUOTED path to the file on which to perform the substitution
-  @param findvar= Macro variable NAME containing the string to search for
-  @param replacevar= Macro variable NAME containing the replacement string
-  @param outfile= (0) Optional QUOTED path to the adjusted output file (to
+  @param [in] infile The QUOTED path to the file on which to perform the
+    substitution.  Note that you can extract the pathname from a fileref using
+    the pathname function, eg: `"%sysfunc(pathname(fref))"`;
+  @param [in] findvar= Macro variable NAME containing the string to search for
+  @param [in] replacevar= Macro variable NAME containing the replacement string
+  @param [out] outfile= (0) Optional QUOTED path to the adjusted output file (to
     avoid overwriting the first file).
 
   <h4> SAS Macros </h4>

base/mp_runddl.sas

@@ -31,9 +31,9 @@
     %mp_runddl(/some/rootlib, exc=LIBREF3) * same as above ;
 
 
-  @param path location of the DDL folder structure
-  @param inc= list of librefs to include
-  @param exc= list of librefs to exclude (takes precedence over inc=)
+  @param [in] path location of the DDL folder structure
+  @param [in] inc= list of librefs to include
+  @param [in] exc= list of librefs to exclude (takes precedence over inc=)
 
   @version 9.3
   @author Allan Bowe

base/mp_searchcols.sas

@@ -9,12 +9,13 @@
 
       %mp_searchcols(libs=sashelp work, cols=name sex age)
 
-  @param libs=(SASHELP) Space separated list of libraries to search for columns
-  @param cols= Space separated list of column names to search for (not case
-    sensitive)
-  @param outds=(mp_searchcols) the table to create with the results.  Will have
-    one line per table match.
-  @param match=(ANY) The match type. Valid values:
+  @param [in] libs= (SASHELP)
+    Space separated list of libraries to search for columns
+  @param [in] cols=
+    Space separated list of column names to search for (not case sensitive)
+  @param [out] outds= (mp_searchcols)
+    The table to create with the results.  Will have one line per table match.
+  @param [in] match= (ANY) The match type. Valid values:
     @li ANY - The table contains at least one of the columns
     @li WILD - The table contains a column with a name that partially matches
 

base/mp_stackdiffs.sas

@@ -197,6 +197,7 @@
   @li mp_coretable.sas
   @li mp_stackdiffs.test.sas
   @li mp_storediffs.sas
+  @li mp_stripdiffs.sas
 
   @todo The current approach assumes that a variable called KEY_HASH is not on
     the base table.  This part will need to be refactored (eg using

base/mp_storediffs.sas

@@ -6,8 +6,7 @@
     data recovery, and change re-application.  This macro is one of many
     data management utilities used in [Data Controller for SAS](
     https:datacontroller.io) - a comprehensive data ingestion solution, which
-    works on any SAS platform (Viya, SAS 9, Foundation) and is free for up to 5
-    users.
+    works on any SAS platform (Viya, SAS 9, Foundation).
 
     NOTE - this macro does not validate the inputs. It is assumed that the
     datasets containing the new / changed / deleted rows are CORRECT, contain
@@ -65,6 +64,7 @@
   <h4> Related Macros </h4>
   @li mp_stackdiffs.sas
   @li mp_storediffs.test.sas
+  @li mp_stripdiffs.sas
 
   @version 9.2
   @author Allan Bowe
@@ -147,9 +147,9 @@ run;
 
 %if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do;
   /* this is a format catalog - cannot query cols directly */
-  %let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH"
-    ,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO"
-    ,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
+  %let vlist="TYPE","FMTNAME","FMTROW","START","END","LABEL","MIN","MAX"
+    ,"DEFAULT","LENGTH","FUZZ","PREFIX","MULT","FILL","NOEDIT","SEXCL"
+    ,"EEXCL","HLO","DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
 %end;
 %else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
 
@@ -165,7 +165,7 @@ data &ds4;
   if upcase(&inds_auto)="&ds2" then tgtvar_type='N';
   else if upcase(&inds_auto)="&ds3" then tgtvar_type='C';
   else do;
-    putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
+    putlog 'ERR' +(-1) "OR: unidentified vartype input!" &inds_auto;
     call symputx('syscc',98);
   end;
 
@@ -174,7 +174,7 @@ data &ds4;
   else if &inds_keep="&modds" then move_type='M';
   else if &inds_keep="&origds" then move_type='O';
   else do;
-    putlog "%str(ERR)OR: unidentified movetype input!" &inds_keep;
+    putlog 'ERR' +(-1) "OR: unidentified movetype input!" &inds_keep;
     call symputx('syscc',99);
   end;
   tgtvar_nm=upcase(tgtvar_nm);
@@ -184,7 +184,7 @@ data &ds4;
 run;
 
 %if "&loadref"="0" %then %let loadref=%sysfunc(uuidgen());
-%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime());
+%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime(),8.6);
 %let libds=%upcase(&libds);
 
 /* join orig vals for modified & deleted */

base/mp_stprequests.sas

@@ -21,9 +21,10 @@
     %mp_stprequests(status_cd=INIT, libds=YOURLIB.DATASET )
 
 
-  @param status_cd= Use INIT for INIT and TERM for TERM events
-  @param libds= Location of base table (library.dataset).  To minimise risk
-    of table locks, we HIGHLY recommend using a database (NOT a SAS dataset).
+  @param [in] status_cd= Use INIT for INIT and TERM for TERM events
+  @param [in] libds= (somelib.stp_requests) Location of base table
+    (library.dataset).  To minimise risk of table locks, we HIGHLY recommend
+    using a database (NOT a SAS dataset).
     THE LIBRARY SHOULD BE ASSIGNED ALREADY - eg in autoexec or earlier in the
     init program proper.
 

base/mp_stripdiffs.sas

@@ -0,0 +1,255 @@
+/**
+  @file
+  @brief Generates a stage dataset to revert diffs tracked in an audit table
+  @details A big benefit of tracking data changes in an audit table is that
+    those changes can be subsequently reverted if necessary!
+
+    This macro prepares a staging dataset containing those differences - eg for:
+
+    @li deleted rows - these are re-inserted
+    @li changed rows - differences are reverted
+    @li added rows - marked with `_____DELETE__THIS__RECORD_____="YES"`
+
+    These changes are NOT applied to the base table - a staging dataset is
+    simply prepared for an ETL process to action.  In Data Controller, this
+    dataset is used directly as an input to the APPROVE process (so that the
+    reversion diffs can be reviewed prior to being applied).
+
+
+  @param [in] libds Base library.dataset (will not be modified).  The library
+    must be assigned.
+  @param [in] loadref Unique identifier for the version to be reverted.  This
+    change, plus ALL SUBSEQUENT CHANGES, will be reverted in the output table.
+  @param [in] difftable The dataset containing the diffs.  Definition available
+    in mddl_dc_difftable.sas
+  @param [in] filtervar= (0) If provided, the contents of this macro variable
+    will be applied as an additional filter against &libds
+  @param [out] outds= (work.mp_stripdiffs) Output table containing the diffs.
+    Has the same format as the base datset, plus a
+    `_____DELETE__THIS__RECORD_____` variable.
+  @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
+  @li mf_islibds.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_abort.sas
+
+  <h4> Related Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mp_stackdiffs.sas
+  @li mp_storediffs.sas
+  @li mp_stripdiffs.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mp_stripdiffs(libds
+  ,loadref
+  ,difftable
+  ,filtervar=0
+  ,outds=work.mp_stripdiffs
+  ,mdebug=0
+)/*/STORE SOURCE*/;
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+%let libds=%upcase(&libds);
+
+/* safety checks */
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc on entry.  Clean session required!)
+)
+%let libds=%upcase(&libds);
+%mp_abort(iftrue= (%mf_islibds(&libds)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid library.dataset reference - %superq(libds))
+)
+
+/* set up unique and temporary vars */
+%local ds1 ds2 ds3 ds4 ds5 fref1 filterstr;
+%let fref1=%mf_getuniquefileref();
+%if &filtervar ne 0 %then %let filterstr=%superq(&filtervar);
+%else %let filterstr=%str(1=1);
+
+/* get timestamp of the diff to be reverted */
+%local ts;
+proc sql noprint;
+select put(processed_dttm,datetime19.6) into: ts
+  from &difftable where load_ref="&loadref";
+%mp_abort(iftrue= (&sqlobs=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Load ref %superq(loadref) not found!)
+)
+
+/* extract diffs for this base table from this timestamp onwards */
+%let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_diffs));
+create table &ds1 (drop=libref dsn) as
+  select * from &difftable
+  where upcase(cats(libref))="%scan(&libds,1,.)"
+  and upcase(cats(dsn))="%scan(&libds,2,.)"
+  and processed_dttm ge "&ts"dt
+  order by processed_dttm desc, key_hash, is_pk;
+
+/* extract key values only */
+%let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_pks));
+%local keyhash processed;
+%let keyhash=%upcase(%mf_getuniquename(prefix=mpsdvar_keyhash));
+%let processed=%upcase(%mf_getuniquename(prefix=mpsdvar_processed));
+create table &ds2 as
+  select key_hash as &keyhash,
+    tgtvar_nm,
+    tgtvar_type,
+    coalescec(oldval_char,newval_char) as charval,
+    coalesce(oldval_num, newval_num) as numval,
+    processed_dttm as &processed
+  from &ds1
+  where is_pk=1
+  order by &keyhash, &processed;
+
+/* grab pk values */
+%local pk;
+select distinct upcase(tgtvar_nm) into: pk separated by ' ' from &ds2;
+
+%let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_keychar));
+proc transpose data=&ds2(where=(tgtvar_type='C'))
+    out=&ds3(drop=_name_);
+  by &keyhash &processed;
+  id TGTVAR_NM;
+  var charval;
+run;
+
+%let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_keynum));
+proc transpose data=&ds2(where=(tgtvar_type='N'))
+    out=&ds4(drop=_name_);
+  by &keyhash &processed;
+  id TGTVAR_NM;
+  var numval;
+run;
+/* shorten the lengths */
+%mp_ds2squeeze(&ds3,outds=&ds3)
+%mp_ds2squeeze(&ds4,outds=&ds4)
+
+/* now merge to get all key values and de-dup */
+%let ds5=%upcase(work.%mf_getuniquename(prefix=mpsd_merged));
+data &ds5;
+  length &keyhash $32 &processed 8;
+  merge &ds3 &ds4;
+  by &keyhash &processed;
+  if not missing(&keyhash);
+run;
+proc sort data=&ds5 nodupkey;
+  by &pk;
+run;
+
+/* join to base table for preliminary stage DS */
+proc sql;
+create table &outds as select "No " as _____DELETE__THIS__RECORD_____
+  %do x=1 %to %sysfunc(countw(&pk,%str( )));
+    ,a.%scan(&pk,&x,%str( ))
+  %end;
+  %local notpkcols;
+  %let notpkcols=%upcase(%mf_getvarlist(&libds));
+  %let notpkcols=%mf_wordsinstr1butnotstr2(str1=&notpkcols,str2=&pk);
+  %do x=1 %to %sysfunc(countw(&notpkcols,%str( )));
+    ,b.%scan(&notpkcols,&x,%str( ))
+  %end;
+  from &ds5 a
+  left join &libds (where=(&filterstr)) b
+  on 1=1
+%do x=1 %to %sysfunc(countw(&pk,%str( )));
+  and a.%scan(&pk,&x,%str( ))=b.%scan(&pk,&x,%str( ))
+%end;
+;
+
+/* create SAS code to apply to stage_ds */
+data _null_;
+  set &ds1;
+  file &fref1 lrecl=33000;
+  length charval $32767;
+  if _n_=1 then put 'proc sql noprint;';
+  by descending processed_dttm key_hash is_pk;
+  if move_type='M' then do;
+    if first.key_hash then do;
+      put "update &outds set " @@;
+    end;
+    if IS_PK=0 then do;
+      put "  " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+      if not last.is_pk then put ',';
+    end;
+    else do;
+      if first.is_pk then put "  where 1=1 " @@;
+      put "  and " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+  end;
+  else if move_type='A' then do;
+    if first.key_hash then do;
+      put "update &outds set _____DELETE__THIS__RECORD_____='Yes' where 1=1 "@@;
+    end;
+    /* gating if - as only need PK now */
+    if is_pk=1;
+    put '  AND ' tgtvar_nm '=' @@;
+    cnt=count(newval_char,'"');
+    charval=quote(trim(substr(newval_char,1,32765-cnt)));
+    if tgtvar_type='C' then put charval @@;
+    else put newval_num @@;
+  end;
+  else if move_type='D' then do;
+    if first.key_hash then do;
+      put "update &outds set _____DELETE__THIS__RECORD_____='No' " @@;
+    end;
+    if IS_PK=0 then do;
+      put "  ," tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+    else do;
+      if first.is_pk then put "  where 1=1 " @@;
+      put "  and " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+  end;
+  if last.key_hash then put ';';
+run;
+
+/* apply the modification statements */
+%inc &fref1/source2 lrecl=33000;
+
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &ds1, &ds2, &ds3, &ds4, &ds5;
+  file &fref1 clear;
+%end;
+%else %do;
+  data _null_;
+    infile &fref1;
+    input;
+    if _n_=1 then putlog "Contents of SQL adjustments";
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_stripdiffs;
+/** @endcond */

base/mp_testjob.sas

@@ -8,8 +8,8 @@
         duration=60*5
       )
 
-  @param [in] duration= the time in seconds which the job should run for. Actual
-  time may vary, as the check is done in between steps.  Default = 30 (seconds).
+  @param [in] duration= (30) The time in seconds which the job should run for.
+    Actual time may vary, as the check is done in between steps.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquelibref.sas

base/mp_testwritespeedlibrary.sas

@@ -11,9 +11,9 @@
         ,outds=work.results
       )
 
-  @param lib= (WORK) The library in which to create the table
-  @param size= (0.1) The size in GB of the table to create
-  @param outds= (WORK.RESULTS) The output dataset to be created.
+  @param [in] lib= (WORK) The library in which to create the table
+  @param [in] size= (0.1) The size in GB of the table to create
+  @param [out] outds= (WORK.RESULTS) The output dataset to be created.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas

base/mp_tree.sas

@@ -15,8 +15,8 @@ https://communities.sas.com/t5/SAS-Programming/listing-all-files-within-a-direct
 https://communities.sas.com/t5/SAS-Programming/listing-all-files-of-all-types-from-all-subdirectories/m-p/334113/highlight/true#M75419
 
 
-  @param dir= Directory to be scanned (default=/tmp)
-  @param outds= Dataset to create (default=work.mp_tree)
+  @param [in] dir= (/tmp) Directory to be scanned
+  @param [out] outds= (work.mp_tree) Dataset to create
 
   @returns outds contains the following variables:
 

base/mp_unzip.sas

@@ -13,11 +13,12 @@
 
       %mp_unzip(ziploc="/some/file.zip",outdir=/some/folder)
 
-  More info:  https://blogs.sas.com/content/sasdummy/2015/05/11/using-filename-zip-to-unzip-and-read-data-files-in-sas/
+  More info:
+  https://blogs.sas.com/content/sasdummy/2015/05/11/using-filename-zip-to-unzip-and-read-data-files-in-sas/
 
-  @param ziploc= Fileref or quoted full path to zip file ("/path/to/file.zip")
-  @param outdir= (%sysfunc(pathname(work))) Directory in which to write the
-    outputs (created if non existant)
+  @param [in] ziploc= Fileref or quoted full path, eg: "/path/to/file.zip"
+  @param [out] outdir= (%sysfunc(pathname(work))) Directory in which to write
+    the outputs (created if needed)
 
   <h4> SAS Macros </h4>
   @li mf_mkdir.sas