fix: handling consecutive add+delete in mp_stripdiffs

feat: mp_stripdiffs macro - closes #373

.all-contributorsrc

@@ -144,8 +144,27 @@
       "contributions": [
         "doc"
       ]
+    },
+    {
+      "login": "rudvfaden",
+      "name": "Rud Faden",
+      "avatar_url": "https://avatars.githubusercontent.com/u/2445577?v=4",
+      "profile": "http://rudvfaden.github.io/",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "andyjessen",
+      "name": "andyjessen",
+      "avatar_url": "https://avatars.githubusercontent.com/u/62343929?v=4",
+      "profile": "https://github.com/andyjessen",
+      "contributions": [
+        "doc"
+      ]
     }
   ],
   "contributorsPerLine": 7,
-  "skipCi": true
+  "skipCi": true,
+  "commitType": "docs"
 }

.git-hooks/pre-commit

@@ -2,7 +2,7 @@
 
 # Ensure lint is passing
 LINT=`sasjs lint`
-if [[ "$LINT" != *"✔ All matched files use @sasjs/lint code style!" ]]; then
+if [[ "$LINT" != *"All matched files use @sasjs/lint code style!" ]]; then
   echo "$LINT"
   echo "To commit in spite of these warnings, use the -n parameter."
   exit 1

.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

.github/CONTRIBUTING.md

@@ -27,5 +27,6 @@ To contribute:
 1.  Create your feature branch (`git checkout -b myfeature`)
 2.  Make your change
 3.  Update the `all.sas` file (`python3 build.py`)
-4.  Push and make a PR
+4.  Commit using a [Conventional Commit](https://www.conventionalcommits.org)
+5.  Push and make a PR
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,4 +15,4 @@ What code changes have been made to achieve the intent.
 - [ ] Code is formatted correctly (`sasjs lint`).
 - [ ] Any new functionality has been unit tested.
 - [ ] All unit tests are passing (`sasjs test`).
-- [ ] `all.sas` has been regenerated (`python3 build.py`)
+- [ ] The PR desc or underlying commits follow the [Conventional Commit](https://www.conventionalcommits.org) standard

.github/workflows/main.yml

@@ -13,10 +13,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Install dependencies
-        run: npm ci
+        run: |
+          npm ci
 
       - name: Check code style (aborts if errors found)
         run: npx @sasjs/cli lint
@@ -36,14 +37,15 @@ jobs:
 
       - name: Install Open VPN
         run: |
-          sudo apt install apt-transport-https
-          sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
-          sudo apt-key add openvpn-repo-pkg-key.pub
-          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
-          sudo apt update
-          sudo apt install openvpn3=17~betaUb22042+jammy
+          #sudo apt install apt-transport-https
+          #sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
+          #sudo apt-key add openvpn-repo-pkg-key.pub
+          #sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
+          #sudo apt update
+          #sudo apt install openvpn3=17~betaUb22042+jammy
       - name: Start Open VPN 3
-        run: openvpn3 session-start --config .github/vpn/config.ovpn
+        run: |
+          # openvpn3 session-start --config .github/vpn/config.ovpn
 
       - name: Add credentials
         run: |
@@ -52,13 +54,13 @@ jobs:
           echo "REFRESH_TOKEN=${{secrets.SAS9_4GL_IO_REFRESH_TOKEN}}" >> .env.server
 
       - name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v3
+        uses: cycjimmy/semantic-release-action@v4
         env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 
       - name: SAS Packages Release
         run: |
           npx @sasjs/cli compile job -s sasjs/utils/create_sas_package.sas -o sasjsbuild -t server
-          # this part depends on https://github.com/sasjs/server/issues/307
-          npx @sasjs/cli run sasjsbuild/jobs/utils/create_sas_package.sas -t server
+          # need long duration token per https://github.com/sasjs/server/issues/307
+          # npx @sasjs/cli run sasjsbuild/jobs/utils/create_sas_package.sas -t server

.github/workflows/notmain.yml

@@ -0,0 +1,32 @@
+# This workflow will run tests using node and then publish a package to GitHub Packages when a release is created
+# For more information see: https://help.github.com/actions/language-and-framework-guides/publishing-nodejs-packages
+
+name: SASjs Core - Update all.sas
+
+on:
+  push:
+    branches-ignore:
+      - main
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          npm ci
+          npm i -g @sasjs/cli@latest
+
+      - name: Ensure all.sas is always up to date
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          python3 build.py
+          git add all.sas
+          git commit -m "chore: updating all.sas" --allow-empty
+          git push
+
+

.gitignore

@@ -13,3 +13,4 @@ mc_*
 *.env*
 
 ~
+

README.md

@@ -212,7 +212,8 @@ When contributing to this library, it is therefore important to ensure that all
 
 ## General Notes
 
-- All macros should be compatible with SAS versions from support level B and above (so currently 9.2 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully (eg `%if %sysevalf(&sysver<9.3) %then %return`).
+- All macros should be compatible with SAS versions from support level B and above (so currently 9.3 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully.
+- It's [best to avoid](https://git.datacontroller.io/dc/dc/issues/50) special / non-ASCII characters for compatibility with the widest variety of SAS installations.
 
 ## Breaking Changes
 
@@ -235,18 +236,19 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 The following repositories are also worth checking out:
 
-* [xieliaing/SAS](https://github.com/xieliaing/SAS)
-* [SASJedi/sas-macros](https://github.com/SASJedi/sas-macros)
 * [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
 * [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)
 * [KatjaGlassConsulting/SMILE-SmartSASMacros](https://github.com/KatjaGlassConsulting/SMILE-SmartSASMacros)
+* [paul-canals/toolbox](https://github.com/paul-canals/toolbox)
 * [rogerjdeangelis](https://github.com/rogerjdeangelis)
+* [SASJedi/sas-macros](https://github.com/SASJedi/sas-macros)
 * [scottbass/sas](https://github.com/scottbass/SAS)
+* [xieliaing/SAS](https://github.com/xieliaing/SAS)
 * [yabwon/sas_packages](https://github.com/yabwon/SAS_PACKAGES)
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-13-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-15-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -271,6 +273,10 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt="Bart Jablonski"/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://bandism.net/"><img src="https://avatars.githubusercontent.com/u/22633385?v=4?s=100" width="100px;" alt="Ikko Ashimine"/><br /><sub><b>Ikko Ashimine</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=eltociear" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/henrik-forsell"><img src="https://avatars.githubusercontent.com/u/109935936?v=4?s=100" width="100px;" alt="Henrik Forsell"/><br /><sub><b>Henrik Forsell</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=henrik-forsell" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://rudvfaden.github.io/"><img src="https://avatars.githubusercontent.com/u/2445577?v=4?s=100" width="100px;" alt="Rud Faden"/><br /><sub><b>Rud Faden</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rudvfaden" title="Code">💻</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/andyjessen"><img src="https://avatars.githubusercontent.com/u/62343929?v=4?s=100" width="100px;" alt="andyjessen"/><br /><sub><b>andyjessen</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=andyjessen" title="Documentation">📖</a></td>
     </tr>
   </tbody>
 </table>

base/mf_abort.sas

@@ -3,6 +3,11 @@
   @brief Abort, ungracefully
   @details Will abort with a straightforward %abort if the condition is true.
 
+  @param [in] mac= (mf_abort.sas) Name of calling macro (is printed to the log)
+  @param [in] msg= ( ) Additional string to print to the log
+  @param [in] iftrue= (%str(1=1)) Conditional logic under which to perform the
+    abort
+
   <h4> Related Macros </h4>
   @li mp_abort.sas
 

base/mf_deletefile.sas

@@ -10,7 +10,7 @@
       %mf_deletefile(&sasjswork/myfile.txt)
 
 
-  @param filepath Full path to the target file
+  @param [in] file Full path to the target file
 
   @returns The return code from the fdelete() invocation
 

base/mf_existds.sas

@@ -10,7 +10,7 @@
     expected results (depending on whether you 'expect' the result to be
     case insensitive in this context!)
 
-  @param libds library.dataset
+  @param [in] libds library.dataset
   @return output returns 1 or 0
 
   <h4> Related Macros </h4>

base/mf_existfileref.sas

@@ -4,7 +4,7 @@
   @details You can probably do without this macro as it is just a one liner.
   Mainly it is here as a convenient way to remember the syntax!
 
-  @param fref the fileref to detect
+  @param [in] fref the fileref to detect
 
   @return output Returns 1 if found and 0 if not found.  Note - it is possible
   that the fileref is found, but the file does not (yet) exist. If you need

base/mf_existfunction.sas

@@ -16,7 +16,7 @@
 https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionexists-macro
   ).
 
-  @param [in] name (positional) - function name
+  @param [in] name function name
 
   @author Allan Bowe
 **/

base/mf_existvarlist.sas

@@ -6,8 +6,8 @@
 
       %put %mf_existVarList(sashelp.class, age sex name dummyvar);
 
-  @param libds 2 part dataset or view reference
-  @param varlist space separated variable names
+  @param [in] libds 2 part dataset or view reference
+  @param [in] varlist space separated variable names
 
   @version 9.2
   @author Allan Bowe

base/mf_getapploc.sas

@@ -31,6 +31,7 @@
       %put %mf_getapploc(/some/location/jobs/extract/somejob/);
       %put %mf_getapploc(/some/location/tests/jobs/somejob/);
 
+  @param [in] pgm The _program value from which to extract the appLoc
 
   @author Allan Bowe
 **/

base/mf_getattrc.sas

@@ -6,8 +6,8 @@
       %put Dataset label = %mf_getattrc(sashelp.class,LABEL);
       %put Member Type = %mf_getattrc(sashelp.class,MTYPE);
 
-  @param libds library.dataset
-  @param attr full list in [documentation](
+  @param [in] libds library.dataset
+  @param [in] attr full list in [documentation](
     https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000147794.htm)
   @return output returns result of the attrc value supplied, or -1 and log
     message if err.

base/mf_getattrn.sas

@@ -6,8 +6,8 @@
       %put Number of observations=%mf_getattrn(sashelp.class,NLOBS);
       %put Number of variables = %mf_getattrn(sashelp.class,NVARS);
 
-  @param libds library.dataset
-  @param attr Common values are NLOBS and NVARS, full list in [documentation](
+  @param [in] libds library.dataset
+  @param [in] attr Common values are NLOBS and NVARS, full list in [documentation](
   http://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000212040.htm)
   @return output returns result of the attrn value supplied, or -1 and log
     message if err.

base/mf_getkeyvalue.sas

@@ -7,8 +7,9 @@
       %put %mf_getkeyvalue(someindex)
 
 
-  @param key Provide a key on which to perform the lookup
-  @param libds= define the target table which holds the parameters
+  @param [in] key Provide a key on which to perform the lookup
+  @param [in] libds= (work.mp_setkeyvalue) The library.dataset which holds the
+    parameters
 
   @version 9.2
   @author Allan Bowe

base/mf_getplatform.sas

@@ -12,7 +12,7 @@
   @li SASJS
   @li BASESAS
 
-  @param switch the param for which to return a platform specific variable
+  @param [in] switch the param for which to return a platform specific variable
 
   <h4> SAS Macros </h4>
   @li mf_mval.sas

base/mf_getschema.sas

@@ -8,7 +8,7 @@
   returns:
   > dbo
 
-  @param libref Library reference (also accepts a 2 level libds ref).
+  @param [in] libref Library reference (also accepts a 2 level libds ref).
 
   @return output returns the library schema for the FIRST library encountered
 

base/mf_getuniquelibref.sas

@@ -19,7 +19,7 @@
   @param [in] prefix= (mclib) first part of the returned libref. As librefs can
     be as long as 8 characters, a maximum length of 7 characters is premitted
     for this prefix.
-  @param [in] maxtries= Deprecated parameter. Remains here to ensure a
+  @param [in] maxtries= (1000) Deprecated parameter. Remains here to ensure a
     non-breaking change.  Will be removed in v5.
 
   @version 9.2

base/mf_getuniquename.sas

@@ -10,7 +10,7 @@
 
 > MCc59c750610321d4c8bf75faadbcd22
 
-  @param prefix= set a prefix for the new name
+  @param [in] prefix= (MC) Sets a prefix for the new name
 
   @version 9.3
   @author Allan Bowe

base/mf_getuser.sas

@@ -13,8 +13,6 @@
         %let user= %mf_getUser();
         %put &user;
 
-  @param type - do not use, may be deprecated in a future release
-
   @return SYSUSERID (if workspace server)
   @return _METAPERSON (if stored process server)
   @return SYS_COMPUTE_SESSION_OWNER (if Viya compute session)

base/mf_getvalue.sas

@@ -13,9 +13,9 @@
   <h4> Related Macros </h4>
   @li mp_setkeyvalue.sas
 
-  @param libds dataset to query
-  @param variable the variable which contains the value to return.
-  @param filter contents of where clause
+  @param [in] libds dataset to query
+  @param [in] variable the variable which contains the value to return.
+  @param [in] filter= (1) contents of where clause
 
   @version 9.2
   @author Allan Bowe

base/mf_getvarformat.sas

@@ -25,7 +25,8 @@
 
   @param [in] libds Two part dataset (or view) reference.
   @param [in] var Variable name for which a format should be returned
-  @param [in] force=(0) Set to 1 to supply a default if the variable has no format
+  @param [in] force= (0) Set to 1 to supply a default if the variable has no
+    format
   @returns outputs format
 
   @author Allan Bowe

base/mf_getvarlen.sas

@@ -18,8 +18,8 @@
       8
       NOTE: Variable renegade does not exist in test
 
-  @param libds Two part dataset (or view) reference.
-  @param var Variable name for which a length should be returned
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] var Variable name for which a length should be returned
   @returns outputs length
 
   @author Allan Bowe

base/mf_getvarnum.sas

@@ -21,8 +21,8 @@ returns:
 
   > NOTE: Variable renegade does not exist in test
 
-  @param libds Two part dataset (or view) reference.
-  @param var Variable name for which a position should be returned
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] var Variable name for which a position should be returned
 
   @author Allan Bowe
   @version 9.2

base/mf_getvartype.sas

@@ -13,8 +13,8 @@ Usage:
 
 
 
-  @param libds Two part dataset (or view) reference.
-  @param var the variable name to be checked
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] var the variable name to be checked
   @return output returns C or N depending on variable type.  If variable
     does not exist then a blank is returned and a note is written to the log.
 

base/mf_getxengine.sas

@@ -10,7 +10,7 @@
   returns:
   > TEMP
 
-  @param fref The fileref to check
+  @param [in] fref The fileref to check
 
   @returns The XENGINE value in sashelp.vextfl or 0 if not found.
 

base/mf_increment.sas

@@ -11,8 +11,8 @@
       %put Now we have run %mf_increment(cnt) lines;
       %put There are %mf_increment(cnt) lines in total;
 
-  @param [in] MACRO_NAME the name of the macro variable to increment
-  @param [in] ITER= The amount to add or subtract to the macro
+  @param [in] macro_name The name of the macro variable to increment
+  @param [in] incr= (1) The amount to add or subtract to the macro
 
   <h4> Related Files </h4>
   @li mf_increment.test.sas

base/mf_isblank.sas

@@ -7,12 +7,12 @@
 
   Usage:
 
-      %put mf_isblank(&var);
+      %put %mf_isblank(&var);
 
   inspiration:
   https://support.sas.com/resources/papers/proceedings09/022-2009.pdf
 
-  @param param VALUE to be checked
+  @param [in] Param VALUE to be checked
 
   @return output returns 1 (if blank) else 0
 

base/mf_isdir.sas

@@ -9,7 +9,7 @@
   With thanks and full credit to Andrea Defronzo -
   https://www.linkedin.com/in/andrea-defronzo-b1a47460/
 
-  @param path full path of the file/directory to be checked
+  @param [in] path Full path of the file/directory to be checked
 
   @return output returns 1 if path is a directory, 0 if it is not
 

base/mf_loc.sas

@@ -6,6 +6,10 @@
 
       %put %mf_loc(POF); %*location of PlatformObjectFramework tools;
 
+  @param [in] loc The item to locate, eg:
+    @li PLAATFORMOBJECTFRAMEWORK (or POF)
+    @li VIYACONFG
+
   @version 9.2
   @author Allan Bowe
 **/
@@ -15,7 +19,8 @@
 %local root;
 
 %if &loc=POF or &loc=PLATFORMOBJECTFRAMEWORK %then %do;
-  %let root=%substr(%sysget(SASROOT),1,%index(%sysget(SASROOT),SASFoundation)-2);
+  %let root=%sysget(SASROOT);
+  %let root=%substr(&root,1,%index(&root,SASFoundation)-2);
   %let root=&root/SASPlatformObjectFramework/&sysver;
   %put Batch tools located at: &root;
   &root

base/mf_mkdir.sas

@@ -7,7 +7,7 @@ Usage:
     %mf_mkdir(/some/path/name)
 
 
-  @param dir relative or absolute pathname.  Unquoted.
+  @param [in] dir Relative or absolute pathname.  Unquoted.
   @version 9.2
 
 **/

base/mf_mval.sas

@@ -8,6 +8,8 @@
 
       %if %mf_mval(maynotexist)=itdid %then %do;
 
+  @param [in] var The macro variable NAME to return the (possible) value for
+
   @version 9.2
   @author Allan Bowe
 **/

base/mf_nobs.sas

@@ -9,7 +9,7 @@
   <h4> SAS Macros </h4>
   @li mf_getattrn.sas
 
-  @param libds library.dataset
+  @param [in] libds library.dataset
 
   @return output returns result of the attrn value supplied, or log message
     if err.

base/mf_trimstr.sas

@@ -11,8 +11,8 @@
   <h4> SAS Macros </h4>
 
 
-  @param basestr The string to be modified
-  @param trimstr The string to be removed from the end of `basestr`, if it
+  @param [in] basestr The string to be modified
+  @param [in] trimstr The string to be removed from the end of `basestr`, if it
     exists
 
   @return output returns result with the value of `trimstr` removed from the end

base/mf_wordsinstr1andstr2.sas

@@ -13,8 +13,8 @@
   returns:
   > blah blaaah brah
 
-  @param str1= string containing words to extract
-  @param str2= used to compare with the extract string
+  @param [in] str1= () string containing words to extract
+  @param [in] str2= () used to compare with the extract string
 
   @warning CASE SENSITIVE!
 

base/mf_wordsinstr1butnotstr2.sas

@@ -16,8 +16,8 @@
   returns:
   > sss bram boo
 
-  @param [in] str1= string containing words to extract
-  @param [in] str2= used to compare with the extract string
+  @param [in] str1= () String containing words to extract
+  @param [in] str2= () Used to compare with the extract string
 
   @version 9.2
   @author Allan Bowe

base/mf_writefile.sas

@@ -21,8 +21,8 @@
   @param [in] mode= (O) Available options are A or O as follows:
     @li A APPEND mode, writes new records after the current end of the file.
     @li O OUTPUT mode, writes new records from the beginning of the file.
-  @param [in] l1= First line
-  @param [in] l2= Second line (etc through to l10)
+  @param [in] l1= () First line
+  @param [in] l2= () Second line (etc through to l10)
 
   <h4> Related Macros </h4>
   @li mf_writefile.test.sas

base/mp_abort.sas

@@ -27,14 +27,15 @@
     currently investigating approaches to deal with this.
 
 
-  @param mac= (mp_abort.sas) To contain the name of the calling macro. Do not
-    use &sysmacroname as this will always resolve to MP_ABORT.
-  @param msg= message to be returned
-  @param iftrue= (1=1) Supply a condition for which the macro should be executed
-  @param errds= (work.mp_abort_errds) There is no clean way to end a process
-    within a %include called within a macro.  Furthermore, there is no way to
-    test if a macro is called within a %include.  To handle this particular
-    scenario, the %include should be switched for the mp_include.sas macro.
+  @param [in] mac= (mp_abort.sas) To contain the name of the calling macro. Do
+    not use &sysmacroname as this will always resolve to MP_ABORT.
+  @param [out] msg= message to be returned
+  @param [in] iftrue= (1=1) Condition under which the macro should be executed
+  @param [in] errds= (work.mp_abort_errds) There is no clean way to end a
+    process within a %include called within a macro.  Furthermore, there is no
+    way to test if a macro is called within a %include.  To handle this
+    particular scenario, the %include should be switched for the mp_include.sas
+    macro.
     This provides an indicator that we are running a macro within a \%include
     (`_SYSINCLUDEFILEDEVICE`) and allows us to provide a dataset with the abort
     values (msg, mac).
@@ -45,8 +46,8 @@
     @li msg (the message)
     @li mac (the mac param)
 
-  @param mode= (REGULAR) If mode=INCLUDE then the &errds dataset is checked for
-    an abort status.
+  @param [in] mode= (REGULAR) If mode=INCLUDE then the &errds dataset is checked
+    for an abort status.
     Valid values:
     @li REGULAR (default)
     @li INCLUDE

base/mp_aligndecimal.sas

@@ -62,8 +62,8 @@
           7998580.8415
 
 
-  @param var The (data step, character) variable to modify
-  @param width= (8) The number of characters BEFORE the decimal point
+  @param [in] var The (data step, character) variable to modify
+  @param [in] width= (8) The number of characters BEFORE the decimal point
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas

base/mp_appendfile.sas

@@ -17,8 +17,8 @@
         %mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3)
 
 
-  @param [in] baseref= Fileref of the base file (should exist)
-  @param [in] appendrefs= One or more filerefs to be appended to the base
+  @param [in] baseref= (0) Fileref of the base file (should exist)
+  @param [in] appendrefs= (0) One or more filerefs to be appended to the base
     fileref.  Space separated.
 
   @version 9.2

base/mp_assertcols.sas

@@ -31,8 +31,8 @@
 
 
   @param [in] inds The input library.dataset to test for values
-  @param [in] cols= The list of columns to check for
-  @param [in] desc= (Testing observations) The user provided test description
+  @param [in] cols= (0) The list of columns to check for
+  @param [in] desc= (0) The user provided test description
   @param [in] test= (ALL) The test to apply.  Valid values are:
     @li ALL - Test is a PASS if ALL columns exist in &inds
     @li ANY - Test is a PASS if ANY of the columns exist in &inds

base/mp_assertcolvals.sas

@@ -36,7 +36,7 @@
 
 
   @param [in] indscol The input library.dataset.column to test for values
-  @param [in] checkvals= A library.dataset.column value containing a UNIQUE
+  @param [in] checkvals= (0) A library.dataset.column value containing a UNIQUE
     list of values to be compared against the source (indscol).
   @param [in] desc= (Testing observations) The user provided test description
   @param [in] test= (ALLVALS) The test to apply.  Valid values are:

base/mp_base64copy.sas

@@ -28,8 +28,8 @@
           put _infile_;
         run;
 
-  @param [in] inref= Fileref of the input file (should exist)
-  @param [out] outref= Output fileref. If it does not exist, it is created.
+  @param [in] inref= (0) Fileref of the input file (should exist)
+  @param [out] outref= (0) Output fileref. If it does not exist, it is created.
   @param [in] action= (ENCODE) The action to take. Valid values:
     @li ENCODE - Convert the file to base64 format
     @li DECODE - Decode the file from base64 format

base/mp_binarycopy.sas

@@ -22,16 +22,17 @@
       %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
 
 
-  @param [in] inloc quoted "path/and/filename.ext" of the file to be copied
-  @param [out] outloc quoted "path/and/filename.ext" of the file to be created
-  @param [in] inref (____in) If provided, this fileref will take precedence over
+  @param [in] inloc= () quoted "path/and/filename.ext" of the file to be copied
+  @param [out] outloc= () quoted "path/and/filename.ext" of the file to create
+  @param [in] inref= (____in) If provided, this fileref takes precedence over
     the `inloc` parameter
-  @param [out] outref (____in) If provided, this fileref will take precedence
+  @param [out] outref= (____in) If provided, this fileref takes precedence
     over the `outloc` parameter.  It must already exist!
-  @param [in] mode (CREATE) Valid values:
+  @param [in] mode= (CREATE) Valid values:
     @li CREATE - Create the file (even if it already exists)
     @li APPEND - Append to the file (don't overwrite)
-  @param iftrue= (1=1) Supply a condition for which the macro should be executed
+  @param [in] iftrue= (1=1)
+    Supply a condition for which the macro should be executed
 
   @returns nothing
 

base/mp_chop.sas

@@ -44,7 +44,7 @@
   For more examples, see mp_chop.test.sas
 
   @param [in] infile The QUOTED path to the file on which to perform the chop
-  @param [in] matchvar= Macro variable NAME containing the string to split by
+  @param [in] matchvar= () Macro variable NAME containing the string to split by
   @param [in] matchpoint= (START) Valid values:
     @li START - chop at the beginning of the string in `matchvar`.
     @li END - chop at the end of the string in `matchvar`.
@@ -54,7 +54,8 @@
     @li FIRST - keep the section of the file before the chop
     @li LAST - keep the section of the file after the chop
   @param [in] mdebug= (0) Set to 1 to provide macro debugging
-  @param outfile= (0) Optional QUOTED path to the adjusted output file (avoids
+  @param [out] outfile= (0)
+    Optional QUOTED path to the adjusted output file (avoids
     overwriting the first file).
 
   <h4> SAS Macros </h4>

base/mp_cleancsv.sas

@@ -12,11 +12,12 @@
       fileref mycsv "/path/your/csv";
       %mp_cleancsv(in=mycsv,out=/path/new.csv)
 
-  @param in= (NOTPROVIDED) Provide path or fileref to input csv. If a period is
+  @param [in] in= (NOTPROVIDED)
+    Provide path or fileref to input csv. If a period is
     found, it is assumed to be a file.
-  @param out= (NOTPROVIDED) Output path or fileref to output csv.  If a period
-    is found, it is assumed to be a file.
-  @param qchar= ('22'x) Quote char - hex code 22 is the double quote.
+  @param [in] out= (NOTPROVIDED) Output path or fileref to output csv.
+    If a period is found, it is assumed to be a file.
+  @param [in] qchar= ('22'x) Quote char - hex code 22 is the double quote.
 
   @version 9.2
   @author Allan Bowe

base/mp_cntlout.sas

@@ -79,8 +79,8 @@ data &cntlout/nonote2err;
   end;
 
   /* create row marker. Data cannot be sorted without it! */
-  if first.fmtname then fmtrow=0;
-  fmtrow+1;
+  if first.fmtname then fmtrow=1;
+  else fmtrow+1;
 
 run;
 proc sort;

base/mp_copyfolder.sas

@@ -18,7 +18,7 @@
 
   @param [in] source Unquoted path to the folder to copy from.
   @param [out] target Unquoted path to the folder to copy to.
-  @param [in] copymax=(MAX) Set to a positive integer to indicate the level of
+  @param [in] copymax= (MAX) Set to a positive integer to indicate the level of
     subdirectory copy recursion - eg 3, to go `./3/levels/deep`.  For unlimited
     recursion, set to MAX.
 

base/mp_createconstraints.sas

@@ -18,11 +18,11 @@
       %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
       %mp_createconstraints(inds=work.constraints,outds=created,execute=YES)
 
-  @param inds= (work.mp_getconstraints) The input table containing the
+  @param [in] inds= (work.mp_getconstraints) The input table containing the
     constraint info
-  @param outds= (work.mp_createconstraints) A table containing the create
+  @param [out] outds= (work.mp_createconstraints) A table containing the create
     statements (create_statement column)
-  @param execute= (NO) To actually create, use YES.
+  @param [in] execute= (NO) To actually create, use YES.
 
   <h4> Related Files </h4>
   @li mp_getconstraints.sas

base/mp_createwebservice.sas

@@ -1,6 +1,6 @@
 /**
   @file mp_createwebservice.sas
-  @brief Create a web service in SAS 9, Viya or SASjs Server
+  @brief Create a web service in SAS 9, Viya or SASjs Server (legacy)
   @details This is actually a wrapper for mx_createwebservice.sas, remaining
   for legacy purposes.  For new apps, use mx_createwebservice.sas.
 

base/mp_csv2ds.sas

@@ -19,11 +19,12 @@
       %mp_csv2ds(inref=mycsv,outds=myds,baseds=sashelp.class)
 
 
-  @param inref= fileref to the CSV
-  @param outds= output ds (lib.ds format)
-  @param view= Set to YES or NO to determine whether the output should be
-    a view or not.  Default is NO (not a view).
-  @param baseds= Template dataset on which to create the input statement.
+  @param [in] inref= (0) Fileref to the CSV
+  @param [out] outds= (0) Output ds (lib.ds format)
+  @param [in] view= (NO) Set to YES or NO to determine whether the output
+    should be a view or not.  Default is NO (not a view).
+  @param [in] baseds= (0)
+    Template dataset on which to create the input statement.
     Is used to determine types, lengths, and any informats.
 
   @version 9.2

base/mp_deleteconstraints.sas

@@ -17,9 +17,11 @@
       %mp_getconstraints(lib=work,ds=example,outds=work.constraints)
       %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
 
-  @param inds= The input table containing the constraint info
-  @param outds= a table containing the drop statements (drop_statement column)
-  @param execute= `YES|NO` - default is NO. To actually drop, use YES.
+  @param [in] inds= (mp_getconstraints)
+    The input table containing constraint info
+  @param [out] outds= (mp_deleteconstraints)
+    Table containing the drop statements (drop_statement column)
+  @param [in] execute= (NO) `YES|NO` - default is NO. To actually drop, use YES.
 
 
   @version 9.2

base/mp_deletefolder.sas

@@ -15,7 +15,7 @@
 
       %mp_deletefolder(&rootdir)
 
-  @param path Unquoted path to the folder to delete.
+  @param [in] folder Unquoted path to the folder to delete.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas

base/mp_dictionary.sas

@@ -29,7 +29,7 @@
 
   ![](https://user-images.githubusercontent.com/4420615/188278365-2987db97-0594-4a39-ac81-dbacdef5cdc8.png)
 
-  @param lib= (WORK) The libref in which to create the views
+  @param [in] lib= (WORK) The libref in which to create the views
 
   <h4> Related Files </h4>
   @li mp_dictionary.test.sas

base/mp_distinctfmtvalues.sas

@@ -7,11 +7,11 @@
 
         %mp_distinctfmtvalues(libds=sashelp.class,var=age,outvar=age,outds=test)
 
-  @param libds input dataset
-  @param var variable to get distinct values for
-  @param outvar variable to create.  Default:  `formatted_value`
-  @param outds dataset to create.  Default:  work.mp_distinctfmtvalues
-  @param varlen length of variable to create (default 200)
+  @param [in] libds= () input dataset
+  @param [in] var= (0) variable to get distinct values for
+  @param [out] outvar= (formatteed_value) variable to create.
+  @param [out] outds= (work.mp_distinctfmtvalues) dataset to create.
+  @param [in] varlen= (2000) length of variable to create
 
   @version 9.2
   @author Allan Bowe

base/mp_ds2csv.sas

@@ -118,13 +118,21 @@ data _null_;
     header = cats(coalescec(varlabel(dsid,i),varnm),dlm);
   %end;
   %else %if &headerformat=SASJS %then %do;
-    if vartype(dsid,i)='C' then header=cats(varnm,':$char',varlen(dsid,i),'.');
+    vlen=varlen(dsid,i);
+    if vartype(dsid,i)='C' then header=cats(varnm,':$char',vlen,'.');
     else do;
       vfmt=coalescec(varfmt(dsid,i),'0');
       fmttype=mcf_getfmttype(vfmt);
       if fmttype='DATE' then header=cats(varnm,':date9.');
       else if fmttype='DATETIME' then header=cats(varnm,':E8601DT26.6');
       else if fmttype='TIME' then header=cats(varnm,':TIME12.');
+      /**
+        * there is not much point importing a short length numeric like this,
+        * eg with best4., as the resulting variable will still be stored as
+        * length 8.  We need a length or format statement to ensure variable
+        * is creatd with the smaller length...
+        **/
+      else if vlen<8 then header=cats(varnm,':best',vlen,'.');
       else header=cats(varnm,':best.');
     end;
   %end;
@@ -151,6 +159,7 @@ data _null_;
   set &ds end=last;
 %do i=1 %to &vcnt;
   %let var=%scan(&varlist,&i);
+  %local vlen&i;
   %if %mf_getvartype(&ds,&var)=C %then %do;
     %let dsv1=%mf_getuniquename(prefix=csvcol1_);
     %let dsv2=%mf_getuniquename(prefix=csvcol2_);

base/mp_ds2ddl.sas

@@ -1,8 +1,20 @@
 /**
   @file
-  @brief A wrapper for mp_getddl.sas
-  @details In the next release, this will be the main version.
+  @brief Fetches DDL for a specific table
+  @details Uses mp_getddl under the hood
 
+  @param [in] libds library.dataset to create ddl for
+  @param [in] fref= (getddl) the fileref to which to _append_ the DDL.  If it
+    does not exist, it will be created.
+  @param [in] flavour= (SAS) The type of DDL to create. Options:
+    @li SAS
+    @li TSQL
+
+  @param [in]showlog= (NO) Set to YES to show the DDL in the log
+  @param [in] schema= () Choose a preferred schema name (default is to use
+    actual schema, else libref)
+  @param [in] applydttm= (NO) For non SAS DDL, choose if columns are created with
+    native datetime2 format or regular decimal type
 
   <h4> SAS Macros </h4>
   @li mp_getddl.sas

base/mp_dsmeta.sas

@@ -49,8 +49,8 @@
   https://support.sas.com/resources/papers/proceedings14/1549-2014.pdf) by
   [Louise Hadden](https://www.linkedin.com/in/louisehadden/).
 
-  @param libds The library.dataset to export the metadata for
-  @param outds= (work.dsmeta) The output table to contain the metadata
+  @param [in] libds The library.dataset to export the metadata for
+  @param [out] outds= (work.dsmeta) The output table to contain the metadata
 
   <h4> Related Files </h4>
   @li mp_dsmeta.test.sas

base/mp_filtercheck.sas

@@ -37,10 +37,12 @@
   @param [in] targetds= The target dataset against which to verify VARIABLE_NM.
     This must be available (ie, the library must be assigned).
   @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
-  @param [out] outds= The output table, which is a copy of the &inds. table
-    plus a REASON_CD column, containing only bad records.  If bad records found,
-    the SYSCC value will be set to 1008 (general data problem).  Downstream
-    processes should check this table (and return code) before continuing.
+  @param [out] outds= (work.badrecords) The output table, which is a copy of the
+    &inds. table plus a REASON_CD column, containing only bad records.
+    If bad records are found, the SYSCC value will be set to 1008
+    (a general data problem).
+    Downstream processes should check this table (and return code) before
+    continuing.
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas
@@ -84,15 +86,14 @@
 /**
   * Sanitise the values based on valid value lists, then strip out
   * quotes, commas, periods and spaces.
-  * Only numeric values should remain
   */
 %local reason_cd nobs;
 %let nobs=0;
 data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
-  set &inds;
-  length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000;
+  set &inds end=last;
+  length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000;
   drop tmp;
 
   /* quick check to ensure column exists */
@@ -108,7 +109,8 @@ data &outds;
   end;
 
   /* need to open the dataset to get the column type */
-  dsid=open("&targetds","i");
+  retain dsid;
+  if _n_=1 then dsid=open("&targetds","i");
   if dsid>0 then do;
     vnum=varnum(dsid,VARIABLE_NM);
     if vnum<1 then do;
@@ -118,11 +120,19 @@ data &outds;
       call symputx('reason_cd',reason_cd,'l');
       call symputx('nobs',_n_,'l');
       output;
-      return;
+      goto endstep;
     end;
     /* now we can get the type */
     else vtype=vartype(dsid,vnum);
   end;
+  else do;
+      REASON_CD=cats("Could not open &targetds");
+      putlog REASON_CD= dsid=;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
+      output;
+      stop;
+  end;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
@@ -157,15 +167,40 @@ data &outds;
   end;
 
   /* special missing logic */
-  if vtype='N'
-    and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE')
-    and cats(upcase(raw_value)) in (
+  if vtype='N' & OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE') then do;
+    if cats(upcase(raw_value)) in (
       '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N'
       '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._'
     )
-  then do;
-    /* valid numeric - exit data step loop */
-    return;
+    then do;
+      /* valid numeric - exit data step loop */
+      return;
+    end;
+    else if subpad(upcase(raw_value),1,1) in (
+      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+    )
+    then do;
+      /* check if the raw_value contains a valid variable NAME */
+      vnum=varnum(dsid,subpad(raw_value,1,32));
+      if vnum>0 then do;
+        /* now we can get the type */
+        vtype2=vartype(dsid,vnum);
+        /* check type matches */
+        if vtype2=vtype then do;
+          /* valid target var - exit loop */
+          return;
+        end;
+        else do;
+          REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
+          putlog REASON_CD= dsid=;
+          call symputx('reason_cd',reason_cd,'l');
+          call symputx('nobs',_n_,'l');
+          output;
+          goto endstep;
+        end;
+      end;
+    end;
   end;
 
   /* special logic */
@@ -187,6 +222,32 @@ data &outds;
     if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
       tmp=scan(raw_value1,i,',');
       if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
+        if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in (
+          'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+          'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+        )
+        then do;
+          /* check if the raw_value contains a valid variable NAME */
+          /* is not valid syntax for IN or NOT IN */
+          vnum=varnum(dsid,subpad(tmp,1,32));
+          if vnum>0 then do;
+            /* now we can get the type */
+            vtype2=vartype(dsid,vnum);
+            /* check type matches */
+            if vtype2=vtype then do;
+              /* valid target var - exit loop */
+              return;
+            end;
+            else do;
+              REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
+              putlog REASON_CD= dsid=;
+              call symputx('reason_cd',reason_cd,'l');
+              call symputx('nobs',_n_,'l');
+              output;
+              goto endstep;
+            end;
+          end;
+        end;
         REASON_CD='Non Numeric value provided';
         putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
         call symputx('reason_cd',reason_cd,'l');
@@ -211,14 +272,42 @@ data &outds;
 
   /* output records that contain values other than digits and spaces */
   if notdigit(compress(raw_value3,' '))>0 then do;
+    if vtype='C' and subpad(upcase(raw_value),1,1) in (
+      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+    )
+    then do;
+      /* check if the raw_value contains a valid variable NAME */
+      vnum=varnum(dsid,subpad(raw_value,1,32));
+      if vnum>0 then do;
+        /* now we can get the type */
+        vtype2=vartype(dsid,vnum);
+        /* check type matches */
+        if vtype2=vtype then do;
+          /* valid target var - exit loop */
+          return;
+        end;
+        else do;
+          REASON_CD=cats("Compared Char Type (",vtype2,") is not (",vtype,")");
+          putlog REASON_CD= dsid=;
+          call symputx('reason_cd',reason_cd,'l');
+          call symputx('nobs',_n_,'l');
+          output;
+          goto endstep;
+        end;
+      end;
+    end;
+
     putlog raw_value3= $hex32.;
     REASON_CD=cats('Invalid RAW_VALUE:',raw_value);
-    putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
+    putlog (_all_)(=);
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');
     output;
   end;
 
+  endstep:
+  if last then rc=close(dsid);
 run;
 
 

base/mp_filtergenerate.sas

@@ -51,8 +51,8 @@
   > )
 
   @param [in] inds The input table with query values
-  @param [out] outref= The output fileref to contain the filter clause.  Will
-    be created (or replaced).
+  @param [out] outref= (filter) The output fileref to contain the filter clause.
+    Will be created (or replaced).
 
   <h4> Related Macros </h4>
   @li mp_filtercheck.sas

base/mp_filterstore.sas

@@ -44,9 +44,10 @@
     mp_coretable.sas as follows:  `mp_coretable(LOCKTABLE)`.
   @param [in] maxkeytable= (0) Optional permanent reference table used for
     retained key tracking.  Described in mp_retainedkey.sas.
-  @param [in] mdebug= set to 1 to enable DEBUG messages
-  @param [out] outresult= The result table with the FILTER_RK
-  @param [out] outquery= The original query, taken as extract after table load
+  @param [in] mdebug= (1) set to 1 to enable DEBUG messages
+  @param [out] outresult= (work.result) The result table with the FILTER_RK
+  @param [out] outquery= (work.query) The original query, taken as extract
+    after table load
 
 
   <h4> SAS Macros </h4>

base/mp_getcols.sas

@@ -12,14 +12,15 @@
 
       %mp_getcols(sashelp.airline,outds=work.myds)
 
-  @param ds The dataset from which to obtain column metadata
-  @param outds= (work.cols) The output dataset to create. Sample data:
+  @param [in] ds The dataset from which to obtain column metadata
+  @param [out] outds= (work.cols) The output dataset to create. Sample data:
 |NAME:$32.|LENGTH:best.|VARNUM:best.|LABEL:$256.|FMTNAME:$32.|FORMAT:$49.|TYPE:$1.|DDTYPE:$9.|
 |---|---|---|---|---|---|---|---|
 |`AIR `|`8 `|`2 `|`international airline travel (thousands) `|` `|`8. `|`N `|`NUMERIC `|
 |`DATE `|`8 `|`1 `|`DATE `|`MONYY `|`MONYY. `|`N `|`DATE `|
 |`REGION `|`3 `|`3 `|`REGION `|` `|`$3. `|`C `|`CHARACTER `|
 
+
   <h4> Related Macros </h4>
   @li mf_getvarlist.sas
   @li mm_getcols.sas

base/mp_getdbml.sas

@@ -23,10 +23,10 @@
   @li mf_getquotedstr.sas
   @li mp_getconstraints.sas
 
-  @param liblist= Space seperated list of librefs to take as
-    input (Default=SASHELP)
-  @param outref= Fileref to contain the DBML (Default=getdbml)
-  @param showlog= set to YES to show the DBML in the log (Default is NO)
+  @param [in] liblist= (SASHELP) Space seperated list of librefs to take as
+    input
+  @param [out] outref= (getdbml) Fileref to contain the DBML
+  @param [in] showlog= (NO) set to YES to show the DBML in the log
 
   @version 9.3
   @author Allan Bowe

base/mp_getddl.sas

@@ -22,16 +22,21 @@
   @li mf_getvarcount.sas
   @li mp_getconstraints.sas
 
-  @param lib libref of the library to create DDL for.  Should be assigned.
-  @param ds dataset to create ddl for (optional)
-  @param fref= the fileref to which to _append_ the DDL.  If it does not exist,
-    it will be created.
-  @param flavour= The type of DDL to create (default=SAS). Supported=TSQL
-  @param showlog= Set to YES to show the DDL in the log
-  @param schema= Choose a preferred schema name (default is to use actual schema
-    ,else libref)
-  @param applydttm= for non SAS DDL, choose if columns are created with native
-    datetime2 format or regular decimal type
+  @param [in] libref Libref of the library to create DDL for. Should already
+    be assigned.
+  @param [in] ds dataset to create ddl for (optional)
+  @param [in] fref= (getddl) the fileref to which to _append_ the DDL.  If it
+    does not exist, it will be created.
+  @param [in] flavour= (SAS) The type of DDL to create. Options:
+    @li SAS
+    @li TSQL
+
+  @param [in]showlog= (NO) Set to YES to show the DDL in the log
+  @param [in] schema= () Choose a preferred schema name (default is to use
+    actual schema, else libref)
+  @param [in] applydttm= (NO) For non SAS DDL, choose if columns are created
+    with native datetime2 format or regular decimal type
+
   @version 9.3
   @author Allan Bowe
 **/

base/mp_gsubfile.sas

@@ -22,15 +22,15 @@
         list;
       run;
 
-  @param file= (0) The file to perform the substitution on
-  @param patternvar= A macro variable containing the Lua
+  @param [in] file= (0) The file to perform the substitution on
+  @param [in] patternvar= A macro variable containing the Lua
     [pattern](https://www.lua.org/pil/20.2.html) to search for.  Due to the use
     of special (magic) characters in Lua patterns, it is safer to pass the NAME
     of the macro variable containing the string, rather than the value itself.
-  @param replacevar= The name of the macro variable containing the replacement
-    _string_.
-  @param outfile= (0) The file to write the output to. If zero, then the file
-    is overwritten in-place.
+  @param [in] replacevar= ()
+    The name of the macro variable containing the replacement _string_.
+  @param [out] outfile= (0) The file to write the output to.
+    If zero, then the file is overwritten in-place.
 
   <h4> SAS Macros </h4>
   @li ml_gsubfile.sas

base/mp_guesspk.sas

@@ -18,7 +18,7 @@
       %mp_guesspk(sashelp.class,outds=classpks)
 
   @param [in] baseds The dataset to analyse
-  @param [out] outds= The output dataset to contain the possible PKs
+  @param [out] outds= (mp_guesspk) Output dataset to contain the possible PKs
   @param [in] max_guesses= (3) The total number of possible primary keys to
     generate. A table may have multiple (unlikely) PKs, so no need to list them
     all.

base/mp_hashdataset.sas

@@ -24,7 +24,7 @@
   @li mp_hashdirectory.sas
 
   @param [in] libds dataset to hash
-  @param [in] salt= Provide a salt (could be, for instance, the dataset name)
+  @param [in] salt= () Provide a salt (could be, for instance, the dataset name)
   @param [in] iftrue= (1=1) A condition under which the macro should be executed
   @param [out] outds= (work._data_) The output dataset to create. This
   will contain one column (hashkey) with one observation (a $hex32.

base/mp_lib2cards.sas

@@ -24,19 +24,19 @@
   @li mf_trimstr.sas
   @li mp_ds2cards.sas
 
-  @param [in] lib= Library in which to convert all datasets
-  @param [out] outloc= Location in which to store output.  Defaults to WORK
-    library. No quotes.
-  @param [out] outfile= Optional output file NAME - if provided, then will create
-  a single output file instead of one file per input table.
-  @param [in] maxobs= limit output to the first <code>maxobs</code> observations
+  @param [in] lib= () Library in which to convert all datasets
+  @param [out] outloc= (%sysfunc(pathname(work))) Location in which to store
+    output. No quotes.
+  @param [out] outfile= (0) Optional output file NAME - if provided, then
+    will create a single output file instead of one file per input table.
+  @param [in] maxobs= (max) limit output to the first <code>maxobs</code> rows
 
   @version 9.2
   @author Allan Bowe
 **/
 
 %macro mp_lib2cards(lib=
-    ,outloc=%sysfunc(pathname(work)) /* without trailing slash */
+    ,outloc=%sysfunc(pathname(work))
     ,maxobs=max
     ,random_sample=NO
     ,outfile=0

base/mp_loadformat.sas

@@ -9,6 +9,9 @@
   format, to prevent loss of data - UNLESS the input dataset contains a marker
   column, specifying that a particular row needs to be deleted (`delete_col=`).
 
+  Positions of formats are made using the FMTROW variable - this must be present
+  and unique (on TYPE / FMTNAME / FMTROW).
+
   This macro can also be used to identify which records would be (or were)
   considered new, modified or deleted (`loadtarget=`) by creating the following
   tables:
@@ -17,7 +20,7 @@
   @li work.outds_del
   @li work.outds_mod
 
-  For example usage, see mp_loadformat.test.sas
+  For example usage, see test (under Related Macros)
 
   @param [in] libcat The format catalog to be loaded
   @param [in] libds The staging table to load
@@ -34,12 +37,15 @@
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_existvar.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
+  @li mp_md5.sas
   @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
@@ -83,6 +89,16 @@
 %let libcat=%scan(&libcat,1,-);
 
 /* perform input validations */
+%mp_abort(
+  iftrue=(%mf_existds(&libds)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(&libds could not be found)
+)
+%mp_abort(
+  iftrue=(%mf_existvar(&libds,FMTROW)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(FMTROW not found in &libds)
+)
 %let err=0;
 %let msg=0;
 data _null_;
@@ -103,13 +119,6 @@ data _null_;
       stop;
     end;
   end;
-  else if name='LIBDS' then do;
-    if exist(value) le 0 then do;
-      call symputx('msg',"Unable to open staging table: "!!value);
-      call symputx('err',1);
-      stop;
-    end;
-  end;
   else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
   and missing(value) then do;
     call symputx('msg',"missing value in var: "!!name);
@@ -117,6 +126,14 @@ data _null_;
     stop;
   end;
 run;
+data _null_;
+  set &libds;
+  if missing(fmtrow) then do;
+    call symputx('msg',"missing fmtrow in format: "!!FMTNAME);
+    call symputx('err',1);
+    stop;
+  end;
+run;
 
 %mp_abort(
   iftrue=(&err ne 0)
@@ -124,6 +141,15 @@ run;
   ,msg=%str(&msg)
 )
 
+%local cnt;
+proc sql noprint;
+select count(distinct catx('|',type,fmtname,fmtrow)) into: cnt from &libds;
+%mp_abort(
+  iftrue=(&cnt ne %mf_nobs(&libds))
+  ,mac=&sysmacroname
+  ,msg=%str(Non-unique primary key on &libds)
+)
+
 /**
   * First, extract only relevant formats from the catalog
   */
@@ -177,12 +203,6 @@ data &inlibds/nonote2err;
     %mp_aligndecimal(end,width=16)
   end;
 
-  /* update row marker - retain new var as fmtrow may already be in libds */
-  if first.fmtname then row=1;
-  else row+1;
-  drop row;
-  fmtrow=row;
-
   fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
 run;
 

base/mp_md5.sas

@@ -28,8 +28,8 @@
   @li Global option:  `options dsoptions=nonote2err;`
   @li Data step option: `data YOURLIB.YOURDATASET /nonote2err;`
 
-  @param cvars= Space seperated list of character variables
-  @param nvars= Space seperated list of numeric variables
+  @param [in] cvars= () Space seperated list of character variables
+  @param [in] nvars= () Space seperated list of numeric variables
 
   <h4> Related Programs </h4>
   @li mp_init.sas

base/mp_perflog.sas

@@ -11,9 +11,9 @@
       %mp_perflog(finished)
 
 
-  @param label Provide label to go into the control dataset
-  @param libds= Provide a dataset in which to store performance stats.  Default
-              name is <code>work.mp_perflog</code>;
+  @param [in] label Provide label to go into the control dataset
+  @param [in] libds= (work.mp_perflog) Provide a dataset in which to store
+    performance stats.  Default name is <code>work.mp_perflog</code>;
 
   @version 9.2
   @author Allan Bowe

base/mp_prevobs.sas

@@ -34,12 +34,12 @@
   Credit is made to `data _null_` for authoring this very helpful paper:
   https://www.lexjansen.com/pharmasug/2008/cc/CC08.pdf
 
-  @param action Either FETCH a current or previous record, or INITialise.
-  @param record The relative (to current) position of the previous observation
+  @param [in] action Either FETCH a current or previous record, or INITialise.
+  @param [in] record The relative (to current) position of the previous row
     to return.
-  @param history= The number of records to retain in the hash table. Default=5
-  @param prefix= the prefix to give to the variables used to store the hash name
-    and index. Default=mp_prevobs
+  @param [in] history= (5) The number of records to retain in the hash table.
+  @param [in] prefix= (mp_prevobs) The prefix to give to the variables used to
+    store the hash name and index.
 
   @version 9.2
   @author Allan Bowe

base/mp_recursivejoin.sas

@@ -23,12 +23,13 @@
         ,childvar=c
         )
 
-  @param base_ds= base table containing hierarchy (not modified)
-  @param outds= the output dataset to create with the generated hierarchy
-  @param matchval= the ultimate parent from which to filter
-  @param parentvar= name of the parent variable
-  @param childvar= name of the child variable (should be same type as parent)
-  @param mdebug= set to 1 to prevent temp tables being dropped
+  @param [in] base_ds= base table containing hierarchy (not modified)
+  @param [out] outds= the output dataset to create with the generated hierarchy
+  @param [in] matchval= the ultimate parent from which to filter
+  @param [in] parentvar= name of the parent variable
+  @param [in] childvar= () name of the child variable (should be same type as
+    parent)
+  @param [in] mdebug= set to 1 to prevent temp tables being dropped
 
 
   @returns outds contains the following variables:

base/mp_replace.sas

@@ -32,10 +32,11 @@
   Note - if you are running a version of SAS that will allow the io package in
   LUA, you can also use this macro: mp_gsubfile.sas
 
-  @param infile The QUOTED path to the file on which to perform the substitution
-  @param findvar= Macro variable NAME containing the string to search for
-  @param replacevar= Macro variable NAME containing the replacement string
-  @param outfile= (0) Optional QUOTED path to the adjusted output file (to
+  @param [in] infile The QUOTED path to the file on which to perform the
+    substitution
+  @param [in] findvar= Macro variable NAME containing the string to search for
+  @param [in] replacevar= Macro variable NAME containing the replacement string
+  @param [out] outfile= (0) Optional QUOTED path to the adjusted output file (to
     avoid overwriting the first file).
 
   <h4> SAS Macros </h4>

base/mp_runddl.sas

@@ -31,9 +31,9 @@
     %mp_runddl(/some/rootlib, exc=LIBREF3) * same as above ;
 
 
-  @param path location of the DDL folder structure
-  @param inc= list of librefs to include
-  @param exc= list of librefs to exclude (takes precedence over inc=)
+  @param [in] path location of the DDL folder structure
+  @param [in] inc= list of librefs to include
+  @param [in] exc= list of librefs to exclude (takes precedence over inc=)
 
   @version 9.3
   @author Allan Bowe

base/mp_searchcols.sas

@@ -9,12 +9,13 @@
 
       %mp_searchcols(libs=sashelp work, cols=name sex age)
 
-  @param libs=(SASHELP) Space separated list of libraries to search for columns
-  @param cols= Space separated list of column names to search for (not case
-    sensitive)
-  @param outds=(mp_searchcols) the table to create with the results.  Will have
-    one line per table match.
-  @param match=(ANY) The match type. Valid values:
+  @param [in] libs= (SASHELP)
+    Space separated list of libraries to search for columns
+  @param [in] cols=
+    Space separated list of column names to search for (not case sensitive)
+  @param [out] outds= (mp_searchcols)
+    The table to create with the results.  Will have one line per table match.
+  @param [in] match= (ANY) The match type. Valid values:
     @li ANY - The table contains at least one of the columns
     @li WILD - The table contains a column with a name that partially matches
 

base/mp_stackdiffs.sas

@@ -197,6 +197,7 @@
   @li mp_coretable.sas
   @li mp_stackdiffs.test.sas
   @li mp_storediffs.sas
+  @li mp_stripdiffs.sas
 
   @todo The current approach assumes that a variable called KEY_HASH is not on
     the base table.  This part will need to be refactored (eg using

base/mp_storediffs.sas

@@ -64,6 +64,7 @@
   <h4> Related Macros </h4>
   @li mp_stackdiffs.sas
   @li mp_storediffs.test.sas
+  @li mp_stripdiffs.sas
 
   @version 9.2
   @author Allan Bowe
@@ -183,7 +184,7 @@ data &ds4;
 run;
 
 %if "&loadref"="0" %then %let loadref=%sysfunc(uuidgen());
-%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime());
+%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime(),8.6);
 %let libds=%upcase(&libds);
 
 /* join orig vals for modified & deleted */

base/mp_stprequests.sas

@@ -21,9 +21,10 @@
     %mp_stprequests(status_cd=INIT, libds=YOURLIB.DATASET )
 
 
-  @param status_cd= Use INIT for INIT and TERM for TERM events
-  @param libds= Location of base table (library.dataset).  To minimise risk
-    of table locks, we HIGHLY recommend using a database (NOT a SAS dataset).
+  @param [in] status_cd= Use INIT for INIT and TERM for TERM events
+  @param [in] libds= (somelib.stp_requests) Location of base table
+    (library.dataset).  To minimise risk of table locks, we HIGHLY recommend
+    using a database (NOT a SAS dataset).
     THE LIBRARY SHOULD BE ASSIGNED ALREADY - eg in autoexec or earlier in the
     init program proper.
 

base/mp_stripdiffs.sas

@@ -0,0 +1,252 @@
+/**
+  @file
+  @brief Generates a stage dataset to revert diffs tracked in an audit table
+  @details A big benefit of tracking data changes in an audit table is that
+    those changes can be subsequently reverted if necessary!
+
+    This macro prepares a staging dataset containing those differences - eg for:
+
+    @li deleted rows - these are re-inserted
+    @li changed rows - differences are reverted
+    @li added rows - marked with `_____DELETE__THIS__RECORD_____="YES"`
+
+    These changes are NOT applied to the base table - a staging dataset is
+    simply prepared for an ETL process to action.  In Data Controller, this
+    dataset is used directly as an input to the APPROVE process (so that the
+    reversion diffs can be reviewed prior to being applied).
+
+
+  @param [in] libds Base library.dataset (will not be modified).  The library
+    must be assigned.
+  @param [in] loadref Unique identifier for the version to be reverted.  This
+    change, plus ALL SUBSEQUENT CHANGES, will be reverted in the output table.
+  @param [in] difftable The dataset containing the diffs.  Definition available
+    in mddl_dc_difftable.sas
+  @param [out] outds= (work.mp_stripdiffs) Output table containing the diffs.
+    Has the same format as the base datset, plus a
+    `_____DELETE__THIS__RECORD_____` variable.
+  @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
+  @li mf_islibds.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_abort.sas
+
+  <h4> Related Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mp_stackdiffs.sas
+  @li mp_storediffs.sas
+  @li mp_stripdiffs.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mp_stripdiffs(libds
+  ,loadref
+  ,difftable
+  ,outds=work.mp_stripdiffs
+  ,mdebug=0
+)/*/STORE SOURCE*/;
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+%let libds=%upcase(&libds);
+
+/* safety checks */
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc on entry.  Clean session required!)
+)
+%let libds=%upcase(&libds);
+%mp_abort(iftrue= (%mf_islibds(&libds)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid library.dataset reference - %superq(libds))
+)
+
+
+
+/* set up unique and temporary vars */
+%local ds1 ds2 ds3 ds4 ds5 fref1;
+%let fref1=%mf_getuniquefileref();
+
+/* get timestamp of the diff to be reverted */
+%local ts;
+proc sql noprint;
+select put(processed_dttm,datetime19.6) into: ts
+  from &difftable where load_ref="&loadref";
+%mp_abort(iftrue= (&sqlobs=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Load ref %superq(loadref) not found!)
+)
+
+/* extract diffs for this base table from this timestamp onwards */
+%let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_diffs));
+create table &ds1 (drop=libref dsn) as
+  select * from &difftable
+  where upcase(cats(libref))="%scan(&libds,1,.)"
+  and upcase(cats(dsn))="%scan(&libds,2,.)"
+  and processed_dttm ge "&ts"dt
+  order by processed_dttm desc, key_hash, is_pk;
+
+/* extract key values only */
+%let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_pks));
+%local keyhash processed;
+%let keyhash=%upcase(%mf_getuniquename(prefix=mpsdvar_keyhash));
+%let processed=%upcase(%mf_getuniquename(prefix=mpsdvar_processed));
+create table &ds2 as
+  select key_hash as &keyhash,
+    tgtvar_nm,
+    tgtvar_type,
+    coalescec(oldval_char,newval_char) as charval,
+    coalesce(oldval_num, newval_num) as numval,
+    processed_dttm as &processed
+  from &ds1
+  where is_pk=1
+  order by &keyhash, &processed;
+
+/* grab pk values */
+%local pk;
+select distinct upcase(tgtvar_nm) into: pk separated by ' ' from &ds2;
+
+%let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_keychar));
+proc transpose data=&ds2(where=(tgtvar_type='C'))
+    out=&ds3(drop=_name_);
+  by &keyhash &processed;
+  id TGTVAR_NM;
+  var charval;
+run;
+
+%let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_keynum));
+proc transpose data=&ds2(where=(tgtvar_type='N'))
+    out=&ds4(drop=_name_);
+  by &keyhash &processed;
+  id TGTVAR_NM;
+  var numval;
+run;
+/* shorten the lengths */
+%mp_ds2squeeze(&ds3,outds=&ds3)
+%mp_ds2squeeze(&ds4,outds=&ds4)
+
+/* now merge to get all key values and de-dup */
+%let ds5=%upcase(work.%mf_getuniquename(prefix=mpsd_merged));
+data &ds5;
+  length &keyhash $32 &processed 8;
+  merge &ds3 &ds4;
+  by &keyhash &processed;
+  if not missing(&keyhash);
+run;
+proc sort data=&ds5 nodupkey;
+  by &pk;
+run;
+
+/* join to base table for preliminary stage DS */
+proc sql;
+create table &outds as select "No " as _____DELETE__THIS__RECORD_____
+  %do x=1 %to %sysfunc(countw(&pk,%str( )));
+    ,a.%scan(&pk,&x,%str( ))
+  %end;
+  %local notpkcols;
+  %let notpkcols=%upcase(%mf_getvarlist(&libds));
+  %let notpkcols=%mf_wordsinstr1butnotstr2(str1=&notpkcols,str2=&pk);
+  %do x=1 %to %sysfunc(countw(&notpkcols,%str( )));
+    ,b.%scan(&notpkcols,&x,%str( ))
+  %end;
+  from &ds5 a
+  left join &libds b
+  on 1=1
+%do x=1 %to %sysfunc(countw(&pk,%str( )));
+  and a.%scan(&pk,&x,%str( ))=b.%scan(&pk,&x,%str( ))
+%end;
+;
+
+/* create SAS code to apply to stage_ds */
+data _null_;
+  set &ds1;
+  file &fref1 lrecl=33000;
+  length charval $32767;
+  if _n_=1 then put 'proc sql noprint;';
+  by descending processed_dttm key_hash is_pk;
+  if move_type='M' then do;
+    if first.key_hash then do;
+      put "update &outds set " @@;
+    end;
+    if IS_PK=0 then do;
+      put "  " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+      if not last.is_pk then put ',';
+    end;
+    else do;
+      if first.is_pk then put "  where 1=1 " @@;
+      put "  and " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+  end;
+  else if move_type='A' then do;
+    if first.key_hash then do;
+      put "update &outds set _____DELETE__THIS__RECORD_____='Yes' where 1=1 "@@;
+    end;
+    /* gating if - as only need PK now */
+    if is_pk=1;
+    put '  AND ' tgtvar_nm '=' @@;
+    cnt=count(newval_char,'"');
+    charval=quote(trim(substr(newval_char,1,32765-cnt)));
+    if tgtvar_type='C' then put charval @@;
+    else put newval_num @@;
+  end;
+  else if move_type='D' then do;
+    if first.key_hash then do;
+      put "update &outds set _____DELETE__THIS__RECORD_____='No' " @@;
+    end;
+    if IS_PK=0 then do;
+      put "  ," tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+    else do;
+      if first.is_pk then put "  where 1=1 " @@;
+      put "  and " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+  end;
+  if last.key_hash then put ';';
+run;
+
+/* apply the modification statements */
+%inc &fref1/source2 lrecl=33000;
+
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &ds1, &ds2, &ds3, &ds4, &ds5;
+  file &fref1 clear;
+%end;
+%else %do;
+  data _null_;
+    infile &fref1;
+    input;
+    if _n_=1 then putlog "Contents of SQL adjustments";
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_stripdiffs;
+/** @endcond */

base/mp_testjob.sas

@@ -8,8 +8,8 @@
         duration=60*5
       )
 
-  @param [in] duration= the time in seconds which the job should run for. Actual
-  time may vary, as the check is done in between steps.  Default = 30 (seconds).
+  @param [in] duration= (30) The time in seconds which the job should run for.
+    Actual time may vary, as the check is done in between steps.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquelibref.sas

base/mp_testwritespeedlibrary.sas

@@ -11,9 +11,9 @@
         ,outds=work.results
       )
 
-  @param lib= (WORK) The library in which to create the table
-  @param size= (0.1) The size in GB of the table to create
-  @param outds= (WORK.RESULTS) The output dataset to be created.
+  @param [in] lib= (WORK) The library in which to create the table
+  @param [in] size= (0.1) The size in GB of the table to create
+  @param [out] outds= (WORK.RESULTS) The output dataset to be created.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas

base/mp_tree.sas

@@ -15,8 +15,8 @@ https://communities.sas.com/t5/SAS-Programming/listing-all-files-within-a-direct
 https://communities.sas.com/t5/SAS-Programming/listing-all-files-of-all-types-from-all-subdirectories/m-p/334113/highlight/true#M75419
 
 
-  @param dir= Directory to be scanned (default=/tmp)
-  @param outds= Dataset to create (default=work.mp_tree)
+  @param [in] dir= (/tmp) Directory to be scanned
+  @param [out] outds= (work.mp_tree) Dataset to create
 
   @returns outds contains the following variables:
 

base/mp_unzip.sas

@@ -13,11 +13,12 @@
 
       %mp_unzip(ziploc="/some/file.zip",outdir=/some/folder)
 
-  More info:  https://blogs.sas.com/content/sasdummy/2015/05/11/using-filename-zip-to-unzip-and-read-data-files-in-sas/
+  More info:
+  https://blogs.sas.com/content/sasdummy/2015/05/11/using-filename-zip-to-unzip-and-read-data-files-in-sas/
 
-  @param ziploc= Fileref or quoted full path to zip file ("/path/to/file.zip")
-  @param outdir= (%sysfunc(pathname(work))) Directory in which to write the
-    outputs (created if non existant)
+  @param [in] ziploc= Fileref or quoted full path, eg: "/path/to/file.zip"
+  @param [out] outdir= (%sysfunc(pathname(work))) Directory in which to write
+    the outputs (created if needed)
 
   <h4> SAS Macros </h4>
   @li mf_mkdir.sas

base/mp_updatevarlength.sas

@@ -14,9 +14,9 @@
         proc sql;
         describe table example;
 
-  @param libds the library.dataset to be modified
-  @param var The variable to modify
-  @param len The new length to apply
+  @param [in] libds the library.dataset to be modified
+  @param [in] var The variable to modify
+  @param [in] len The new length to apply
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas

base/mp_validatecol.sas

@@ -57,7 +57,7 @@
 %end;
 %else %if &rule=ISNUM %then %do;
   /*
-    credit SØREN LASSEN
+    credit SOREN LASSEN
     https://sasmacro.blogspot.com/2009/06/welcome-isnum-macro.html
   */
   &tempcol=input(&incol,?? best32.);

base/mp_wait4file.sas

@@ -11,7 +11,7 @@
   @param [in] file The file to wait for.  Must be provided.
   @param [in] maxwait= (0) Number of seconds to wait.  If set to zero, will
     loop indefinitely (to a maximum of 46 days, per SAS [documentation](
-      https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a001418809.htm
+    https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a001418809.htm
     )).  Otherwise, execution will proceed upon sleep expiry.
   @param [in] interval= (1) The wait period between sleeps, in seconds
 

base/mp_webin.sas

@@ -18,7 +18,7 @@
 
   Usage:
 
-    %mp_webin()
+      %mp_webin()
 
   This was created as a macro procedure (over a macro function) as it will also
   use the filename statement in Viya environments (where `_webin_fileuri` is

base/mp_zip.sas

@@ -15,14 +15,14 @@
   <h4> SAS Macros </h4>
   @li mp_dirlist.sas
 
-  @param in= unquoted filepath, dataset of files or directory to zip
-  @param type= (FILE) Valid values:
+  @param [in] in= unquoted filepath, dataset of files or directory to zip
+  @param [in] type= (FILE) Valid values:
     @li FILE - /full/path/and/filename.extension to a particular file
     @li DATASET - a dataset containing a list of files to zip (see `incol`)
     @li DIRECTORY - a directory to zip
-  @param outname= (FILE) Output file to create, _without_ .zip extension
-  @param outpath= (%sysfunc(pathname(WORK))) Parent folder for output zip file
-  @param incol= if DATASET input, say which column contains the filepath
+  @param [out] outname= (FILE) Output file to create, _without_ .zip extension
+  @param [out] outpath= (%sysfunc(pathname(WORK))) Parent folder for zip file
+  @param [in] incol= () If DATASET input, say which column contains the filepath
 
   <h4> Related Macros </h4>
   @li mp_unzip.sas

ddl/mddl_sas_cntlout.sas

@@ -6,6 +6,22 @@
   (given various practical restrictions) are described here to enable
   consistency when dealing with format data.
 
+  The HLO variable may have a number of values, documented here due to the
+  256 char label description length limit:
+
+    F=Standard format/informat.
+    H=Range ending value is HIGH.
+    I=Numeric informat.
+    J=Justification for an informat.
+    L=Range starting value is LOW.
+    M=MultiLabel.
+    N=Format or informat has no ranges, including no OTHER= range.
+    O=Range is OTHER.
+    R=ROUND option is in effect.
+    S=Specifies that NOTSORTED is in effect.
+    U=Specifies that the UPCASE option for an informat be used.
+
+
 **/
 
 
@@ -13,9 +29,11 @@
 
   proc sql;
   create table &libds(
-    TYPE char(1) label='Type of format - either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
+    TYPE char(1) label=
+'Format Type: either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
     ,FMTNAME char(32)     label='Format name'
-    ,FMTROW num label='CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
+    ,FMTROW num label=
+'CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
     ,START char(32767)    label='Starting value for format'
     /*
       Keep lengths of START and END the same to avoid this err:
@@ -35,7 +53,8 @@
     ,NOEDIT num length=3  label='Is picture string noedit?'
     ,SEXCL char(1)        label='Start exclusion'
     ,EEXCL char(1)        label='End exclusion'
-    ,HLO char(13)         label='Additional information. M=MultiLabel'
+    ,HLO char(13) label=
+'More info: https://core.sasjs.io/mddl__sas__cntlout_8sas_source.html'
     ,DECSEP char(1)       label='Decimal separator'
     ,DIG3SEP char(1)      label='Three-digit separator'
     ,DATATYPE char(8)     label='Date/time/datetime?'

meta/mm_adduser2group.sas

@@ -13,9 +13,9 @@
         ,group=someGroup)
 
 
-  @param user= the user name (not displayname)
-  @param group= the group to which to add the user
-  @param mdebug= (0) set to 1 to show debug info in log
+  @param [in] user= the user name (not displayname)
+  @param [in] group= the group to which to add the user
+  @param [in] mdebug= (0) set to 1 to show debug info in log
 
   <h4> Related Files </h4>
   @li ms_adduser2group.sas

meta/mm_assigndirectlib.sas

@@ -18,12 +18,13 @@
   @li mf_getengine.sas
   @li mp_abort.sas
 
-  @param libref the libref (not name) of the metadata library
-  @param open_passthrough= provide an alias to produce the CONNECT TO statement
-    for the relevant external database
-  @param sql_options= an override default output fileref to avoid naming clash
-  @param mDebug= set to 1 to show debug messages in the log
-  @param mAbort= set to 1 to call %mp_abort().
+  @param [in] libref the libref (not name) of the metadata library
+  @param [in] open_passthrough= () Provide an alias to produce the CONNECT TO
+    statement for the relevant external database
+  @param [in] sql_options= () Add any options to add to proc sql statement,
+    eg outobs= (only valid for pass through)
+  @param [in] mDebug= (0) set to 1 to show debug messages in the log
+  @param [in] mAbort= (0) set to 1 to call %mp_abort().
 
   @returns libname statement
 
@@ -33,12 +34,9 @@
 **/
 
 %macro mm_assigndirectlib(
-    libref /* libref to assign from metadata */
-    ,open_passthrough= /* provide an alias to produce the
-                          CONNECT TO statement for the
-                          relevant external database */
-    ,sql_options= /* add any options to add to proc sql statement eg outobs=
-                      (only valid for pass through) */
+    libref
+    ,open_passthrough=
+    ,sql_options=
     ,mDebug=0
     ,mAbort=0
 )/*/STORE SOURCE*/;
@@ -212,9 +210,9 @@ run;
   %end;
 %end;
 %else %if &engine=ODBC %then %do;
-  &mD.%put NOTE: Retrieving ODBC connection details;
+  %&mD.put NOTE: Retrieving ODBC connection details;
   data _null_;
-    length connx_uri conprop_uri value datasource up_uri schema $256.;
+    length connx_uri conprop_uri value datasource up_uri schema domprop_uri authdomain $256.;
     call missing (of _all_);
     /* get source connection ID */
     rc=metadata_getnasn("&liburi",'LibraryConnection',1,connx_uri);
@@ -229,6 +227,13 @@ run;
         rc2=-1;
       end;
     end;
+
+    /* get auth domain */
+    autrc=metadata_getnasn(connx_uri,"Domain",1,domprop_uri);
+    arc=metadata_getattr(domprop_uri,"Name",authdomain);
+    if not missing(authdomain) then authdomain=cats('AUTHDOMAIN=',authdomain);
+    call symputx('authdomain',authdomain,'l');
+
     /* get SCHEMA */
     rc6=metadata_getnasn("&liburi",'UsingPackages',1,up_uri);
     rc7=metadata_getattr(up_uri,'SchemaName',schema);
@@ -245,7 +250,7 @@ run;
       (INSERT_SQL=YES DATASRC=&sql_dsn. CONNECTION=global);
   %end;
   %else %do;
-    libname &libref ODBC DATASRC=&sql_dsn SCHEMA=&sql_schema;
+    libname &libref ODBC DATASRC=&sql_dsn SCHEMA=&sql_schema &authdomain;
   %end;
 %end;
 %else %if &engine=POSTGRES %then %do;

meta/mm_assignlib.sas

@@ -14,8 +14,8 @@
   @li mp_abort.sas
 
   @param [in] libref The libref (not name) of the metadata library
-  @param [in] mAbort= If not assigned, HARD will call %mp_abort(), SOFT will
-    silently return
+  @param [in] mAbort= (HARD) If not assigned, HARD will call %mp_abort(), SOFT
+    will silently return
 
   @returns libname statement
 

meta/mm_createapplication.sas

@@ -21,21 +21,24 @@
   @li mp_abort.sas
   @li mf_verifymacvars.sas
 
-  @param tree= The metadata folder uri, or the metadata path, in which to
+  @param [in] tree= The metadata folder uri, or the metadata path, in which to
     create the object.  This must exist.
-  @param name= Application object name.  Avoid spaces.
-  @param ClassIdentifier= the class of applications to which this app belongs
-  @param params= name=value pairs which will become public properties of the
+  @param [in] name= Application object name.  Avoid spaces.
+  @param [in] ClassIdentifier=
+    The class of applications to which this app belongs
+  @param [in] params=
+    name=value pairs which will become public properties of the
     application object. These are delimited using 
 (newline character)
 
-  @param desc= Application description (optional).  Avoid ampersands as these
+  @param [in] desc= Application description (optional).
+    Avoid ampersands as these
     are illegal characters (unless they are escapted- eg &)
-  @param version= version number of application
-  @param frefin= fileref to use (enables change if there is a conflict).  The
-    filerefs are left open, to enable inspection after running the
+  @param [in] version= version number of application
+  @param [in] frefin= fileref to use (enables change if there is a conflict).
+    The filerefs are left open, to enable inspection after running the
     macro (or importing into an xmlmap if needed).
-  @param frefout= fileref to use (enables change if there is a conflict)
-  @param mDebug= set to 1 to show debug messages in the log
+  @param [out] frefout= fileref to use (enables change if there is a conflict)
+  @param [in] mDebug= set to 1 to show debug messages in the log
 
   @author Allan Bowe
 

meta/mm_createdataset.sas

@@ -22,12 +22,12 @@
   @li mm_gettables.sas
   @li mm_getcols.sas
 
-  @param libds= library.dataset metadata source.  Note - table names in metadata
+  @param [in] libds= library.dataset metadata source.  Note - table names in metadata
     can be longer than 32 chars (just fyi, not an issue here)
-  @param tableuri= Metadata URI of the table to be created
-  @param outds= (work.mm_createdataset) The dataset to create.  The table name
-    needs to be 32 chars or less as per SAS naming rules.
-  @param mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] tableuri= Metadata URI of the table to be created
+  @param [out] outds= (work.mm_createdataset) The dataset to create.  The table
+    name needs to be 32 chars or less as per SAS naming rules.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
 
   @version 9.4
   @author Allan Bowe