fix: handling consecutive add+delete in mp_stripdiffs

feat: mp_stripdiffs macro - closes #373

.all-contributorsrc

@@ -144,8 +144,27 @@
       "contributions": [
         "doc"
       ]
+    },
+    {
+      "login": "rudvfaden",
+      "name": "Rud Faden",
+      "avatar_url": "https://avatars.githubusercontent.com/u/2445577?v=4",
+      "profile": "http://rudvfaden.github.io/",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "andyjessen",
+      "name": "andyjessen",
+      "avatar_url": "https://avatars.githubusercontent.com/u/62343929?v=4",
+      "profile": "https://github.com/andyjessen",
+      "contributions": [
+        "doc"
+      ]
     }
   ],
   "contributorsPerLine": 7,
-  "skipCi": true
+  "skipCi": true,
+  "commitType": "docs"
 }

.git-hooks/pre-commit

@@ -2,7 +2,7 @@
 
 # Ensure lint is passing
 LINT=`sasjs lint`
-if [[ "$LINT" != *"✔ All matched files use @sasjs/lint code style!" ]]; then
+if [[ "$LINT" != *"All matched files use @sasjs/lint code style!" ]]; then
   echo "$LINT"
   echo "To commit in spite of these warnings, use the -n parameter."
   exit 1

.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

.github/CONTRIBUTING.md

@@ -27,5 +27,6 @@ To contribute:
 1.  Create your feature branch (`git checkout -b myfeature`)
 2.  Make your change
 3.  Update the `all.sas` file (`python3 build.py`)
-4.  Push and make a PR
+4.  Commit using a [Conventional Commit](https://www.conventionalcommits.org)
+5.  Push and make a PR
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,4 +15,4 @@ What code changes have been made to achieve the intent.
 - [ ] Code is formatted correctly (`sasjs lint`).
 - [ ] Any new functionality has been unit tested.
 - [ ] All unit tests are passing (`sasjs test`).
-- [ ] `all.sas` has been regenerated (`python3 build.py`)
+- [ ] The PR desc or underlying commits follow the [Conventional Commit](https://www.conventionalcommits.org) standard

.github/workflows/main.yml

@@ -13,10 +13,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Install dependencies
-        run: npm ci
+        run: |
+          npm ci
 
       - name: Check code style (aborts if errors found)
         run: npx @sasjs/cli lint
@@ -36,14 +37,15 @@ jobs:
 
       - name: Install Open VPN
         run: |
-          sudo apt install apt-transport-https
+          #sudo apt install apt-transport-https
-          sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
+          #sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
-          sudo apt-key add openvpn-repo-pkg-key.pub
+          #sudo apt-key add openvpn-repo-pkg-key.pub
-          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
+          #sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
-          sudo apt update
+          #sudo apt update
-          sudo apt install openvpn3=17~betaUb22042+jammy
+          #sudo apt install openvpn3=17~betaUb22042+jammy
       - name: Start Open VPN 3
-        run: openvpn3 session-start --config .github/vpn/config.ovpn
+        run: |
+          # openvpn3 session-start --config .github/vpn/config.ovpn
 
       - name: Add credentials
         run: |
@@ -52,13 +54,13 @@ jobs:
           echo "REFRESH_TOKEN=${{secrets.SAS9_4GL_IO_REFRESH_TOKEN}}" >> .env.server
 
       - name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v3
+        uses: cycjimmy/semantic-release-action@v4
         env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 
       - name: SAS Packages Release
         run: |
           npx @sasjs/cli compile job -s sasjs/utils/create_sas_package.sas -o sasjsbuild -t server
-          # this part depends on https://github.com/sasjs/server/issues/307
+          # need long duration token per https://github.com/sasjs/server/issues/307
-          npx @sasjs/cli run sasjsbuild/jobs/utils/create_sas_package.sas -t server
+          # npx @sasjs/cli run sasjsbuild/jobs/utils/create_sas_package.sas -t server

.github/workflows/notmain.yml

@@ -0,0 +1,32 @@
+# This workflow will run tests using node and then publish a package to GitHub Packages when a release is created
+# For more information see: https://help.github.com/actions/language-and-framework-guides/publishing-nodejs-packages
+
+name: SASjs Core - Update all.sas
+
+on:
+  push:
+    branches-ignore:
+      - main
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          npm ci
+          npm i -g @sasjs/cli@latest
+
+      - name: Ensure all.sas is always up to date
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          python3 build.py
+          git add all.sas
+          git commit -m "chore: updating all.sas" --allow-empty
+          git push
+
+

.gitignore

@@ -13,3 +13,4 @@ mc_*
 *.env*
 
 ~
+

README.md

@@ -212,7 +212,8 @@ When contributing to this library, it is therefore important to ensure that all
 
 ## General Notes
 
-- All macros should be compatible with SAS versions from support level B and above (so currently 9.2 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully (eg `%if %sysevalf(&sysver<9.3) %then %return`).
+- All macros should be compatible with SAS versions from support level B and above (so currently 9.3 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully.
+- It's [best to avoid](https://git.datacontroller.io/dc/dc/issues/50) special / non-ASCII characters for compatibility with the widest variety of SAS installations.
 
 ## Breaking Changes
 
@@ -235,18 +236,19 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 The following repositories are also worth checking out:
 
-* [xieliaing/SAS](https://github.com/xieliaing/SAS)
-* [SASJedi/sas-macros](https://github.com/SASJedi/sas-macros)
 * [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
 * [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)
 * [KatjaGlassConsulting/SMILE-SmartSASMacros](https://github.com/KatjaGlassConsulting/SMILE-SmartSASMacros)
+* [paul-canals/toolbox](https://github.com/paul-canals/toolbox)
 * [rogerjdeangelis](https://github.com/rogerjdeangelis)
+* [SASJedi/sas-macros](https://github.com/SASJedi/sas-macros)
 * [scottbass/sas](https://github.com/scottbass/SAS)
+* [xieliaing/SAS](https://github.com/xieliaing/SAS)
 * [yabwon/sas_packages](https://github.com/yabwon/SAS_PACKAGES)
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-13-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-15-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -271,6 +273,10 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt="Bart Jablonski"/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://bandism.net/"><img src="https://avatars.githubusercontent.com/u/22633385?v=4?s=100" width="100px;" alt="Ikko Ashimine"/><br /><sub><b>Ikko Ashimine</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=eltociear" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/henrik-forsell"><img src="https://avatars.githubusercontent.com/u/109935936?v=4?s=100" width="100px;" alt="Henrik Forsell"/><br /><sub><b>Henrik Forsell</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=henrik-forsell" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://rudvfaden.github.io/"><img src="https://avatars.githubusercontent.com/u/2445577?v=4?s=100" width="100px;" alt="Rud Faden"/><br /><sub><b>Rud Faden</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rudvfaden" title="Code">💻</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/andyjessen"><img src="https://avatars.githubusercontent.com/u/62343929?v=4?s=100" width="100px;" alt="andyjessen"/><br /><sub><b>andyjessen</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=andyjessen" title="Documentation">📖</a></td>
     </tr>
   </tbody>
 </table>

base/mf_abort.sas

@@ -3,6 +3,11 @@
   @brief Abort, ungracefully
   @details Will abort with a straightforward %abort if the condition is true.
 
+  @param [in] mac= (mf_abort.sas) Name of calling macro (is printed to the log)
+  @param [in] msg= ( ) Additional string to print to the log
+  @param [in] iftrue= (%str(1=1)) Conditional logic under which to perform the
+    abort
+
   <h4> Related Macros </h4>
   @li mp_abort.sas
 

base/mf_deletefile.sas

@@ -10,7 +10,7 @@
       %mf_deletefile(&sasjswork/myfile.txt)
 
 
-  @param filepath Full path to the target file
+  @param [in] file Full path to the target file
 
   @returns The return code from the fdelete() invocation
 

base/mf_existds.sas

@@ -10,7 +10,7 @@
     expected results (depending on whether you 'expect' the result to be
     case insensitive in this context!)
 
-  @param libds library.dataset
+  @param [in] libds library.dataset
   @return output returns 1 or 0
 
   <h4> Related Macros </h4>

base/mf_existfileref.sas

@@ -4,7 +4,7 @@
   @details You can probably do without this macro as it is just a one liner.
   Mainly it is here as a convenient way to remember the syntax!
 
-  @param fref the fileref to detect
+  @param [in] fref the fileref to detect
 
   @return output Returns 1 if found and 0 if not found.  Note - it is possible
   that the fileref is found, but the file does not (yet) exist. If you need

base/mf_existfunction.sas

@@ -16,7 +16,7 @@
 https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionexists-macro
   ).
 
-  @param [in] name (positional) - function name
+  @param [in] name function name
 
   @author Allan Bowe
 **/

base/mf_existvarlist.sas

@@ -6,8 +6,8 @@
 
       %put %mf_existVarList(sashelp.class, age sex name dummyvar);
 
-  @param libds 2 part dataset or view reference
+  @param [in] libds 2 part dataset or view reference
-  @param varlist space separated variable names
+  @param [in] varlist space separated variable names
 
   @version 9.2
   @author Allan Bowe

base/mf_getapploc.sas

@@ -31,6 +31,7 @@
       %put %mf_getapploc(/some/location/jobs/extract/somejob/);
       %put %mf_getapploc(/some/location/tests/jobs/somejob/);
 
+  @param [in] pgm The _program value from which to extract the appLoc
 
   @author Allan Bowe
 **/

base/mf_getattrc.sas

@@ -6,8 +6,8 @@
       %put Dataset label = %mf_getattrc(sashelp.class,LABEL);
       %put Member Type = %mf_getattrc(sashelp.class,MTYPE);
 
-  @param libds library.dataset
+  @param [in] libds library.dataset
-  @param attr full list in [documentation](
+  @param [in] attr full list in [documentation](
     https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000147794.htm)
   @return output returns result of the attrc value supplied, or -1 and log
     message if err.

base/mf_getattrn.sas

@@ -6,8 +6,8 @@
       %put Number of observations=%mf_getattrn(sashelp.class,NLOBS);
       %put Number of variables = %mf_getattrn(sashelp.class,NVARS);
 
-  @param libds library.dataset
+  @param [in] libds library.dataset
-  @param attr Common values are NLOBS and NVARS, full list in [documentation](
+  @param [in] attr Common values are NLOBS and NVARS, full list in [documentation](
   http://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000212040.htm)
   @return output returns result of the attrn value supplied, or -1 and log
     message if err.

base/mf_getkeyvalue.sas

@@ -7,8 +7,9 @@
       %put %mf_getkeyvalue(someindex)
 
 
-  @param key Provide a key on which to perform the lookup
+  @param [in] key Provide a key on which to perform the lookup
-  @param libds= define the target table which holds the parameters
+  @param [in] libds= (work.mp_setkeyvalue) The library.dataset which holds the
+    parameters
 
   @version 9.2
   @author Allan Bowe

base/mf_getplatform.sas

@@ -12,7 +12,7 @@
   @li SASJS
   @li BASESAS
 
-  @param switch the param for which to return a platform specific variable
+  @param [in] switch the param for which to return a platform specific variable
 
   <h4> SAS Macros </h4>
   @li mf_mval.sas

base/mf_getschema.sas

@@ -8,7 +8,7 @@
   returns:
   > dbo
 
-  @param libref Library reference (also accepts a 2 level libds ref).
+  @param [in] libref Library reference (also accepts a 2 level libds ref).
 
   @return output returns the library schema for the FIRST library encountered
 

base/mf_getuniquelibref.sas

@@ -19,7 +19,7 @@
   @param [in] prefix= (mclib) first part of the returned libref. As librefs can
     be as long as 8 characters, a maximum length of 7 characters is premitted
     for this prefix.
-  @param [in] maxtries= Deprecated parameter. Remains here to ensure a
+  @param [in] maxtries= (1000) Deprecated parameter. Remains here to ensure a
     non-breaking change.  Will be removed in v5.
 
   @version 9.2

base/mf_getuniquename.sas

@@ -10,7 +10,7 @@
 
 > MCc59c750610321d4c8bf75faadbcd22
 
-  @param prefix= set a prefix for the new name
+  @param [in] prefix= (MC) Sets a prefix for the new name
 
   @version 9.3
   @author Allan Bowe

base/mf_getuser.sas

@@ -13,8 +13,6 @@
         %let user= %mf_getUser();
         %put &user;
 
-  @param type - do not use, may be deprecated in a future release
-
   @return SYSUSERID (if workspace server)
   @return _METAPERSON (if stored process server)
   @return SYS_COMPUTE_SESSION_OWNER (if Viya compute session)

base/mf_getvalue.sas

@@ -13,9 +13,9 @@
   <h4> Related Macros </h4>
   @li mp_setkeyvalue.sas
 
-  @param libds dataset to query
+  @param [in] libds dataset to query
-  @param variable the variable which contains the value to return.
+  @param [in] variable the variable which contains the value to return.
-  @param filter contents of where clause
+  @param [in] filter= (1) contents of where clause
 
   @version 9.2
   @author Allan Bowe

base/mf_getvarformat.sas

@@ -25,7 +25,8 @@
 
   @param [in] libds Two part dataset (or view) reference.
   @param [in] var Variable name for which a format should be returned
-  @param [in] force=(0) Set to 1 to supply a default if the variable has no format
+  @param [in] force= (0) Set to 1 to supply a default if the variable has no
+    format
   @returns outputs format
 
   @author Allan Bowe

base/mf_getvarlen.sas

@@ -18,8 +18,8 @@
       8
       NOTE: Variable renegade does not exist in test
 
-  @param libds Two part dataset (or view) reference.
+  @param [in] libds Two part dataset (or view) reference.
-  @param var Variable name for which a length should be returned
+  @param [in] var Variable name for which a length should be returned
   @returns outputs length
 
   @author Allan Bowe

base/mf_getvarnum.sas

@@ -21,8 +21,8 @@ returns:
 
   > NOTE: Variable renegade does not exist in test
 
-  @param libds Two part dataset (or view) reference.
+  @param [in] libds Two part dataset (or view) reference.
-  @param var Variable name for which a position should be returned
+  @param [in] var Variable name for which a position should be returned
 
   @author Allan Bowe
   @version 9.2

base/mf_getvartype.sas

@@ -13,8 +13,8 @@ Usage:
 
 
 
-  @param libds Two part dataset (or view) reference.
+  @param [in] libds Two part dataset (or view) reference.
-  @param var the variable name to be checked
+  @param [in] var the variable name to be checked
   @return output returns C or N depending on variable type.  If variable
     does not exist then a blank is returned and a note is written to the log.
 

base/mf_getxengine.sas

@@ -10,7 +10,7 @@
   returns:
   > TEMP
 
-  @param fref The fileref to check
+  @param [in] fref The fileref to check
 
   @returns The XENGINE value in sashelp.vextfl or 0 if not found.
 

base/mf_increment.sas

@@ -11,8 +11,8 @@
       %put Now we have run %mf_increment(cnt) lines;
       %put There are %mf_increment(cnt) lines in total;
 
-  @param [in] MACRO_NAME the name of the macro variable to increment
+  @param [in] macro_name The name of the macro variable to increment
-  @param [in] ITER= The amount to add or subtract to the macro
+  @param [in] incr= (1) The amount to add or subtract to the macro
 
   <h4> Related Files </h4>
   @li mf_increment.test.sas

base/mf_isblank.sas

@@ -7,12 +7,12 @@
 
   Usage:
 
-      %put mf_isblank(&var);
+      %put %mf_isblank(&var);
 
   inspiration:
   https://support.sas.com/resources/papers/proceedings09/022-2009.pdf
 
-  @param param VALUE to be checked
+  @param [in] Param VALUE to be checked
 
   @return output returns 1 (if blank) else 0
 

base/mf_isdir.sas

@@ -9,7 +9,7 @@
   With thanks and full credit to Andrea Defronzo -
   https://www.linkedin.com/in/andrea-defronzo-b1a47460/
 
-  @param path full path of the file/directory to be checked
+  @param [in] path Full path of the file/directory to be checked
 
   @return output returns 1 if path is a directory, 0 if it is not
 

base/mf_loc.sas

@@ -6,6 +6,10 @@
 
       %put %mf_loc(POF); %*location of PlatformObjectFramework tools;
 
+  @param [in] loc The item to locate, eg:
+    @li PLAATFORMOBJECTFRAMEWORK (or POF)
+    @li VIYACONFG
+
   @version 9.2
   @author Allan Bowe
 **/
@@ -15,7 +19,8 @@
 %local root;
 
 %if &loc=POF or &loc=PLATFORMOBJECTFRAMEWORK %then %do;
-  %let root=%substr(%sysget(SASROOT),1,%index(%sysget(SASROOT),SASFoundation)-2);
+  %let root=%sysget(SASROOT);
+  %let root=%substr(&root,1,%index(&root,SASFoundation)-2);
   %let root=&root/SASPlatformObjectFramework/&sysver;
   %put Batch tools located at: &root;
   &root

base/mf_mkdir.sas

@@ -7,7 +7,7 @@ Usage:
     %mf_mkdir(/some/path/name)
 
 
-  @param dir relative or absolute pathname.  Unquoted.
+  @param [in] dir Relative or absolute pathname.  Unquoted.
   @version 9.2
 
 **/

base/mf_mval.sas

@@ -8,6 +8,8 @@
 
       %if %mf_mval(maynotexist)=itdid %then %do;
 
+  @param [in] var The macro variable NAME to return the (possible) value for
+
   @version 9.2
   @author Allan Bowe
 **/

base/mf_nobs.sas

@@ -9,7 +9,7 @@
   <h4> SAS Macros </h4>
   @li mf_getattrn.sas
 
-  @param libds library.dataset
+  @param [in] libds library.dataset
 
   @return output returns result of the attrn value supplied, or log message
     if err.

base/mf_trimstr.sas

@@ -11,8 +11,8 @@
   <h4> SAS Macros </h4>
 
 
-  @param basestr The string to be modified
+  @param [in] basestr The string to be modified
-  @param trimstr The string to be removed from the end of `basestr`, if it
+  @param [in] trimstr The string to be removed from the end of `basestr`, if it
     exists
 
   @return output returns result with the value of `trimstr` removed from the end

base/mf_wordsinstr1andstr2.sas

@@ -13,8 +13,8 @@
   returns:
   > blah blaaah brah
 
-  @param str1= string containing words to extract
+  @param [in] str1= () string containing words to extract
-  @param str2= used to compare with the extract string
+  @param [in] str2= () used to compare with the extract string
 
   @warning CASE SENSITIVE!
 

base/mf_wordsinstr1butnotstr2.sas

@@ -16,8 +16,8 @@
   returns:
   > sss bram boo
 
-  @param [in] str1= string containing words to extract
+  @param [in] str1= () String containing words to extract
-  @param [in] str2= used to compare with the extract string
+  @param [in] str2= () Used to compare with the extract string
 
   @version 9.2
   @author Allan Bowe

base/mf_writefile.sas

@@ -21,8 +21,8 @@
   @param [in] mode= (O) Available options are A or O as follows:
     @li A APPEND mode, writes new records after the current end of the file.
     @li O OUTPUT mode, writes new records from the beginning of the file.
-  @param [in] l1= First line
+  @param [in] l1= () First line
-  @param [in] l2= Second line (etc through to l10)
+  @param [in] l2= () Second line (etc through to l10)
 
   <h4> Related Macros </h4>
   @li mf_writefile.test.sas

base/mp_abort.sas

@@ -27,14 +27,15 @@
     currently investigating approaches to deal with this.
 
 
-  @param mac= (mp_abort.sas) To contain the name of the calling macro. Do not
+  @param [in] mac= (mp_abort.sas) To contain the name of the calling macro. Do
-    use &sysmacroname as this will always resolve to MP_ABORT.
+    not use &sysmacroname as this will always resolve to MP_ABORT.
-  @param msg= message to be returned
+  @param [out] msg= message to be returned
-  @param iftrue= (1=1) Supply a condition for which the macro should be executed
+  @param [in] iftrue= (1=1) Condition under which the macro should be executed
-  @param errds= (work.mp_abort_errds) There is no clean way to end a process
+  @param [in] errds= (work.mp_abort_errds) There is no clean way to end a
-    within a %include called within a macro.  Furthermore, there is no way to
+    process within a %include called within a macro.  Furthermore, there is no
-    test if a macro is called within a %include.  To handle this particular
+    way to test if a macro is called within a %include.  To handle this
-    scenario, the %include should be switched for the mp_include.sas macro.
+    particular scenario, the %include should be switched for the mp_include.sas
+    macro.
     This provides an indicator that we are running a macro within a \%include
     (`_SYSINCLUDEFILEDEVICE`) and allows us to provide a dataset with the abort
     values (msg, mac).
@@ -45,8 +46,8 @@
     @li msg (the message)
     @li mac (the mac param)
 
-  @param mode= (REGULAR) If mode=INCLUDE then the &errds dataset is checked for
+  @param [in] mode= (REGULAR) If mode=INCLUDE then the &errds dataset is checked
-    an abort status.
+    for an abort status.
     Valid values:
     @li REGULAR (default)
     @li INCLUDE

base/mp_aligndecimal.sas

@@ -62,8 +62,8 @@
           7998580.8415
 
 
-  @param var The (data step, character) variable to modify
+  @param [in] var The (data step, character) variable to modify
-  @param width= (8) The number of characters BEFORE the decimal point
+  @param [in] width= (8) The number of characters BEFORE the decimal point
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas

base/mp_appendfile.sas

@@ -17,8 +17,8 @@
         %mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3)
 
 
-  @param [in] baseref= Fileref of the base file (should exist)
+  @param [in] baseref= (0) Fileref of the base file (should exist)
-  @param [in] appendrefs= One or more filerefs to be appended to the base
+  @param [in] appendrefs= (0) One or more filerefs to be appended to the base
     fileref.  Space separated.
 
   @version 9.2

base/mp_assertcols.sas

@@ -31,8 +31,8 @@
 
 
   @param [in] inds The input library.dataset to test for values
-  @param [in] cols= The list of columns to check for
+  @param [in] cols= (0) The list of columns to check for
-  @param [in] desc= (Testing observations) The user provided test description
+  @param [in] desc= (0) The user provided test description
   @param [in] test= (ALL) The test to apply.  Valid values are:
     @li ALL - Test is a PASS if ALL columns exist in &inds
     @li ANY - Test is a PASS if ANY of the columns exist in &inds

base/mp_assertcolvals.sas

@@ -36,7 +36,7 @@
 
 
   @param [in] indscol The input library.dataset.column to test for values
-  @param [in] checkvals= A library.dataset.column value containing a UNIQUE
+  @param [in] checkvals= (0) A library.dataset.column value containing a UNIQUE
     list of values to be compared against the source (indscol).
   @param [in] desc= (Testing observations) The user provided test description
   @param [in] test= (ALLVALS) The test to apply.  Valid values are:

base/mp_base64copy.sas

@@ -28,8 +28,8 @@
           put _infile_;
         run;
 
-  @param [in] inref= Fileref of the input file (should exist)
+  @param [in] inref= (0) Fileref of the input file (should exist)
-  @param [out] outref= Output fileref. If it does not exist, it is created.
+  @param [out] outref= (0) Output fileref. If it does not exist, it is created.
   @param [in] action= (ENCODE) The action to take. Valid values:
     @li ENCODE - Convert the file to base64 format
     @li DECODE - Decode the file from base64 format

base/mp_binarycopy.sas

@@ -22,16 +22,17 @@
       %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
 
 
-  @param [in] inloc quoted "path/and/filename.ext" of the file to be copied
+  @param [in] inloc= () quoted "path/and/filename.ext" of the file to be copied
-  @param [out] outloc quoted "path/and/filename.ext" of the file to be created
+  @param [out] outloc= () quoted "path/and/filename.ext" of the file to create
-  @param [in] inref (____in) If provided, this fileref will take precedence over
+  @param [in] inref= (____in) If provided, this fileref takes precedence over
     the `inloc` parameter
-  @param [out] outref (____in) If provided, this fileref will take precedence
+  @param [out] outref= (____in) If provided, this fileref takes precedence
     over the `outloc` parameter.  It must already exist!
-  @param [in] mode (CREATE) Valid values:
+  @param [in] mode= (CREATE) Valid values:
     @li CREATE - Create the file (even if it already exists)
     @li APPEND - Append to the file (don't overwrite)
-  @param iftrue= (1=1) Supply a condition for which the macro should be executed
+  @param [in] iftrue= (1=1)
+    Supply a condition for which the macro should be executed
 
   @returns nothing
 

base/mp_chop.sas

@@ -44,7 +44,7 @@
   For more examples, see mp_chop.test.sas
 
   @param [in] infile The QUOTED path to the file on which to perform the chop
-  @param [in] matchvar= Macro variable NAME containing the string to split by
+  @param [in] matchvar= () Macro variable NAME containing the string to split by
   @param [in] matchpoint= (START) Valid values:
     @li START - chop at the beginning of the string in `matchvar`.
     @li END - chop at the end of the string in `matchvar`.
@@ -54,7 +54,8 @@
     @li FIRST - keep the section of the file before the chop
     @li LAST - keep the section of the file after the chop
   @param [in] mdebug= (0) Set to 1 to provide macro debugging
-  @param outfile= (0) Optional QUOTED path to the adjusted output file (avoids
+  @param [out] outfile= (0)
+    Optional QUOTED path to the adjusted output file (avoids
     overwriting the first file).
 
   <h4> SAS Macros </h4>

base/mp_cleancsv.sas

@@ -12,11 +12,12 @@
       fileref mycsv "/path/your/csv";
       %mp_cleancsv(in=mycsv,out=/path/new.csv)
 
-  @param in= (NOTPROVIDED) Provide path or fileref to input csv. If a period is
+  @param [in] in= (NOTPROVIDED)
+    Provide path or fileref to input csv. If a period is
     found, it is assumed to be a file.
-  @param out= (NOTPROVIDED) Output path or fileref to output csv.  If a period
+  @param [in] out= (NOTPROVIDED) Output path or fileref to output csv.
-    is found, it is assumed to be a file.
+    If a period is found, it is assumed to be a file.
-  @param qchar= ('22'x) Quote char - hex code 22 is the double quote.
+  @param [in] qchar= ('22'x) Quote char - hex code 22 is the double quote.
 
   @version 9.2
   @author Allan Bowe

base/mp_cntlout.sas

@@ -79,8 +79,8 @@ data &cntlout/nonote2err;
   end;
 
   /* create row marker. Data cannot be sorted without it! */
-  if first.fmtname then fmtrow=0;
+  if first.fmtname then fmtrow=1;
-  fmtrow+1;
+  else fmtrow+1;
 
 run;
 proc sort;

base/mp_createconstraints.sas

@@ -18,11 +18,11 @@
       %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
       %mp_createconstraints(inds=work.constraints,outds=created,execute=YES)
 
-  @param inds= (work.mp_getconstraints) The input table containing the
+  @param [in] inds= (work.mp_getconstraints) The input table containing the
     constraint info
-  @param outds= (work.mp_createconstraints) A table containing the create
+  @param [out] outds= (work.mp_createconstraints) A table containing the create
     statements (create_statement column)
-  @param execute= (NO) To actually create, use YES.
+  @param [in] execute= (NO) To actually create, use YES.
 
   <h4> Related Files </h4>
   @li mp_getconstraints.sas

base/mp_createwebservice.sas

@@ -1,6 +1,6 @@
 /**
   @file mp_createwebservice.sas
-  @brief Create a web service in SAS 9, Viya or SASjs Server
+  @brief Create a web service in SAS 9, Viya or SASjs Server (legacy)
   @details This is actually a wrapper for mx_createwebservice.sas, remaining
   for legacy purposes.  For new apps, use mx_createwebservice.sas.
 

base/mp_csv2ds.sas

@@ -19,11 +19,12 @@
       %mp_csv2ds(inref=mycsv,outds=myds,baseds=sashelp.class)
 
 
-  @param inref= fileref to the CSV
+  @param [in] inref= (0) Fileref to the CSV
-  @param outds= output ds (lib.ds format)
+  @param [out] outds= (0) Output ds (lib.ds format)
-  @param view= Set to YES or NO to determine whether the output should be
+  @param [in] view= (NO) Set to YES or NO to determine whether the output
-    a view or not.  Default is NO (not a view).
+    should be a view or not.  Default is NO (not a view).
-  @param baseds= Template dataset on which to create the input statement.
+  @param [in] baseds= (0)
+    Template dataset on which to create the input statement.
     Is used to determine types, lengths, and any informats.
 
   @version 9.2

base/mp_deleteconstraints.sas

@@ -17,9 +17,11 @@
       %mp_getconstraints(lib=work,ds=example,outds=work.constraints)
       %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
 
-  @param inds= The input table containing the constraint info
+  @param [in] inds= (mp_getconstraints)
-  @param outds= a table containing the drop statements (drop_statement column)
+    The input table containing constraint info
-  @param execute= `YES|NO` - default is NO. To actually drop, use YES.
+  @param [out] outds= (mp_deleteconstraints)
+    Table containing the drop statements (drop_statement column)
+  @param [in] execute= (NO) `YES|NO` - default is NO. To actually drop, use YES.
 
 
   @version 9.2

base/mp_deletefolder.sas

@@ -15,7 +15,7 @@
 
       %mp_deletefolder(&rootdir)
 
-  @param path Unquoted path to the folder to delete.
+  @param [in] folder Unquoted path to the folder to delete.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas

base/mp_dictionary.sas

@@ -29,7 +29,7 @@
 
   ![](https://user-images.githubusercontent.com/4420615/188278365-2987db97-0594-4a39-ac81-dbacdef5cdc8.png)
 
-  @param lib= (WORK) The libref in which to create the views
+  @param [in] lib= (WORK) The libref in which to create the views
 
   <h4> Related Files </h4>
   @li mp_dictionary.test.sas

base/mp_distinctfmtvalues.sas

@@ -7,11 +7,11 @@
 
         %mp_distinctfmtvalues(libds=sashelp.class,var=age,outvar=age,outds=test)
 
-  @param libds input dataset
+  @param [in] libds= () input dataset
-  @param var variable to get distinct values for
+  @param [in] var= (0) variable to get distinct values for
-  @param outvar variable to create.  Default:  `formatted_value`
+  @param [out] outvar= (formatteed_value) variable to create.
-  @param outds dataset to create.  Default:  work.mp_distinctfmtvalues
+  @param [out] outds= (work.mp_distinctfmtvalues) dataset to create.
-  @param varlen length of variable to create (default 200)
+  @param [in] varlen= (2000) length of variable to create
 
   @version 9.2
   @author Allan Bowe

base/mp_ds2csv.sas

@@ -118,13 +118,21 @@ data _null_;
     header = cats(coalescec(varlabel(dsid,i),varnm),dlm);
   %end;
   %else %if &headerformat=SASJS %then %do;
-    if vartype(dsid,i)='C' then header=cats(varnm,':$char',varlen(dsid,i),'.');
+    vlen=varlen(dsid,i);
+    if vartype(dsid,i)='C' then header=cats(varnm,':$char',vlen,'.');
     else do;
       vfmt=coalescec(varfmt(dsid,i),'0');
       fmttype=mcf_getfmttype(vfmt);
       if fmttype='DATE' then header=cats(varnm,':date9.');
       else if fmttype='DATETIME' then header=cats(varnm,':E8601DT26.6');
       else if fmttype='TIME' then header=cats(varnm,':TIME12.');
+      /**
+        * there is not much point importing a short length numeric like this,
+        * eg with best4., as the resulting variable will still be stored as
+        * length 8.  We need a length or format statement to ensure variable
+        * is creatd with the smaller length...
+        **/
+      else if vlen<8 then header=cats(varnm,':best',vlen,'.');
       else header=cats(varnm,':best.');
     end;
   %end;
@@ -151,6 +159,7 @@ data _null_;
   set &ds end=last;
 %do i=1 %to &vcnt;
   %let var=%scan(&varlist,&i);
+  %local vlen&i;
   %if %mf_getvartype(&ds,&var)=C %then %do;
     %let dsv1=%mf_getuniquename(prefix=csvcol1_);
     %let dsv2=%mf_getuniquename(prefix=csvcol2_);

base/mp_ds2ddl.sas

@@ -1,8 +1,20 @@
 /**
   @file
-  @brief A wrapper for mp_getddl.sas
+  @brief Fetches DDL for a specific table
-  @details In the next release, this will be the main version.
+  @details Uses mp_getddl under the hood
 
+  @param [in] libds library.dataset to create ddl for
+  @param [in] fref= (getddl) the fileref to which to _append_ the DDL.  If it
+    does not exist, it will be created.
+  @param [in] flavour= (SAS) The type of DDL to create. Options:
+    @li SAS
+    @li TSQL
+
+  @param [in]showlog= (NO) Set to YES to show the DDL in the log
+  @param [in] schema= () Choose a preferred schema name (default is to use
+    actual schema, else libref)
+  @param [in] applydttm= (NO) For non SAS DDL, choose if columns are created with
+    native datetime2 format or regular decimal type
 
   <h4> SAS Macros </h4>
   @li mp_getddl.sas

base/mp_dsmeta.sas

@@ -49,8 +49,8 @@
   https://support.sas.com/resources/papers/proceedings14/1549-2014.pdf) by
   [Louise Hadden](https://www.linkedin.com/in/louisehadden/).
 
-  @param libds The library.dataset to export the metadata for
+  @param [in] libds The library.dataset to export the metadata for
-  @param outds= (work.dsmeta) The output table to contain the metadata
+  @param [out] outds= (work.dsmeta) The output table to contain the metadata
 
   <h4> Related Files </h4>
   @li mp_dsmeta.test.sas

base/mp_filtercheck.sas

@@ -37,10 +37,12 @@
   @param [in] targetds= The target dataset against which to verify VARIABLE_NM.
     This must be available (ie, the library must be assigned).
   @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
-  @param [out] outds= The output table, which is a copy of the &inds. table
+  @param [out] outds= (work.badrecords) The output table, which is a copy of the
-    plus a REASON_CD column, containing only bad records.  If bad records found,
+    &inds. table plus a REASON_CD column, containing only bad records.
-    the SYSCC value will be set to 1008 (general data problem).  Downstream
+    If bad records are found, the SYSCC value will be set to 1008
-    processes should check this table (and return code) before continuing.
+    (a general data problem).
+    Downstream processes should check this table (and return code) before
+    continuing.
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas
@@ -84,15 +86,14 @@
 /**
   * Sanitise the values based on valid value lists, then strip out
   * quotes, commas, periods and spaces.
-  * Only numeric values should remain
   */
 %local reason_cd nobs;
 %let nobs=0;
 data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
-  set &inds;
+  set &inds end=last;
-  length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000;
+  length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000;
   drop tmp;
 
   /* quick check to ensure column exists */
@@ -108,7 +109,8 @@ data &outds;
   end;
 
   /* need to open the dataset to get the column type */
-  dsid=open("&targetds","i");
+  retain dsid;
+  if _n_=1 then dsid=open("&targetds","i");
   if dsid>0 then do;
     vnum=varnum(dsid,VARIABLE_NM);
     if vnum<1 then do;
@@ -118,11 +120,19 @@ data &outds;
       call symputx('reason_cd',reason_cd,'l');
       call symputx('nobs',_n_,'l');
       output;
-      return;
+      goto endstep;
     end;
     /* now we can get the type */
     else vtype=vartype(dsid,vnum);
   end;
+  else do;
+      REASON_CD=cats("Could not open &targetds");
+      putlog REASON_CD= dsid=;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
+      output;
+      stop;
+  end;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
@@ -157,9 +167,8 @@ data &outds;
   end;
 
   /* special missing logic */
-  if vtype='N'
+  if vtype='N' & OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE') then do;
-    and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE')
+    if cats(upcase(raw_value)) in (
-    and cats(upcase(raw_value)) in (
       '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N'
       '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._'
     )
@@ -167,6 +176,32 @@ data &outds;
       /* valid numeric - exit data step loop */
       return;
     end;
+    else if subpad(upcase(raw_value),1,1) in (
+      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+    )
+    then do;
+      /* check if the raw_value contains a valid variable NAME */
+      vnum=varnum(dsid,subpad(raw_value,1,32));
+      if vnum>0 then do;
+        /* now we can get the type */
+        vtype2=vartype(dsid,vnum);
+        /* check type matches */
+        if vtype2=vtype then do;
+          /* valid target var - exit loop */
+          return;
+        end;
+        else do;
+          REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
+          putlog REASON_CD= dsid=;
+          call symputx('reason_cd',reason_cd,'l');
+          call symputx('nobs',_n_,'l');
+          output;
+          goto endstep;
+        end;
+      end;
+    end;
+  end;
 
   /* special logic */
   if OPERATOR_NM in ('IN','NOT IN','BETWEEN') then do;
@@ -187,6 +222,32 @@ data &outds;
     if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
       tmp=scan(raw_value1,i,',');
       if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
+        if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in (
+          'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+          'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+        )
+        then do;
+          /* check if the raw_value contains a valid variable NAME */
+          /* is not valid syntax for IN or NOT IN */
+          vnum=varnum(dsid,subpad(tmp,1,32));
+          if vnum>0 then do;
+            /* now we can get the type */
+            vtype2=vartype(dsid,vnum);
+            /* check type matches */
+            if vtype2=vtype then do;
+              /* valid target var - exit loop */
+              return;
+            end;
+            else do;
+              REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
+              putlog REASON_CD= dsid=;
+              call symputx('reason_cd',reason_cd,'l');
+              call symputx('nobs',_n_,'l');
+              output;
+              goto endstep;
+            end;
+          end;
+        end;
         REASON_CD='Non Numeric value provided';
         putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
         call symputx('reason_cd',reason_cd,'l');
@@ -211,14 +272,42 @@ data &outds;
 
   /* output records that contain values other than digits and spaces */
   if notdigit(compress(raw_value3,' '))>0 then do;
+    if vtype='C' and subpad(upcase(raw_value),1,1) in (
+      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+    )
+    then do;
+      /* check if the raw_value contains a valid variable NAME */
+      vnum=varnum(dsid,subpad(raw_value,1,32));
+      if vnum>0 then do;
+        /* now we can get the type */
+        vtype2=vartype(dsid,vnum);
+        /* check type matches */
+        if vtype2=vtype then do;
+          /* valid target var - exit loop */
+          return;
+        end;
+        else do;
+          REASON_CD=cats("Compared Char Type (",vtype2,") is not (",vtype,")");
+          putlog REASON_CD= dsid=;
+          call symputx('reason_cd',reason_cd,'l');
+          call symputx('nobs',_n_,'l');
+          output;
+          goto endstep;
+        end;
+      end;
+    end;
+
     putlog raw_value3= $hex32.;
     REASON_CD=cats('Invalid RAW_VALUE:',raw_value);
-    putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
+    putlog (_all_)(=);
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');
     output;
   end;
 
+  endstep:
+  if last then rc=close(dsid);
 run;
 
 

base/mp_filtergenerate.sas

@@ -51,8 +51,8 @@
   > )
 
   @param [in] inds The input table with query values
-  @param [out] outref= The output fileref to contain the filter clause.  Will
+  @param [out] outref= (filter) The output fileref to contain the filter clause.
-    be created (or replaced).
+    Will be created (or replaced).
 
   <h4> Related Macros </h4>
   @li mp_filtercheck.sas

base/mp_filterstore.sas

@@ -44,9 +44,10 @@
     mp_coretable.sas as follows:  `mp_coretable(LOCKTABLE)`.
   @param [in] maxkeytable= (0) Optional permanent reference table used for
     retained key tracking.  Described in mp_retainedkey.sas.
-  @param [in] mdebug= set to 1 to enable DEBUG messages
+  @param [in] mdebug= (1) set to 1 to enable DEBUG messages
-  @param [out] outresult= The result table with the FILTER_RK
+  @param [out] outresult= (work.result) The result table with the FILTER_RK
-  @param [out] outquery= The original query, taken as extract after table load
+  @param [out] outquery= (work.query) The original query, taken as extract
+    after table load
 
 
   <h4> SAS Macros </h4>

base/mp_getcols.sas

@@ -12,14 +12,15 @@
 
       %mp_getcols(sashelp.airline,outds=work.myds)
 
-  @param ds The dataset from which to obtain column metadata
+  @param [in] ds The dataset from which to obtain column metadata
-  @param outds= (work.cols) The output dataset to create. Sample data:
+  @param [out] outds= (work.cols) The output dataset to create. Sample data:
 |NAME:$32.|LENGTH:best.|VARNUM:best.|LABEL:$256.|FMTNAME:$32.|FORMAT:$49.|TYPE:$1.|DDTYPE:$9.|
 |---|---|---|---|---|---|---|---|
 |`AIR `|`8 `|`2 `|`international airline travel (thousands) `|` `|`8. `|`N `|`NUMERIC `|
 |`DATE `|`8 `|`1 `|`DATE `|`MONYY `|`MONYY. `|`N `|`DATE `|
 |`REGION `|`3 `|`3 `|`REGION `|` `|`$3. `|`C `|`CHARACTER `|
 
+
   <h4> Related Macros </h4>
   @li mf_getvarlist.sas
   @li mm_getcols.sas

base/mp_getdbml.sas

@@ -23,10 +23,10 @@
   @li mf_getquotedstr.sas
   @li mp_getconstraints.sas
 
-  @param liblist= Space seperated list of librefs to take as
+  @param [in] liblist= (SASHELP) Space seperated list of librefs to take as
-    input (Default=SASHELP)
+    input
-  @param outref= Fileref to contain the DBML (Default=getdbml)
+  @param [out] outref= (getdbml) Fileref to contain the DBML
-  @param showlog= set to YES to show the DBML in the log (Default is NO)
+  @param [in] showlog= (NO) set to YES to show the DBML in the log
 
   @version 9.3
   @author Allan Bowe

base/mp_getddl.sas

@@ -22,16 +22,21 @@
   @li mf_getvarcount.sas
   @li mp_getconstraints.sas
 
-  @param lib libref of the library to create DDL for.  Should be assigned.
+  @param [in] libref Libref of the library to create DDL for. Should already
-  @param ds dataset to create ddl for (optional)
+    be assigned.
-  @param fref= the fileref to which to _append_ the DDL.  If it does not exist,
+  @param [in] ds dataset to create ddl for (optional)
-    it will be created.
+  @param [in] fref= (getddl) the fileref to which to _append_ the DDL.  If it
-  @param flavour= The type of DDL to create (default=SAS). Supported=TSQL
+    does not exist, it will be created.
-  @param showlog= Set to YES to show the DDL in the log
+  @param [in] flavour= (SAS) The type of DDL to create. Options:
-  @param schema= Choose a preferred schema name (default is to use actual schema
+    @li SAS
-    ,else libref)
+    @li TSQL
-  @param applydttm= for non SAS DDL, choose if columns are created with native
+
-    datetime2 format or regular decimal type
+  @param [in]showlog= (NO) Set to YES to show the DDL in the log
+  @param [in] schema= () Choose a preferred schema name (default is to use
+    actual schema, else libref)
+  @param [in] applydttm= (NO) For non SAS DDL, choose if columns are created
+    with native datetime2 format or regular decimal type
+
   @version 9.3
   @author Allan Bowe
 **/

base/mp_gsubfile.sas

@@ -22,15 +22,15 @@
         list;
       run;
 
-  @param file= (0) The file to perform the substitution on
+  @param [in] file= (0) The file to perform the substitution on
-  @param patternvar= A macro variable containing the Lua
+  @param [in] patternvar= A macro variable containing the Lua
     [pattern](https://www.lua.org/pil/20.2.html) to search for.  Due to the use
     of special (magic) characters in Lua patterns, it is safer to pass the NAME
     of the macro variable containing the string, rather than the value itself.
-  @param replacevar= The name of the macro variable containing the replacement
+  @param [in] replacevar= ()
-    _string_.
+    The name of the macro variable containing the replacement _string_.
-  @param outfile= (0) The file to write the output to. If zero, then the file
+  @param [out] outfile= (0) The file to write the output to.
-    is overwritten in-place.
+    If zero, then the file is overwritten in-place.
 
   <h4> SAS Macros </h4>
   @li ml_gsubfile.sas

base/mp_guesspk.sas

@@ -18,7 +18,7 @@
       %mp_guesspk(sashelp.class,outds=classpks)
 
   @param [in] baseds The dataset to analyse
-  @param [out] outds= The output dataset to contain the possible PKs
+  @param [out] outds= (mp_guesspk) Output dataset to contain the possible PKs
   @param [in] max_guesses= (3) The total number of possible primary keys to
     generate. A table may have multiple (unlikely) PKs, so no need to list them
     all.

base/mp_hashdataset.sas

@@ -24,7 +24,7 @@
   @li mp_hashdirectory.sas
 
   @param [in] libds dataset to hash
-  @param [in] salt= Provide a salt (could be, for instance, the dataset name)
+  @param [in] salt= () Provide a salt (could be, for instance, the dataset name)
   @param [in] iftrue= (1=1) A condition under which the macro should be executed
   @param [out] outds= (work._data_) The output dataset to create. This
   will contain one column (hashkey) with one observation (a $hex32.

base/mp_lib2cards.sas

@@ -24,19 +24,19 @@
   @li mf_trimstr.sas
   @li mp_ds2cards.sas
 
-  @param [in] lib= Library in which to convert all datasets
+  @param [in] lib= () Library in which to convert all datasets
-  @param [out] outloc= Location in which to store output.  Defaults to WORK
+  @param [out] outloc= (%sysfunc(pathname(work))) Location in which to store
-    library. No quotes.
+    output. No quotes.
-  @param [out] outfile= Optional output file NAME - if provided, then will create
+  @param [out] outfile= (0) Optional output file NAME - if provided, then
-  a single output file instead of one file per input table.
+    will create a single output file instead of one file per input table.
-  @param [in] maxobs= limit output to the first <code>maxobs</code> observations
+  @param [in] maxobs= (max) limit output to the first <code>maxobs</code> rows
 
   @version 9.2
   @author Allan Bowe
 **/
 
 %macro mp_lib2cards(lib=
-    ,outloc=%sysfunc(pathname(work)) /* without trailing slash */
+    ,outloc=%sysfunc(pathname(work))
     ,maxobs=max
     ,random_sample=NO
     ,outfile=0

base/mp_loadformat.sas

@@ -9,6 +9,9 @@
   format, to prevent loss of data - UNLESS the input dataset contains a marker
   column, specifying that a particular row needs to be deleted (`delete_col=`).
 
+  Positions of formats are made using the FMTROW variable - this must be present
+  and unique (on TYPE / FMTNAME / FMTROW).
+
   This macro can also be used to identify which records would be (or were)
   considered new, modified or deleted (`loadtarget=`) by creating the following
   tables:
@@ -17,7 +20,7 @@
   @li work.outds_del
   @li work.outds_mod
 
-  For example usage, see mp_loadformat.test.sas
+  For example usage, see test (under Related Macros)
 
   @param [in] libcat The format catalog to be loaded
   @param [in] libds The staging table to load
@@ -34,12 +37,15 @@
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_existvar.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
+  @li mp_md5.sas
   @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
@@ -83,6 +89,16 @@
 %let libcat=%scan(&libcat,1,-);
 
 /* perform input validations */
+%mp_abort(
+  iftrue=(%mf_existds(&libds)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(&libds could not be found)
+)
+%mp_abort(
+  iftrue=(%mf_existvar(&libds,FMTROW)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(FMTROW not found in &libds)
+)
 %let err=0;
 %let msg=0;
 data _null_;
@@ -103,13 +119,6 @@ data _null_;
       stop;
     end;
   end;
-  else if name='LIBDS' then do;
-    if exist(value) le 0 then do;
-      call symputx('msg',"Unable to open staging table: "!!value);
-      call symputx('err',1);
-      stop;
-    end;
-  end;
   else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
   and missing(value) then do;
     call symputx('msg',"missing value in var: "!!name);
@@ -117,6 +126,14 @@ data _null_;
     stop;
   end;
 run;
+data _null_;
+  set &libds;
+  if missing(fmtrow) then do;
+    call symputx('msg',"missing fmtrow in format: "!!FMTNAME);
+    call symputx('err',1);
+    stop;
+  end;
+run;
 
 %mp_abort(
   iftrue=(&err ne 0)
@@ -124,6 +141,15 @@ run;
   ,msg=%str(&msg)
 )
 
+%local cnt;
+proc sql noprint;
+select count(distinct catx('|',type,fmtname,fmtrow)) into: cnt from &libds;
+%mp_abort(
+  iftrue=(&cnt ne %mf_nobs(&libds))
+  ,mac=&sysmacroname
+  ,msg=%str(Non-unique primary key on &libds)
+)
+
 /**
   * First, extract only relevant formats from the catalog
   */
@@ -177,12 +203,6 @@ data &inlibds/nonote2err;
     %mp_aligndecimal(end,width=16)
   end;
 
-  /* update row marker - retain new var as fmtrow may already be in libds */
-  if first.fmtname then row=1;
-  else row+1;
-  drop row;
-  fmtrow=row;
-
   fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
 run;
 

base/mp_md5.sas

@@ -28,8 +28,8 @@
   @li Global option:  `options dsoptions=nonote2err;`
   @li Data step option: `data YOURLIB.YOURDATASET /nonote2err;`
 
-  @param cvars= Space seperated list of character variables
+  @param [in] cvars= () Space seperated list of character variables
-  @param nvars= Space seperated list of numeric variables
+  @param [in] nvars= () Space seperated list of numeric variables
 
   <h4> Related Programs </h4>
   @li mp_init.sas

base/mp_perflog.sas

@@ -11,9 +11,9 @@
       %mp_perflog(finished)
 
 
-  @param label Provide label to go into the control dataset
+  @param [in] label Provide label to go into the control dataset
-  @param libds= Provide a dataset in which to store performance stats.  Default
+  @param [in] libds= (work.mp_perflog) Provide a dataset in which to store
-              name is <code>work.mp_perflog</code>;
+    performance stats.  Default name is <code>work.mp_perflog</code>;
 
   @version 9.2
   @author Allan Bowe

base/mp_prevobs.sas

@@ -34,12 +34,12 @@
   Credit is made to `data _null_` for authoring this very helpful paper:
   https://www.lexjansen.com/pharmasug/2008/cc/CC08.pdf
 
-  @param action Either FETCH a current or previous record, or INITialise.
+  @param [in] action Either FETCH a current or previous record, or INITialise.
-  @param record The relative (to current) position of the previous observation
+  @param [in] record The relative (to current) position of the previous row
     to return.
-  @param history= The number of records to retain in the hash table. Default=5
+  @param [in] history= (5) The number of records to retain in the hash table.
-  @param prefix= the prefix to give to the variables used to store the hash name
+  @param [in] prefix= (mp_prevobs) The prefix to give to the variables used to
-    and index. Default=mp_prevobs
+    store the hash name and index.
 
   @version 9.2
   @author Allan Bowe

base/mp_recursivejoin.sas

@@ -23,12 +23,13 @@
         ,childvar=c
         )
 
-  @param base_ds= base table containing hierarchy (not modified)
+  @param [in] base_ds= base table containing hierarchy (not modified)
-  @param outds= the output dataset to create with the generated hierarchy
+  @param [out] outds= the output dataset to create with the generated hierarchy
-  @param matchval= the ultimate parent from which to filter
+  @param [in] matchval= the ultimate parent from which to filter
-  @param parentvar= name of the parent variable
+  @param [in] parentvar= name of the parent variable
-  @param childvar= name of the child variable (should be same type as parent)
+  @param [in] childvar= () name of the child variable (should be same type as
-  @param mdebug= set to 1 to prevent temp tables being dropped
+    parent)
+  @param [in] mdebug= set to 1 to prevent temp tables being dropped
 
 
   @returns outds contains the following variables:

base/mp_replace.sas

@@ -32,10 +32,11 @@
   Note - if you are running a version of SAS that will allow the io package in
   LUA, you can also use this macro: mp_gsubfile.sas
 
-  @param infile The QUOTED path to the file on which to perform the substitution
+  @param [in] infile The QUOTED path to the file on which to perform the
-  @param findvar= Macro variable NAME containing the string to search for
+    substitution
-  @param replacevar= Macro variable NAME containing the replacement string
+  @param [in] findvar= Macro variable NAME containing the string to search for
-  @param outfile= (0) Optional QUOTED path to the adjusted output file (to
+  @param [in] replacevar= Macro variable NAME containing the replacement string
+  @param [out] outfile= (0) Optional QUOTED path to the adjusted output file (to
     avoid overwriting the first file).
 
   <h4> SAS Macros </h4>

base/mp_runddl.sas

@@ -31,9 +31,9 @@
     %mp_runddl(/some/rootlib, exc=LIBREF3) * same as above ;
 
 
-  @param path location of the DDL folder structure
+  @param [in] path location of the DDL folder structure
-  @param inc= list of librefs to include
+  @param [in] inc= list of librefs to include
-  @param exc= list of librefs to exclude (takes precedence over inc=)
+  @param [in] exc= list of librefs to exclude (takes precedence over inc=)
 
   @version 9.3
   @author Allan Bowe

base/mp_searchcols.sas

@@ -9,12 +9,13 @@
 
       %mp_searchcols(libs=sashelp work, cols=name sex age)
 
-  @param libs=(SASHELP) Space separated list of libraries to search for columns
+  @param [in] libs= (SASHELP)
-  @param cols= Space separated list of column names to search for (not case
+    Space separated list of libraries to search for columns
-    sensitive)
+  @param [in] cols=
-  @param outds=(mp_searchcols) the table to create with the results.  Will have
+    Space separated list of column names to search for (not case sensitive)
-    one line per table match.
+  @param [out] outds= (mp_searchcols)
-  @param match=(ANY) The match type. Valid values:
+    The table to create with the results.  Will have one line per table match.
+  @param [in] match= (ANY) The match type. Valid values:
     @li ANY - The table contains at least one of the columns
     @li WILD - The table contains a column with a name that partially matches
 

base/mp_stackdiffs.sas

@@ -197,6 +197,7 @@
   @li mp_coretable.sas
   @li mp_stackdiffs.test.sas
   @li mp_storediffs.sas
+  @li mp_stripdiffs.sas
 
   @todo The current approach assumes that a variable called KEY_HASH is not on
     the base table.  This part will need to be refactored (eg using

base/mp_storediffs.sas

@@ -64,6 +64,7 @@
   <h4> Related Macros </h4>
   @li mp_stackdiffs.sas
   @li mp_storediffs.test.sas
+  @li mp_stripdiffs.sas
 
   @version 9.2
   @author Allan Bowe
@@ -183,7 +184,7 @@ data &ds4;
 run;
 
 %if "&loadref"="0" %then %let loadref=%sysfunc(uuidgen());
-%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime());
+%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime(),8.6);
 %let libds=%upcase(&libds);
 
 /* join orig vals for modified & deleted */

base/mp_stprequests.sas

@@ -21,9 +21,10 @@
     %mp_stprequests(status_cd=INIT, libds=YOURLIB.DATASET )
 
 
-  @param status_cd= Use INIT for INIT and TERM for TERM events
+  @param [in] status_cd= Use INIT for INIT and TERM for TERM events
-  @param libds= Location of base table (library.dataset).  To minimise risk
+  @param [in] libds= (somelib.stp_requests) Location of base table
-    of table locks, we HIGHLY recommend using a database (NOT a SAS dataset).
+    (library.dataset).  To minimise risk of table locks, we HIGHLY recommend
+    using a database (NOT a SAS dataset).
     THE LIBRARY SHOULD BE ASSIGNED ALREADY - eg in autoexec or earlier in the
     init program proper.
 

base/mp_stripdiffs.sas

@@ -0,0 +1,252 @@
+/**
+  @file
+  @brief Generates a stage dataset to revert diffs tracked in an audit table
+  @details A big benefit of tracking data changes in an audit table is that
+    those changes can be subsequently reverted if necessary!
+
+    This macro prepares a staging dataset containing those differences - eg for:
+
+    @li deleted rows - these are re-inserted
+    @li changed rows - differences are reverted
+    @li added rows - marked with `_____DELETE__THIS__RECORD_____="YES"`
+
+    These changes are NOT applied to the base table - a staging dataset is
+    simply prepared for an ETL process to action.  In Data Controller, this
+    dataset is used directly as an input to the APPROVE process (so that the
+    reversion diffs can be reviewed prior to being applied).
+
+
+  @param [in] libds Base library.dataset (will not be modified).  The library
+    must be assigned.
+  @param [in] loadref Unique identifier for the version to be reverted.  This
+    change, plus ALL SUBSEQUENT CHANGES, will be reverted in the output table.
+  @param [in] difftable The dataset containing the diffs.  Definition available
+    in mddl_dc_difftable.sas
+  @param [out] outds= (work.mp_stripdiffs) Output table containing the diffs.
+    Has the same format as the base datset, plus a
+    `_____DELETE__THIS__RECORD_____` variable.
+  @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
+  @li mf_islibds.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_abort.sas
+
+  <h4> Related Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mp_stackdiffs.sas
+  @li mp_storediffs.sas
+  @li mp_stripdiffs.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mp_stripdiffs(libds
+  ,loadref
+  ,difftable
+  ,outds=work.mp_stripdiffs
+  ,mdebug=0
+)/*/STORE SOURCE*/;
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+%let libds=%upcase(&libds);
+
+/* safety checks */
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc on entry.  Clean session required!)
+)
+%let libds=%upcase(&libds);
+%mp_abort(iftrue= (%mf_islibds(&libds)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid library.dataset reference - %superq(libds))
+)
+
+
+
+/* set up unique and temporary vars */
+%local ds1 ds2 ds3 ds4 ds5 fref1;
+%let fref1=%mf_getuniquefileref();
+
+/* get timestamp of the diff to be reverted */
+%local ts;
+proc sql noprint;
+select put(processed_dttm,datetime19.6) into: ts
+  from &difftable where load_ref="&loadref";
+%mp_abort(iftrue= (&sqlobs=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Load ref %superq(loadref) not found!)
+)
+
+/* extract diffs for this base table from this timestamp onwards */
+%let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_diffs));
+create table &ds1 (drop=libref dsn) as
+  select * from &difftable
+  where upcase(cats(libref))="%scan(&libds,1,.)"
+  and upcase(cats(dsn))="%scan(&libds,2,.)"
+  and processed_dttm ge "&ts"dt
+  order by processed_dttm desc, key_hash, is_pk;
+
+/* extract key values only */
+%let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_pks));
+%local keyhash processed;
+%let keyhash=%upcase(%mf_getuniquename(prefix=mpsdvar_keyhash));
+%let processed=%upcase(%mf_getuniquename(prefix=mpsdvar_processed));
+create table &ds2 as
+  select key_hash as &keyhash,
+    tgtvar_nm,
+    tgtvar_type,
+    coalescec(oldval_char,newval_char) as charval,
+    coalesce(oldval_num, newval_num) as numval,
+    processed_dttm as &processed
+  from &ds1
+  where is_pk=1
+  order by &keyhash, &processed;
+
+/* grab pk values */
+%local pk;
+select distinct upcase(tgtvar_nm) into: pk separated by ' ' from &ds2;
+
+%let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_keychar));
+proc transpose data=&ds2(where=(tgtvar_type='C'))
+    out=&ds3(drop=_name_);
+  by &keyhash &processed;
+  id TGTVAR_NM;
+  var charval;
+run;
+
+%let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_keynum));
+proc transpose data=&ds2(where=(tgtvar_type='N'))
+    out=&ds4(drop=_name_);
+  by &keyhash &processed;
+  id TGTVAR_NM;
+  var numval;
+run;
+/* shorten the lengths */
+%mp_ds2squeeze(&ds3,outds=&ds3)
+%mp_ds2squeeze(&ds4,outds=&ds4)
+
+/* now merge to get all key values and de-dup */
+%let ds5=%upcase(work.%mf_getuniquename(prefix=mpsd_merged));
+data &ds5;
+  length &keyhash $32 &processed 8;
+  merge &ds3 &ds4;
+  by &keyhash &processed;
+  if not missing(&keyhash);
+run;
+proc sort data=&ds5 nodupkey;
+  by &pk;
+run;
+
+/* join to base table for preliminary stage DS */
+proc sql;
+create table &outds as select "No " as _____DELETE__THIS__RECORD_____
+  %do x=1 %to %sysfunc(countw(&pk,%str( )));
+    ,a.%scan(&pk,&x,%str( ))
+  %end;
+  %local notpkcols;
+  %let notpkcols=%upcase(%mf_getvarlist(&libds));
+  %let notpkcols=%mf_wordsinstr1butnotstr2(str1=&notpkcols,str2=&pk);
+  %do x=1 %to %sysfunc(countw(&notpkcols,%str( )));
+    ,b.%scan(&notpkcols,&x,%str( ))
+  %end;
+  from &ds5 a
+  left join &libds b
+  on 1=1
+%do x=1 %to %sysfunc(countw(&pk,%str( )));
+  and a.%scan(&pk,&x,%str( ))=b.%scan(&pk,&x,%str( ))
+%end;
+;
+
+/* create SAS code to apply to stage_ds */
+data _null_;
+  set &ds1;
+  file &fref1 lrecl=33000;
+  length charval $32767;
+  if _n_=1 then put 'proc sql noprint;';
+  by descending processed_dttm key_hash is_pk;
+  if move_type='M' then do;
+    if first.key_hash then do;
+      put "update &outds set " @@;
+    end;
+    if IS_PK=0 then do;
+      put "  " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+      if not last.is_pk then put ',';
+    end;
+    else do;
+      if first.is_pk then put "  where 1=1 " @@;
+      put "  and " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+  end;
+  else if move_type='A' then do;
+    if first.key_hash then do;
+      put "update &outds set _____DELETE__THIS__RECORD_____='Yes' where 1=1 "@@;
+    end;
+    /* gating if - as only need PK now */
+    if is_pk=1;
+    put '  AND ' tgtvar_nm '=' @@;
+    cnt=count(newval_char,'"');
+    charval=quote(trim(substr(newval_char,1,32765-cnt)));
+    if tgtvar_type='C' then put charval @@;
+    else put newval_num @@;
+  end;
+  else if move_type='D' then do;
+    if first.key_hash then do;
+      put "update &outds set _____DELETE__THIS__RECORD_____='No' " @@;
+    end;
+    if IS_PK=0 then do;
+      put "  ," tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+    else do;
+      if first.is_pk then put "  where 1=1 " @@;
+      put "  and " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+  end;
+  if last.key_hash then put ';';
+run;
+
+/* apply the modification statements */
+%inc &fref1/source2 lrecl=33000;
+
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &ds1, &ds2, &ds3, &ds4, &ds5;
+  file &fref1 clear;
+%end;
+%else %do;
+  data _null_;
+    infile &fref1;
+    input;
+    if _n_=1 then putlog "Contents of SQL adjustments";
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_stripdiffs;
+/** @endcond */

base/mp_testjob.sas

@@ -8,8 +8,8 @@
         duration=60*5
       )
 
-  @param [in] duration= the time in seconds which the job should run for. Actual
+  @param [in] duration= (30) The time in seconds which the job should run for.
-  time may vary, as the check is done in between steps.  Default = 30 (seconds).
+    Actual time may vary, as the check is done in between steps.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquelibref.sas

base/mp_testwritespeedlibrary.sas

@@ -11,9 +11,9 @@
         ,outds=work.results
       )
 
-  @param lib= (WORK) The library in which to create the table
+  @param [in] lib= (WORK) The library in which to create the table
-  @param size= (0.1) The size in GB of the table to create
+  @param [in] size= (0.1) The size in GB of the table to create
-  @param outds= (WORK.RESULTS) The output dataset to be created.
+  @param [out] outds= (WORK.RESULTS) The output dataset to be created.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas

base/mp_tree.sas

@@ -15,8 +15,8 @@ https://communities.sas.com/t5/SAS-Programming/listing-all-files-within-a-direct
 https://communities.sas.com/t5/SAS-Programming/listing-all-files-of-all-types-from-all-subdirectories/m-p/334113/highlight/true#M75419
 
 
-  @param dir= Directory to be scanned (default=/tmp)
+  @param [in] dir= (/tmp) Directory to be scanned
-  @param outds= Dataset to create (default=work.mp_tree)
+  @param [out] outds= (work.mp_tree) Dataset to create
 
   @returns outds contains the following variables:
 

base/mp_unzip.sas

@@ -13,11 +13,12 @@
 
       %mp_unzip(ziploc="/some/file.zip",outdir=/some/folder)
 
-  More info:  https://blogs.sas.com/content/sasdummy/2015/05/11/using-filename-zip-to-unzip-and-read-data-files-in-sas/
+  More info:
+  https://blogs.sas.com/content/sasdummy/2015/05/11/using-filename-zip-to-unzip-and-read-data-files-in-sas/
 
-  @param ziploc= Fileref or quoted full path to zip file ("/path/to/file.zip")
+  @param [in] ziploc= Fileref or quoted full path, eg: "/path/to/file.zip"
-  @param outdir= (%sysfunc(pathname(work))) Directory in which to write the
+  @param [out] outdir= (%sysfunc(pathname(work))) Directory in which to write
-    outputs (created if non existant)
+    the outputs (created if needed)
 
   <h4> SAS Macros </h4>
   @li mf_mkdir.sas

base/mp_updatevarlength.sas

@@ -14,9 +14,9 @@
         proc sql;
         describe table example;
 
-  @param libds the library.dataset to be modified
+  @param [in] libds the library.dataset to be modified
-  @param var The variable to modify
+  @param [in] var The variable to modify
-  @param len The new length to apply
+  @param [in] len The new length to apply
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas

base/mp_validatecol.sas

@@ -57,7 +57,7 @@
 %end;
 %else %if &rule=ISNUM %then %do;
   /*
-    credit SØREN LASSEN
+    credit SOREN LASSEN
     https://sasmacro.blogspot.com/2009/06/welcome-isnum-macro.html
   */
   &tempcol=input(&incol,?? best32.);

base/mp_zip.sas

@@ -15,14 +15,14 @@
   <h4> SAS Macros </h4>
   @li mp_dirlist.sas
 
-  @param in= unquoted filepath, dataset of files or directory to zip
+  @param [in] in= unquoted filepath, dataset of files or directory to zip
-  @param type= (FILE) Valid values:
+  @param [in] type= (FILE) Valid values:
     @li FILE - /full/path/and/filename.extension to a particular file
     @li DATASET - a dataset containing a list of files to zip (see `incol`)
     @li DIRECTORY - a directory to zip
-  @param outname= (FILE) Output file to create, _without_ .zip extension
+  @param [out] outname= (FILE) Output file to create, _without_ .zip extension
-  @param outpath= (%sysfunc(pathname(WORK))) Parent folder for output zip file
+  @param [out] outpath= (%sysfunc(pathname(WORK))) Parent folder for zip file
-  @param incol= if DATASET input, say which column contains the filepath
+  @param [in] incol= () If DATASET input, say which column contains the filepath
 
   <h4> Related Macros </h4>
   @li mp_unzip.sas

ddl/mddl_sas_cntlout.sas

@@ -6,6 +6,22 @@
   (given various practical restrictions) are described here to enable
   consistency when dealing with format data.
 
+  The HLO variable may have a number of values, documented here due to the
+  256 char label description length limit:
+
+    F=Standard format/informat.
+    H=Range ending value is HIGH.
+    I=Numeric informat.
+    J=Justification for an informat.
+    L=Range starting value is LOW.
+    M=MultiLabel.
+    N=Format or informat has no ranges, including no OTHER= range.
+    O=Range is OTHER.
+    R=ROUND option is in effect.
+    S=Specifies that NOTSORTED is in effect.
+    U=Specifies that the UPCASE option for an informat be used.
+
+
 **/
 
 
@@ -13,9 +29,11 @@
 
   proc sql;
   create table &libds(
-    TYPE char(1) label='Type of format - either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
+    TYPE char(1) label=
+'Format Type: either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
     ,FMTNAME char(32)     label='Format name'
-    ,FMTROW num label='CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
+    ,FMTROW num label=
+'CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
     ,START char(32767)    label='Starting value for format'
     /*
       Keep lengths of START and END the same to avoid this err:
@@ -35,7 +53,8 @@
     ,NOEDIT num length=3  label='Is picture string noedit?'
     ,SEXCL char(1)        label='Start exclusion'
     ,EEXCL char(1)        label='End exclusion'
-    ,HLO char(13)         label='Additional information. M=MultiLabel'
+    ,HLO char(13) label=
+'More info: https://core.sasjs.io/mddl__sas__cntlout_8sas_source.html'
     ,DECSEP char(1)       label='Decimal separator'
     ,DIG3SEP char(1)      label='Three-digit separator'
     ,DATATYPE char(8)     label='Date/time/datetime?'

meta/mm_adduser2group.sas

@@ -13,9 +13,9 @@
         ,group=someGroup)
 
 
-  @param user= the user name (not displayname)
+  @param [in] user= the user name (not displayname)
-  @param group= the group to which to add the user
+  @param [in] group= the group to which to add the user
-  @param mdebug= (0) set to 1 to show debug info in log
+  @param [in] mdebug= (0) set to 1 to show debug info in log
 
   <h4> Related Files </h4>
   @li ms_adduser2group.sas

meta/mm_assigndirectlib.sas

@@ -18,12 +18,13 @@
   @li mf_getengine.sas
   @li mp_abort.sas
 
-  @param libref the libref (not name) of the metadata library
+  @param [in] libref the libref (not name) of the metadata library
-  @param open_passthrough= provide an alias to produce the CONNECT TO statement
+  @param [in] open_passthrough= () Provide an alias to produce the CONNECT TO
-    for the relevant external database
+    statement for the relevant external database
-  @param sql_options= an override default output fileref to avoid naming clash
+  @param [in] sql_options= () Add any options to add to proc sql statement,
-  @param mDebug= set to 1 to show debug messages in the log
+    eg outobs= (only valid for pass through)
-  @param mAbort= set to 1 to call %mp_abort().
+  @param [in] mDebug= (0) set to 1 to show debug messages in the log
+  @param [in] mAbort= (0) set to 1 to call %mp_abort().
 
   @returns libname statement
 
@@ -33,12 +34,9 @@
 **/
 
 %macro mm_assigndirectlib(
-    libref /* libref to assign from metadata */
+    libref
-    ,open_passthrough= /* provide an alias to produce the
+    ,open_passthrough=
-                          CONNECT TO statement for the
+    ,sql_options=
-                          relevant external database */
-    ,sql_options= /* add any options to add to proc sql statement eg outobs=
-                      (only valid for pass through) */
     ,mDebug=0
     ,mAbort=0
 )/*/STORE SOURCE*/;
@@ -212,9 +210,9 @@ run;
   %end;
 %end;
 %else %if &engine=ODBC %then %do;
-  &mD.%put NOTE: Retrieving ODBC connection details;
+  %&mD.put NOTE: Retrieving ODBC connection details;
   data _null_;
-    length connx_uri conprop_uri value datasource up_uri schema $256.;
+    length connx_uri conprop_uri value datasource up_uri schema domprop_uri authdomain $256.;
     call missing (of _all_);
     /* get source connection ID */
     rc=metadata_getnasn("&liburi",'LibraryConnection',1,connx_uri);
@@ -229,6 +227,13 @@ run;
         rc2=-1;
       end;
     end;
+
+    /* get auth domain */
+    autrc=metadata_getnasn(connx_uri,"Domain",1,domprop_uri);
+    arc=metadata_getattr(domprop_uri,"Name",authdomain);
+    if not missing(authdomain) then authdomain=cats('AUTHDOMAIN=',authdomain);
+    call symputx('authdomain',authdomain,'l');
+
     /* get SCHEMA */
     rc6=metadata_getnasn("&liburi",'UsingPackages',1,up_uri);
     rc7=metadata_getattr(up_uri,'SchemaName',schema);
@@ -245,7 +250,7 @@ run;
       (INSERT_SQL=YES DATASRC=&sql_dsn. CONNECTION=global);
   %end;
   %else %do;
-    libname &libref ODBC DATASRC=&sql_dsn SCHEMA=&sql_schema;
+    libname &libref ODBC DATASRC=&sql_dsn SCHEMA=&sql_schema &authdomain;
   %end;
 %end;
 %else %if &engine=POSTGRES %then %do;

meta/mm_assignlib.sas

@@ -14,8 +14,8 @@
   @li mp_abort.sas
 
   @param [in] libref The libref (not name) of the metadata library
-  @param [in] mAbort= If not assigned, HARD will call %mp_abort(), SOFT will
+  @param [in] mAbort= (HARD) If not assigned, HARD will call %mp_abort(), SOFT
-    silently return
+    will silently return
 
   @returns libname statement
 

meta/mm_createapplication.sas

@@ -21,21 +21,24 @@
   @li mp_abort.sas
   @li mf_verifymacvars.sas
 
-  @param tree= The metadata folder uri, or the metadata path, in which to
+  @param [in] tree= The metadata folder uri, or the metadata path, in which to
     create the object.  This must exist.
-  @param name= Application object name.  Avoid spaces.
+  @param [in] name= Application object name.  Avoid spaces.
-  @param ClassIdentifier= the class of applications to which this app belongs
+  @param [in] ClassIdentifier=
-  @param params= name=value pairs which will become public properties of the
+    The class of applications to which this app belongs
+  @param [in] params=
+    name=value pairs which will become public properties of the
     application object. These are delimited using 
 (newline character)
 
-  @param desc= Application description (optional).  Avoid ampersands as these
+  @param [in] desc= Application description (optional).
+    Avoid ampersands as these
     are illegal characters (unless they are escapted- eg &)
-  @param version= version number of application
+  @param [in] version= version number of application
-  @param frefin= fileref to use (enables change if there is a conflict).  The
+  @param [in] frefin= fileref to use (enables change if there is a conflict).
-    filerefs are left open, to enable inspection after running the
+    The filerefs are left open, to enable inspection after running the
     macro (or importing into an xmlmap if needed).
-  @param frefout= fileref to use (enables change if there is a conflict)
+  @param [out] frefout= fileref to use (enables change if there is a conflict)
-  @param mDebug= set to 1 to show debug messages in the log
+  @param [in] mDebug= set to 1 to show debug messages in the log
 
   @author Allan Bowe
 

meta/mm_createdataset.sas

@@ -22,12 +22,12 @@
   @li mm_gettables.sas
   @li mm_getcols.sas
 
-  @param libds= library.dataset metadata source.  Note - table names in metadata
+  @param [in] libds= library.dataset metadata source.  Note - table names in metadata
     can be longer than 32 chars (just fyi, not an issue here)
-  @param tableuri= Metadata URI of the table to be created
+  @param [in] tableuri= Metadata URI of the table to be created
-  @param outds= (work.mm_createdataset) The dataset to create.  The table name
+  @param [out] outds= (work.mm_createdataset) The dataset to create.  The table
-    needs to be 32 chars or less as per SAS naming rules.
+    name needs to be 32 chars or less as per SAS naming rules.
-  @param mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
 
   @version 9.4
   @author Allan Bowe

meta/mm_createdocument.sas

@@ -17,15 +17,15 @@
   @li mf_verifymacvars.sas
 
 
-  @param tree= The metadata folder uri, or the metadata path, in which to
+  @param [in] tree= The metadata folder uri, or the metadata path, in which to
     create the document.  This must exist.
-  @param name= Document object name.  Avoid spaces.
+  @param [in] name= Document object name.  Avoid spaces.
 
-  @param desc= Document description (optional)
+  @param [in] desc= Document description (optional)
-  @param textrole= TextRole property (optional)
+  @param [in] textrole= TextRole property (optional)
-  @param frefin= fileref to use (enables change if there is a conflict)
+  @param [in] frefin= fileref to use (enables change if there is a conflict)
-  @param frefout= fileref to use (enables change if there is a conflict)
+  @param [out] frefout= fileref to use (enables change if there is a conflict)
-  @param mDebug= set to 1 to show debug messages in the log
+  @param [in] mDebug= set to 1 to show debug messages in the log
 
   @author Allan Bowe
 

meta/mm_createlibrary.sas

@@ -24,30 +24,33 @@
   @li mm_createfolder.sas
 
 
-  @param libname= Library name (as displayed to user, 256 chars). Duplicates
+  @param [in] libname= (My New Library) Library name (as displayed to user,
-    are not created (case sensitive).
+    256 chars). Duplicates are not created (case sensitive).
-  @param libref= Library libref (8 chars).  Duplicate librefs are not created,
+  @param [in] libref= (mynewlib) Library libref (8 chars).  Duplicate
-    HOWEVER- the check is not case sensitive - if *libref* exists, *LIBREF*
+    librefs are not created, HOWEVER- the check is not case sensitive - if
-    will still be created.   Librefs created will always be uppercased.
+    *libref* exists, *LIBREF* will still be created.
-  @param engine= Library engine (currently only BASE supported)
+    Librefs created will always be uppercased.
-  @param tree= The metadata folder uri, or the metadata path, in which to
+  @param [in] engine= Library engine (currently only BASE supported)
+  @param [in] tree= The metadata folder uri, or the metadata path, in which to
     create the library.
-  @param servercontext= The SAS server against which the library is registered.
+  @param [in] servercontext= (SASApp) The SAS server against which
-  @param IsPreassigned= set to 1 if the library should be pre-assigned.
+    the library is registered.
+  @param [in] IsPreassigned= set to 1 if the library should be pre-assigned.
 
-  @param libdesc= Library description (optional)
+  @param [in] libdesc= Library description (optional)
-  @param directory= Required for the BASE engine. The metadata directory objects
+  @param [in] directory= (/tmp/somelib) Required for the BASE engine.
-    are searched to find an existing one with a matching physical path.
+    The metadata directory objects are searched to find an existing
+    one with a matching physical path.
     If more than one uri found with that path, then the first one will be used.
     If no URI is found, a new directory object will be created.  The physical
     path will also be created, if it doesn't exist.
 
 
-  @param mDebug= set to 1 to show debug messages in the log
+  @param [in] mDebug= set to 1 to show debug messages in the log
-  @param frefin= fileref to use (enables change if there is a conflict).  The
+  @param [in] frefin= fileref to use (enables change if there is a conflict).
-    filerefs are left open, to enable inspection after running the
+    The filerefs are left open, to enable inspection after running the
     macro (or importing into an xmlmap if needed).
-  @param frefout= fileref to use (enables change if there is a conflict)
+  @param [out] frefout= fileref to use (enables change if there is a conflict)
 
 
   @version 9.3

meta/mm_createstp.sas

@@ -40,29 +40,38 @@
         ,Server=SASApp
         ,stptype=2)
 
-  @param stpname= Stored Process name.  Avoid spaces - testing has shown that
+  @param [in] stpname= (SASjs Default STP) Stored Process name.
+    Avoid spaces - testing has shown that
     the check to avoid creating multiple STPs in the same folder with the same
     name does not work when the name contains spaces.
-  @param stpdesc= Stored Process description (optional)
+  @param [in] stpdesc= Stored Process description (optional)
-  @param filename= the name of the .sas program to run
+  @param [in] filename= the name of the .sas program to run
-  @param directory= The directory uri, or the actual path to the sas program
+  @param [in] directory= (SASEnvironment/sascode)
-    (no trailing slash).  If more than uri is found with that path, then the
+    The directory uri or the actual path to the sas program (no trailing slash).
-    first one will be used.
+    If more than uri is found with that path, then the first one will be used.
-  @param tree= The metadata folder uri, or the metadata path, in which to
+  @param [in] tree= The metadata folder uri, or the metadata path, in which to
     create the STP.
-  @param server= The server which will run the STP.  Server name or uri is fine.
+  @param [in] server= (SASApp) The server which will run the STP.
-  @param outds= The two level name of the output dataset.  Will contain all the
+    Server name or uri is fine.
-    meta uris. Defaults to work.mm_createstp.
+  @param [out] outds= (work.mm_createstp)
-  @param mDebug= set to 1 to show debug messages in the log
+    The two level name of the output dataset.  Will contain all the meta uris.
-  @param stptype= Default is 1 (STP code saved on filesystem).  Set to 2 if
+  @param [in] mDebug= set to 1 to show debug messages in the log
+  @param [in] stptype= Default is 1 (STP code saved on filesystem).  Set to 2 if
     source code is to be saved in metadata (9.3 and above feature).
-  @param minify= set to YES to strip comments / blank lines etc
+  @param [in] minify= set to YES to strip comments / blank lines etc
-  @param frefin= fileref to use (enables change if there is a conflict).  The
+  @param [in] frefin= (mm_in) fileref to use (enables change if there is
-    filerefs are left open, to enable inspection after running the
+    a conflict).
+    The filerefs are left open, to enable inspection after running the
     macro (or importing into an xmlmap if needed).
-  @param frefout= fileref to use (enables change if there is a conflict)
+  @param [out] frefout= (mm_out) fileref to use (enables change if there is
-  @param repo= ServerContext is tied to a repo, if you are not using the
+    a conflict)
+  @param [in] repo= ServerContext is tied to a repo, if you are not using the
     foundation repo then select a different one here
+  @param [in] LogicalServerType= (Sps) Server Type to use. Valid options:
+    @li Any - Uses the default server.
+    @li Sps - Stored Process Server, best choice for web app development. Runs
+      under a system account identity (eg sassrv).
+    @li Wks - Workspace Server.  Runs under the end user identity.
 
   @returns outds  dataset containing the following columns:
     - stpuri
@@ -88,7 +97,7 @@
 **/
 
 %macro mm_createstp(
-    stpname=Macro People STP
+    stpname=SASjs Default STP
     ,stpdesc=This stp was created automatically by the mm_createstp macro
     ,filename=mm_createstp.sas
     ,directory=SASEnvironment/SASCode
@@ -102,6 +111,7 @@
     ,minify=NO
     ,frefin=mm_in
     ,frefout=mm_out
+    ,LogicalServerType=Sps
 )/*/STORE SOURCE*/;
 
 %local mD;
@@ -118,6 +128,17 @@
 
 %mp_dropmembers(%scan(&outds,2,.))
 
+/* check LogicalServerType validity */
+%mp_abort(
+  iftrue=(
+    &LogicalServerType ne Sps
+    and &LogicalServerType ne Wks
+    and &LogicalServerType ne Any
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for LogicalServerType (&LogicalServerType))
+)
+
 /**
   * check tree exists
   */
@@ -350,8 +371,9 @@ run;
     '  <TextStore IsHidden="0" Name="Stored Process" UsageVersion="0" '/
     '    TextRole="StoredProcessConfiguration" TextType="XML" '/
     '    StoredText="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&qu'@@
-    'ot;?&gt;&lt;StoredProcess&gt;&lt;ServerContext LogicalServerType=&quot;S'@@
+    'ot;?&gt;&lt;StoredProcess&gt;&lt;ServerContext LogicalServerType=&quot;'@@
-    'ps&quot; OtherAllowed=&quot;false&quot;/&gt;&lt;ResultCapabilities Packa'@@
+    "&LogicalServerType"@@
+    '&quot; OtherAllowed=&quot;false&quot;/&gt;&lt;ResultCapabilities Packa'@@
     'ge=&quot;' @@ "&package" @@ '&quot; Streaming=&quot;' @@ "&streaming" @@
     '&quot;/&gt;&lt;OutputParameters/&gt;&lt;/StoredProcess&gt;" />' /
     "  </Notes> "/