Merge pull request #184 from sasjs/issue181

feat: ms_getfile service (and test).  Closes #181

.all-contributorsrc

@@ -117,6 +117,15 @@
       "contributions": [
         "bug"
       ]
+    },
+    {
+      "login": "yabwon",
+      "name": "Bart Jablonski",
+      "avatar_url": "https://avatars.githubusercontent.com/u/9314894?v=4",
+      "profile": "https://github.com/yabwon",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.github/CONTRIBUTING.md

@@ -12,7 +12,7 @@ This repository makes use of the [SASjs](https://sasjs.io) framework for code or
 * [VSCode](https://sasjs.io/windows/#vscode) - feature packed IDE for code editing (warning - highly effective!)
 * [GIT](https://sasjs.io/windows/#git) - a safety net you cannot (and should not) do without.
 
-For generating the documentation (`sasjs doc`) it is also necessary to install [doxygen](https://www.doxygen.nl/manual/install.html).
+For generating the documentation (`sasjs doc`) it is also necessary to install [doxygen](https://www.doxygen.nl/manual/install.html) and GraphViz (`sudo port install graphviz` on mac, or `sudo apt-get install graphviz` on Ubuntu).
 
 
 To get configured:

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [sasjs]

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+  - package-ecosystem: npm
+    directory: '/'
+    schedule:
+      interval: monthly
+    open-pull-requests-limit: 3
+    allow:
+      - dependency-type: "production"

.gitpod.dockerfile

@@ -1,6 +1,6 @@
 FROM gitpod/workspace-full
 
 RUN sudo apt-get update \
- && sudo apt-get install -y \
-    doxygen \
+ && sudo apt-get install -y doxygen \
+ && sudo apt-get install -y graphviz \
  && sudo rm -rf /var/lib/apt/lists/*

.gitpod.yml

@@ -1,5 +1,7 @@
 tasks:
-  - init: nvm install --lts && npm i -g @sasjs/cli
+  - init: |
+      nvm install --lts
+      npm i -g @sasjs/cli
 
 image:
   file: .gitpod.dockerfile

README.md

@@ -31,19 +31,28 @@ Documentation: https://core.sasjs.io
 
 ## Components
 
-### BASE library (SAS9/Viya)
+### BASE library (All Platforms)
 
 - OS independent
-- Not metadata aware
+- Works on all SAS Platforms
 - No X command
 - Prefixes: _mf_, _mp_
 
-#### FCMP library (SAS9/Viya)
+### DDL library (All Platforms)
+
+- OS independent
+- Works on all SAS Platforms
+- No X command
+- Prefixes: _mddl_(lib)_ -> where lib can be "SAS" (in relation to a SAS component) or "DC" (in relation to a Data Controller component)
+
+This library will not be used for storing data entries (such as formats or datalines).  Where this becomes necessary in the future, a new repo will be created, in order to keep the NPM bundle size down (for the benefit of those looking to embed purely macros in their applications).
+
+### FCMP library (All Platforms)
+
 - Function and macro names are identical, except for special cases
 - Prefixes: _mcf_
 
-The fcmp macros are used to generate fcmp functions, and can be used with or
-without the `proc fcmp` wrapper.
+The fcmp macros are used to generate fcmp functions, and can be used with or without the `proc fcmp` wrapper.
 
 ### META library (SAS9 only)
 
@@ -81,7 +90,7 @@ Macros used for interfacing with SAS Viya.
 
 Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
 
-To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert your LUA into a data step with put statements, and create the macro wrapper with a `ml_` prefix.  You can then use your module in any program by running:
+To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert all files with a ".lua" extension into a macro wrapper with an `ml_` prefix (embedding the necessary data step put statements).  You can then use your module in any program by running:
 
 ```sas
 /* compile the lua module */
@@ -95,8 +104,7 @@ endsubmit;
 run;
 ```
 
-- X command enabled
-- Prefixes: _mmw_,_mmu_,_mmx_
+- Prefixes: _ml_
 
 ## Installation
 
@@ -126,14 +134,15 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 - one macro per file
 - prefixes:
   - _mcf_ for macro compiled functions (proc fcmp)
+  - _mddl_ for macros containing DDL (Data Definition Language)
   - _mf_ for macro functions (can be used in open code).
   - _ml_ for macros that are used to compile LUA modules
   - _mm_ for metadata macros (interface with the metadata server).
-  - _mmx_ for macros that use metadata and are XCMD enabled
+  - _mmx_ for macros that use metadata and are XCMD enabled (working on both windows and unix)
   - _mp_ for macro procedures (which generate sas code)
   - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
   - _mv_ for macro procedures that will only work in Viya
-  - _mx_ for macros that are XCMD enabled
+  - _mx_ for macros that are XCMD enabled (working on both windows and unix)
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
@@ -181,7 +190,7 @@ When contributing to this library, it is therefore important to ensure that all
 - The closing `%mend;` should **not** contain the macro name.
 - All macros should be defined with brackets, even if no variables are needed - ie `%macro x();` not `%macro x;`
 - Mandatory parameters should be positional, all optional parameters should be keyword (var=) style.
-- All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect.
+- All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect, or the [USER](https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/lrcon/n18m1vkqmeo4esn1moikt23zhp8s.htm) library is active.
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
 - Where global macro variables are absolutely necessary, they should make use of `&sasjs_prefix` - see mp_init.sas
 - The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
@@ -193,9 +202,14 @@ When contributing to this library, it is therefore important to ensure that all
 
 ## Breaking Changes
 
-We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major (breaking) release becomes necessary:
+We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major/breaking release (v5) becomes necessary:
 
-* (None as yet)
+* mf_getuniquelibref.sas to have the deprecated maxtried parameter removed (no longer needed)
+* mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
+* `insert_cmplib` option of mcf_xxx macros will be deprecated (the option is now checked automatically with value inserted only if needed)
+* mcf_xxx macros to have `wrap=` option defaulted to YES for convenience.  Set this option explicitly to avoid issues.
+* mp_getddl.sas to be renamed to mp_ds2ddl.sas (consistent with other ds2xxx macros).  A wrapper macro is already in place, and you are able to use this immediately.  The default for SHOWLOG will also be YES instead of NO.
+* mp_coretable.sas will be replaced by the standalone macros in the `ddl` folder (which are already available)
 
 ## Star Gazing
 
@@ -204,10 +218,9 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 ![](https://starchart.cc/sasjs/core.svg)
 
 
-
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-10-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-11-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -228,6 +241,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
     <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
     <td align="center"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
+    <td align="center"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
   </tr>
 </table>
 

base/mf_dedup.sas

@@ -23,7 +23,6 @@
 
   <h4> Related Macros </h4>
   @li mf_trimstr.sas
-  @li mf_wordsinstr1butnotstr2.sas
 
   @version 9.2
   @author Allan Bowe

base/mf_existvar.sas

@@ -1,11 +1,11 @@
 /**
   @file
   @brief Checks if a variable exists in a data set.
-  @details Returns 0 if the variable does NOT exist, and return the position of
-    the var if it does.
-    Usage:
+  @details Returns 0 if the variable does NOT exist, and the position of the var
+  if it does.
+  Usage:
 
-        %put %mf_existvar(work.someds, somevar)
+      %put %mf_existvar(work.someds, somevar)
 
   @param [in] libds 2 part dataset or view reference
   @param [in] var variable name

base/mf_getapploc.sas

@@ -15,6 +15,7 @@
   @li /data
   @li /jobs
   @li /services
+  @li /tests
   @li /tests/jobs
   @li /tests/services
   @li /tests/macros
@@ -46,9 +47,13 @@
 
 /**
   * First check we are not in the tests/macros folder (which has no subfolders)
+  * or specifically in the testsetup or testteardown services
   */
-%if %index(&pgm,/tests/macros/) %then %do;
-  %let root=%substr(&pgm,1,%index(&pgm,/tests/macros)-1);
+%if %index(&pgm,/tests/macros/)
+or %index(&pgm,/tests/testsetup)
+or %index(&pgm,/tests/testteardown)
+%then %do;
+  %let root=%substr(&pgm,1,%index(&pgm,/tests)-1);
   &root
   %return;
 %end;

base/mf_getfilesize.sas

@@ -5,18 +5,19 @@
 
       %put %mf_getfilesize(fpath=C:\temp\myfile.txt);
 
-      or
+  or, provide a libds value as follows:
 
       data x;do x=1 to 100000;y=x;output;end;run;
       %put %mf_getfilesize(libds=work.x,format=yes);
 
-      gives:
+  Which gives:
 
-      2mb
+  > 2mb
+
+  @param [in] fpath= Full path and filename.  Provide this OR the libds value.
+  @param [in] libds= (0) Library.dataset value (assumes library is BASE engine)
+  @param [in] format= (NO) Set to yes to apply sizekmg. format
 
-  @param fpath= full path and filename.  Provide this OR the libds value.
-  @param libds= library.dataset value (assumes library is BASE engine)
-  @param format=  set to yes to apply sizekmg. format
   @returns bytes
 
   @version 9.2
@@ -26,16 +27,32 @@
 %macro mf_getfilesize(fpath=,libds=0,format=NO
 )/*/STORE SOURCE*/;
 
-  %if &libds ne 0 %then %do;
-    %let fpath=%sysfunc(pathname(%scan(&libds,1,.)))/%scan(&libds,2,.).sas7bdat;
-  %end;
+  %local rc fid fref bytes dsid lib vnum;
 
-  %local rc fid fref bytes;
-  %let rc=%sysfunc(filename(fref,&fpath));
-  %let fid=%sysfunc(fopen(&fref));
-  %let bytes=%sysfunc(finfo(&fid,File Size (bytes)));
-  %let rc=%sysfunc(fclose(&fid));
-  %let rc=%sysfunc(filename(fref));
+  %if &libds ne 0 %then %do;
+    %let libds=%upcase(&libds);
+    %if %index(&libds,.)=0 %then %let lib=WORK;
+    %else %let lib=%scan(&libds,1,.);
+    %let dsid=%sysfunc(open(
+      sashelp.vtable(where=(libname="&lib" and memname="%scan(&libds,-1,.)")
+        keep=libname memname filesize
+      )
+    ));
+    %if (&dsid ^= 0) %then %do;
+      %let vnum=%sysfunc(varnum(&dsid,FILESIZE));
+      %let rc=%sysfunc(fetch(&dsid));
+      %let bytes=%sysfunc(getvarn(&dsid,&vnum));
+      %let rc= %sysfunc(close(&dsid));
+    %end;
+    %else %put &sysmacroname: &libds could not be opened! %sysfunc(sysmsg());
+  %end;
+  %else %do;
+    %let rc=%sysfunc(filename(fref,&fpath));
+    %let fid=%sysfunc(fopen(&fref));
+    %let bytes=%sysfunc(finfo(&fid,File Size (bytes)));
+    %let rc=%sysfunc(fclose(&fid));
+    %let rc=%sysfunc(filename(fref));
+  %end;
 
   %if &format=NO %then %do;
     &bytes

base/mf_getplatform.sas

@@ -22,6 +22,12 @@
 )/*/STORE SOURCE*/;
 %local a b c;
 %if &switch.NONE=NONE %then %do;
+  %if %symexist(sasjsprocessmode) %then %do;
+    %if &sasjsprocessmode=Stored Program %then %do;
+      SASJS
+      %return;
+    %end;
+  %end;
   %if %symexist(sysprocessmode) %then %do;
     %if "&sysprocessmode"="SAS Object Server"
     or "&sysprocessmode"= "SAS Compute Server" %then %do;

base/mf_getuniquelibref.sas

@@ -14,27 +14,44 @@
 
 > mclib3
 
-  @param prefix= first part of libref.  Remember that librefs can only be 8 characters,
-    so a 7 letter prefix would mean that maxtries should be 10.
-  @param maxtries= the last part of the libref.  Provide an integer value.
+  A blank value is returned if no usable libname is determined.
+
+  @param [in] prefix= (mclib) first part of the returned libref. As librefs can
+    be as long as 8 characters, a maximum length of 7 characters is premitted
+    for this prefix.
+  @param [in] maxtries= Deprecated parameter. Remains here to ensure a
+    non-breaking change.  Will be removed in v5.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-
 %macro mf_getuniquelibref(prefix=mclib,maxtries=1000);
-  %local x libref;
-  %let x=0;
-  %do x=0 %to &maxtries;
-  %if %sysfunc(libref(&prefix&x)) ne 0 %then %do;
-    %let libref=&prefix&x;
-    %let rc=%sysfunc(libname(&libref,%sysfunc(pathname(work))));
-    %if &rc %then %put %sysfunc(sysmsg());
-    &prefix&x
-    %*put &sysmacroname: Libref &libref assigned as WORK and returned;
+  %local x;
+
+  %if ( %length(&prefix) gt 7 ) %then %do;
+    %put %str(ERR)OR: The prefix parameter cannot exceed 7 characters.;
+    0
     %return;
   %end;
+  %else %if (%sysfunc(NVALID(&prefix,v7))=0) %then %do;
+    %put %str(ERR)OR: Invalid prefix (&prefix);
+    0
+    %return;
   %end;
-  %put unable to find available libref in range &prefix.0-&maxtries;
+
+  /* Set maxtries equal to '10 to the power of [# unused characters] - 1' */
+  %let maxtries=%eval(10**(8-%length(&prefix))-1);
+
+  %do x = 0 %to &maxtries;
+    %if %sysfunc(libref(&prefix&x)) ne 0 %then %do;
+      &prefix&x
+      %return;
+    %end;
+    %let x = %eval(&x + 1);
+  %end;
+
+  %put %str(ERR)OR: No usable libref in range &prefix.0-&maxtries;
+  %put %str(ERR)OR- Try reducing the prefix or deleting some libraries!;
+  0
 %mend mf_getuniquelibref;

base/mf_verifymacvars.sas

@@ -1,6 +1,6 @@
 /**
   @file
-  @brief Checks if a set of macro variables exist / contain values.
+  @brief Checks if a set of macro variables exist AND contain values.
   @details Writes ERROR to log if abortType is SOFT, else will call %mf_abort.
   Usage:
 
@@ -14,10 +14,11 @@
   <h4> SAS Macros </h4>
   @li mf_abort.sas
 
-  @param verifyvars space separated list of macro variable names
-  @param makeupcase= set to YES to convert all variable VALUES to
+  @param [in] verifyvars Space separated list of macro variable names
+  @param [in] makeupcase= (NO) Set to YES to convert all variable VALUES to
     uppercase.
-  @param mAbort= Abort Type.  Default is SOFT (writes err to log).
+  @param [in] mAbort= (SOFT) Abort Type.  When SOFT, simply writes an err
+    message to the log.
     Set to any other value to call mf_abort (which can be configured to abort in
     various fashions according to context).
 
@@ -58,8 +59,14 @@
 
   %goto exit_success;
   %exit_err:
-    %if &mAbort=SOFT %then %put %str(ERR)OR: &abortmsg;
-    %else %mf_abort(mac=mf_verifymacvars,type=&mabort,msg=&abortmsg);
+    %put %str(ERR)OR: &abortmsg;
+    %mf_abort(iftrue=(&mabort ne SOFT),
+      mac=mf_verifymacvars,
+      msg=%str(&abortmsg)
+    )
+    0
+    %return;
   %exit_success:
+  1
 
 %mend mf_verifymacvars;

base/mf_wordsinstr1andstr2.sas

@@ -30,7 +30,6 @@
 
 %local count_base count_extr i i2 extr_word base_word match outvar;
 %if %length(&str1)=0 or %length(&str2)=0 %then %do;
-  %put %str(WARN)ING: empty string provided!;
   %put base string (str1)= &str1;
   %put compare string (str2) = &str2;
   %return;

base/mf_wordsinstr1butnotstr2.sas

@@ -3,6 +3,9 @@
   @brief Returns words that are in string 1 but not in string 2
   @details  Compares two space separated strings and returns the words that are
   in the first but not in the second.
+
+  Note - case sensitive!
+
   Usage:
 
       %let x= %mf_wordsInStr1ButNotStr2(
@@ -13,10 +16,8 @@
   returns:
   > sss bram boo
 
-  @param str1= string containing words to extract
-  @param str2= used to compare with the extract string
-
-  @warning CASE SENSITIVE!
+  @param [in] str1= string containing words to extract
+  @param [in] str2= used to compare with the extract string
 
   @version 9.2
   @author Allan Bowe
@@ -30,7 +31,6 @@
 
 %local count_base count_extr i i2 extr_word base_word match outvar;
 %if %length(&str1)=0 or %length(&str2)=0 %then %do;
-  %put %str(WARN)ING: empty string provided!;
   %put base string (str1)= &str1;
   %put compare string (str2) = &str2;
   %return;

base/mp_abort.sas

@@ -66,7 +66,10 @@
   %if %length(&mac)>0 %then %put NOTE- called by &mac;
   %put NOTE - &msg;
 
-  %if %symexist(_SYSINCLUDEFILEDEVICE) %then %do;
+  %if %symexist(_SYSINCLUDEFILEDEVICE)
+  /* abort cancel FILE does not restart outside the INCLUDE on Viya 3.5 */
+  and "&SYSPROCESSNAME " ne "Compute Server "
+  %then %do;
     %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do;
       data &errds;
         iftrue='1=1';

base/mp_assertdsobs.sas

@@ -12,10 +12,6 @@
 
       %mp_assertdsobs(sashelp.class,test=ATMOST 20) %* pass if <21 obs present;
 
-  <h4> SAS Macros </h4>
-  @li mf_nobs.sas
-  @li mp_abort.sas
-
 
   @param [in] inds input dataset to test for presence of observations
   @param [in] desc= (Testing observations) The user provided test description
@@ -33,6 +29,11 @@
   |---|---|---|
   |User Provided description|PASS|Dataset &inds has XX obs|
 
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
   <h4> Related Macros </h4>
   @li mp_assertcolvals.sas
   @li mp_assert.sas
@@ -49,9 +50,10 @@
   outds=work.test_results
 )/*/STORE SOURCE*/;
 
-  %local nobs;
+  %local nobs ds;
   %let nobs=%mf_nobs(&inds);
   %let test=%upcase(&test);
+  %let ds=%mf_getuniquename(prefix=mp_assertdsobs);
 
   %if %substr(&test.xxxxx,1,6)=EQUALS %then %do;
     %let val=%scan(&test,2,%str( ));
@@ -84,7 +86,7 @@
     )
   %end;
 
-  data;
+  data &ds;
     length test_description $256 test_result $4 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
@@ -110,9 +112,6 @@
   %end;
   run;
 
-  %local ds;
-  %let ds=&syslast;
-
   proc append base=&outds data=&ds;
   run;
 

base/mp_assertscope.sas

@@ -1,18 +1,25 @@
 /**
   @file
   @brief Used to capture scope leakage of macro variables
-  @details A common 'difficult to detect' bug in macros is where a nested
-    macro over-writes variables in a higher level macro.
+  @details
 
-    This assertion takes a snapshot of the macro variables before and after
-    a macro invocation.  This makes it easy to detect whether any macro
-    variables were modified or changed.
+  A common 'difficult to detect' bug in macros is where a nested macro
+  over-writes variables in a higher level macro.
 
-    Currently, the macro only checks for global scope variables.  In the future
-    it may be extended to work at multiple levels of nesting.
+  This assertion takes a snapshot of the macro variables before and after
+  a macro invocation.  Differences are captured in the `&outds` table. This
+  makes it easy to detect whether any macro variables were modified or
+  changed.
 
-    If you would like this feature, feel free to contribute / raise an issue /
-    engage the SASjs team directly.
+  The following variables are NOT tested (as they are known, global variables
+  used in SASjs):
+
+  @li &sasjs_prefix._FUNCTIONS
+
+  Global variables are initialised in mp_init.sas - which will also trigger
+  "strict mode" in your SAS session.  Whilst this is a default in SASjs
+  produced apps, if you prefer not to use this mode, simply instantiate the
+  following variable to prevent the macro from running:  `SASJS_PREFIX`
 
   Example usage:
 
@@ -24,12 +31,17 @@
         desc=Checking macro variables against previous snapshot
       )
 
+  This macro is designed to work alongside `sasjs test` - for more information
+  about this facility, visit [cli.sasjs.io/test](https://cli.sasjs.io/test).
+
   @param [in] action (SNAPSHOT) The action to take.  Valid values:
     @li SNAPSHOT - take a copy of the current macro variables
     @li COMPARE - compare the current macro variables against previous values
   @param [in] scope= (GLOBAL) The scope of the variables to be checked.  This
     corresponds to the values in the SCOPE column in `sashelp.vmacro`.
   @param [in] desc= (Testing scope leakage) The user provided test description
+  @param [in] ignorelist= Provide a list of macro variable names to ignore from
+    the comparison
   @param [in,out] scopeds= (work.mp_assertscope) The dataset to contain the
     scope snapshot
   @param [out] outds= (work.test_results) The output dataset to contain the
@@ -38,6 +50,10 @@
   |---|---|---|
   |User Provided description|PASS|No out of scope variables created or modified|
 
+  <h4> SAS Macros </h4>
+  @li mf_getquotedstr.sas
+  @li mp_init.sas
+
   <h4> Related Macros </h4>
   @li mp_assert.sas
   @li mp_assertcols.sas
@@ -54,9 +70,19 @@
   desc=Testing Scope Leakage,
   scope=GLOBAL,
   scopeds=work.mp_assertscope,
+  ignorelist=,
   outds=work.test_results
 )/*/STORE SOURCE*/;
-%local ds test_result test_comments del add mod;
+%local ds test_result test_comments del add mod ilist;
+%let ilist=%upcase(&sasjs_prefix._FUNCTIONS SYS_PROCHTTP_STATUS_CODE
+  SYS_PROCHTTP_STATUS_CODE SYS_PROCHTTP_STATUS_PHRASE &ignorelist);
+
+/**
+  * this sets up the global vars, it will also enter STRICT mode.  If this
+  * behaviour is not desired, simply initiate the following global macro
+  * variable to prevent the macro from running: SASJS_PREFIX
+  */
+%mp_init()
 
 /* get current variables */
 %if &action=SNAPSHOT %then %do;
@@ -64,7 +90,7 @@
   create table &scopeds as
     select name,offset,value
     from dictionary.macros
-    where scope="&scope"
+    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
     order by name,offset;
 %end;
 %else %if &action=COMPARE %then %do;
@@ -73,12 +99,14 @@
   create table _data_ as
     select name,offset,value
     from dictionary.macros
-    where scope="&scope"
+    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
     order by name,offset;
 
   %let ds=&syslast;
 
-  proc compare base=&scopeds compare=&ds;
+  proc compare
+    base=&scopeds(where=(upcase(name) not in (%mf_getquotedstr(&ilist))))
+    compare=&ds;
   run;
 
   %if &sysinfo=0 %then %do;
@@ -116,4 +144,4 @@
   drop table &ds;
 %end;
 
-%mend mp_assertscope;
+%mend mp_assertscope;

base/mp_cntlout.sas

@@ -0,0 +1,85 @@
+/**
+  @file mp_cntlout.sas
+  @brief Creates a cntlout dataset in a consistent format
+  @details The dataset produced by proc format in the cntlout option will vary
+  according to its contents.
+
+  When dealing with formats from an ETL perspective (eg in [Data Controller for
+  SAS](https://datacontroller.io)), it is important that the output dataset
+  has a consistent model (and compariable values).
+
+  This macro makes use of mddl_sas_cntlout.sas to provide the consistent model,
+  and will left-align the start and end values when dealing with numeric ranges
+  to enable consistency when checking for differences.
+
+  usage:
+
+      %mp_cntlout(libcat=yourlib.cat,cntlout=work.formatexport)
+
+  @param [in] libcat The library.catalog reference
+  @param [in] fmtlist= (0) provide a space separated list of specific formats to
+    extract
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [out] cntlout= (work.fmtextract) Libds reference for the output dataset
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mf_getvarformat.sas
+  @li mp_getformats.sas
+  @li mp_loadformat.sas
+  @li mp_ds2fmtds.sas
+
+  @version 9.2
+  @author Allan Bowe
+  @cond
+**/
+
+%macro mp_cntlout(
+  iftrue=(1=1)
+  ,libcat=
+  ,cntlout=work.fmtextract
+  ,fmtlist=0
+)/*/STORE SOURCE*/;
+%local ddlds cntlds i;
+
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+%let ddlds=%mf_getuniquename();
+%let cntlds=%mf_getuniquename();
+
+%mddl_sas_cntlout(libds=&ddlds)
+
+%if %index(&libcat,-)>0 and %scan(&libcat,2,-)=FC %then %do;
+  %let libcat=%scan(&libcat,1,-);
+%end;
+
+proc format lib=&libcat cntlout=&cntlds;
+%if "&fmtlist" ne "0" %then %do;
+  select
+  %do i=1 %to %sysfunc(countw(&fmtlist));
+    %scan(&fmtlist,&i,%str( ))
+  %end;
+  ;
+%end;
+run;
+
+data &cntlout;
+  if 0 then set &ddlds;
+  set &cntlds;
+  if type="N" then do;
+    start=cats(start);
+    end=cats(end);
+  end;
+run;
+proc sort;
+  by fmtname start;
+run;
+
+proc sql;
+drop table &ddlds,&cntlds;
+
+%mend mp_cntlout;
+/** @endcond */

base/mp_coretable.sas

@@ -5,26 +5,38 @@
     make use of permanent tables.  To avoid duplication in definitions, this
     macro provides a central location for managing the corresponding DDL.
 
+  Note - this macro is likely to be deprecated in future in favour of a
+  dedicated "datamodel" folder (prefix mddl)
+
+  Any corresponding data would go in a seperate repo, to avoid this one
+  ballooning in size!
+
   Example usage:
 
       %mp_coretable(LOCKTABLE,libds=work.locktable)
 
   @param [in] table_ref The type of table to create.  Example values:
-    @li FILTER_DETAIL - For storing detailed filter values.  Used by
-      mp_filterstore.sas.
-    @li FILTER_SUMMARY - For storing summary filter values.  Used by
-      mp_filterstore.sas.
-    @li LOCKANYTABLE - For "locking" tables prior to multipass loads. Used by
-      mp_lockanytable.sas
-    @li MAXKEYTABLE - For storing the maximum retained key information.  Used
-      by mp_retainedkey.sas
+    @li DIFFTABLE
+    @li FILTER_DETAIL
+    @li FILTER_SUMMARY
+    @li LOCKANYTABLE
+    @li MAXKEYTABLE
   @param [in] libds= (0) The library.dataset reference used to create the table.
     If not provided, then the DDL is simply printed to the log.
 
+  <h4> SAS Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mddl_dc_filterdetail.sas
+  @li mddl_dc_filtersummary.sas
+  @li mddl_dc_locktable.sas
+  @li mddl_dc_maxkeytable.sas
+
   <h4> Related Macros </h4>
   @li mp_filterstore.sas
   @li mp_lockanytable.sas
   @li mp_retainedkey.sas
+  @li mp_storediffs.sas
+  @li mp_stackdiffs.sas
 
   @version 9.2
   @author Allan Bowe
@@ -36,55 +48,22 @@
 %local outds ;
 %let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
 proc sql;
-%if &table_ref=LOCKTABLE %then %do;
-  create table &outds(
-      lock_lib char(8),
-      lock_ds char(32),
-      lock_status_cd char(10) not null,
-      lock_user_nm char(100) not null ,
-      lock_ref char(200),
-      lock_pid char(10),
-      lock_start_dttm num format=E8601DT26.6,
-      lock_end_dttm num format=E8601DT26.6,
-    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+%if &table_ref=DIFFTABLE %then %do;
+  %mddl_dc_difftable(libds=&outds)
+%end;
+%else %if &table_ref=LOCKTABLE %then %do;
+  %mddl_dc_locktable(libds=&outds)
 %end;
 %else %if &table_ref=FILTER_SUMMARY %then %do;
-  create table &outds(
-      filter_rk num not null,
-      filter_hash char(32) not null,
-      filter_table char(41) not null,
-      processed_dttm num not null format=E8601DT26.6,
-    constraint pk_mpe_filteranytable
-      primary key(filter_rk));
+  %mddl_dc_filtersummary(libds=&outds)
 %end;
 %else %if &table_ref=FILTER_DETAIL %then %do;
-  create table &outds(
-      filter_hash char(32) not null,
-      filter_line num not null,
-      group_logic char(3) not null,
-      subgroup_logic char(3) not null,
-      subgroup_id num not null,
-      variable_nm varchar(32) not null,
-      operator_nm varchar(12) not null,
-      raw_value varchar(4000) not null,
-      processed_dttm num not null format=E8601DT26.6,
-    constraint pk_mpe_filteranytable
-      primary key(filter_hash,filter_line));
+  %mddl_dc_filterdetail(libds=&outds)
 %end;
 %else %if &table_ref=MAXKEYTABLE %then %do;
-  create table &outds(
-      keytable varchar(41) label='Base table in libref.dataset format',
-      keycolumn char(32) format=$32.
-        label='The Retained key field containing the key values.',
-      max_key num label=
-        'Integer representing current max RK or SK value in the KEYTABLE',
-      processed_dttm num format=E8601DT26.6
-        label='Datetime this value was last updated',
-    constraint pk_mpe_maxkeyvalues
-        primary key(keytable));
+  %mddl_dc_maxkeytable(libds=&outds)
 %end;
 
-
 %if &libds=0 %then %do;
   describe table &syslast;
   drop table &syslast;

base/mp_createwebservice.sas

@@ -24,20 +24,22 @@ Usage:
         %webout(OBJ,example2) * Object format, easier to work with ;
         %webout(CLOSE)
     ;;;;
-    %mp_createwebservice(path=/Public/app/common,name=appInit,code=ft15f001,replace=YES)
+    %mp_createwebservice(path=/Public/app/common,name=appInit,replace=YES)
 
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
   @li mm_createwebservice.sas
   @li mv_createwebservice.sas
 
-  @param path= The full folder path where the service will be created
-  @param name= Service name.  Avoid spaces.
-  @param desc= The description of the service (optional)
-  @param precode= Space separated list of filerefs, pointing to the code that
-    needs to be attached to the beginning of the service (optional)
-  @param code= Space seperated fileref(s) of the actual code to be added
-  @param replace= select YES to replace any existing service in that location
+  @param [in,out] path= The full folder path where the service will be created
+  @param [in,out] name= Service name.  Avoid spaces.
+  @param [in] desc= The description of the service (optional)
+  @param [in] precode= Space separated list of filerefs, pointing to the code
+    that needs to be attached to the beginning of the service (optional)
+  @param [in] code= (ft15f001) Space seperated fileref(s) of the actual code to
+    be added
+  @param [in] replace= (YES) Select YES to replace any existing service in that
+    location
 
 
   @version 9.2

base/mp_csv2ds.sas

@@ -49,10 +49,6 @@
   ,mac=&sysmacroname
   ,msg=%str(the BASEDS variable must be provided)
 )
-%mp_abort(iftrue=( &baseds=0 )
-  ,mac=&sysmacroname
-  ,msg=%str(the BASEDS variable must be provided)
-)
 %mp_abort(iftrue=( %mf_existds(&baseds)=0 )
   ,mac=&sysmacroname
   ,msg=%str(the BASEDS dataset (&baseds) needs to be assigned, and to exist)

base/mp_deletefolder.sas

@@ -51,6 +51,7 @@
     data _null_;
       set work.&tempds end=last;
       length fref $8;
+      fref='';
       rc=filename(fref,filepath);
       rc=fdelete(fref);
       if rc then do;

base/mp_dirlist.sas

@@ -82,7 +82,8 @@ data &out_ds(compress=no
     keep=file_or_folder filepath filename ext msg directory level
   );
   length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
-    ext $20 msg $200;
+    ext $20 msg $200 foption $16;
+  if _n_=1 then call missing(of _all_);
   retain level &level;
   %if &fref=0 %then %do;
     rc = filename(fref, "&path");
@@ -93,7 +94,13 @@ data &out_ds(compress=no
   %end;
   if rc = 0 then do;
     did = dopen(fref);
-    directory=dinfo(did,'Directory');
+    /* attribute is OS-dependent - could be "Directory" or "Directory Name" */
+    numopts=doptnum(did);
+    do i=1 to numopts;
+      foption=doptname(did,i);
+      if foption=:'Directory' then i=numopts;
+    end;
+    directory=dinfo(did,foption);
     if did=0 then do;
       putlog "NOTE: This directory is empty - " directory;
       msg=sysmsg();

base/mp_ds2csv.sas

@@ -1,23 +1,82 @@
 /**
   @file
-  @brief Export a dataset to a CSV file
-  @details Export to a file or a fileref
+  @brief Export a dataset to a CSV file WITH leading blanks
+  @details Export a dataset to a file or fileref, retaining leading blanks.
+
+  When using SASJS headerformat, the input statement is provided in the first
+  row of the CSV.
+
   Usage:
 
       %mp_ds2csv(sashelp.class,outref="%sysfunc(pathname(work))/file.csv")
 
-  @param ds The dataset to be exported
-  @param outfile= The output filename - should be quoted.
-  @param outref= The output fileref (takes precedence if provided)
-  @param outencoding= The output encoding to use (unquoted)
+      filename example temp;
+      %mp_ds2csv(sashelp.air,outref=example,headerformat=SASJS)
+      data; infile example; input;put _infile_; if _n_>5 then stop;run;
+
+      data _null_;
+        infile example;
+        input;
+        call symputx('stmnt',_infile_);
+        stop;
+      run;
+      data work.want;
+        infile example dsd firstobs=2;
+        input &stmnt;
+      run;
+
+  Why use mp_ds2csv over, say, proc export?
+
+  1. Ability to retain leading blanks (this is a major one)
+  2. Control the header format
+  3. Simple one-liner
+
+  @param [in] ds The dataset to be exported
+  @param [in] dlm= (COMMA) The delimeter to apply.  For SASJS, will always be
+    COMMA. Supported values:
+    @li COMMA
+    @li SEMICOLON
+  @param [in] headerformat= (LABEL) The format to use for the header section.
+    Valid values:
+    @li LABEL - Use the variable label (or name, if blank)
+    @li NAME - Use the variable name
+    @li SASJS - Used to create sasjs-formatted input CSVs, eg for use in
+      mp_testservice.sas.  This format will supply an input statement in the
+      first row, making ingestion by datastep a breeze.  Special misisng values
+      will be prefixed with a period (eg `.A`) to enable ingestion on both SAS 9
+      and Viya.  Dates / Datetimes etc are identified by the format type (lookup
+      with mcf_getfmttype.sas) and converted to human readable formats (not
+      numbers).
+  @param [out] outfile= The output filename - should be quoted.
+  @param [out] outref= (0) The output fileref (takes precedence if provided)
+  @param [in] outencoding= (0) The (quoted) output encoding to use, eg `"UTF-8"`
+  @param [in] termstr= (CRLF) The line seperator to use.  For SASJS, will
+    always be CRLF.  Valid values:
+    @li CRLF
+    @li LF
+
+  <h4> SAS Macros </h4>
+  @li mcf_getfmttype.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarformat.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
 
   @version 9.2
   @author Allan Bowe (credit mjsq)
 **/
 
-%macro mp_ds2csv(ds, outref=0, outfile=, outencoding=0
+%macro mp_ds2csv(ds
+  ,dlm=COMMA
+  ,outref=0
+  ,outfile=
+  ,outencoding=0
+  ,headerformat=LABEL
+  ,termstr=CRLF
 )/*/STORE SOURCE*/;
 
+%local outloc delim i varlist var vcnt vat dsv vcom vmiss fmttype vfmt;
+
 %if not %sysfunc(exist(&ds)) %then %do;
   %put %str(WARN)ING:  &ds does not exist;
   %return;
@@ -26,33 +85,128 @@
 %if %index(&ds,.)=0 %then %let ds=WORK.&ds;
 
 %if &outencoding=0 %then %let outencoding=;
-%else %let outencoding=encoding="&outencoding";
+%else %let outencoding=encoding=&outencoding;
 
-%local outloc;
 %if &outref=0 %then %let outloc=&outfile;
 %else %let outloc=&outref;
 
+%if &headerformat=SASJS %then %do;
+  %let delim=",";
+  %let termstr=CRLF;
+  %mcf_getfmttype(wrap=YES)
+%end;
+%else %if &dlm=COMMA %then %let delim=",";
+%else %let delim=";";
+
 /* credit to mjsq - https://stackoverflow.com/a/55642267 */
 
 /* first get headers */
 data _null_;
-  file &outloc dlm=',' dsd &outencoding lrecl=32767;
-  length header $ 2000;
+  file &outloc &outencoding lrecl=32767 termstr=&termstr;
+  length header $ 2000 varnm vfmt $32 dlm $1 fmttype $8;
+  call missing(of _all_);
   dsid=open("&ds.","i");
   num=attrn(dsid,"nvars");
+  dlm=&delim;
   do i=1 to num;
-    header = cats(coalescec(varlabel(dsid,i),varname(dsid,i)));
+    varnm=upcase(varname(dsid,i));
+    if i=num then dlm='';
+  %if &headerformat=NAME %then %do;
+    header=cats(varnm,dlm);
+  %end;
+  %else %if &headerformat=LABEL %then %do;
+    header = cats(coalescec(varlabel(dsid,i),varnm),dlm);
+  %end;
+  %else %if &headerformat=SASJS %then %do;
+    if vartype(dsid,i)='C' then header=cats(varnm,':$char',varlen(dsid,i),'.');
+    else do;
+      vfmt=coalescec(varfmt(dsid,i),'0');
+      fmttype=mcf_getfmttype(vfmt);
+      if fmttype='DATE' then header=cats(varnm,':date9.');
+      else if fmttype='DATETIME' then header=cats(varnm,':E8601DT26.6');
+      else if fmttype='TIME' then header=cats(varnm,':TIME12.');
+      else header=cats(varnm,':best.');
+    end;
+  %end;
+  %else %do;
+    %put &sysmacroname: Invalid headerformat value (&headerformat);
+    %return;
+  %end;
     put header @;
   end;
   rc=close(dsid);
 run;
 
+%let varlist=%mf_getvarlist(&ds);
+%let vcnt=%sysfunc(countw(&varlist));
+
+/**
+  * The $quote modifier (without a width) will take the length from the variable
+  * and increase by two.  However this will lead to truncation where the value
+  * contains double quotes (which are doubled up).  To get around this, scan the
+  * data to see the max number of double quotes, so that the appropriate width
+  * can be applied in the subsequent step.
+  */
+data _null_;
+  set &ds end=last;
+%do i=1 %to &vcnt;
+  %let var=%scan(&varlist,&i);
+  %if %mf_getvartype(&ds,&var)=C %then %do;
+    %let dsv1=%mf_getuniquename(prefix=csvcol1_);
+    %let dsv2=%mf_getuniquename(prefix=csvcol2_);
+    retain &dsv1 0;
+    &dsv2=length(&var)+countc(&var,'"');
+    if &dsv2>&dsv1 then &dsv1=&dsv2;
+    if last then call symputx(
+      "vlen&i"
+      /* should be no shorter than varlen, and no longer than 32767 */
+      ,cats('$quote',min(&dsv1+2,32767),'.')
+      ,'l'
+    );
+  %end;
+%end;
+
+%let vat=@;
+%let vcom=&delim;
+%let vmiss=%mf_getuniquename(prefix=csvcol3_);
 /* next, export data */
 data _null_;
   set &ds.;
-  file &outloc mod dlm=',' dsd &outencoding lrecl=32767;
-  put (_all_) (+0);
+  file &outloc mod dlm=&delim dsd &outencoding lrecl=32767 termstr=&termstr;
+  if _n_=1 then &vmiss='  ';
+  %do i=1 %to &vcnt;
+    %let var=%scan(&varlist,&i);
+    %if &i=&vcnt %then %do;
+      %let vat=;
+      %let vcom=;
+    %end;
+    %if %mf_getvartype(&ds,&var)=N %then %do;
+      %if &headerformat = SASJS %then %do;
+        %let vcom=&delim;
+        %let fmttype=%sysfunc(mcf_getfmttype(%mf_getvarformat(&ds,&var)0));
+        %if &fmttype=DATE %then %let vfmt=DATE9.;
+        %else %if &fmttype=DATETIME %then %let vfmt=E8601DT26.6;
+        %else %if &fmttype=TIME %then %let vfmt=TIME12.;
+        %else %do;
+          %let vfmt=;
+          %let vcom=;
+        %end;
+      %end;
+      %else %let vcom=;
+
+      /* must use period - in order to work in both 9.4 and Viya 3.5 */
+      if missing(&var) and &var ne %sysfunc(getoption(MISSING)) then do;
+        &vmiss=cats('.',&var);
+        put &vmiss &vat;
+      end;
+      else put &var &vfmt &vcom &vat;
+
+    %end;
+    %else %do;
+      %if &i ne &vcnt %then %let vcom=&delim;
+      put &var &&vlen&i &vcom &vat;
+    %end;
+  %end;
 run;
 
-
 %mend mp_ds2csv;

base/mp_ds2ddl.sas

@@ -0,0 +1,30 @@
+/**
+  @file
+  @brief A wrapper for mp_getddl.sas
+  @details In the next release, this will be the main version.
+
+
+  <h4> SAS Macros </h4>
+  @li mp_getddl.sas
+
+**/
+
+%macro mp_ds2ddl(libds,fref=getddl,flavour=SAS,showlog=YES,schema=
+  ,applydttm=NO
+)/*/STORE SOURCE*/;
+
+%local libref;
+%let libds=%upcase(&libds);
+%let libref=%scan(&libds,1,.);
+%if &libref=&libds %then %let libds=WORK.&libds;
+
+%mp_getddl(%scan(&libds,1,.)
+  ,%scan(&libds,2,.)
+  ,fref=&fref
+  ,flavour=SAS
+  ,showlog=&showlog
+  ,schema=&schema
+  ,applydttm=&applydttm
+)
+
+%mend mp_ds2ddl;

base/mp_ds2squeeze.sas

@@ -49,7 +49,7 @@
 
 %macro mp_ds2squeeze(
   libds,
-  outds=work.work.mp_ds2squeeze,
+  outds=work.mp_ds2squeeze,
   mdebug=0
 )/*/STORE SOURCE*/;
 %local dbg source;
@@ -62,9 +62,10 @@
   %let source=/source2;
 %end;
 
-%local optval ds fref;
+%local optval ds fref startsize;
 %let ds=%mf_getuniquename();
 %let fref=%mf_getuniquefileref();
+%let startsize=%mf_getfilesize(libds=&libds,format=yes);
 
 %mp_getmaxvarlengths(&libds,outds=&ds)
 
@@ -113,7 +114,7 @@ options varlenchk=&optval;
   filename &fref clear;
 %end;
 
-%put &sysmacroname: &libds was %mf_getfilesize(libds=&libds,format=yes);
+%put &sysmacroname: &libds was &startsize;
 %put &sysmacroname: &outds is %mf_getfilesize(libds=&outds,format=yes);
 
 %mend mp_ds2squeeze;

base/mp_filtercheck.sas

@@ -6,7 +6,8 @@
   SYSCC to 1008 if bad records are found, and call mp_abort.sas for a
   graceful service exit (configurable).
 
-  Used for dynamic filtering in [Data Controller for SAS®](https://datacontroller.io).
+  Used for dynamic filtering in [Data Controller for SAS®](
+  https://datacontroller.io).
 
   Usage:
 
@@ -125,7 +126,7 @@ data &outds;
     output;
   end;
   if OPERATOR_NM not in
-  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
+  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE')
   then do;
     REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;

base/mp_filterstore.sas

@@ -8,8 +8,13 @@
   https://sasapps.io)).  This macro is also used in [Data Controller for SAS](
   https://datacontroller.io).
 
+  A more recent feature of this macro is the ability to support filter queries
+  on Format Catalogs.  This is achieved by adding a `-FC` suffix to the `libds`
+  parameter - where the "ds" in this case is the catalog name.
 
-  @param [in] libds= The target dataset to be filtered (lib should be assigned)
+
+  @param [in] libds= The target dataset to be filtered (lib should be assigned).
+    If filtering a format catalog, add the following suffix:  `-FC`.
   @param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to
     be validated.  Has the following format:
 |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
@@ -45,6 +50,7 @@
 
 
   <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
   @li mf_getuniquename.sas
   @li mf_getvalue.sas
   @li mf_islibds.sas
@@ -78,7 +84,10 @@
 %put &sysmacroname entry vars:;
 %put _local_;
 
-%local ds1 ds2 ds3 ds4 filter_hash;
+%local ds0 ds1 ds2 ds3 ds4 filter_hash orig_libds;
+%let libds=%upcase(&libds);
+%let orig_libds=&libds;
+
 %mp_abort(iftrue= (&syscc ne 0)
   ,mac=mp_filterstore
   ,msg=%str(syscc=&syscc on macro entry)
@@ -96,12 +105,27 @@
   ,msg=%str(Invalid lock_table value: &lock_table)
 )
 
-/* validate query */
+/**
+  * validate query
+  * use format catalog export, if a format
+  */
+%if "%substr(&libds,%length(&libds)-2,3)"="-FC" %then %do;
+  %let libds=%scan(&libds,1,-); /* chop off -FC extension */
+  %let ds0=%mf_getuniquename(prefix=fmtds_);
+  %let libds=&ds0;
+  /*
+    There is no need to export the entire format catalog here - the validations
+    are done against the data model, not the data values.  So we can simply
+    hardcode the structure based on the cntlout dataset.
+  */
+  %mddl_sas_cntlout(libds=&ds0)
+
+%end;
 %mp_filtercheck(&queryds,targetds=&libds,abort=YES)
 
 /* hash the result */
 %let ds1=%mf_getuniquename(prefix=hashds);
-%mp_hashdataset(&queryds,outds=&ds1,salt=&libds)
+%mp_hashdataset(&queryds,outds=&ds1,salt=&orig_libds)
 %let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
 %if &mdebug=1 %then %do;
   data _null_;
@@ -132,7 +156,7 @@ run;
   %let ds3=%mf_getuniquename(prefix=filtersum);
   data work.&ds3;
     if 0 then set &filter_summary;
-    filter_table=symget('libds');
+    filter_table="&orig_libds";
     filter_hash="&filter_hash";
     PROCESSED_DTTM=%sysfunc(datetime());
     output;

base/mp_filtervalidate.sas

@@ -96,8 +96,7 @@ filename &fref1 clear;
     run;
     %mp_abort(
       mac=&sysmacroname,
-      msg=%str(Filter validation issues. ERR=%superq(SYSERRORTEXT)
-        , WARN=%superq(SYSWARNINGTEXT) )
+      msg=%str(Filter validation issues.)
     )
   %end;
   %let syscc=1008;

base/mp_getformats.sas

@@ -40,6 +40,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
   |`WHICHPATH `|`**OTHER** `|`**OTHER** `|`big fat problem if not path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|`O `|` `|` `|` `|` `|
 
   <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
   @li mf_dedup.sas
   @li mf_getfmtlist.sas
   @li mf_getfmtname.sas
@@ -94,30 +95,7 @@ create table &outsummary as
 
 %if "&outdetail" ne "0" %then %do;
   /* ensure base table always exists */
-  proc sql;
-  create table &outdetail(
-      FMTNAME char(32)     label='Format name'
-      ,START char(16)     label='Starting value for format'
-      ,END char(16)     label='Ending value for format'
-      ,LABEL char(256)     label='Format value label'
-      ,MIN num length=3     label='Minimum length'
-      ,MAX num length=3     label='Maximum length'
-      ,DEFAULT num length=3     label='Default length'
-      ,LENGTH num length=3     label='Format length'
-      ,FUZZ num     label='Fuzz value'
-      ,PREFIX char(2)     label='Prefix characters'
-      ,MULT num     label='Multiplier'
-      ,FILL char(1)     label='Fill character'
-      ,NOEDIT num length=3     label='Is picture string noedit?'
-      ,TYPE char(1)     label='Type of format'
-      ,SEXCL char(1)     label='Start exclusion'
-      ,EEXCL char(1)     label='End exclusion'
-      ,HLO char(13)     label='Additional information'
-      ,DECSEP char(1)     label='Decimal separator'
-      ,DIG3SEP char(1)     label='Three-digit separator'
-      ,DATATYPE char(8)     label='Date/time/datetime?'
-      ,LANGUAGE char(8)     label='Language for date strings'
-  );
+  %mddl_sas_cntlout(libds=&outdetail)
   /* grab the location of each format */
   %let fmtcnt=0;
   data _null_;
@@ -134,7 +112,11 @@ create table &outsummary as
     proc format library=&&fmtloc&i CNTLOUT=&tempds;
       select &&fmtname&i;
     run;
-    proc append base=&outdetail data=&tempds;
+    data &tempds;
+      if 0 then set &outdetail;
+      set &tempds;
+    run;
+    proc append base=&outdetail data=&tempds ;
     run;
   %end;
 %end;

base/mp_guesspk.sas

@@ -17,16 +17,21 @@
       %inc mc;
       %mp_guesspk(sashelp.class,outds=classpks)
 
-  @param baseds The dataset to analyse
-  @param outds= The output dataset to contain the possible PKs
-  @param max_guesses= (3) The total number of possible primary keys to generate.
-    A table may have multiple unlikely PKs, so no need to list them all.
-  @param min_rows= (5) The minimum number of rows a table should have in order
-    to try and guess the PK.
+  @param [in] baseds The dataset to analyse
+  @param [out] outds= The output dataset to contain the possible PKs
+  @param [in] max_guesses= (3) The total number of possible primary keys to
+    generate. A table may have multiple (unlikely) PKs, so no need to list them
+    all.
+  @param [in] min_rows= (5) The minimum number of rows a table should have in
+    order to try and guess the PK.
+  @param [in] ignore_cols (0) Space seperated list of columns which you are
+    sure are not part of the primary key (helps to avoid false positives)
+  @param [in] mdebug= Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
   @li mf_getvarlist.sas
   @li mf_getuniquename.sas
+  @li mf_wordsInstr1butnotstr2.sas
   @li mf_nobs.sas
 
   <h4> Related Macros </h4>
@@ -38,179 +43,226 @@
 **/
 
 %macro mp_guesspk(baseds
-      ,outds=mp_guesspk
-      ,max_guesses=3
-      ,min_rows=5
+  ,outds=mp_guesspk
+  ,max_guesses=3
+  ,min_rows=5
+  ,ignore_cols=0
+  ,mdebug=0
 )/*/STORE SOURCE*/;
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
 
-  /* declare local vars */
-  %local var vars vcnt i j k l tmpvar tmpds rows posspks ppkcnt;
-  %let vars=%mf_getvarlist(&baseds);
-  %let vcnt=%sysfunc(countw(&vars));
+/* declare local vars */
+%local var vars vcnt i j k l tmpvar tmpds rows posspks ppkcnt;
+%let vars=%upcase(%mf_getvarlist(&baseds));
+%let vars=%mf_wordsInStr1ButNotStr2(str1=&vars,str2=%upcase(&ignore_cols));
+%let vcnt=%sysfunc(countw(&vars));
 
-  %if &vcnt=0 %then %do;
-    %put &sysmacroname: &baseds has no variables!  Exiting.;
-    %return;
+%if &vcnt=0 %then %do;
+  %put &sysmacroname: &baseds has no variables!  Exiting.;
+  %return;
+%end;
+
+/* get null count and row count */
+%let tmpvar=%mf_getuniquename();
+proc sql noprint;
+create table _data_ as select
+  count(*) as &tmpvar
+%do i=1 %to &vcnt;
+  %let var=%scan(&vars,&i);
+  ,sum(case when &var is missing then 1 else 0 end) as &var
+%end;
+  from &baseds;
+
+/* transpose table and scan for not null cols */
+proc transpose;
+data _null_;
+  set &syslast end=last;
+  length vars $32767;
+  retain vars ;
+  if _name_="&tmpvar" then call symputx('rows',col1,'l');
+  else if col1=0 then vars=catx(' ',vars,_name_);
+  if last then call symputx('posspks',vars,'l');
+run;
+
+%let ppkcnt=%sysfunc(countw(&posspks));
+%if &ppkcnt=0 %then %do;
+  %put &sysmacroname: &baseds has no non-missing variables!  Exiting.;
+  %return;
+%end;
+
+proc sort data=&baseds(keep=&posspks) out=_data_ noduprec;
+  by _all_;
+run;
+%local pkds; %let pkds=&syslast;
+
+%if &rows > %mf_nobs(&pkds) %then %do;
+  %put &sysmacroname: &baseds has no combination of unique records! Exiting.;
+  %return;
+%end;
+
+/* now check cardinality */
+proc sql noprint;
+create table _data_ as select
+%do i=1 %to &ppkcnt;
+  %let var=%scan(&posspks,&i);
+  count(distinct &var) as &var
+  %if &i<&ppkcnt %then ,;
+%end;
+  from &pkds;
+
+/* transpose and sort by cardinality */
+proc transpose;
+proc sort; by descending col1;
+run;
+
+/* create initial PK list and re-order posspks list */
+data &outds(keep=pkguesses);
+  length pkguesses $5000 vars $5000;
+  set &syslast end=last;
+  retain vars ;
+  vars=catx(' ',vars,_name_);
+  if col1=&rows then do;
+    pkguesses=_name_;
+    output;
+  end;
+  if last then call symputx('posspks',vars,'l');
+run;
+
+%if %mf_nobs(&outds) ge &max_guesses %then %do;
+  %put &sysmacroname: %mf_nobs(&outds) possible primary key values found;
+  %return;
+%end;
+
+%if &ppkcnt=1 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %return;
+%end;
+
+/* begin scanning for uniques on pairs of PKs */
+%let tmpds=%mf_getuniquename();
+%local lev1 lev2;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do;
+      /* check for two level uniqueness */
+      proc sort data=&pkds(keep=&lev1 &lev2) out=&tmpds noduprec;
+        by _all_;
+      run;
+      %if %mf_nobs(&tmpds)=&rows %then %do;
+        proc sql;
+        insert into &outds values("&lev1 &lev2");
+        %if %mf_nobs(&outds) ge &max_guesses %then %do;
+          %put &sysmacroname: Max PKs reached at Level 2 for &baseds;
+          %goto exit;
+        %end;
+      %end;
+    %end;
   %end;
+%end;
 
-  /* get null count and row count */
-  %let tmpvar=%mf_getuniquename();
-  proc sql noprint;
-  create table _data_ as select
-    count(*) as &tmpvar
-  %do i=1 %to &vcnt;
-    %let var=%scan(&vars,&i);
-    ,sum(case when &var is missing then 1 else 0 end) as &var
-  %end;
-    from &baseds;
+%if &ppkcnt=2 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* transpose table and scan for not null cols */
-  proc transpose;
-  data _null_;
-    set &syslast end=last;
-    length vars $32767;
-    retain vars ;
-    if _name_="&tmpvar" then call symputx('rows',col1,'l');
-    else if col1=0 then vars=catx(' ',vars,_name_);
-    if last then call symputx('posspks',vars,'l');
-  run;
-
-  %let ppkcnt=%sysfunc(countw(&posspks));
-  %if &ppkcnt=0 %then %do;
-    %put &sysmacroname: &baseds has no non-missing variables!  Exiting.;
-    %return;
-  %end;
-
-  proc sort data=&baseds(keep=&posspks) out=_data_ noduprec;
-    by _all_;
-  run;
-  %local pkds; %let pkds=&syslast;
-
-  %if &rows > %mf_nobs(&pkds) %then %do;
-    %put &sysmacroname: &baseds has no combination of unique records! Exiting.;
-    %return;
-  %end;
-
-  /* now check cardinality */
-  proc sql noprint;
-  create table _data_ as select
-  %do i=1 %to &ppkcnt;
-    %let var=%scan(&posspks,&i);
-    count(distinct &var) as &var
-    %if &i<&ppkcnt %then ,;
-  %end;
-    from &pkds;
-
-  /* transpose and sort by cardinality */
-  proc transpose;
-  proc sort; by descending col1;
-  run;
-
-  /* create initial PK list and re-order posspks list */
-  data &outds(keep=pkguesses);
-    length pkguesses $5000 vars $5000;
-    set &syslast end=last;
-    retain vars ;
-    vars=catx(' ',vars,_name_);
-    if col1=&rows then do;
-      pkguesses=_name_;
-      output;
-    end;
-    if last then call symputx('posspks',vars,'l');
-  run;
-
-  %if %mf_nobs(&outds) ge &max_guesses %then %do;
-    %put &sysmacroname: %mf_nobs(&outds) possible primary key values found;
-    %return;
-  %end;
-
-  %if &ppkcnt=1 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
-
-  /* begin scanning for uniques on pairs of PKs */
-  %let tmpds=%mf_getuniquename();
-  %local lev1 lev2;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do;
-        /* check for two level uniqueness */
-        proc sort data=&pkds(keep=&lev1 &lev2) out=&tmpds noduprec;
+/* begin scanning for uniques on PK triplets */
+%local lev3;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
+      %let lev3=%scan(&posspks,&k);
+      %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do;
+        /* check for three level uniqueness */
+        proc sort data=&pkds(keep=&lev1 &lev2 &lev3) out=&tmpds noduprec;
           by _all_;
         run;
         %if %mf_nobs(&tmpds)=&rows %then %do;
           proc sql;
-          insert into &outds values("&lev1 &lev2");
+          insert into &outds values("&lev1 &lev2 &lev3");
           %if %mf_nobs(&outds) ge &max_guesses %then %do;
-            %put &sysmacroname: Max PKs reached at Level 2 for &baseds;
-            %return;
+            %put &sysmacroname: Max PKs reached at Level 3 for &baseds;
+            %goto exit;
           %end;
         %end;
       %end;
     %end;
   %end;
+%end;
 
-  %if &ppkcnt=2 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%if &ppkcnt=3 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* begin scanning for uniques on PK triplets */
-  %local lev3;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
-        %let lev3=%scan(&posspks,&k);
-        %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do;
-          /* check for three level uniqueness */
-          proc sort data=&pkds(keep=&lev1 &lev2 &lev3) out=&tmpds noduprec;
+/* scan for uniques on up to 4 PK fields */
+%local lev4;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
+      %let lev3=%scan(&posspks,&k);
+      %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
+        %let lev4=%scan(&posspks,&l);
+        %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then %do;
+          /* check for four level uniqueness */
+          proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4)
+              out=&tmpds noduprec;
             by _all_;
           run;
           %if %mf_nobs(&tmpds)=&rows %then %do;
             proc sql;
-            insert into &outds values("&lev1 &lev2 &lev3");
+            insert into &outds values("&lev1 &lev2 &lev3 &lev4");
             %if %mf_nobs(&outds) ge &max_guesses %then %do;
-              %put &sysmacroname: Max PKs reached at Level 3 for &baseds;
-              %return;
+              %put &sysmacroname: Max PKs reached at Level 4 for &baseds;
+              %goto exit;
             %end;
           %end;
         %end;
       %end;
     %end;
   %end;
+%end;
 
-  %if &ppkcnt=3 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%if &ppkcnt=4 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* scan for uniques on up to 4 PK fields */
-  %local lev4;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
-        %let lev3=%scan(&posspks,&k);
-        %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
-          %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then %do;
+/* scan for uniques on up to 4 PK fields */
+%local lev5 m;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
+      %let lev3=%scan(&posspks,&k);
+      %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
+        %let lev4=%scan(&posspks,&l);
+        %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
+        %do m=5 %to &ppkcnt;
+          %let lev5=%scan(&posspks,&m);
+          %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then %do;
             /* check for four level uniqueness */
-            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4)
+            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5)
                 out=&tmpds noduprec;
               by _all_;
             run;
             %if %mf_nobs(&tmpds)=&rows %then %do;
               proc sql;
-              insert into &outds values("&lev1 &lev2 &lev3 &lev4");
+              insert into &outds values("&lev1 &lev2 &lev3 &lev4 &lev5");
               %if %mf_nobs(&outds) ge &max_guesses %then %do;
-                %put &sysmacroname: Max PKs reached at Level 4 for &baseds;
-                %return;
+                %put &sysmacroname: Max PKs reached at Level 5 for &baseds;
+                %goto exit;
               %end;
             %end;
           %end;
@@ -218,37 +270,44 @@
       %end;
     %end;
   %end;
+%end;
 
-  %if &ppkcnt=4 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%if &ppkcnt=5 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* scan for uniques on up to 4 PK fields */
-  %local lev5 m;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
-        %let lev3=%scan(&posspks,&k);
-        %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
-          %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
-          %do m=5 %to &ppkcnt;
-            %let lev5=%scan(&posspks,&m);
-            %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then %do;
+/* scan for uniques on up to 4 PK fields */
+%local lev6 n;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
+      %let lev3=%scan(&posspks,&k);
+      %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
+        %let lev4=%scan(&posspks,&l);
+        %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
+        %do m=5 %to &ppkcnt;
+          %let lev5=%scan(&posspks,&m);
+          %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5
+          %then %do n=6 %to &ppkcnt;
+            %let lev6=%scan(&posspks,&n);
+            %if &lev1 ne &lev6 & &lev2 ne &lev6 & &lev3 ne &lev6
+            & &lev4 ne &lev6 & &lev5 ne &lev6 %then
+            %do;
               /* check for four level uniqueness */
-              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5)
-                  out=&tmpds noduprec;
+              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5 &lev6)
+                out=&tmpds noduprec;
                 by _all_;
               run;
               %if %mf_nobs(&tmpds)=&rows %then %do;
                 proc sql;
-                insert into &outds values("&lev1 &lev2 &lev3 &lev4 &lev5");
+                insert into &outds
+                  values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
                 %if %mf_nobs(&outds) ge &max_guesses %then %do;
-                  %put &sysmacroname: Max PKs reached at Level 5 for &baseds;
-                  %return;
+                  %put &sysmacroname: Max PKs reached at Level 6 for &baseds;
+                  %goto exit;
                 %end;
               %end;
             %end;
@@ -257,56 +316,17 @@
       %end;
     %end;
   %end;
+%end;
 
-  %if &ppkcnt=5 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%if &ppkcnt=6 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* scan for uniques on up to 4 PK fields */
-  %local lev6 n;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
-        %let lev3=%scan(&posspks,&k);
-        %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
-          %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
-          %do m=5 %to &ppkcnt;
-            %let lev5=%scan(&posspks,&m);
-            %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then
-            %do n=6 %to &ppkcnt;
-              %let lev6=%scan(&posspks,&n);
-              %if &lev1 ne &lev6 & &lev2 ne &lev6 & &lev3 ne &lev6
-              & &lev4 ne &lev6 & &lev5 ne &lev6 %then
-              %do;
-                /* check for four level uniqueness */
-                proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5 &lev6)
-                  out=&tmpds noduprec;
-                  by _all_;
-                run;
-                %if %mf_nobs(&tmpds)=&rows %then %do;
-                  proc sql;
-                  insert into &outds
-                    values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
-                  %if %mf_nobs(&outds) ge &max_guesses %then %do;
-                    %put &sysmacroname: Max PKs reached at Level 6 for &baseds;
-                    %return;
-                  %end;
-                %end;
-              %end;
-            %end;
-          %end;
-        %end;
-      %end;
-    %end;
-  %end;
-
-  %if &ppkcnt=6 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%exit:
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &tmpds;
+%end;
 
 %mend mp_guesspk;

base/mp_include.sas

@@ -51,9 +51,10 @@ https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/mcrolref/n1j5tcc0n2xczyn1
     this dataset.
     It will then run an abort cancel FILE to stop the include running, and pass
     the dataset back.
-    NOTE - it is NOT possible to read this dataset as part of _this_ macro -
-    when running abort cancel FILE, ALL macros are closed, so instead it is
-    necessary to invoke "%mp_abort(mode=INCLUDE)" OUTSIDE of any macro wrappers.
+
+    IMPORTANT NOTE - it is NOT possible to read this dataset as part of _this_
+    macro! When running abort cancel FILE, ALL macros are closed, so instead it
+    is necessary to invoke "%mp_abort(mode=INCLUDE)" OUTSIDE of macro wrappers.
 
 
   @version 9.4

base/mp_init.sas

@@ -37,6 +37,7 @@
 
 %global
   SASJS_PREFIX       /* the ONLY hard-coded global macro variable in SASjs    */
+  &prefix._FUNCTIONS /* used in mcf_init() to track core function compilation */
   &prefix._INIT_NUM  /* initialisation time as numeric                        */
   &prefix._INIT_DTTM /* initialisation time in E8601DT26.6 format             */
   &prefix.WORK       /* avoid typing %sysfunc(pathname(work)) every time      */
@@ -52,11 +53,9 @@ data _null_;
 run;
 
 options
-  noautocorrect           /* disallow misspelled procedure names            */
   compress=CHAR           /* default is none so ensure we have something!   */
   datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
-  dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
-  %str(err)orcheck=STRICT /* catch errs in libname/filename statements      */
+  errorcheck=STRICT       /* catch errs in libname/filename statements      */
   fmterr                  /* ensure err when a format cannot be found       */
   mergenoby=%str(ERR)OR   /* throw err when a merge has no BY variables     */
   missing=.               /* changing this can cause hard to detect errs    */
@@ -68,6 +67,10 @@ options
   validvarname=V7         /* avoid special characters etc in variable names */
   varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
   varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
+%if %substr(&sysver,1,1) ne 4 %then %do;
+  noautocorrect           /* disallow misspelled procedure names            */
+  dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
+%end;
 ;
 
 %mend mp_init;

base/mp_jsonout.sas

@@ -131,7 +131,7 @@
       %put &sysmacroname: Switching to DATASTEP engine;
       %goto datastep;
     %end;
-    data &tempds /view=&tempds;set &ds;
+    data &tempds;set &ds;
     %if &fmt=N %then format _numeric_ best32.;;
     /* PRETTY is necessary to avoid line truncation in large files */
     proc json out=&jref pretty
@@ -157,7 +157,12 @@
         ));
       %do i=1 %to &numcols;
         length &&name&i $&&len&i;
-        &&name&i=left(put(&&newname&i,&&fmt&i));
+        %if &&typelong&i=num %then %do;
+          &&name&i=left(put(&&newname&i,&&fmt&i));
+        %end;
+        %else %do;
+          &&name&i=put(&&newname&i,&&fmt&i);
+        %end;
         drop &&newname&i;
       %end;
         if _error_ then call symputx('syscc',1012);
@@ -177,7 +182,7 @@
     %end;
       other = [best.];
 
-    data &tempds/view=&tempds;
+    data &tempds;
       attrib _all_ label='';
       %do i=1 %to &numcols;
         %if &&typelong&i=char or &fmt=Y %then %do;
@@ -239,8 +244,7 @@
   %end;
 
   proc sql;
-  drop view &tempds;
-  drop table &colinfo;
+  drop table &colinfo, &tempds;
 
   %if &showmeta=YES %then %do;
     data _null_; file &jref encoding='utf-8' mod;

base/mp_loadformat.sas

@@ -0,0 +1,298 @@
+/**
+  @file
+  @brief Loads a format catalog from a staging dataset
+  @details When loading staged data, it is common to receive only the records
+  that have actually changed.  However, when loading a format catalog, if
+  records are missing they are presumed to be no longer required.
+
+  This macro will augment a staging dataset with other records from the same
+  format, to prevent loss of data - UNLESS the input dataset contains a marker
+  column, specifying that a particular row needs to be deleted (`delete_col=`).
+
+  This macro can also be used to identify which records would be (or were)
+  considered new, modified or deleted (`loadtarget=`) by creating the following
+  tables:
+
+  @li work.outds_add
+  @li work.outds_del
+  @li work.outds_mod
+
+  For example usage, see mp_loadformat.test.sas
+
+  @param [in] libcat The format catalog to be loaded
+  @param [in] libds The staging table to load
+  @param [in] loadtarget= (NO) Set to YES to actually load the target catalog
+  @param [in] delete_col= (_____DELETE__THIS__RECORD_____) The column used to
+    mark a record for deletion.  Values should be "Yes" or "No".
+  @param [out] auditlibds= (0) For change tracking, set to the libds of an audit
+    table as defined in mddl_dc_difftable.sas
+  @param [in] locklibds= (0) For multi-user (parallel) situations, set to the
+    libds of the DC lock table as defined in the mddl_dc_locktable.sas macro.
+  @param [out] outds_add= (0) Set a libds here to see the new records added
+  @param [out] outds_del= (0) Set a libds here to see the records deleted
+  @param [out] outds_mod= (0) Set a libds here to see the modified records
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_cntlout.sas
+  @li mp_lockanytable.sas
+
+  <h4> Related Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mddl_dc_locktable.sas
+  @li mp_loadformat.test.sas
+  @li mp_lockanytable.sas
+  @li mp_storediffs.sas
+  @li mp_stackdiffs.sas
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_loadformat(libcat,libds
+  ,loadtarget=NO
+  ,auditlibds=0
+  ,locklibds=0
+  ,delete_col=_____DELETE__THIS__RECORD_____
+  ,outds_add=0
+  ,outds_del=0
+  ,outds_mod=0
+  ,mdebug=0
+);
+/* set up local macro variables and temporary tables (with a prefix) */
+%local err msg prefix dslist i var fmtlist ibufsize;
+%let dslist=base_fmts template inlibds ds1 stagedata storediffs;
+%if &outds_add=0 %then %let dslist=&dslist outds_add;
+%if &outds_del=0 %then %let dslist=&dslist outds_del;
+%if &outds_mod=0 %then %let dslist=&dslist outds_mod;
+%let prefix=%substr(%mf_getuniquename(),1,21);
+%do i=1 %to %sysfunc(countw(&dslist));
+  %let var=%scan(&dslist,&i);
+  %local &var;
+  %let &var=%upcase(&prefix._&var);
+%end;
+
+/*
+format values can be up to 32767 wide.  SQL joins on such a wide column can
+cause buffer issues.  Update ibufsize and reset at the end.
+*/
+%let ibufsize=%sysfunc(getoption(ibufsize));
+options ibufsize=32767 ;
+
+/* in DC, format catalogs maybe specified in the libds with a -FC extension */
+%let libcat=%scan(&libcat,1,-);
+
+/* perform input validations */
+%let err=0;
+%let msg=0;
+data _null_;
+  if _n_=1 then putlog "&sysmacroname entry vars:";
+  set sashelp.vmacro;
+  where scope="&sysmacroname";
+  value=upcase(value);
+  if &mdebug=0 then put name '=' value;
+  if name=:'LOAD' and value not in ('YES','NO') then do;
+    call symputx('msg',"invalid value for "!!name!!":"!!value);
+    call symputx('err',1);
+    stop;
+  end;
+  else if name='LIBCAT' then do;
+    if exist(value,'CATALOG') le 0 then do;
+      call symputx('msg',"Unable to open catalog: "!!value);
+      call symputx('err',1);
+      stop;
+    end;
+  end;
+  else if name='LIBDS' then do;
+    if exist(value) le 0 then do;
+      call symputx('msg',"Unable to open staging table: "!!value);
+      call symputx('err',1);
+      stop;
+    end;
+  end;
+  else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
+  and missing(value) then do;
+    call symputx('msg',"missing value in var: "!!name);
+    call symputx('err',1);
+    stop;
+  end;
+run;
+
+%mp_abort(
+  iftrue=(&err ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(&msg)
+)
+
+/**
+  * First, extract only relevant formats from the catalog
+  */
+proc sql noprint;
+select distinct fmtname into: fmtlist separated by ' ' from &libds;
+
+%mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
+
+
+/**
+  * Ensure input table and base_formats have consistent lengths and types
+  */
+%mddl_sas_cntlout(libds=&template)
+data &inlibds;
+  if 0 then set &template;
+  set &libds;
+  if missing(type) then do;
+    if substr(fmtname,1,1)='$' then type='C';
+    else type='N';
+  end;
+  if type='N' then do;
+    start=cats(start);
+    end=cats(end);
+  end;
+run;
+
+
+/**
+  * Identify new records
+  */
+proc sql;
+create table &outds_add(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  left join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where b.fmtname is null
+    and upcase(a.&delete_col) ne "YES"
+  order by fmtname, start;;
+
+/**
+  * Identify deleted records
+  */
+create table &outds_del(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where upcase(a.&delete_col)="YES"
+  order by fmtname, start;
+
+/**
+  * Identify modified records
+  */
+create table &outds_mod (drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where upcase(a.&delete_col) ne "YES"
+  order by fmtname, start;
+
+options ibufsize=&ibufsize;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc prior to load prep)
+)
+
+%if &loadtarget=YES %then %do;
+  data &ds1;
+    merge &base_fmts(in=base)
+      &outds_mod(in=mod)
+      &outds_add(in=add)
+      &outds_del(in=del);
+    if not del and not mod;
+    by fmtname start;
+  run;
+  data &stagedata;
+    set &ds1 &outds_mod;
+  run;
+  proc sort;
+    by fmtname start;
+  run;
+%end;
+/* mp abort needs to run outside of conditional blocks */
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc prior to actual load)
+)
+%if &loadtarget=YES %then %do;
+  %if %mf_nobs(&stagedata)=0 %then %do;
+    %put There are no changes to load in &libcat!;
+    %return;
+  %end;
+  %if &locklibds ne 0 %then %do;
+    /* prevent parallel updates */
+    %mp_lockanytable(LOCK
+      ,lib=%scan(&libcat,1,.)
+      ,ds=%scan(&libcat,2,.)-FC
+      ,ref=MP_LOADFORMAT commencing format load
+      ,ctl_ds=&locklibds
+    )
+  %end;
+  /* do the actual load */
+  proc format lib=&libcat cntlin=&stagedata;
+  run;
+  %if &locklibds ne 0 %then %do;
+    /* unlock the table */
+    %mp_lockanytable(UNLOCK
+      ,lib=%scan(&libcat,1,.)
+      ,ds=%scan(&libcat,2,.)-FC
+      ,ref=MP_LOADFORMAT completed format load
+      ,ctl_ds=&locklibds
+    )
+  %end;
+  /* track the changes */
+  %if &auditlibds ne 0 %then %do;
+    %if &locklibds ne 0 %then %do;
+      %mp_lockanytable(LOCK
+        ,lib=%scan(&auditlibds,1,.)
+        ,ds=%scan(&auditlibds,2,.)
+        ,ref=MP_LOADFORMAT commencing audit table load
+        ,ctl_ds=&locklibds
+      )
+    %end;
+
+    %mp_storediffs(&libcat-FC
+      ,&inlibds
+      ,FMTNAME START
+      ,delds=&outds_del
+      ,modds=&outds_mod
+      ,appds=&outds_add
+      ,outds=&storediffs
+      ,mdebug=&mdebug
+    )
+
+    %if &locklibds ne 0 %then %do;
+      %mp_lockanytable(UNLOCK
+        ,lib=%scan(&auditlibds,1,.)
+        ,ds=%scan(&auditlibds,2,.)
+        ,ref=MP_LOADFORMAT commencing audit table load
+        ,ctl_ds=&locklibds
+      )
+    %end;
+  %end;
+%end;
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc after load)
+)
+
+%if &mdebug=0 %then %do;
+  proc datasets lib=work;
+    delete &prefix:;
+  run;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%mend mp_loadformat;

base/mp_lockanytable.sas

@@ -1,7 +1,8 @@
 /**
   @file
   @brief Mechanism for locking tables to prevent parallel modifications
-  @details Uses a control table to enable ANY table to be locked for updates.
+  @details Uses a control table to enable ANY table to be locked for updates
+  (not just SAS datasets).
   Only useful if every update uses the macro!   Used heavily within
   [Data Controller for SAS](https://datacontroller.io).
 
@@ -15,7 +16,7 @@
     length is 200 characters.
   @param [out] ctl_ds= (0) The control table which controls the actual locking.
     Should already be assigned and available.  The definition is available by
-    running mp_coretable.sas as follows:  `%mp_coretable(LOCKTABLE)`.
+    running the mddl_dc_locktable.sas macro.
 
   @param [in] loops= (25) Number of times to check for a lock.
   @param [in] loop_secs= (1) Seconds to wait between each lock attempt
@@ -48,7 +49,7 @@ data _null_;
   put name '=' value;
 run;
 
-%mp_abort(iftrue= (&ds=0 and &action ne MAKETABLE)
+%mp_abort(iftrue= ("&ds"="0" and &action ne MAKETABLE)
   ,mac=&sysmacroname
   ,msg=%str(dataset was not provided)
 )
@@ -85,7 +86,7 @@ run;
 /* do not proceed if no observations can be processed */
 %mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)
   ,mac=&sysmacroname
-  ,msg=%str(options obs = 0. syserrortext=&syserrortext)
+  ,msg=%str(cannot continue when options obs = 0)
 )
 
 %if &ACTION=LOCK %then %do;

base/mp_lockfilecheck.sas

@@ -37,7 +37,7 @@ run;
   ,mac=checklock.sas
   ,msg=Aborting with syscc=&syscc on entry.
 )
-%mp_abort(iftrue= (&libds=0)
+%mp_abort(iftrue= ("&libds"="0")
   ,mac=&sysmacroname
   ,msg=%str(libds not provided)
 )
@@ -46,6 +46,12 @@ run;
 %let lib=%upcase(%scan(&libds,1,.));
 %let ds=%upcase(%scan(&libds,2,.));
 
+/* in DC, format catalogs are passed with a -FC suffix. No saslock here! */
+%if %scan(&libds,2,-)=FC %then %do;
+  %put &sysmacroname: Format Catalog detected, no lockfile applied to &libds;
+  %return;
+%end;
+
 /* do not proceed if no observations can be processed */
 %let msg=options obs = 0. syserrortext=%superq(syserrortext);
 %mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)

base/mp_md5.sas

@@ -0,0 +1,58 @@
+/**
+  @file
+  @brief Generates an md5 expression for hashing a set of variables
+  @details This is the same algorithm used to hash records in
+  [Data Controller for SAS](https://datacontroller.io) (free for up
+  to 5 users).
+
+  It is not designed to be efficient - it is designed to be effective,
+  given the range of edge cases (large floating points, special missing
+  numerics, thousands of columns, very wide columns).
+
+  It can be used only in data step, eg as follows:
+
+      data _null_;
+        set sashelp.class;
+        hashvar=%mp_md5(cvars=name sex, nvars=age height weight);
+        put hashvar=;
+      run;
+
+  Unfortunately it will not run in SQL - it fails with the following message:
+
+  > The width value for HEX is out of bounds. It should be between 1 and 16
+
+  The macro will also cause errors if the data contains (non-special) missings
+  and the (undocumented) `options dsoptions=nonote2err;` is in effect.
+
+  This can be avoided in two ways:
+
+  @li Global option:  `options dsoptions=nonote2err;`
+  @li Data step option: `data YOURLIB.YOURDATASET /nonote2err;`
+
+  @param cvars= Space seperated list of character variables
+  @param nvars= Space seperated list of numeric variables
+
+  <h4> Related Programs </h4>
+  @li mp_init.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_md5(cvars=,nvars=);
+%local i var sep;
+put(md5(
+  %do i=1 %to %sysfunc(countw(&cvars));
+    %let var=%scan(&cvars,&i,%str( ));
+    &sep put(md5(trim(&var)),$hex32.)
+    %let sep=!!;
+  %end;
+  %do i=1 %to %sysfunc(countw(&nvars));
+    %let var=%scan(&nvars,&i,%str( ));
+    /* multiply by 1 to strip precision errors (eg 0 != 0) */
+    /* but ONLY if not missing, else will lose any special missing values */
+    &sep put(md5(trim(put(ifn(missing(&var),&var,&var*1),binary64.))),$hex32.)
+    %let sep=!!;
+  %end;
+),hex32.)
+%mend mp_md5;

base/mp_replace.sas

@@ -0,0 +1,149 @@
+/**
+  @file
+  @brief Performs a text substitution on a file
+  @details Performs a find and replace on a file, either in place or to a new
+  file. Can be used on files where lines are longer than 32767.
+
+  Works by reading in the file byte by byte, then marking the beginning and end
+  of each matched string, before finally doing the replace.
+
+  Full credit for this highly efficient and syntactically satisfying SAS logic
+  goes to [Bartosz Jabłoński](https://www.linkedin.com/in/yabwon), founder of
+  the [SAS Packages](https://github.com/yabwon/SAS_PACKAGES) framework.
+
+  Usage:
+
+      %let file="%sysfunc(pathname(work))/file.txt";
+      %let str=replace/me;
+      %let rep=with/this;
+      data _null_;
+        file &file;
+        put 'blahblah';
+        put "blahblah&str.blah";
+        put 'blahblahblah';
+      run;
+      %mp_replace(&file, findvar=str, replacevar=rep)
+      data _null_;
+        infile &file;
+        input;
+        list;
+      run;
+
+  Note - if you are running a version of SAS that will allow the io package in
+  LUA, you can also use this macro: mp_gsubfile.sas
+
+  @param infile The QUOTED path to the file on which to perform the substitution
+  @param findvar= Macro variable NAME containing the string to search for
+  @param replacevar= Macro variable NAME containing the replacement string
+  @param outfile= (0) Optional QUOTED path to an the adjusted output file (to
+    avoid overwriting the first file).
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mp_gsubfile.test.sas
+
+  @version 9.4
+  @author Bartosz Jabłoński
+  @author Allan Bowe
+**/
+
+%macro mp_replace(infile,
+  findvar=,
+  replacevar=,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+%local inref dttm ds1;
+%let inref=%mf_getuniquefileref();
+%let outref=%mf_getuniquefileref();
+%if &outfile=0 %then %let outfile=&infile;
+%let ds1=%mf_getuniquename(prefix=allchars);
+%let ds2=%mf_getuniquename(prefix=startmark);
+
+/* START */
+%let dttm=%sysfunc(datetime());
+
+filename &inref &infile lrecl=1 recfm=n;
+
+data &ds1;
+  infile &inref;
+  input sourcechar $ 1. @@;
+  format sourcechar hex2.;
+run;
+
+data &ds2;
+  /* set find string to length in bytes to cover trailing spaces */
+  length string $ %length(%superq(&findvar));
+  string =symget("&findvar");
+  drop string;
+
+  firstchar=char(string,1);
+  findlen=lengthm(string); /* <- for trailing bytes */
+
+  do _N_=1 to nobs;
+    set &ds1 nobs=nobs point=_N_;
+    if sourcechar=firstchar then do;
+      pos=1;
+      s=0;
+      do point=_N_ to min(_N_ + findlen -1,nobs);
+        set &ds1 point=point;
+        if sourcechar=char(string, pos) then s + 1;
+        else goto _leave_;
+        pos+1;
+      end;
+      _leave_:
+      if s=findlen then do;
+        START =_N_;
+        _N_ =_N_+ s - 1;
+        STOP =_N_;
+        output;
+      end;
+    end;
+  end;
+  stop;
+  keep START STOP;
+run;
+
+data &ds1;
+  declare hash HS(dataset:"&ds2(keep=start)");
+  HS.defineKey("start");
+  HS.defineDone();
+  declare hash HE(dataset:"&ds2(keep=stop)");
+  HE.defineKey("stop");
+  HE.defineDone();
+  do until(eof);
+    set &ds1 end=eof curobs =n;
+    start = ^HS.check(key:n);
+    stop  = ^HE.check(key:n);
+    length strt $ 1;
+    strt =put(start,best. -L);
+    retain out 1;
+    if out   then output;
+    if start then out=0;
+    if stop  then out=1;
+  end;
+  stop;
+  keep sourcechar strt;
+run;
+
+filename &outref &outfile recfm=n;
+
+data _null_;
+  length replace $ %length(%superq(&replacevar));
+  replace=symget("&replacevar");
+  file &outref;
+  do until(eof);
+    set &ds1 end=eof;
+    if strt ="1" then put replace char.;
+    else put sourcechar char1.;
+  end;
+  stop;
+run;
+
+/* END */
+%put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) seconds to run;
+
+%mend mp_replace;

base/mp_stackdiffs.sas

@@ -0,0 +1,592 @@
+/**
+  @file
+  @brief Prepares an audit table for stacking (re-applying) the changes.
+  @details When the underlying data from a Base Table is refreshed, it can be
+    helpful to have any previously-applied changes, re-applied.
+
+    Such situation might arise if you are applying those changes using a tool
+    like [Data Controller for SAS®](https://datacontroller.io) - which records
+    all such changes in an audit table.
+    It may also apply if you are preparing a series of specific cell-level
+    transactions, that you would like to apply to multiple sets of (similarly
+    structured) Base Tables.
+
+    In both cases, it is necessary that the transactions are stored using
+    the mp_storediffs.sas macro, or at least that the underlying table is
+    structured as per the definition in mp_coretable.sas (DIFFTABLE entry)
+
+    <b>This</b> macro is used to convert the stored changes (tall format) into
+    staged changes (wide format), with base table values incorporated (in the
+    case of modified rows), ready for the subsequent load process.
+
+    Essentially then, what this macro does, is turn a table like this:
+
+|KEY_HASH:$32.|MOVE_TYPE:$1.|TGTVAR_NM:$32.|IS_PK:best.|IS_DIFF:best.|TGTVAR_TYPE:$1.|OLDVAL_NUM:best32.|NEWVAL_NUM:best32.|OLDVAL_CHAR:$32765.|NEWVAL_CHAR:$32765.|
+|---|---|---|---|---|---|---|---|---|---|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`NAME `|`1 `|`-1 `|`C `|`. `|`. `|` `|`Newbie `|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`AGE `|`0 `|`-1 `|`N `|`. `|`13 `|` `|` `|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`HEIGHT `|`0 `|`-1 `|`N `|`. `|`65.3 `|` `|` `|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`SEX `|`0 `|`-1 `|`C `|`. `|`. `|` `|`F `|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`WEIGHT `|`0 `|`-1 `|`N `|`. `|`98 `|` `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`NAME `|`1 `|`-1 `|`C `|`. `|`. `|`Alfred `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`AGE `|`0 `|`-1 `|`N `|`14 `|`. `|` `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`HEIGHT `|`0 `|`-1 `|`N `|`69 `|`. `|` `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`SEX `|`0 `|`-1 `|`C `|`. `|`. `|`M `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`WEIGHT `|`0 `|`-1 `|`N `|`112.5 `|`. `|` `|` `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`NAME `|`1 `|`0 `|`C `|`. `|`. `|`Alice `|`Alice `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`AGE `|`0 `|`1 `|`N `|`13 `|`99 `|` `|` `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`HEIGHT `|`0 `|`0 `|`N `|`56.5 `|`56.5 `|` `|` `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`SEX `|`0 `|`0 `|`C `|`. `|`. `|`F `|`F `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`WEIGHT `|`0 `|`0 `|`N `|`84 `|`84 `|` `|` `|
+
+    Into three tables like this:
+
+  <b> `work.outmod`: </b>
+  |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.|
+  |---|---|---|---|---|
+  |`Alice `|`F `|`99 `|`56.5 `|`84 `|
+
+  <b> `work.outadd`: </b>
+  |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.|
+  |---|---|---|---|---|
+  |`Newbie `|`F `|`13 `|`65.3 `|`98 `|
+
+  <b> `work.outdel`: </b>
+  |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.|
+  |---|---|---|---|---|
+  |`Alfred `|`M `|`14 `|`69 `|`112.5 `|
+
+    As you might expect, there are a bunch of extra features and checks.
+
+    The macro supports both SCD2 (TXTEMPORAL) and UPDATE loadtypes. If the
+    base table contains a PROCESSED_DTTM column (or similar), this can be
+    ignored by declaring it in the `processed_dttm_var` parameter.
+
+    The macro is also flexible where columns have been added or removed from
+    the base table UNLESS there is a change to the primary key.
+
+    Changes to the primary key fields are NOT supported, and are likely to cause
+    unexpected results.
+
+    The following pre-flight checks are made:
+
+    @li All primary key columns exist on the base table
+    @li There is no change in variable TYPE for any of the columns
+    @li There is no reduction in variable LENGTH below the max-length of the
+      supplied values
+
+    Rules for stacking changes are as follows:
+
+    <table>
+    <tr>
+      <th>Transaction Type</th><th>Key Behaviour</th><th>Column Behaviour</th>
+    </tr>
+    <tr>
+      <td>Deletes</td>
+      <td>
+        The row is added to `&outDEL.` UNLESS it no longer exists
+        in the base table, in which case it is added to `&errDS.` instead.
+      </td>
+      <td>
+        Deletes are unaffected by the addition or removal of non Primary-Key
+        columns.
+      </td>
+    </tr>
+    <tr>
+      <td>Inserts</td>
+      <td>
+        Previously newly added rows are added to the `outADD` table UNLESS they
+        are present in the Base table.<br>In this case they are added to the
+        `&errDS.` table instead.
+      </td>
+      <td>
+        Inserts are unaffected by the addition of columns in the Base Table
+        (they are padded with blanks).  Deleted columns are only a problem if
+        they appear on the previous insert - in which case the record is added
+        to `&errDS.`.
+      </td>
+    </tr>
+    <tr>
+      <td>Updates</td>
+      <td>
+        Previously modified rows are merged with base table values such that
+        only the individual cells that were _previously_ changed are re-applied.
+        Where the row contains cells that were not marked as having changed in
+        the prior transaction, the 'blanks' are filled with base table values in
+        the `outMOD` table.<br>
+        If the row no longer exists on the base table, then the row is added to
+        the `errDS` table instead.
+      </td>
+      <td>
+        Updates are unaffected by the addition of columns in the Base Table -
+        the new cells are simply populated with Base Table values.  Deleted
+        columns are only a problem if they relate to a modified cell
+        (`is_diff=1`) - in which case the record is added to `&errDS.`.
+      </td>
+    </tr>
+    </table>
+
+    To illustrate the above with a diagram:
+
+  @dot
+    digraph {
+      rankdir="TB"
+      start[label="Transaction Type?" shape=Mdiamond]
+      del[label="Does Base Row exist?" shape=rectangle]
+      add [label="Does Base Row exist?" shape=rectangle]
+      mod [label="Does Base Row exist?" shape=rectangle]
+      chkmod [label="Do all modified\n(is_diff=1) cells exist?" shape=rectangle]
+      chkadd [label="Do all inserted cells exist?" shape=rectangle]
+      outmod [label="outMOD\nTable" shape=Msquare style=filled]
+      outadd [label="outADD\nTable" shape=Msquare style=filled]
+      outdel [label="outDEL\nTable" shape=Msquare style=filled]
+      outerr [label="ErrDS Table" shape=Msquare fillcolor=Orange style=filled]
+      start -> del [label="Delete"]
+      start -> add [label="Insert"]
+      start -> mod [label="Update"]
+
+      del -> outdel [label="Yes"]
+      del -> outerr [label="No" color="Red" fontcolor="Red"]
+      add -> chkadd [label="No"]
+      add -> outerr [label="Yes" color="Red" fontcolor="Red"]
+      mod -> outerr [label="No" color="Red" fontcolor="Red"]
+      mod -> chkmod [label="Yes"]
+      chkmod -> outerr [label="No" color="Red" fontcolor="Red"]
+      chkmod -> outmod [label="Yes"]
+      chkadd -> outerr [label="No" color="Red" fontcolor="Red"]
+      chkadd -> outadd [label="Yes"]
+
+    }
+  @enddot
+
+    For examples of usage, check out the mp_stackdiffs.test.sas program.
+
+
+  @param [in] baselibds Base Table against which the changes will be applied,
+    in libref.dataset format.
+  @param [in] auditlibds Dataset with previously applied transactions, to be
+    re-applied. Use libref.dataset format.
+    DDL as follows:  %mp_coretable(DIFFTABLE)
+  @param [in] key Space seperated list of key variables
+  @param [in] mdebug= Set to 1 to enable DEBUG messages and preserve outputs
+  @param [in] processed_dttm_var= (0) If a variable is being used to mark
+    the processed datetime, put the name of the variable here.  It will NOT
+    be included in the staged dataset (the load process is expected to
+    provide this)
+  @param [out] errds= (work.errds) Output table containing problematic records.
+    The columns of this table are:
+    @li PK_VARS - Space separated list of primary key variable names
+    @li PK_VALS - Slash separted list of PK variable values
+    @li ERR_MSG - Explanation of why this record is problematic
+  @param [out] outmod= (work.outmod) Output table containing modified records
+  @param [out] outadd= (work.outadd) Output table containing additional records
+  @param [out] outdel= (work.outdel) Output table containing deleted records
+
+  <h4> SAS Macros </h4>
+  @li mf_existvarlist.sas
+  @li mf_getquotedstr.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_islibds.sas
+  @li mf_nobs.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_abort.sas
+  @li mp_ds2squeeze.sas
+
+  <h4> Related Macros </h4>
+  @li mp_coretable.sas
+  @li mp_stackdiffs.test.sas
+  @li mp_storediffs.sas
+
+  @todo The current approach assumes that a variable called KEY_HASH is not on
+    the base table.  This part will need to be refactored (eg using
+    mf_getuniquename.sas) when such a use case arises.
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mp_stackdiffs(baselibds
+  ,auditlibds
+  ,key
+  ,mdebug=0
+  ,processed_dttm_var=0
+  ,errds=work.errds
+  ,outmod=work.outmod
+  ,outadd=work.outadd
+  ,outdel=work.outdel
+)/*/STORE SOURCE*/;
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+/* input parameter validations */
+%mp_abort(iftrue= (%mf_islibds(&baselibds) ne 1)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid baselibds: &baselibds)
+)
+%mp_abort(iftrue= (%mf_islibds(&auditlibds) ne 1)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid auditlibds: &auditlibds)
+)
+%mp_abort(iftrue= (%length(&key)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Missing key variables!)
+)
+%mp_abort(iftrue= (
+    %mf_existVarList(&auditlibds,LIBREF DSN MOVE_TYPE KEY_HASH TGTVAR_NM IS_PK
+      IS_DIFF TGTVAR_TYPE OLDVAL_NUM NEWVAL_NUM OLDVAL_CHAR NEWVAL_CHAR)=0
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Input &auditlibds is missing required columns!)
+)
+
+
+/* set up macro vars */
+%local prefix dslist x var keyjoin commakey keepvars missvars fref;
+%let prefix=%substr(%mf_getuniquename(),1,25);
+%let dslist=ds1d ds2d ds3d ds1a ds2a ds3a ds1m ds2m ds3m pks dups base
+  delrec delerr addrec adderr modrec moderr;
+%do x=1 %to %sysfunc(countw(&dslist));
+  %let var=%scan(&dslist,&x);
+  %local &var;
+  %let &var=%upcase(&prefix._&var);
+%end;
+
+%let key=%upcase(&key);
+%let commakey=%mf_getquotedstr(&key,quote=N);
+
+%let keyjoin=1=1;
+%do x=1 %to %sysfunc(countw(&key));
+  %let var=%scan(&key,&x);
+  %let keyjoin=&keyjoin and a.&var=b.&var;
+%end;
+
+data &errds;
+  length pk_vars $256 pk_vals $4098 err_msg $512;
+  call missing (of _all_);
+  stop;
+run;
+
+/**
+  * Prepare raw DELETE table
+  * Records are in the OLDVAL_xxx columns
+  */
+%let keepvars=MOVE_TYPE KEY_HASH TGTVAR_NM TGTVAR_TYPE IS_PK
+              OLDVAL_NUM OLDVAL_CHAR
+              NEWVAL_NUM NEWVAL_CHAR;
+proc sort data=&auditlibds(where=(move_type='D') keep=&keepvars)
+  out=&ds1d(drop=move_type);
+by KEY_HASH TGTVAR_NM;
+run;
+proc transpose data=&ds1d(where=(tgtvar_type='N'))
+    out=&ds2d(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var OLDVAL_NUM;
+run;
+proc transpose data=&ds1d(where=(tgtvar_type='C'))
+    out=&ds3d(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var OLDVAL_CHAR;
+run;
+%mp_ds2squeeze(&ds2d,outds=&ds2d)
+%mp_ds2squeeze(&ds3d,outds=&ds3d)
+data &outdel;
+  if 0 then set &baselibds;
+  set &ds2d;
+  set &ds3d;
+  drop key_hash;
+  if not missing(%scan(&key,1));
+run;
+proc sort;
+  by &key;
+run;
+
+/**
+  * Prepare raw APPEND table
+  * Records are in the NEWVAL_xxx columns
+  */
+proc sort data=&auditlibds(where=(move_type='A') keep=&keepvars)
+    out=&ds1a(drop=move_type);
+  by KEY_HASH TGTVAR_NM;
+run;
+proc transpose data=&ds1a(where=(tgtvar_type='N'))
+    out=&ds2a(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var NEWVAL_NUM;
+run;
+proc transpose data=&ds1a(where=(tgtvar_type='C'))
+    out=&ds3a(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var NEWVAL_CHAR;
+run;
+%mp_ds2squeeze(&ds2a,outds=&ds2a)
+%mp_ds2squeeze(&ds3a,outds=&ds3a)
+data &outadd;
+  if 0 then set &baselibds;
+  set &ds2a;
+  set &ds3a;
+  drop key_hash;
+  if not missing(%scan(&key,1));
+run;
+proc sort;
+  by &key;
+run;
+
+/**
+  * Prepare raw MODIFY table
+  * Keep only primary key - will add modified values later
+  */
+proc sort data=&auditlibds(
+      where=(move_type='M' and is_pk=1) keep=&keepvars
+    ) out=&ds1m(drop=move_type);
+  by KEY_HASH TGTVAR_NM;
+run;
+proc transpose data=&ds1m(where=(tgtvar_type='N'))
+    out=&ds2m(drop=_name_);
+  by KEY_HASH ;
+  id TGTVAR_NM;
+  var NEWVAL_NUM;
+run;
+proc transpose data=&ds1m(where=(tgtvar_type='C'))
+    out=&ds3m(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var NEWVAL_CHAR;
+run;
+%mp_ds2squeeze(&ds2m,outds=&ds2m)
+%mp_ds2squeeze(&ds3m,outds=&ds3m)
+data &outmod;
+  if 0 then set &baselibds;
+  set &ds2m;
+  set &ds3m;
+  if not missing(%scan(&key,1));
+run;
+proc sort;
+  by &key;
+run;
+
+/**
+  * Extract matching records from the base table
+  * Do this in one join for efficiency.
+  * At a later date, this should be optimised for large database tables by using
+  * passthrough and a temporary table.
+  */
+data &pks;
+  if 0 then set &baselibds;
+  set &outadd &outmod &outdel;
+  keep &key;
+run;
+
+proc sort noduprec dupout=&dups;
+by &key;
+run;
+data _null_;
+  set &dups;
+  putlog (_all_)(=);
+run;
+%mp_abort(iftrue= (%mf_nobs(&dups) ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(duplicates (%mf_nobs(&dups)) found on &auditlibds!)
+)
+
+proc sql;
+create table &base as
+  select a.*
+  from &baselibds a, &pks b
+  where &keyjoin;
+
+/**
+  * delete check
+  * This is straightforward as it relates to records only
+  */
+proc sql;
+create table &delrec as
+  select a.*
+  from &outdel a
+  left join &base b
+  on &keyjoin
+  where b.%scan(&key,1) is null
+  order by &commakey;
+
+data &delerr;
+  if 0 then set &errds;
+  set &delrec;
+  PK_VARS="&key";
+  PK_VALS=catx('/',&commakey);
+  ERR_MSG="Rows cannot be deleted as they do not exist on the Base dataset";
+  keep PK_VARS PK_VALS ERR_MSG;
+run;
+proc append base=&errds data=&delerr;
+run;
+
+data &outdel;
+  merge &outdel (in=a) &delrec (in=b);
+  by &key;
+  if not b;
+run;
+
+/**
+  * add check
+  * Problems - where record already exists, or base table has columns missing
+  */
+%let missvars=%mf_wordsinstr1butnotstr2(
+  Str1=%upcase(%mf_getvarlist(&outadd)),
+  Str2=%upcase(%mf_getvarlist(&baselibds))
+);
+%if %length(&missvars)>0 %then %do;
+    /* add them to the err table */
+  data &adderr;
+    if 0 then set &errds;
+    set &outadd;
+    PK_VARS="&key";
+    PK_VALS=catx('/',&commakey);
+    ERR_MSG="Rows cannot be added due to missing base vars: &missvars";
+    keep PK_VARS PK_VALS ERR_MSG;
+  run;
+  proc append base=&errds data=&adderr;
+  run;
+  proc sql;
+  delete * from &outadd;
+%end;
+%else %do;
+  proc sql;
+  /* find records that already exist on base table */
+  create table &addrec as
+    select a.*
+    from &outadd a
+    inner join &base b
+    on &keyjoin
+    order by &commakey;
+
+  /* add them to the err table */
+  data &adderr;
+    if 0 then set &errds;
+    set &addrec;
+    PK_VARS="&key";
+    PK_VALS=catx('/',&commakey);
+    ERR_MSG="Rows cannot be added as they already exist on the Base dataset";
+    keep PK_VARS PK_VALS ERR_MSG;
+  run;
+  proc append base=&errds data=&adderr;
+  run;
+
+  /* remove invalid rows from the outadd table */
+  data &outadd;
+    merge &outadd (in=a) &addrec (in=b);
+    by &key;
+    if not b;
+  run;
+%end;
+
+/**
+  * mod check
+  * Problems - where record does not exist or baseds has modified cols missing
+  */
+proc sql noprint;
+select distinct tgtvar_nm into: missvars separated by ' '
+  from &auditlibds
+  where move_type='M' and is_diff=1;
+%let missvars=%mf_wordsinstr1butnotstr2(
+  Str1=&missvars,
+  Str2=%upcase(%mf_getvarlist(&baselibds))
+);
+%if %length(&missvars)>0 %then %do;
+    /* add them to the err table */
+  data &moderr;
+    if 0 then set &errds;
+    set &outmod;
+    PK_VARS="&key";
+    PK_VALS=catx('/',&commakey);
+    ERR_MSG="Rows cannot be modified due to missing base vars: &missvars";
+    keep PK_VARS PK_VALS ERR_MSG;
+  run;
+  proc append base=&errds data=&moderr;
+  run;
+  proc sql;
+  delete * from &outmod;
+%end;
+%else %do;
+  /* now check for records that do not exist (therefore cannot be modified) */
+  proc sql;
+  create table &modrec as
+    select a.*
+    from &outmod a
+    left join &base b
+    on &keyjoin
+    where b.%scan(&key,1) is null
+    order by &commakey;
+  data &moderr;
+    if 0 then set &errds;
+    set &modrec;
+    PK_VARS="&key";
+    PK_VALS=catx('/',&commakey);
+    ERR_MSG="Rows cannot be modified as they do not exist on the Base dataset";
+    keep PK_VARS PK_VALS ERR_MSG;
+  run;
+  proc append base=&errds data=&moderr;
+  run;
+  /* delete the above records from the outmod table */
+  data &outmod;
+    merge &outmod (in=a) &modrec (in=b);
+    by &key;
+    if not b;
+  run;
+  /* now - we can prepare the final MOD table (which is currently PK only) */
+  proc sql undo_policy=none;
+  create table &outmod as
+    select a.key_hash
+      ,b.*
+    from &outmod a
+    inner join &base b
+    on &keyjoin
+    order by &commakey;
+  /* now - to update outmod with modified (is_diff=1) values */
+  %let fref=%mf_getuniquefileref();
+  data _null_;
+    file &fref;
+    set &auditlibds(where=(move_type='M')) end=lastobs;
+    by key_hash;
+    retain comma 'N';
+    if _n_=1 then put 'proc sql;';
+    if first.key_hash then do;
+      comma='N';
+      put "update &outmod set " @@;
+    end;
+    if is_diff=1 then do;
+      if comma='N' then do;
+        put '  '@@;
+        comma='Y';
+      end;
+      else put '  ,'@@;
+      if tgtvar_type='C' then do;
+        length qstr $32767;
+        qstr=quote(trim(NEWVAL_CHAR));
+        put tgtvar_nm '=' qstr;
+      end;
+      else put tgtvar_nm '=' newval_num;
+      if comma='  ' then comma='  ,';
+    end;
+    if last.key_hash then put '  where key_hash=trim("' key_hash '");';
+    if lastobs then put "alter table &outmod drop key_hash;";
+  run;
+  %inc &fref/source2;
+%end;
+
+%if &mdebug=0 %then %do;
+  proc datasets lib=work;
+    delete &prefix:;
+  run;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%mend mp_stackdiffs;
+/** @endcond */

base/mp_storediffs.sas

@@ -49,41 +49,23 @@
   @param [in] appds= (0) Dataset with appended records
   @param [in] modds= (0) Dataset with modified records
   @param [out] outds= (work.mp_storediffs) Output table containing stored data.
-    Has the following format:
+    DDL as follows:  %mp_coretable(DIFFTABLE)
 
-        proc sql;
-        create table &outds(
-          load_ref char(36) label='unique load reference',
-          processed_dttm num format=E8601DT26.6 label='Processed at timestamp',
-          libref char(8) label='Library Reference (8 chars)',
-          dsn char(32) label='Dataset Name (32 chars)',
-          key_hash char(32) label=
-            'MD5 Hash of primary key values (pipe seperated)',
-          move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified',
-          is_pk num label='Is Primary Key Field? (1/0)',
-          is_diff num label=
-            'Did value change? (1/0/-1).  Always -1 for appends and deletes.',
-          tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
-          tgtvar_nm char(32) label='Target variable name (32 chars)',
-          oldval_num num format=best32. label='Old (numeric) value',
-          newval_num num format=best32. label='New (numeric) value',
-          oldval_char char(32765) label='Old (character) value',
-          newval_char char(32765) label='New (character) value',
-          constraint pk_mpe_audit
-            primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
-        );
-
-    @param [in] processed_dttm= (0) Provide a datetime constant in relation to
-      the actual load time.  If not provided, current timestamp is used.
-    @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
-    @param [out] loadref= (0) Provide a unique key to reference the load,
-      otherwise a UUID will be generated.
+  @param [in] processed_dttm= (0) Provide a datetime constant in relation to
+    the actual load time.  If not provided, current timestamp is used.
+  @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
+  @param [out] loadref= (0) Provide a unique key to reference the load,
+    otherwise a UUID will be generated.
 
   <h4> SAS Macros </h4>
   @li mf_getquotedstr.sas
   @li mf_getuniquename.sas
   @li mf_getvarlist.sas
 
+  <h4> Related Macros </h4>
+  @li mp_stackdiffs.sas
+  @li mp_storediffs.test.sas
+
   @version 9.2
   @author Allan Bowe
 **/
@@ -108,7 +90,7 @@
 %else %let dbg=*;
 
 /* set up unique and temporary vars */
-%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist;
+%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist vlist;
 %let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_ds1));
 %let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_ds2));
 %let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_ds3));
@@ -162,12 +144,21 @@ proc transpose data=&ds1
   by &inds_keep &hashkey;
   var _character_;
 run;
+
+%if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do;
+  /* this is a format catalog - cannot query cols directly */
+  %let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH"
+    ,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO"
+    ,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
+%end;
+%else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
+
 data &ds4;
   length &inds_keep $41 tgtvar_nm $32;
   set &ds2 &ds3 indsname=&inds_auto;
 
   tgtvar_nm=upcase(tgtvar_nm);
-  if tgtvar_nm in (%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE)));
+  if tgtvar_nm in (%upcase(&vlist));
 
   if &inds_auto="&ds2" then tgtvar_type='N';
   else if &inds_auto="&ds3" then tgtvar_type='C';

base/mp_streamfile.sas

@@ -12,17 +12,20 @@
 
       %mp_streamfile(contenttype=csv,inloc=/some/where.txt,outname=myfile.txt)
 
+  @param [in] contenttype= (TEXT) Either TEXT, ZIP, CSV, EXCEL
+  @param [in] inloc= /path/to/file.ext to be sent
+  @param [in] inref= fileref of file to be sent (if provided, overrides `inloc`)
+  @param [in] iftrue= (1=1) Provide a condition under which to execute.
+  @param [out] outname= the name of the file, as downloaded by the browser
+  @param [out] outref= (_webout) The destination where the file should be
+    streamed.
+
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
+  @li mfs_httpheader.sas
   @li mp_binarycopy.sas
 
-  @param contenttype= Either TEXT, ZIP, CSV, EXCEL (default TEXT)
-  @param inloc= /path/to/file.ext to be sent
-  @param inref= fileref of file to be sent (if provided, overrides `inloc`)
-  @param outname= the name of the file, as downloaded by the browser
-
   @author Allan Bowe
-  @source https://github.com/sasjs/core
 
 **/
 
@@ -30,12 +33,16 @@
   contenttype=TEXT
   ,inloc=
   ,inref=0
+  ,iftrue=%str(1=1)
   ,outname=
+  ,outref=_webout
 )/*/STORE SOURCE*/;
 
-%let contentype=%upcase(&contenttype);
-%local platform; %let platform=%mf_getplatform();
+%if not(%eval(%unquote(&iftrue))) %then %return;
 
+%let contentype=%upcase(&contenttype);
+%let outref=%upcase(&outref);
+%local platform; %let platform=%mf_getplatform();
 
 /**
   * check engine type to avoid the below err message:
@@ -44,39 +51,101 @@
 %local streamweb;
 %let streamweb=0;
 data _null_;
-  set sashelp.vextfl(where=(upcase(fileref)="_WEBOUT"));
+  set sashelp.vextfl(where=(upcase(fileref)="&outref"));
   if xengine='STREAM' then call symputx('streamweb',1,'l');
 run;
 
-%if &contentype=ZIP %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+%if &contentype=CSV %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/zip');
+      rc=stpsrv_header('Content-type','application/csv');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.zip'
-      contenttype='application/zip'
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
+      contenttype='application/csv'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,application/csv)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
 %else %if &contentype=EXCEL %then %do;
   /* suitable for XLS format */
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/vnd.ms-excel');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
       contenttype='application/vnd.ms-excel'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,application/vnd.ms-excel)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
+%end;
+%else %if &contentype=GIF or &contentype=JPEG or &contentype=PNG %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type',"image/%lowcase(&contenttype)");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI"
+      contenttype="image/%lowcase(&contenttype)";
+  %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,image/%lowcase(&contenttype))
+  %end;
+%end;
+%else %if &contentype=HTML %then %do;
+  %if &platform=SASVIYA %then %do;
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
+      contenttype="text/html";
+  %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,text/html)
+  %end;
+%end;
+%else %if &contentype=TEXT %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type','application/text');
+      rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
+      contenttype='application/text'
+      contentdisp="attachment; filename=&outname";
+  %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,application/text)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
+%end;
+%else %if &contentype=WOFF or &contentype=WOFF2 or &contentype=TTF %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type',"font/%lowcase(&contenttype)");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI"
+      contenttype="font/%lowcase(&contenttype)";
+  %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,font/%lowcase(&contenttype))
+  %end;
 %end;
 %else %if &contentype=XLSX %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
       rc=stpsrv_header('Content-type',
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
@@ -84,54 +153,44 @@ run;
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
       contenttype=
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type
+      ,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+    )
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
-%else %if &contentype=TEXT %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+%else %if &contentype=ZIP %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/text');
+      rc=stpsrv_header('Content-type','application/zip');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
-      contenttype='application/text'
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.zip'
+      contenttype='application/zip'
       contentdisp="attachment; filename=&outname";
   %end;
-%end;
-%else %if &contentype=CSV %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
-    data _null_;
-      rc=stpsrv_header('Content-type','application/csv');
-      rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
-    run;
-  %end;
-  %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
-      contenttype='application/csv'
-      contentdisp="attachment; filename=&outname";
-  %end;
-%end;
-%else %if &contentype=HTML %then %do;
-  %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
-      contenttype="text/html";
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,application/zip)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
   %end;
 %end;
 %else %do;
   %put %str(ERR)OR: Content Type &contenttype NOT SUPPORTED by &sysmacroname!;
-  %return;
 %end;
 
 %if &inref ne 0 %then %do;
-  %mp_binarycopy(inref=&inref,outref=_webout)
+  %mp_binarycopy(inref=&inref,outref=&outref)
 %end;
 %else %do;
-  %mp_binarycopy(inloc="&inloc",outref=_webout)
+  %mp_binarycopy(inloc="&inloc",outref=&outref)
 %end;
 
 %mend mp_streamfile;

base/mp_testservice.sas

@@ -1,12 +1,10 @@
 /**
-  @file mp_testservice.sas
-  @brief Will execute a test against a SASjs web service on SAS 9 or Viya
+  @file
+  @brief Will execute a SASjs web service on SAS 9 or Viya
   @details Prepares the input files and retrieves the resulting datasets from
   the response JSON.
 
-      %mp_testjob(
-        duration=60*5
-      )
+
 
   Note - the _webout fileref should NOT be assigned prior to running this macro.
 
@@ -14,6 +12,10 @@
   @param [in] inputfiles=(0) A list of space seperated fileref:filename pairs as
     follows:
         inputfiles=inref:filename inref2:filename2
+  @param [in] inputdatasets= (0) All datasets in this space seperated list are
+    converted into SASJS-formatted CSVs (see mp_ds2csv.sas) files and added to
+    the list of `inputfiles` for ingestion.  The dataset will be sent with the
+    same name (no need for a colon modifier).
   @param [in] inputparams=(0) A dataset containing name/value pairs in the
     following format:
     |name:$32|value:$1000|
@@ -38,9 +40,13 @@
   @li mf_getuniquename.sas
   @li mp_abort.sas
   @li mp_binarycopy.sas
+  @li mp_ds2csv.sas
   @li mv_getjobresult.sas
   @li mv_jobflow.sas
 
+  <h4> Related Programs </h4>
+  @li mp_testservice.test.sas
+
   @version 9.4
   @author Allan Bowe
 
@@ -48,6 +54,7 @@
 
 %macro mp_testservice(program,
   inputfiles=0,
+  inputdatasets=0,
   inputparams=0,
   debug=log,
   mdebug=0,
@@ -56,7 +63,7 @@
   viyaresult=WEBOUT_JSON,
   viyacontext=SAS Job Execution compute context
 )/*/STORE SOURCE*/;
-%local dbg;
+%local dbg pcnt fref1 webref i webcount var platform;
 %if &mdebug=1 %then %do;
   %put &sysmacroname entry vars:;
   %put _local_;
@@ -64,7 +71,6 @@
 %else %let dbg=*;
 
 /* sanitise inputparams */
-%local pcnt;
 %let pcnt=0;
 %if &inputparams ne 0 %then %do;
   data _null_;
@@ -86,17 +92,25 @@
   )
 %end;
 
+/* convert inputdatasets to filerefs */
+%if "&inputdatasets" ne "0" %then %do;
+  %if %quote(&inputfiles)=0 %then %let inputfiles=;
+  %do i=1 %to %sysfunc(countw(&inputdatasets,%str( )));
+    %let var=%scan(&inputdatasets,&i,%str( ));
+    %local dsref&i;
+    %let dsref&i=%mf_getuniquefileref();
+    %mp_ds2csv(&var,outref=&&dsref&i,headerformat=SASJS)
+    %let inputfiles=&inputfiles &&dsref&i:%scan(&var,-1,.);
+  %end;
+%end;
+
 
-%local fref1 webref;
 %let fref1=%mf_getuniquefileref();
 %let webref=%mf_getuniquefileref();
-
-%local platform;
 %let platform=%mf_getplatform();
 %if &platform=SASMETA %then %do;
 
   /* parse the input files */
-  %local webcount i var;
   %if %quote(&inputfiles) ne 0 %then %do;
     %let webcount=%sysfunc(countw(&inputfiles));
     %put &=webcount;

base/mp_wait4file.sas

@@ -4,9 +4,7 @@
   @details Loops with a `sleep()` command until a file arrives or the max wait
   period expires.
 
-  @example
-
-  Wait 3 minutes OR for /tmp/flag.txt to appear
+  Example: Wait 3 minutes OR for /tmp/flag.txt to appear
 
     %mp_wait4file(/tmp/flag.txt , maxwait=60*3)
 

build.py

@@ -84,7 +84,7 @@ options noquotelenmax;
 """
 f = open('all.sas', "w")             # r / r+ / rb / rb+ / w / wb
 f.write(header)
-folders=['base','meta','metax','server','viya','lua','fcmp']
+folders=['base','ddl','meta','metax','server','viya','lua','fcmp']
 for folder in folders:
     filenames = [fn for fn in Path('./' + folder).iterdir() if fn.match("*.sas")]
     filenames.sort()

ddl/mddl_dc_difftable.sas

@@ -0,0 +1,34 @@
+/**
+  @file
+  @brief Difftable DDL
+  @details Used to store changes to tables.  Used by mp_storediffs.sas
+      and mp_stackdiffs.sas
+
+**/
+
+
+%macro mddl_dc_difftable(libds=WORK.DIFFTABLE);
+
+  proc sql;
+  create table &libds(
+      load_ref char(36) label='unique load reference',
+      processed_dttm num format=E8601DT26.6 label='Processed at timestamp',
+      libref char(8) label='Library Reference (8 chars)',
+      dsn char(32) label='Dataset Name (32 chars)',
+      key_hash char(32) label=
+        'MD5 Hash of primary key values (pipe seperated)',
+      move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified',
+      is_pk num label='Is Primary Key Field? (1/0)',
+      is_diff num label=
+        'Did value change? (1/0/-1).  Always -1 for appends and deletes.',
+      tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
+      tgtvar_nm char(32) label='Target variable name (32 chars)',
+      oldval_num num format=best32. label='Old (numeric) value',
+      newval_num num format=best32. label='New (numeric) value',
+      oldval_char char(32765) label='Old (character) value',
+      newval_char char(32765) label='New (character) value',
+    constraint pk_mpe_audit
+      primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
+  );
+
+%mend mddl_dc_difftable;

ddl/mddl_dc_filterdetail.sas

@@ -0,0 +1,27 @@
+/**
+  @file
+  @brief Filtertable DDL
+  @details For storing detailed filter values.  Used by
+      mp_filterstore.sas.
+
+**/
+
+
+%macro mddl_dc_filterdetail(libds=WORK.FILTER_DETAIL);
+
+  proc sql;
+  create table &libds(
+      filter_hash char(32) not null,
+      filter_line num not null,
+      group_logic char(3) not null,
+      subgroup_logic char(3) not null,
+      subgroup_id num not null,
+      variable_nm varchar(32) not null,
+      operator_nm varchar(12) not null,
+      raw_value varchar(4000) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_hash,filter_line)
+  );
+
+%mend mddl_dc_filterdetail;

ddl/mddl_dc_filtersummary.sas

@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Filtersummary DDL
+  @details For storing summary filter values.  Used by
+      mp_filterstore.sas.
+
+**/
+
+
+%macro mddl_dc_filtersummary(libds=WORK.FILTER_SUMMARY);
+
+  proc sql;
+  create table &libds(
+      filter_rk num not null,
+      filter_hash char(32) not null,
+      filter_table char(41) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_rk)
+  );
+
+%mend mddl_dc_filtersummary;

ddl/mddl_dc_locktable.sas

@@ -0,0 +1,25 @@
+/**
+  @file
+  @brief Locktable DDL
+  @details For "locking" tables prior to multipass loads. Used by
+      mp_lockanytable.sas
+
+**/
+
+
+%macro mddl_dc_locktable(libds=WORK.LOCKTABLE);
+
+  proc sql;
+  create table &libds(
+      lock_lib char(8),
+      lock_ds char(32),
+      lock_status_cd char(10) not null,
+      lock_user_nm char(100) not null ,
+      lock_ref char(200),
+      lock_pid char(10),
+      lock_start_dttm num format=E8601DT26.6,
+      lock_end_dttm num format=E8601DT26.6,
+    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds)
+  );
+
+%mend mddl_dc_locktable;

ddl/mddl_dc_maxkeytable.sas

@@ -0,0 +1,24 @@
+/**
+  @file
+  @brief Maxkeytable DDL
+  @details For storing the maximum retained key information.  Used
+      by mp_retainedkey.sas
+
+**/
+
+
+%macro mddl_dc_maxkeytable(libds=WORK.MAXKEYTABLE);
+
+  proc sql;
+  create table &libds(
+      keytable varchar(41) label='Base table in libref.dataset format',
+      keycolumn char(32) format=$32.
+        label='The Retained key field containing the key values.',
+      max_key num label=
+        'Integer representing current max RK or SK value in the KEYTABLE',
+      processed_dttm num format=E8601DT26.6
+        label='Datetime this value was last updated',
+    constraint pk_mpe_maxkeyvalues
+        primary key(keytable));
+
+%mend mddl_dc_maxkeytable;

ddl/mddl_sas_cntlout.sas

@@ -0,0 +1,43 @@
+/**
+  @file
+  @brief The CNTLOUT table generated by proc format
+  @details This table will actually change format depending on the data values,
+  therefore the max possible lengths are described here to enable consistency
+  when dealing with format data.
+
+**/
+
+
+%macro mddl_sas_cntlout(libds=WORK.CNTLOUT);
+
+proc sql;
+create table &libds(
+    FMTNAME char(32)      label='Format name'
+    /*
+      to accomodate larger START values, mp_loadformat.sas will need the
+      SQL dependency removed (proc sql needs to accomodate 3 index values in
+      a 32767 ibufsize limit)
+    */
+    ,START char(10000)    label='Starting value for format'
+    ,END char(32767)      label='Ending value for format'
+    ,LABEL char(32767)    label='Format value label'
+    ,MIN num length=3     label='Minimum length'
+    ,MAX num length=3     label='Maximum length'
+    ,DEFAULT num length=3 label='Default length'
+    ,LENGTH num length=3  label='Format length'
+    ,FUZZ num             label='Fuzz value'
+    ,PREFIX char(2)       label='Prefix characters'
+    ,MULT num             label='Multiplier'
+    ,FILL char(1)         label='Fill character'
+    ,NOEDIT num length=3  label='Is picture string noedit?'
+    ,TYPE char(1)         label='Type of format'
+    ,SEXCL char(1)        label='Start exclusion'
+    ,EEXCL char(1)        label='End exclusion'
+    ,HLO char(13)         label='Additional information'
+    ,DECSEP char(1)       label='Decimal separator'
+    ,DIG3SEP char(1)      label='Three-digit separator'
+    ,DATATYPE char(8)     label='Date/time/datetime?'
+    ,LANGUAGE char(8)     label='Language for date strings'
+);
+
+%mend mddl_sas_cntlout;

fcmp/mcf_getfmttype.sas

@@ -0,0 +1,118 @@
+/**
+  @file
+  @brief Returns the type of the format
+  @details
+  Returns the type, eg DATE / DATETIME / TIME (based on hard-coded lookup)
+  else CHAR / NUM.
+
+  This macro may be extended in the future to support custom formats - this
+  would necessitate a call to `dosubl()` for running a proc format with cntlout.
+
+  The function itself takes the following (positional) parameters:
+
+  | PARAMETER | DESCRIPTION |
+  |---|---|
+  |fmtnm| Format name to be tested.  Can be with or without the w.d extension.|
+
+  Usage:
+
+      %mcf_getfmttype(wrap=YES, insert_cmplib=YES)
+
+      data _null_;
+        fmt1=mcf_getfmttype('DATE9.');
+        fmt2=mcf_getfmttype('DATETIME');
+        put (fmt:)(=);
+      run;
+      %put fmt3=%sysfunc(mcf_getfmttype(TIME9.));
+
+  Returns:
+
+  > fmt1=DATE fmt2=DATETIME
+  > fmt3=TIME
+
+  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
+  @param [out] lib= (work) The output library in which to create the catalog.
+  @param [out] cat= (sasjs) The output catalog in which to create the package.
+  @param [out] pkg= (utils) The output package in which to create the function.
+    Uses a 3 part format:  libref.catalog.package
+  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
+    values inserted only if needed.
+
+  <h4> SAS Macros </h4>
+  @li mcf_init.sas
+
+  <h4> Related Programs </h4>
+  @li mcf_getfmttype.test.sas
+  @li mp_init.sas
+
+  @todo "Custom Format Lookups" To enable site-specific formats, make
+  use of a set of SASJS_FMTLIST_(DATATYPE) global variables.
+
+**/
+
+%macro mcf_getfmttype(wrap=NO
+  ,insert_cmplib=DEPRECATED
+  ,lib=WORK
+  ,cat=SASJS
+  ,pkg=UTILS
+)/*/STORE SOURCE*/;
+%local i var cmpval found;
+
+%if %mcf_init(mcf_getfmttype)=1 %then %return;
+
+%if &wrap=YES  %then %do;
+  proc fcmp outlib=&lib..&cat..&pkg;
+%end;
+
+function mcf_getfmttype(fmtnm $) $8;
+  if substr(fmtnm,1,1)='$' then return('CHAR');
+  else do;
+    /* extract NAME */
+    length fmt $32;
+    fmt=scan(fmtnm,1,'.');
+    do while (
+      substr(fmt,length(fmt),1) in ('1','2','3','4','5','6','7','8','9','0')
+      );
+      if length(fmt)=1 then fmt='W';
+      else fmt=substr(fmt,1,length(fmt)-1);
+    end;
+
+    /* apply lookups */
+    if cats(fmt) in ('DATETIME','B8601DN','B8601DN','B8601DT','B8601DT'
+      ,'B8601DZ','B8601DZ','DATEAMPM','DTDATE','DTMONYY','DTWKDATX','DTYEAR'
+      ,'DTYYQC','E8601DN','E8601DN','E8601DT','E8601DT','E8601DZ','E8601DZ')
+      then return('DATETIME');
+    else if fmt in ('DATE','YYMMDD','B8601DA','B8601DA','DAY','DDMMYY'
+      ,'DDMMYYB','DDMMYYC','DDMMYYD','DDMMYYN','DDMMYYP','DDMMYYS','DDMMYYx'
+      ,'DOWNAME','E8601DA','E8601DA','JULDAY','JULIAN','MMDDYY','MMDDYYB'
+      ,'MMDDYYC','MMDDYYD','MMDDYYN','MMDDYYP','MMDDYYS','MMDDYYx','MMYY'
+      ,'MMYYC','MMYYD','MMYYN','MMYYP','MMYYS','MMYYx','MONNAME','MONTH'
+      ,'MONYY','PDJULG','PDJULI','QTR','QTRR','WEEKDATE','WEEKDATX','WEEKDAY'
+      ,'WEEKU','WEEKV','WEEKW','WORDDATE','WORDDATX','YEAR','YYMM','YYMMC'
+      ,'YYMMD','YYMMDDB','YYMMDDC','YYMMDDD','YYMMDDN','YYMMDDP','YYMMDDS'
+      ,'YYMMDDx','YYMMN','YYMMP','YYMMS','YYMMx','YYMON','YYQ','YYQC','YYQD'
+      ,'YYQN','YYQP','YYQR','YYQRC','YYQRD','YYQRN','YYQRP','YYQRS','YYQRx'
+      ,'YYQS','YYQx','YYQZ') then return('DATE');
+    else if fmt in ('TIME','B8601LZ','B8601LZ','B8601TM','B8601TM','B8601TZ'
+      ,'B8601TZ','E8601LZ','E8601LZ','E8601TM','E8601TM','E8601TZ','E8601TZ'
+      ,'HHMM','HOUR','MMSS','TIMEAMPM','TOD') then return('TIME');
+    else return('NUM');
+  end;
+endsub;
+
+%if &wrap=YES %then %do;
+  quit;
+%end;
+
+/* insert the CMPLIB if not already there */
+%let cmpval=%sysfunc(getoption(cmplib));
+%let found=0;
+%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
+  %let var=%scan(&cmpval,&i,%str( %(%)));
+  %if &var=&lib..&cat %then %let found=1;
+%end;
+%if &found=0 %then %do;
+  options insert=(CMPLIB=(&lib..&cat));
+%end;
+
+%mend mcf_getfmttype;

fcmp/mcf_init.sas

@@ -0,0 +1,44 @@
+/**
+  @file
+  @brief Sets up the mcf_xx functions
+  @details
+  There is no (efficient) way to determine if an mcf_xx macro has already been
+  invoked.  So, we make use of a global macro variable list to keep track.
+
+  Usage:
+
+      %mcf_init(MCF_LENGTH)
+
+  Returns:
+
+  > 1 (if already initialised) else 0
+
+  @param [in] func The function to be initialised
+
+  <h4> Related Macros </h4>
+  @li mcf_init.test.sas
+
+**/
+
+%macro mcf_init(func
+)/*/STORE SOURCE*/;
+
+%if not (%symexist(SASJS_PREFIX)) %then %do;
+  %global SASJS_PREFIX;
+  %let SASJS_PREFIX=SASJS;
+%end;
+
+%let func=%upcase(&func);
+
+/* the / character is just a seperator */
+%global &sasjs_prefix._FUNCTIONS;
+%if %index(&&&sasjs_prefix._FUNCTIONS,&func/)>0 %then %do;
+  1
+  %return;
+%end;
+%else %do;
+  %let &sasjs_prefix._FUNCTIONS=&&&sasjs_prefix._FUNCTIONS &func/;
+  0
+%end;
+
+%mend mcf_init;

fcmp/mcf_length.sas

@@ -32,29 +32,30 @@
   > outa=3 outb=4 outc=5 outd=0
 
   @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
-  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
-    CMPLIB reference.
   @param [out] lib= (work) The output library in which to create the catalog.
   @param [out] cat= (sasjs) The output catalog in which to create the package.
   @param [out] pkg= (utils) The output package in which to create the function.
     Uses a 3 part format:  libref.catalog.package
+  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
+    values inserted only if needed.
 
   <h4> SAS Macros </h4>
-  @li mf_existfunction.sas
+  @li mcf_init.sas
 
-  <h4> Related Macros </h4>
+  <h4> Related Programs </h4>
   @li mcf_length.test.sas
+  @li mp_init.sas
 
 **/
 
 %macro mcf_length(wrap=NO
-  ,insert_cmplib=NO
+  ,insert_cmplib=DEPRECATED
   ,lib=WORK
   ,cat=SASJS
   ,pkg=UTILS
 )/*/STORE SOURCE*/;
-
-%if %mf_existfunction(mcf_length)=1 %then %return;
+%local i var cmpval found;
+%if %mcf_init(mcf_length)=1 %then %return;
 
 %if &wrap=YES  %then %do;
   proc fcmp outlib=&lib..&cat..&pkg;
@@ -75,7 +76,14 @@ endsub;
   quit;
 %end;
 
-%if &insert_cmplib=YES %then %do;
+/* insert the CMPLIB if not already there */
+%let cmpval=%sysfunc(getoption(cmplib));
+%let found=0;
+%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
+  %let var=%scan(&cmpval,&i,%str( %(%)));
+  %if &var=&lib..&cat %then %let found=1;
+%end;
+%if &found=0 %then %do;
   options insert=(CMPLIB=(&lib..&cat));
 %end;
 

fcmp/mcf_stpsrv_header.sas

@@ -47,26 +47,30 @@
 
 
   @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
-  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
-    CMPLIB reference.
   @param [out] lib= (work) The output library in which to create the catalog.
   @param [out] cat= (sasjs) The output catalog in which to create the package.
   @param [out] pkg= (utils) The output package in which to create the function.
     Uses a 3 part format:  libref.catalog.package
+  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
+    values inserted only if needed.
 
   <h4> SAS Macros </h4>
-  @li mf_existfunction.sas
+  @li mcf_init.sas
+
+  <h4> Related Programs </h4>
+  @li mcf_stpsrv_header.test.sas
+  @li mp_init.sas
 
 **/
 
 %macro mcf_stpsrv_header(wrap=NO
-  ,insert_cmplib=NO
+  ,insert_cmplib=DEPRECATED
   ,lib=WORK
   ,cat=SASJS
   ,pkg=UTILS
 )/*/STORE SOURCE*/;
-
-%if %mf_existfunction(stpsrv_header)=1 %then %return;
+%local i var cmpval found;
+%if %mcf_init(stpsrv_header)=1 %then %return;
 
 %if &wrap=YES  %then %do;
   proc fcmp outlib=&lib..&cat..&pkg;
@@ -92,7 +96,14 @@ endsub;
   quit;
 %end;
 
-%if &insert_cmplib=YES %then %do;
+/* insert the CMPLIB if not already there */
+%let cmpval=%sysfunc(getoption(cmplib));
+%let found=0;
+%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
+  %let var=%scan(&cmpval,&i,%str( %(%)));
+  %if &var=&lib..&cat %then %let found=1;
+%end;
+%if &found=0 %then %do;
   options insert=(CMPLIB=(&lib..&cat));
 %end;
 

fcmp/mcf_string2file.sas

@@ -32,26 +32,30 @@
       run;
 
   @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
-  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
-    CMPLIB reference.
   @param [out] lib= (work) The output library in which to create the catalog.
   @param [out] cat= (sasjs) The output catalog in which to create the package.
   @param [out] pkg= (utils) The output package in which to create the function.
     Uses a 3 part format:  libref.catalog.package
+  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
+    values inserted only if needed.
 
   <h4> SAS Macros </h4>
-  @li mf_existfunction.sas
+  @li mcf_init.sas
+
+  <h4> Related Programs </h4>
+  @li mcf_stpsrv_header.test.sas
+  @li mp_init.sas
 
 **/
 
 %macro mcf_string2file(wrap=NO
-  ,insert_cmplib=NO
+  ,insert_cmplib=DEPRECATED
   ,lib=WORK
   ,cat=SASJS
   ,pkg=UTILS
 )/*/STORE SOURCE*/;
-
-%if %mf_existfunction(mcf_string2file)=1 %then %return;
+%local i var cmpval found;
+%if %mcf_init(mcf_string2file)=1 %then %return;
 
 %if &wrap=YES  %then %do;
   proc fcmp outlib=&lib..&cat..&pkg;
@@ -77,7 +81,14 @@ endsub;
   quit;
 %end;
 
-%if &insert_cmplib=YES %then %do;
+/* insert the CMPLIB if not already there */
+%let cmpval=%sysfunc(getoption(cmplib));
+%let found=0;
+%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
+  %let var=%scan(&cmpval,&i,%str( %(%)));
+  %if &var=&lib..&cat %then %let found=1;
+%end;
+%if &found=0 %then %do;
   options insert=(CMPLIB=(&lib..&cat));
 %end;
 

meta/mm_assignlib.sas

@@ -40,6 +40,12 @@
       /* now try and assign it */
       if libname("&libref",,'meta',cats('liburi="',liburi,'";')) ne 0 then do;
         putlog "&libref could not be assigned";
+        putlog liburi=;
+        /**
+          * Fetch the system message for display in the abort modal.  This is
+          * not always helpful though.  One example, previously received:
+          * NOTE: Libref XX refers to the same library metadata as libref XX.
+          */
         call symputx('msg',sysmsg(),'l');
         if "&mabort"='HARD' then call symputx('mp_abort',1,'l');
       end;
@@ -61,7 +67,7 @@
 
   %if &mp_abort=1 %then %do;
     %mp_abort(iftrue= (&mp_abort=1)
-      ,mac=&sysmacroname
+      ,mac=mm_assignlib.sas
       ,msg=&msg
     )
     %return;

meta/mm_createapplication.sas

@@ -59,7 +59,10 @@
 %&mD.put Executing &sysmacroname..sas;
 %&mD.put _local_;
 
-%mf_verifymacvars(tree name)
+%mp_abort(iftrue= (%mf_verifymacvars(tree name)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Empty inputs: tree name)
+)
 
 /**
   * check tree exists

meta/mm_createdocument.sas

@@ -47,7 +47,10 @@
 %&mD.put Executing &sysmacroname..sas;
 %&mD.put _local_;
 
-%mf_verifymacvars(tree name)
+%mp_abort(iftrue= (%mf_verifymacvars(tree name)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Empty inputs: tree name)
+)
 
 /**
   * check tree exists

meta/mm_createfolder.sas

@@ -12,13 +12,14 @@
     The macro is idempotent - if you run it twice, it will only create a folder
     once.
 
-  usage:
+  Usage:
 
       %mm_createfolder(path=/some/meta/folder)
 
   @param [in] path= Name of the folder to create.
   @param [in] mdebug= set DBG to 1 to disable DEBUG messages
 
+
   @version 9.4
   @author Allan Bowe
 

meta/mm_createlibrary.sas

@@ -133,12 +133,14 @@ run;
 filename &frefin temp;
 filename &frefout temp;
 
+%mp_abort(iftrue= (
+    &engine=BASE & %mf_verifymacvars(libname libref engine servercontext tree)=0
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Empty inputs: libname libref engine servercontext tree)
+)
+
 %if &engine=BASE %then %do;
-
-  %mf_verifymacvars(libname libref engine servercontext tree)
-
-
-
   /**
     * Check that the ServerContext exists
     */

meta/mm_createstp.sas

@@ -12,7 +12,7 @@
     This macro is idempotent - if you run it twice, it will only create an STP
     once.
 
-  usage (type 1 STP):
+  Usage (type 1 STP):
 
       %mm_createstp(stpname=MyNewSTP
         ,filename=mySpecialProgram.sas
@@ -31,7 +31,8 @@
         putlog (_all_)(=);
       run;
 
-  usage (type 2 STP):
+  Usage (type 2 STP):
+
       %mm_createstp(stpname=MyNewType2STP
         ,filename=mySpecialProgram.sas
         ,directory=SASEnvironment/SASCode/STPs
@@ -74,8 +75,9 @@
   @li mf_verifymacvars.sas
   @li mm_getdirectories.sas
   @li mm_updatestpsourcecode.sas
-  @li mp_dropmembers.sas
   @li mm_getservercontexts.sas
+  @li mp_abort.sas
+  @li mp_dropmembers.sas
 
   <h4> Related Macros </h4>
   @li mm_createwebservice.sas
@@ -108,7 +110,12 @@
 %&mD.put Executing mm_CreateSTP.sas;
 %&mD.put _local_;
 
-%mf_verifymacvars(stpname filename directory tree)
+%mp_abort(
+  iftrue=(%mf_verifymacvars(stpname filename directory tree)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Empty inputs: stpname filename directory tree)
+)
+
 %mp_dropmembers(%scan(&outds,2,.))
 
 /**
@@ -177,6 +184,7 @@ run;
 
   data &outds (keep=stpuri prompturi fileuri texturi);
     length stpuri prompturi fileuri texturi serveruri $256 ;
+    if _n_=1 then call missing (of _all_);
     set &outds;
 
     /* final checks on uris */

meta/mm_createwebservice.sas

@@ -164,7 +164,7 @@ data _null_;
   put '      %put &sysmacroname: Switching to DATASTEP engine; ';
   put '      %goto datastep; ';
   put '    %end; ';
-  put '    data &tempds /view=&tempds;set &ds; ';
+  put '    data &tempds;set &ds; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
   put '    /* PRETTY is necessary to avoid line truncation in large files */ ';
   put '    proc json out=&jref pretty ';
@@ -190,7 +190,12 @@ data _null_;
   put '        )); ';
   put '      %do i=1 %to &numcols; ';
   put '        length &&name&i $&&len&i; ';
-  put '        &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '        %if &&typelong&i=num %then %do; ';
+  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '        %end; ';
+  put '        %else %do; ';
+  put '          &&name&i=put(&&newname&i,&&fmt&i); ';
+  put '        %end; ';
   put '        drop &&newname&i; ';
   put '      %end; ';
   put '        if _error_ then call symputx(''syscc'',1012); ';
@@ -210,7 +215,7 @@ data _null_;
   put '    %end; ';
   put '      other = [best.]; ';
   put ' ';
-  put '    data &tempds/view=&tempds; ';
+  put '    data &tempds; ';
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
@@ -272,8 +277,7 @@ data _null_;
   put '  %end; ';
   put ' ';
   put '  proc sql; ';
-  put '  drop view &tempds; ';
-  put '  drop table &colinfo; ';
+  put '  drop table &colinfo, &tempds; ';
   put ' ';
   put '  %if &showmeta=YES %then %do; ';
   put '    data _null_; file &jref encoding=''utf-8'' mod; ';

meta/mm_getauthinfo.sas

@@ -1,16 +1,21 @@
 /**
   @file mm_getauthinfo.sas
-  @brief extracts authentication info
-  @details usage:
+  @brief Extracts authentication info for each user in metadata
+  @details
+  Usage:
 
-    %mm_getauthinfo(outds=auths)
+      %mm_getauthinfo(outds=auths)
 
-  @param outds= the ONE LEVEL work dataset to create
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
+  @param [out] outds= (mm_getauthinfo) The output dataset to create
 
   <h4> SAS Macros </h4>
-  @li mm_getobjects.sas
   @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
   @li mm_getdetails.sas
+  @li mm_getobjects.sas
+
 
   @version 9.4
   @author Allan Bowe
@@ -18,67 +23,69 @@
 **/
 
 %macro mm_getauthinfo(outds=mm_getauthinfo
+  ,mdebug=0
 )/*/STORE SOURCE*/;
+%local prefix fileref;
+%let prefix=%substr(%mf_getuniquename(),1,25);
 
-%if %length(&outds)>30 %then %do;
-  %put %str(ERR)OR: Temp tables are created with the &outds prefix, which
-    therefore needs to be 30 characters or less;
-  %return;
-%end;
-%if %index(&outds,'.')>0 %then %do;
-  %put %str(ERR)OR: Table &outds should be ONE LEVEL (no library);
-  %return;
-%end;
-
-%mm_getobjects(type=Login,outds=&outds.0)
+%mm_getobjects(type=Login,outds=&prefix.0)
 
 %local fileref;
 %let fileref=%mf_getuniquefileref();
 
 data _null_;
   file &fileref;
-  set &outds.0 end=last;
+  set &prefix.0 end=last;
   /* run macro */
-  str=cats('%mm_getdetails(uri=',id,",outattrs=&outds.d",_n_
-    ,",outassocs=&outds.a",_n_,")");
+  str=cats('%mm_getdetails(uri=',id,",outattrs=&prefix.d",_n_
+    ,",outassocs=&prefix.a",_n_,")");
   put str;
   /* transpose attributes */
-  str=cats("proc transpose data=&outds.d",_n_,"(drop=type) out=&outds.da"
+  str=cats("proc transpose data=&prefix.d",_n_,"(drop=type) out=&prefix.da"
     ,_n_,"(drop=_name_);var value;id name;run;");
   put str;
   /* add extra info to attributes */
-  str=cats("data &outds.da",_n_,";length login_id login_name $256; login_id="
-    ,quote(trim(id)),";set &outds.da",_n_
+  str=cats("data &prefix.da",_n_,";length login_id login_name $256; login_id="
+    ,quote(trim(id)),";set &prefix.da",_n_
     ,";login_name=trim(subpad(name,1,256));drop name;run;");
   put str;
   /* add extra info to associations */
-  str=cats("data &outds.a",_n_,";length login_id login_name $256; login_id="
+  str=cats("data &prefix.a",_n_,";length login_id login_name $256; login_id="
     ,quote(trim(id)),";login_name=",quote(trim(name))
-    ,";set &outds.a",_n_,";run;");
+    ,";set &prefix.a",_n_,";run;");
   put str;
   if last then do;
     /* collate attributes */
-    str=cats("data &outds._logat; set &outds.da1-&outds.da",_n_,";run;");
+    str=cats("data &prefix._logat; set &prefix.da1-&prefix.da",_n_,";run;");
     put str;
     /* collate associations */
-    str=cats("data &outds._logas; set &outds.a1-&outds.a",_n_,";run;");
+    str=cats("data &prefix._logas; set &prefix.a1-&prefix.a",_n_,";run;");
     put str;
     /* tidy up */
-    str=cats("proc delete data=&outds.da1-&outds.da",_n_,";run;");
+    str=cats("proc delete data=&prefix.da1-&prefix.da",_n_,";run;");
     put str;
-    str=cats("proc delete data=&outds.d1-&outds.d",_n_,";run;");
+    str=cats("proc delete data=&prefix.d1-&prefix.d",_n_,";run;");
     put str;
-    str=cats("proc delete data=&outds.a1-&outds.a",_n_,";run;");
+    str=cats("proc delete data=&prefix.a1-&prefix.a",_n_,";run;");
     put str;
   end;
 run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fileref;
+    if _n_=1 then putlog // "Now executing the following code:" //;
+    input; putlog _infile_;
+  run;
+%end;
 %inc &fileref;
+filename &fileref clear;
 
 /* get libraries */
-proc sort data=&outds._logas(where=(assoc='Libraries')) out=&outds._temp;
+proc sort data=&prefix._logas(where=(assoc='Libraries')) out=&prefix._temp;
   by login_id;
-data &outds._temp;
-  set &outds._temp;
+data &prefix._temp;
+  set &prefix._temp;
   by login_id;
   length library_list $32767;
   retain library_list;
@@ -86,31 +93,27 @@ data &outds._temp;
   else library_list=catx(' !! ',library_list,name);
 proc sql;
 /* get auth domain */
-create table &outds._dom as
+create table &prefix._dom as
   select login_id,name as domain
-  from &outds._logas
+  from &prefix._logas
   where assoc='Domain';
-create unique index login_id on &outds._dom(login_id);
+create unique index login_id on &prefix._dom(login_id);
 /* join it all together */
-create table &outds._logins as
+create table &outds as
   select a.*
     ,c.domain
     ,b.library_list
-  from &outds._logat (drop=ishidden lockedby usageversion publictype) a
-  left join &outds._temp b
+  from &prefix._logat (drop=ishidden lockedby usageversion publictype) a
+  left join &prefix._temp b
   on a.login_id=b.login_id
-  left join &outds._dom c
+  left join &prefix._dom c
   on a.login_id=c.login_id;
-drop table &outds._temp;
-drop table &outds._logat;
-drop table &outds._logas;
 
-data _null_;
-  infile &fileref;
-  if _n_=1 then putlog // "Now executing the following code:" //;
-  input; putlog _infile_;
-run;
+%if &mdebug=0 %then %do;
+  proc datasets lib=work;
+    delete &prefix:;
+  run;
+%end;
 
-filename &fileref clear;
 
 %mend mm_getauthinfo;

meta/mm_getgroupmembers.sas

@@ -1,21 +1,28 @@
 /**
   @file
   @brief Creates dataset with all members of a metadata group
-  @details
+  @details This macro will query SAS metadata and return all the members
+    of a particular group.
 
-  usage:
+  Usage:
 
-    %mm_getgroupmembers(someGroupName
-      ,outds=work.mm_getgroupmembers
-      ,emails=YES)
+      %mm_getgroupmembers(someGroupName
+        ,outds=work.mm_getgroupmembers
+        ,emails=YES
+      )
 
   @param group metadata group for which to bring back members
-  @param outds= the dataset to create that contains the list of members
-  @param emails= set to YES to bring back email addresses
-  @param id= set to yes if passing an ID rather than a group name
+  @param outds= (work.mm_getgroupmembers) The dataset to create that contains
+    the list of members
+  @param emails= (NO) Set to YES to bring back email addresses
+  @param id= (NO) Set to yes if passing an ID rather than a group name
 
   @returns outds  dataset containing all members of the metadata group
 
+  <h4> Related Macros </h4>
+  @li mm_getgorups.sas
+  @li mm_adduser2group.sas
+
   @version 9.2
   @author Allan Bowe
 

meta/mm_getservercontexts.sas

@@ -41,7 +41,11 @@ run;
 
 filename __mc1 temp;
 filename __mc2 temp;
-data &outds; length serveruri servername $200; stop;run;
+data &outds;
+  length serveruri servername $200;
+  call missing (of _all_);
+  stop;
+run;
 %do x=1 %to &repocnt;
   options metarepository=&&repo&x;
   proc metadata in=
@@ -60,13 +64,16 @@ data &outds; length serveruri servername $200; stop;run;
   data _null_;
     file __mc2;
     put '<SXLEMAP version="1.2" name="SASContexts"><TABLE name="SASContexts">';
-    put "<TABLE-PATH syntax='XPath'>/GetMetadataObjects/Objects/ServerContext</TABLE-PATH>";
+    put "<TABLE-PATH syntax='XPath'>/GetMetadataObjects/Objects/ServerContext";
+    put "</TABLE-PATH>";
     put '<COLUMN name="serveruri">';
-    put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/ServerContext/@Id</PATH>";
+    put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/ServerContext/@Id";
+    put "</PATH>";
     put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>200</LENGTH>";
     put '</COLUMN>';
     put '<COLUMN name="servername">';
-    put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/ServerContext/@Name</PATH>";
+    put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/ServerContext/@Name";
+    put "</PATH>";
     put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>200</LENGTH>";
     put '</COLUMN>';
     put '</TABLE></SXLEMAP>';

package.json

@@ -33,6 +33,7 @@
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "3.6.0"
+    "@sasjs/cli": "3.6.0",
+    "@sasjs/core": "4.4.7"
   }
 }

sasjs/doxy/Doxyfile

@@ -8,6 +8,7 @@ FILE_PATTERNS          = *.sas \
                          *.dox
 GENERATE_LATEX         = NO
 GENERATE_TREEVIEW      = YES
+HAVE_DOT               = YES
 HIDE_FRIEND_COMPOUNDS  = YES
 HIDE_IN_BODY_DOCS      = YES
 HIDE_SCOPE_NAMES       = YES

sasjs/doxy/new_header.html

@@ -17,9 +17,7 @@
   <meta name="description" content="$projectbrief" />
   <meta name="og:description" content="$projectbrief" />
   <!--END PROJECT_NAME-->
-  <!--BEGIN !PROJECT_NAME-->
   <title>$title</title>
-  <!--END !PROJECT_NAME-->
   <link href="$relpath^tabs.css" rel="stylesheet" type="text/css" />
   <script type="text/javascript" src="$relpath^jquery.js"></script>
   <script type="text/javascript" src="$relpath^dynsections.js"></script>

sasjs/sasjsconfig.json

@@ -2,19 +2,21 @@
   "$schema": "https://raw.githubusercontent.com/sasjs/utils/main/src/types/sasjsconfig-schema.json",
   "macroFolders": [
     "base",
+    "ddl",
     "fcmp",
     "meta",
     "metax",
     "server",
     "viya",
     "lua",
-    "tests/crossplatform"
+    "tests/crossplatform",
+    "tests/ddl"
   ],
   "docConfig": {
     "displayMacroCore": false,
     "enableLineage": false,
     "doxyContent": {
-      "favIcon": "runningman.jpg",
+      "favIcon": "favicon.ico",
       "logo": "Macro_core_website_1.png",
       "readMe": "../../README.md"
     }
@@ -40,6 +42,7 @@
         "tests/viyaonly"
       ],
       "programFolders": [],
+      "binaryFolders": [],
       "deployConfig": {
         "deployServicePack": true,
         "deployScripts": []
@@ -58,6 +61,7 @@
         "tests/sas9only"
       ],
       "programFolders": [],
+      "binaryFolders": [],
       "deployConfig": {
         "deployServicePack": true,
         "deployScripts": []
@@ -67,19 +71,26 @@
     },
     {
       "name": "server",
-      "serverUrl": "https://sas.analytium.co.uk:5001",
+      "serverUrl": "https://sas.analytium.co.uk:5000",
       "serverType": "SASJS",
-      "appLoc": "/Shared Data/temp/macrocore",
+      "httpsAgentOptions": {
+        "allowInsecureRequests": false
+      },
+      "appLoc": "/sasjs/core",
       "macroFolders": [
         "tests/serveronly"
       ],
+      "programFolders": [],
+      "binaryFolders": [],
       "deployConfig": {
-        "deployServicePack": true
+        "deployServicePack": true,
+        "deployScripts": []
       }
     },
     {
       "name": "docsonly",
       "serverType": "SAS9",
+      "appLoc": "dummy",
       "macroFolders": [
         "tests/sas9only",
         "tests/viyaonly"

server/mfs_httpheader.sas

@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief Sets the http headers in the SASjs/server response
+  @details For GET requests, SASjs server will use the file generated by this
+  macro for setting the appropriate http headers in the response.
+
+  It works by writing a file to the session directory, that is then ingested by
+  the server.
+
+  The location of this file is driven by the global variable
+  `sasjs_stpsrv_header_loc` which is made available in the autoexec.
+
+  Usage:
+
+      %mfs_httpheader(Content-type,application/csv)
+
+  @param [in] header_name Name of the http header to set
+  @param [in] header_value Value of the http header to set
+
+  <h4> Related Macros </h4>
+  @li mcf_stpsrv_header.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mfs_httpheader(header_name
+  ,header_value
+)/*/STORE SOURCE*/;
+%local fref fid i;
+
+%if %sysfunc(filename(fref,&sasjs_stpsrv_header_loc)) ne 0 %then %do;
+  %put &=fref &=sasjs_stpsrv_header_loc;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let fid=%sysfunc(fopen(&fref,A));
+
+%if &fid=0 %then %do;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let rc=%sysfunc(fput(&fid,%str(&header_name): %str(&header_value)));
+%let rc=%sysfunc(fwrite(&fid));
+
+%let rc=%sysfunc(fclose(&fid));
+%let rc=%sysfunc(filename(&fref));
+
+%mend mfs_httpheader;

server/ms_createfile.sas

@@ -0,0 +1,89 @@
+/**
+  @file
+  @brief Creates a file on SASjs Drive
+  @details Creates a file on SASjs Drive. To use the file as a Stored Program,
+  it must have a ".sas" extension.
+
+  Example:
+
+      filename stpcode temp;
+      data _null_;
+        file stpcode;
+        put '%put hello world;';
+      run;
+      %ms_createfile(/some/stored/program.sas, inref=stpcode)
+
+  @param [in] driveloc The full path to the file in SASjs Drive
+  @param [in] inref= (0) The fileref containing the file to create.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+
+**/
+
+%macro ms_createfile(driveloc
+    ,inref=0
+    ,mdebug=0
+  );
+
+%local fname0 fname1 boundary fname statcd msg;
+%let fname0=%mf_getuniquefileref();
+%let fname1=%mf_getuniquefileref();
+%let boundary=%mf_getuniquename();
+
+data _null_;
+  file &fname0 termstr=crlf;
+  infile &inref end=eof;
+  if _n_ = 1 then do;
+    put "--&boundary.";
+    put 'Content-Disposition: form-data; name="filePath"';
+    put ;
+    put "&driveloc";
+    put "--&boundary";
+    put 'Content-Disposition: form-data; name="file"; filename="ignore.sas"';
+    put "Content-Type: text/plain";
+    put ;
+  end;
+  input;
+  put _infile_; /* add the actual file to be sent */
+  if eof then do;
+    put ;
+    put "--&boundary--";
+  end;
+run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fname0;
+    input;
+    put _infile_;
+  run;
+%end;
+
+proc http method='POST' in=&fname0 out=&fname1
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file";
+  headers "Content-Type"="multipart/form-data; boundary=&boundary";
+%if &mdebug=1 %then %do;
+  debug level=1;
+%end;
+run;
+
+%let statcd=0;
+data _null_;
+  infile &fname1;
+  input;
+  putlog _infile_;
+  if _infile_='{"status":"success"}' then call symputx('statcd',1,'l');
+  else call symputx('msg',_infile_,'l');
+run;
+
+%mp_abort(
+  iftrue=(&statcd=0)
+  ,mac=ms_createfile.sas
+  ,msg=%superq(msg)
+)
+
+%mend ms_createfile;

server/ms_getfile.sas

@@ -0,0 +1,32 @@
+/**
+  @file
+  @brief Gets a file from SASjs Drive
+  @details Fetches a file on SASjs Drive and stores it in the output fileref.
+
+  Example:
+
+      %ms_getfile(/some/stored/file.ext, outref=myfile)
+
+  @param [in] driveloc The full path to the file in SASjs Drive
+  @param [out] outref= (msgetfil) The fileref to contain the file.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+
+**/
+
+%macro ms_getfile(driveloc
+    ,outref=msgetfil
+    ,mdebug=0
+  );
+
+filename &outref temp;
+
+proc http method='GET' out=&outref
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file?filePath=&driveloc";
+%if &mdebug=1 %then %do;
+  debug level=2;
+%end;
+run;
+
+
+%mend ms_getfile;

server/ms_runstp.sas

@@ -0,0 +1,77 @@
+/**
+  @file
+  @brief Executes a SASjs Server Stored Program
+  @details Runs a Stored Program (using POST method) and extracts the webout and
+  log from the response JSON.
+
+  Example:
+
+      %ms_runstp(/some/stored/program
+        ,debug=131
+        ,outref=weboot
+      )
+
+  @param [in] pgm The full path to the Stored Program in SASjs Drive (_program
+    parameter)
+  @param [in] debug= (131) The value to supply to the _debug URL parameter
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outref= (outweb) The output fileref to contain the response JSON
+    (will be created using temp engine)
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_abort.sas
+
+**/
+
+%macro ms_runstp(pgm
+    ,debug=131
+    ,outref=outweb
+    ,mdebug=0
+  );
+%local dbg fname1;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+%let fname1=%mf_getuniquefileref();
+
+%mp_abort(iftrue=("&pgm"="")
+  ,mac=&sysmacroname
+  ,msg=%str(Program not provided)
+)
+
+data _null_;
+  file &fname1;
+  infile "&_sasjs_tokenfile";
+  input;
+  put 'Authorization: Bearer' _infile_;
+run;
+
+filename &outref temp;
+
+/* prepare request*/
+proc http method='POST' headerin=&fname1 out=&outref
+  url="&_sasjs_apiserverurl.&_sasjs_apipath?_program=&pgm%str(&)_debug=131";
+run;
+%if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  or &mdebug=1 %then %do;
+  data _null_;infile &outref;input;putlog _infile_;run;
+%end;
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  ,mac=&sysmacroname
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+
+%if &mdebug=1 %then %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%else %do;
+  /* clear refs */
+  filename &fname1 clear;
+%end;
+%mend ms_runstp;

server/ms_webout.sas

@@ -33,8 +33,9 @@
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
 
   <h4> SAS Macros </h4>
-  @li mp_jsonout.sas
   @li mf_getuser.sas
+  @li mp_jsonout.sas
+  @li mfs_httpheader.sas
 
   <h4> Related Macros </h4>
   @li mv_webout.sas
@@ -86,11 +87,11 @@
   /* fix encoding */
   OPTIONS NOBOMFILE;
 
+  /* set the header */
+  %mfs_httpheader(Content-type,application/json)
+
   /* setup json */
   data _null_;file &fref encoding='utf-8' termstr=lf;
-  %if %str(&_debug) ge 131 %then %do;
-    put '>>weboutBEGIN<<';
-  %end;
     put '{"SYSDATE" : "' "&SYSDATE" '"';
     put ',"SYSTIME" : "' "&SYSTIME" '"';
   run;
@@ -170,9 +171,6 @@
     memsize=quote(cats(memsize));
     put ',"MEMSIZE" : ' memsize;
     put "}" @;
-  %if %str(&_debug) ge 131 %then %do;
-    put '>>weboutEND<<';
-  %end;
   run;
 %end;
 

tests/crossplatform/mcf_getfmttype.test.sas

@@ -0,0 +1,39 @@
+/**
+  @file
+  @brief Testing mcf_getfmttype.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mcf_getfmttype.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%mp_assertscope(SNAPSHOT)
+%mcf_getfmttype(wrap=YES, insert_cmplib=YES)
+%mp_assertscope(COMPARE,ignorelist=SASJS_FUNCTIONS)
+
+%mp_assert(
+  iftrue=(%sysfunc(mcf_getfmttype(DATE9.))=DATE),
+  desc=Check DATE format
+)
+%mp_assert(
+  iftrue=(%sysfunc(mcf_getfmttype($6))=CHAR),
+  desc=Check CHAR format
+)
+%mp_assert(
+  iftrue=(%sysfunc(mcf_getfmttype(8.))=NUM),
+  desc=Check NUM format
+)
+%mp_assert(
+  iftrue=(%sysfunc(mcf_getfmttype(E8601DT))=DATETIME),
+  desc=Check DATETIME format
+)
+
+/* test 2 - compile again test for warnings */
+%mcf_getfmttype(wrap=YES, insert_cmplib=YES)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check syscc=0 after re-initialisation
+)

tests/crossplatform/mcf_init.test.sas

@@ -0,0 +1,46 @@
+/**
+  @file
+  @brief Testing mcf_init.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mcf_init.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(%mcf_init(test)=0),
+  desc=Check if new func returns 0
+)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errs on basic invocation
+)
+%mp_assert(
+  iftrue=(%mcf_init(test)=1),
+  desc=Check if second invocation returns 1
+)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errs on second invocation
+)
+%mp_assert(
+  iftrue=(%mcf_init(test2)=0),
+  desc=Check if new invocation returns 0
+)
+%mp_assert(
+  iftrue=(%mcf_init(test2)=1),
+  desc=Check if second new invocation returns 1
+)
+%mp_assert(
+  iftrue=(%mcf_init(test)=1),
+  desc=Check original returns 1
+)
+%mp_assert(
+  iftrue=(%mcf_init(t)=1),
+  desc=Check subset returns 1
+)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errs at end
+)

tests/crossplatform/mf_existvar.test.sas

@@ -10,13 +10,11 @@
 
 
 %mp_assert(
-  iftrue=(%mf_existvar(sashelp.class,age)=1),
-  desc=Checking existing var exists,
-  outds=work.test_results
+  iftrue=(%mf_existvar(sashelp.class,age)>0),
+  desc=Checking existing var exists
 )
 
 %mp_assert(
   iftrue=(%mf_existvar(sashelp.class,isjustanumber)=0),
-  desc=Checking non existing var does not exist,
-  outds=work.test_results
+  desc=Checking non existing var does not exist
 )

tests/crossplatform/mf_getapploc.test.sas

@@ -46,4 +46,20 @@
   ),
   desc=Checking tests/macros appLoc matches (which has no subfolder),
   outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/tests/testsetup)"="/some/area"
+  ),
+  desc=Checking tests/testsetup operation,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/tests/testteardown)"="/some/area"
+  ),
+  desc=Checking tests/teardown operation,
+  outds=work.test_results
 )

tests/crossplatform/mf_getfilesize.test.sas

@@ -0,0 +1,30 @@
+/**
+  @file
+  @brief Testing mf_getfilesize macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getfilesize.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+data test;
+  x=1;
+run;
+
+%mp_assertscope(SNAPSHOT)
+%put %mf_getfilesize(libds=work.test)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking syscc
+)
+
+%put %mf_getfilesize(libds=test);
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking syscc with one level name
+)

tests/crossplatform/mf_getuniquelibref.test.sas

@@ -0,0 +1,54 @@
+/**
+  @file
+  @brief Testing mf_getuniquelibref macro
+  @details To test performance you can also use the following macro:
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquelibref.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+/* check valid libs */
+%mp_assertscope(SNAPSHOT)
+%let libshort=%mf_getuniquelibref(prefix=lib);
+%mp_assertscope(COMPARE,ignorelist=LIBSHORT)
+libname &libshort (work);
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking for valid libref &libshort,
+  outds=work.test_results
+)
+
+%let lib7=%mf_getuniquelibref(prefix=libref7);
+libname &lib7 (work);
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking for valid libref &lib7,
+  outds=work.test_results
+)
+
+
+/* check for invalid libs */
+
+%let lib8=%mf_getuniquelibref(prefix=lib8char);
+%mp_assert(
+  iftrue=(&lib8=0),
+  desc=Invalid prefix (8 chars),
+  outds=work.test_results
+)
+
+%let liblong=%mf_getuniquelibref(prefix=invalidlib);
+%mp_assert(
+  iftrue=(&liblong=0),
+  desc=Checking for invalid libref (long),
+  outds=work.test_results
+)
+
+%let badlib=%mf_getuniquelibref(prefix=8adlib);
+%mp_assert(
+  iftrue=(&badlib=0),
+  desc=Checking for invalid libref (8adlib),
+  outds=work.test_results
+)

tests/crossplatform/mf_verifymacvars.test.sas

@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Testing mf_verifymacvars macro
+
+  <h4> SAS Macros </h4>
+  @li mf_verifymacvars.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%let var1=x;
+%let var2=y;
+
+%mp_assertscope(SNAPSHOT)
+%mp_assert(
+  iftrue=(%mf_verifymacvars(var1 var2)=1),
+  desc=Checking macvars exist,
+  outds=work.test_results
+)
+%mp_assertscope(COMPARE)
+

tests/crossplatform/mp_applyformats.test.sas

@@ -34,7 +34,7 @@ run;
 %mp_applyformats(work.cols2)
 
 %mp_assert(
-  iftrue=("&orig_fmt"=""),
+  iftrue=("&origfmt"=""),
   desc=Check that formats were cleared,
   outds=work.test_results
 )

tests/crossplatform/mp_cntlout.test.sas

@@ -0,0 +1,39 @@
+/**
+  @file
+  @brief Testing mp_cntlout.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_cntlout.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+libname perm (work);
+data work.loadfmts;
+  length fmtname $32;
+  eexcl='Y';
+  type='N';
+  do i=1 to 100;
+    fmtname=cats('SASJS_',i,'X');
+    do j=1 to 100;
+      start=cats(j);
+      end=cats(j+1);
+      label= cats('Dummy ',start);
+      output;
+    end;
+  end;
+run;
+proc format cntlin=work.loadfmts library=perm.testcat;
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mp_cntlout(libcat=perm.testcat,cntlout=work.cntlout)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.cntlout)=10000),
+  desc=Checking first hash diff,
+  outds=work.test_results
+)

tests/crossplatform/mp_ds2csv.test.sas

@@ -0,0 +1,166 @@
+/**
+  @file
+  @brief Testing mp_ds2csv.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2csv.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+data work.somedata;
+  x=1;
+  y='  t"w"o';
+  z=.z;
+  label x='x factor';
+run;
+
+/**
+  * Test 1 - default CSV
+  */
+%mp_assertscope(SNAPSHOT)
+%mp_ds2csv(work.somedata,outfile="&sasjswork/test1.csv")
+%mp_assertscope(COMPARE)
+
+%let test1b=FAIL;
+data _null_;
+  infile "&sasjswork/test1.csv";
+  input;
+  list;
+  if _n_=1 then call symputx('test1a',_infile_);
+  else if _infile_=:'1,"  t""w""o",.Z' then call symputx('test1b','PASS');
+run;
+
+%mp_assert(
+  iftrue=("&test1a"="x factor, Y, Z"),
+  desc=Checking header row Test 1,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test1b"="PASS"),
+  desc=Checking data row Test 1,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - NAME header with fileref and semicolons
+  */
+filename test2 "&sasjswork/test2.csv";
+%mp_ds2csv(work.somedata,outref=test2,dlm=SEMICOLON,headerformat=NAME)
+
+%let test2b=FAIL;
+data _null_;
+  infile test2;
+  input;
+  list;
+  if _n_=1 then call symputx('test2a',_infile_);
+  else if _infile_=:'1;"  t""w""o";.Z' then call symputx('test2b','PASS');
+run;
+
+%mp_assert(
+  iftrue=("&test2a"="X; Y; Z"),
+  desc=Checking header row Test 2,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test2b"="PASS"),
+  desc=Checking data row Test 2,
+  outds=work.test_results
+)
+
+/**
+  * Test 3 - SASjs format
+  */
+filename test3 "&sasjswork/test3.csv";
+%mp_ds2csv(work.somedata,outref=test3,headerformat=SASJS)
+
+%let test3b=FAIL;
+data _null_;
+  infile test3;
+  input;
+  list;
+  if _n_=1 then call symputx('test3a',_infile_);
+  else if _infile_=:'1,"  t""w""o",.Z' then call symputx('test3b','PASS');
+run;
+
+%mp_assert(
+  iftrue=("&test3a"="X:best. Y:$char7. Z:best."),
+  desc=Checking header row Test 3,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test3b"="PASS"),
+  desc=Checking data row Test 3,
+  outds=work.test_results
+)
+
+/* test 4 - sasjs with compare */
+filename example temp;
+%mp_ds2csv(sashelp.air,outref=example,headerformat=SASJS)
+data _null_; infile example; input;put _infile_; if _n_>5 then stop;run;
+
+data _null_;
+  infile example;
+  input;
+  call symputx('stmnt',_infile_);
+  stop;
+run;
+data work.want;
+  infile example dsd firstobs=2;
+  input &stmnt;
+run;
+
+%mp_assert(
+  iftrue=(&syscc =0),
+  desc=Checking syscc prior to compare of sashelp.air,
+  outds=work.test_results
+)
+
+proc compare base=want compare=sashelp.air;
+run;
+%mp_assert(
+  iftrue=(&sysinfo le 41),
+  desc=Checking compare of sashelp.air,
+  outds=work.test_results
+)
+
+/* test 5 - sasjs with time/datetime/date */
+filename f2 temp;
+data work.test5;
+  do x=1 to 5;
+    y=x;
+    z=x;
+  end;
+  format x date9. y datetime19. z time.;
+run;
+%mp_ds2csv(work.test5,outref=f2,headerformat=SASJS)
+data _null_; infile example; input;put _infile_; if _n_>5 then stop;run;
+
+data _null_;
+  infile f2;
+  input;
+  putlog _infile_;
+  call symputx('stmnt2',_infile_);
+  stop;
+run;
+data work.want5;
+  infile f2 dsd firstobs=2;
+  input &stmnt2;
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking syscc prior to compare of test5,
+  outds=work.test_results
+)
+
+proc compare base=want5 compare=work.test5;
+run;
+%mp_assert(
+  iftrue=(&sysinfo le 41),
+  desc=Checking compare of work.test5,
+  outds=work.test_results
+)
+

tests/crossplatform/mp_filterstore.test.1.sas

@@ -41,7 +41,7 @@ run;
   outquery=work.query,
   mdebug=1
 )
-%mp_assert(iftrue=(&syscc>0),
+%mp_assert(iftrue=(&syscc=0),
   desc=Ensure macro runs without errors,
   outds=work.test_results
 )

tests/crossplatform/mp_filterstore.test.2.sas

@@ -0,0 +1,112 @@
+/**
+  @file
+  @brief Testing mp_filterstore macro with a format catalog
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
+  @li mp_coretable.sas
+  @li mp_filterstore.sas
+
+**/
+
+libname permlib (work);
+
+%mp_coretable(LOCKTABLE,libds=permlib.locktable)
+%mp_coretable(FILTER_SUMMARY,libds=permlib.filtsum)
+%mp_coretable(FILTER_DETAIL,libds=permlib.filtdet)
+%mp_coretable(MAXKEYTABLE,libds=permlib.maxkey)
+
+/* valid filter */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$12. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,Start,>,"'2'"
+AND,AND,1,Fmtname,=,"'MORDOR'"
+OR,OR,2,Label,IN,"('Dragon1','Dragon2')"
+OR,OR,2,End,=,"'6'"
+OR,OR,2,Start,GE,"'10'"
+;;;;
+run;
+
+/* make some formats */
+PROC FORMAT library=permlib.testfmts;
+  picture MyMSdt other='%0Y-%0m-%0dT%0H:%0M:%0S' (datatype=datetime);
+RUN;
+data work.fmts;
+  length fmtname $32;
+  do fmtname='SMAUG','MORDOR','GOLLUM';
+    do start=1 to 10;
+      label= cats('Dragon ',start);
+      output;
+    end;
+  end;
+run;
+proc sort data=work.fmts nodupkey;
+  by fmtname start;
+run;
+proc format cntlin=work.fmts library=permlib.testfmts;
+run;
+proc format library=permlib.testfmts;
+  invalue indays (default=13) other=42;
+run;
+
+
+%mp_assertscope(SNAPSHOT)
+%mp_filterstore(libds=permlib.testfmts-fc,
+  queryds=work.inds,
+  filter_summary=permlib.filtsum,
+  filter_detail=permlib.filtdet,
+  lock_table=permlib.locktable,
+  maxkeytable=permlib.maxkey,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+)
+%mp_assertscope(COMPARE)
+
+%mp_assert(iftrue=(&syscc=0),
+  desc=Ensure macro runs without errors,
+  outds=work.test_results
+)
+/* ensure only one record created */
+%mp_assertdsobs(permlib.filtsum,
+  desc=Initial query,
+  test=ATMOST 1,
+  outds=work.test_results
+)
+/* check RK is correct */
+proc sql noprint;
+select max(filter_rk) into: test1 from work.result;
+%mp_assert(iftrue=(&test1=1),
+  desc=Ensure filter rk is correct,
+  outds=work.test_results
+)
+
+/* Test 2 - load same table again and ensure we get the same RK */
+%mp_filterstore(libds=permlib.testfmts-fc,
+  queryds=work.inds,
+  filter_summary=permlib.filtsum,
+  filter_detail=permlib.filtdet,
+  lock_table=permlib.locktable,
+  maxkeytable=permlib.maxkey,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+)
+/* ensure only one record created */
+%mp_assertdsobs(permlib.filtsum,
+  desc=Initial query - same obs,
+  test=ATMOST 1,
+  outds=work.test_results
+)
+/* check RK is correct */
+proc sql noprint;
+select max(filter_rk) into: test2 from work.result;
+%mp_assert(iftrue=(&test2=1),
+  desc=Ensure filter rk is correct for second run,
+  outds=work.test_results
+)

tests/crossplatform/mp_getpk.test.sas

@@ -52,14 +52,13 @@ create table work.example2(
 %mp_getpk(work,ds=example2,outds=test2)
 
 data _null_;
-  set work.test1;
+  set work.test2;
   call symputx('test2',pk_fields);
 run;
 
 %mp_assert(
   iftrue=("&test2"="TX_FROM DD_TYPE"),
-  desc=mp_getpk gets unique constraint with NOT NULL in correct order,
-  outds=work.test_results
+  desc=mp_getpk gets unique constraint with NOT NULL in correct order
 )
 
 /* unique key without NOT NULL NOT captured */
@@ -71,13 +70,17 @@ create table work.example3(
   DD_SHORTDESC char(256),
   constraint unq1 unique(tx_from, dd_type),
   constraint unq2 unique(tx_from, dd_type, dd_source),
-  constraint nnn not null(tx_from),
-  constraint nnnn not null(dd_type)
+  constraint nnn not null(tx_from)
 );
 %mp_getpk(work,ds=example3,outds=test3)
 
+data _null_;
+  set work.test3;
+  call symputx('test3',pk_fields);
+run;
+
 %mp_assert(
-  iftrue=(%mf_nobs(work.test3)=0),
+  iftrue=("&test3 "=" "),
   desc=mp_getpk does not capture unique constraint without NOT NULL,
   outds=work.test_results
 )

tests/crossplatform/mp_loadformat.test.sas

@@ -0,0 +1,76 @@
+/**
+  @file
+  @brief Testing mp_loadformat.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_loadformat.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+/* prep format catalog */
+libname perm (work);
+
+data work.loadfmts;
+  length fmtname $32;
+  eexcl='Y';
+  type='N';
+  do i=1 to 100;
+    fmtname=cats('SASJS_',i,'X');
+    do j=1 to 100;
+      start=cats(j);
+      end=cats(j+1);
+      label= cats('Dummy ',start);
+      output;
+    end;
+  end;
+run;
+proc format cntlin=work.loadfmts library=perm.testcat;
+run;
+
+/* make some test data */
+data work.stagedata;
+  set work.loadfmts;
+  type='N';
+  eexcl='Y';
+  if _n_<150 then deleteme='Yes';
+  else if _n_<250 then label='mod'!!cats(_n_);
+  else if _n_<350 then do;
+    start=cats(_n_);
+    end=cats(_n_+1);
+    label='newval'!!cats(_N_);
+  end;
+  else stop;
+run;
+
+/* load the above */
+%mp_assertscope(SNAPSHOT)
+%mp_loadformat(perm.testcat
+  ,work.stagedata
+  ,loadtarget=YES
+  ,auditlibds=0
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_test1
+  ,outds_del=del_test1
+  ,outds_mod=mod_test1
+  ,mdebug=1
+)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(%mf_nobs(del_test1)=149),
+  desc=Test 1 - delete obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(add_test1)=100),
+  desc=Test 1 - add obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(mod_test1)=100),
+  desc=Test 1 - mod obs,
+  outds=work.test_results
+)

tests/crossplatform/mp_md5.test.sas

@@ -0,0 +1,41 @@
+/**
+  @file
+  @brief Testing mp_md5.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_md5.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+%global hash1 hash2 hash3;
+
+%mp_assertscope(SNAPSHOT)
+data work.test1 /nonote2err;
+  c1='';
+  c2=repeat('x',32767);
+  c3='  f';
+  n1=.a;
+  n2=.;
+  n3=1.0000000001;
+  hash=%mp_md5(cvars=c1 c2 c3,nvars=n1 n2 n3);
+  call symputx('hash1',hash);
+  n1=.b;
+  hash=%mp_md5(cvars=c1 c2 c3,nvars=n1 n2 n3);
+  call symputx('hash2',hash);
+  c3='f';
+  hash=%mp_md5(cvars=c1 c2 c3,nvars=n1 n2 n3);
+  call symputx('hash3',hash);
+run;
+%mp_assertscope(COMPARE,ignorelist=HASH1 HASH2 HASH3)
+
+%mp_assert(
+  iftrue=("&hash1" ne "&hash2"),
+  desc=Checking first hash diff,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&hash2" ne "&hash3"),
+  desc=Checking first hash diff,
+  outds=work.test_results
+)

tests/crossplatform/mp_replace.test.sas

@@ -0,0 +1,65 @@
+/**
+  @file
+  @brief Testing mp_replace.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_replace.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+
+%let test1="&sasjswork/file.txt";
+%let str=replace/me;
+%let rep=with/this;
+data _null_;
+  file &test1;
+  put 'blahblah';
+  put "blahblah&str.blah";
+  put 'blahblahblah';
+run;
+%mp_assertscope(SNAPSHOT)
+%mp_replace(&test1, findvar=str, replacevar=rep)
+%mp_assertscope(COMPARE)
+data _null_;
+  infile &test1;
+  input;
+  if _n_=2 then call symputx('test1result',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&test1result" = "blahblah&rep.blah"),
+  desc=Checking first replace,
+  outds=work.test_results
+)
+
+
+%let test2="&sasjswork/file2.txt";
+%let str=%str(replacewith trailing spaces   );
+%let rep=%str( with more spaces  );
+data _null_;
+  file &test2;
+  put 'blahblah';
+  put "blahblah&str.blah&str. replace &str.X";
+  put "blahbreplacewith&str.spacesahblah";
+run;
+%mp_replace(&test2, findvar=str, replacevar=rep)
+
+data _null_;
+  infile &test2;
+  input;
+  if _n_=2 then call symputx('test2resulta',_infile_);
+  if _n_=3 then call symputx('test2resultb',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&test2resulta" = "blahblah&rep.blah&rep. replace &rep.X"),
+  desc=Checking second replace 2nd row,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test2resultb" = "blahbreplacewith&rep.spacesahblah"),
+  desc=Checking second replace 3rd row,
+  outds=work.test_results
+)

tests/crossplatform/mp_stackdiffs.test.sas

@@ -0,0 +1,302 @@
+/**
+  @file
+  @brief Testing mp_storediffs macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
+  @li mp_stackdiffs.sas
+  @li mp_storediffs.sas
+
+**/
+
+/* first, make some data */
+
+data work.orig work.deleted work.changed work.appended;
+  set sashelp.electric;
+  if _n_ le 10 then do;
+    output work.deleted;
+  end;
+  else if _n_ le 20 then do;
+    output work.orig;
+    coal=-1;
+    coaltip='modified';
+    output work.changed;
+  end;
+  else if _n_ le 30 then do;
+    year=_n_;
+    output work.appended;
+  end;
+  else stop;
+run;
+
+%mp_storediffs(sashelp.electric
+  ,work.orig
+  ,CUSTOMER YEAR
+  ,delds=work.deleted
+  ,modds=work.changed
+  ,appds=work.appended
+  ,outds=work.final
+  ,mdebug=1
+)
+
+
+%mp_assertscope(SNAPSHOT)
+
+/**
+  * Deletions test - where record does exist
+  */
+data work.orig1;
+  set sashelp.electric;
+  if _n_ le 10;
+run;
+data work.final1;
+  set work.final;
+  where move_type='D';
+run;
+%mp_stackdiffs(work.orig1
+  ,work.final1
+  ,CUSTOMER YEAR
+  ,mdebug=1
+  ,errds=work.errds1
+  ,outmod=work.mod1
+  ,outadd=work.add1
+  ,outdel=work.del1
+)
+%mp_assertdsobs(work.errds1,
+  desc=Delete1 - no errs,
+  test=EQUALS 0
+)
+%mp_assertdsobs(work.del1,
+  desc=Delete1 - records populated,
+  test=EQUALS 10
+)
+/**
+  * Deletions test - where record does NOT exist
+  */
+data work.orig2;
+  set work.orig;
+  stop; /* empty table */
+run;
+data work.final2;
+  set work.final;
+  where move_type='D';
+run;
+%mp_stackdiffs(work.orig2
+  ,work.final2
+  ,CUSTOMER YEAR
+  ,mdebug=1
+  ,errds=work.errds2
+  ,outmod=work.mod2
+  ,outadd=work.add2
+  ,outdel=work.del2
+)
+%mp_assertdsobs(work.errds2,
+  desc=Delete2 - has errs,
+  test=EQUALS 10
+)
+%mp_assertdsobs(work.del2,
+  desc=Delete2 - records not populated,
+  test=EQUALS 0
+)
+
+/**
+  * Additions test - where record does not exist
+  */
+data work.orig3;
+  set work.orig;
+  stop;
+run;
+data work.final3;
+  set work.final;
+  where move_type='A';
+run;
+%mp_stackdiffs(work.orig3
+  ,work.final3
+  ,CUSTOMER YEAR
+  ,mdebug=1
+  ,errds=work.errds3
+  ,outmod=work.mod3
+  ,outadd=work.add3
+  ,outdel=work.del3
+)
+%mp_assertdsobs(work.errds3,
+  desc=Add3 - no errs,
+  test=EQUALS 0
+)
+%mp_assertdsobs(work.add3,
+  desc=Add3 - records populated,
+  test=EQUALS 10
+)
+
+/**
+  * Additions test - where record does exist
+  */
+data work.orig4;
+  set sashelp.electric;
+  if _n_ ge 20;
+  year=_n_;
+  if _n_>25 then stop;
+run;
+data work.final4;
+  set work.final;
+  where move_type='A';
+run;
+%mp_stackdiffs(work.orig4
+  ,work.final4
+  ,CUSTOMER YEAR
+  ,mdebug=1
+  ,errds=work.errds4
+  ,outmod=work.mod4
+  ,outadd=work.add4
+  ,outdel=work.del4
+)
+%mp_assertdsobs(work.errds4,
+  desc=Add4 - 5 errs,
+  test=EQUALS 5
+)
+%mp_assertdsobs(work.add4,
+  desc=Add4 - records populated,
+  test=EQUALS 5
+)
+
+/**
+  * Additions test - where base table has missing vars
+  */
+data work.orig5;
+  set work.orig;
+  drop Coal;
+run;
+data work.final5;
+  set work.final;
+  where move_type='A';
+run;
+%mp_stackdiffs(work.orig5
+  ,work.final5
+  ,CUSTOMER YEAR
+  ,mdebug=1
+  ,errds=work.errds5
+  ,outmod=work.mod5
+  ,outadd=work.add5
+  ,outdel=work.del5
+)
+%mp_assertdsobs(work.errds5,
+  desc=Add5 - 10 errs,
+  test=EQUALS 10
+)
+%mp_assertdsobs(work.add5,
+  desc=Add5 - 0 records populated due to structure change,
+  test=EQUALS 0
+)
+
+/**
+  * Additions test - where append table has missing vars
+  */
+data work.final6;
+  set work.final;
+  where tgtvar_nm ne 'COAL' and move_type='A';
+run;
+%mp_stackdiffs(work.orig
+  ,work.final6
+  ,CUSTOMER YEAR
+  ,mdebug=1
+  ,errds=work.errds6
+  ,outmod=work.mod6
+  ,outadd=work.add6
+  ,outdel=work.del6
+)
+%mp_assertdsobs(work.errds6,
+  desc=Add6 - 0 errs,
+  test=EQUALS 0
+)
+%mp_assertdsobs(work.add6,
+  desc=Add6 - 10 records populated (structure change irrelevant),
+  test=EQUALS 10
+)
+
+/**
+  * Modifications test - where base table has missing vars
+  */
+data work.orig7;
+  set work.orig;
+  drop Coal;
+run;
+data work.final7;
+  set work.final;
+  where move_type='M';
+run;
+%mp_stackdiffs(work.orig7
+  ,work.final7
+  ,CUSTOMER YEAR
+  ,mdebug=1
+  ,errds=work.errds7
+  ,outmod=work.mod7
+  ,outadd=work.add7
+  ,outdel=work.del7
+)
+%mp_assertdsobs(work.errds7,
+  desc=Mod7 - 10 errs,
+  test=EQUALS 10
+)
+%mp_assertdsobs(work.Mod7,
+  desc=Mod7 - 0 records populated (structure change relevant),
+  test=EQUALS 0
+)
+%mp_assertdsobs(work.add7,
+  desc=add7 - 0 records populated ,
+  test=EQUALS 0
+)
+%mp_assertdsobs(work.del7,
+  desc=del7 - 0 records populated ,
+  test=EQUALS 0
+)
+/**
+  * Modifications (big) test - where base table has missing rows
+  * Also used as a full integration test (all move_types)
+  * And a test if the actual values were applied
+  */
+data work.orig8;
+  set sashelp.electric;
+  if _n_ le 16;
+run;
+%mp_stackdiffs(work.orig8
+  ,work.final
+  ,CUSTOMER YEAR
+  ,mdebug=1
+  ,errds=work.errds8
+  ,outmod=work.mod8
+  ,outadd=work.add8
+  ,outdel=work.del8
+)
+%mp_assertdsobs(work.errds8,
+  desc=Mod4 - 4 errs,
+  test=EQUALS 4
+)
+%mp_assertdsobs(work.Mod8,
+  desc=Mod8 - 6 records populated (missing rows relevant),
+  test=EQUALS 6
+)
+
+/**
+  * Modifications test - were diffs actually applied?
+  */
+data work.checkds;
+  charchk='modified';
+  numchk=-1;
+  output;
+run;
+%mp_assertcolvals(work.mod8.coal,
+  checkvals=work.checkds.numchk,
+  desc=Modified numeric value matches,
+  test=ALLVALS
+)
+%mp_assertcolvals(work.mod8.coaltip,
+  checkvals=work.checkds.charchk,
+  desc=Modified char value matches,
+  test=ALLVALS
+)
+
+
+%mp_assertscope(COMPARE,ignorelist=SASJS_FUNCTIONS)

tests/crossplatform/mp_streamfile.test.sas

@@ -0,0 +1,28 @@
+/**
+  @file
+  @brief Testing mp_streamfile.sas macro
+  @details This is tricky to test as it streams to webout.  For now just
+  check the compilation, and for macro leakage.
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mp_streamfile.sas
+
+**/
+
+%mp_assertscope(SNAPSHOT)
+
+%mp_streamfile(iftrue=(1=0)
+  ,contenttype=csv,inloc=/some/where.txt
+  ,outname=myfile.txt
+)
+
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking error condition,
+  outds=work.test_results
+)
+

tests/crossplatform/mp_testservice.test.sas

@@ -0,0 +1,91 @@
+/**
+  @file
+  @brief Testing mp_testservice.sas macro
+
+  Be sure to run <code>%let mcTestAppLoc=/Public/temp/macrocore;</code> when
+  runnin in Studio
+
+  <h4> SAS Macros </h4>
+  @li mp_createwebservice.sas
+  @li mp_testservice.sas
+  @li mp_assert.sas
+
+**/
+
+filename ft15f001 temp;
+parmcards4;
+  %put Initialising sendObj: ;
+  %put _all_;
+  %webout(FETCH)
+  %webout(OPEN)
+  %macro x();
+  %if %symexist(sasjs_tables) %then %do i=1 %to %sysfunc(countw(&sasjs_tables));
+    %let table=%scan(&sasjs_tables,&i);
+    %webout(OBJ,&table,missing=STRING)
+  %end;
+  %else %do i=1 %to &_webin_file_count;
+    %webout(OBJ,&&_webin_name&i,missing=STRING)
+  %end;
+  %mend x; %x()
+  %webout(CLOSE)
+;;;;
+%mp_createwebservice(path=&mcTestAppLoc/services,name=sendObj)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errors after service creation,
+  outds=work.test_results
+)
+
+/**
+  * Test 1 - send a dataset
+  */
+data work.somedata1 work.somedata2;
+  x=1;
+  y='  t"w"o';
+  z=.z;
+  label x='x factor';
+  output;
+run;
+
+%mp_testservice(&mcTestAppLoc/services/sendObj,
+  inputdatasets=work.somedata1 work.somedata2,
+  debug=log,
+  mdebug=1,
+  outlib=testlib1,
+  outref=test1
+)
+
+%global test1a test1b test1c test1d;
+data _null_;
+  infile test1;
+  input;
+  putlog _n_ _infile_;
+  if _infile_=', "somedata1":' then call symputx('test1a','PASS');
+  if _infile_='{"X":1 ,"Y":"  t\"w\"o" ,"Z":"Z" }' then
+    call symputx('test1b','PASS');
+  if _infile_='], "somedata2":' then call symputx('test1c','PASS');
+  if _infile_='{"X":1 ,"Y":"  t\"w\"o" ,"Z":"Z" }' then
+      call symputx('test1d','PASS');
+run;
+
+%mp_assert(
+  iftrue=(&test1a=PASS),
+  desc=Test 1 table 1 name,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&test1b=PASS),
+  desc=Test 1 table 1 values,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&test1c=PASS),
+  desc=Test 1 table 2 name,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&test1d=PASS),
+  desc=Test 1 table 2 values,
+  outds=work.test_results
+)

tests/ddl/mddl_dc_difftable.test.sas

@@ -0,0 +1,19 @@
+/**
+  @file
+  @brief Difftable DDL test
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mf_existds.sas
+  @li mp_assert.sas
+
+**/
+
+
+%mddl_dc_difftable(libds=WORK.DIFFTABLE)
+
+%mp_assert(
+  iftrue=(%mf_existds(WORK.DIFFTABLE)=1),
+  desc=Checking table was created,
+  outds=work.test_results
+)

tests/ddl/mddl_dc_filterdetail.test.sas

@@ -0,0 +1,18 @@
+/**
+  @file
+  @brief Filtertable DDL test
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_filterdetail.sas
+  @li mf_existds.sas
+  @li mp_assert.sas
+
+**/
+
+%mddl_dc_filterdetail(libds=WORK.TEST)
+
+%mp_assert(
+  iftrue=(%mf_existds(WORK.TEST)=1),
+  desc=Checking table was created,
+  outds=work.test_results
+)