Merge pull request #183 from sasjs/feature/mf_getuniquelibref_safe_default

fix: Closes #182 - update to mf_getuniquelibref.sas

.all-contributorsrc

@@ -117,6 +117,15 @@
       "contributions": [
         "bug"
       ]
+    },
+    {
+      "login": "yabwon",
+      "name": "Bart Jablonski",
+      "avatar_url": "https://avatars.githubusercontent.com/u/9314894?v=4",
+      "profile": "https://github.com/yabwon",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.github/CONTRIBUTING.md

@@ -12,7 +12,7 @@ This repository makes use of the [SASjs](https://sasjs.io) framework for code or
 * [VSCode](https://sasjs.io/windows/#vscode) - feature packed IDE for code editing (warning - highly effective!)
 * [GIT](https://sasjs.io/windows/#git) - a safety net you cannot (and should not) do without.
 
-For generating the documentation (`sasjs doc`) it is also necessary to install [doxygen](https://www.doxygen.nl/manual/install.html).
+For generating the documentation (`sasjs doc`) it is also necessary to install [doxygen](https://www.doxygen.nl/manual/install.html) and GraphViz (`sudo port install graphviz` on mac, or `sudo apt-get install graphviz` on Ubuntu).
 
 
 To get configured:

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [sasjs]

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+  - package-ecosystem: npm
+    directory: '/'
+    schedule:
+      interval: monthly
+    open-pull-requests-limit: 3
+    allow:
+      - dependency-type: "production"

.gitpod.dockerfile

@@ -1,6 +1,6 @@
 FROM gitpod/workspace-full
 
 RUN sudo apt-get update \
- && sudo apt-get install -y \
-    doxygen \
+ && sudo apt-get install -y doxygen \
+ && sudo apt-get install -y graphviz \
  && sudo rm -rf /var/lib/apt/lists/*

.gitpod.yml

@@ -1,5 +1,7 @@
 tasks:
-  - init: nvm install --lts && npm i -g @sasjs/cli
+  - init: |
+      nvm install --lts
+      npm i -g @sasjs/cli
 
 image:
   file: .gitpod.dockerfile

README.md

@@ -31,19 +31,28 @@ Documentation: https://core.sasjs.io
 
 ## Components
 
-### BASE library (SAS9/Viya)
+### BASE library (All Platforms)
 
 - OS independent
-- Not metadata aware
+- Works on all SAS Platforms
 - No X command
 - Prefixes: _mf_, _mp_
 
-#### FCMP library (SAS9/Viya)
+### DDL library (All Platforms)
+
+- OS independent
+- Works on all SAS Platforms
+- No X command
+- Prefixes: _mddl_(lib)_ -> where lib can be "SAS" (in relation to a SAS component) or "DC" (in relation to a Data Controller component)
+
+This library will not be used for storing data entries (such as formats or datalines).  Where this becomes necessary in the future, a new repo will be created, in order to keep the NPM bundle size down (for the benefit of those looking to embed purely macros in their applications).
+
+### FCMP library (All Platforms)
+
 - Function and macro names are identical, except for special cases
 - Prefixes: _mcf_
 
-The fcmp macros are used to generate fcmp functions, and can be used with or
-without the `proc fcmp` wrapper.
+The fcmp macros are used to generate fcmp functions, and can be used with or without the `proc fcmp` wrapper.
 
 ### META library (SAS9 only)
 
@@ -81,7 +90,7 @@ Macros used for interfacing with SAS Viya.
 
 Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
 
-To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert your LUA into a data step with put statements, and create the macro wrapper with a `ml_` prefix.  You can then use your module in any program by running:
+To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert all files with a ".lua" extension into a macro wrapper with an `ml_` prefix (embedding the necessary data step put statements).  You can then use your module in any program by running:
 
 ```sas
 /* compile the lua module */
@@ -95,8 +104,7 @@ endsubmit;
 run;
 ```
 
-- X command enabled
-- Prefixes: _mmw_,_mmu_,_mmx_
+- Prefixes: _ml_
 
 ## Installation
 
@@ -125,14 +133,16 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 - macro names must be lowercase
 - one macro per file
 - prefixes:
+  - _mcf_ for macro compiled functions (proc fcmp)
+  - _mddl_ for macros containing DDL (Data Definition Language)
   - _mf_ for macro functions (can be used in open code).
   - _ml_ for macros that are used to compile LUA modules
   - _mm_ for metadata macros (interface with the metadata server).
-  - _mmx_ for macros that use metadata and are XCMD enabled
+  - _mmx_ for macros that use metadata and are XCMD enabled (working on both windows and unix)
   - _mp_ for macro procedures (which generate sas code)
   - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
   - _mv_ for macro procedures that will only work in Viya
-  - _mx_ for macros that are XCMD enabled
+  - _mx_ for macros that are XCMD enabled (working on both windows and unix)
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
@@ -166,7 +176,7 @@ SAS code can contain one of two types of dependency - SAS Macros, and SAS Includ
   @li someprogram.sas FREFTWO
 ```
 
-The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Includes) when creating SAS Jobs and Services.
+The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Includes) when creating SAS Jobs and Services (and Tests).
 
 When contributing to this library, it is therefore important to ensure that all dependencies are listed in the header in this format.
 
@@ -180,14 +190,27 @@ When contributing to this library, it is therefore important to ensure that all
 - The closing `%mend;` should **not** contain the macro name.
 - All macros should be defined with brackets, even if no variables are needed - ie `%macro x();` not `%macro x;`
 - Mandatory parameters should be positional, all optional parameters should be keyword (var=) style.
-- All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect.
+- All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect, or the [USER](https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/lrcon/n18m1vkqmeo4esn1moikt23zhp8s.htm) library is active.
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
+- Where global macro variables are absolutely necessary, they should make use of `&sasjs_prefix` - see mp_init.sas
 - The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
+- Use [sasjs lint](https://github.com/sasjs/lint)!
 
 ## General Notes
 
 - All macros should be compatible with SAS versions from support level B and above (so currently 9.2 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully (eg `%if %sysevalf(&sysver<9.3) %then %return`).
 
+## Breaking Changes
+
+We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major/breaking release (v5) becomes necessary:
+
+* mf_getuniquelibref.sas to have the deprecated maxtried parameter removed (no longer needed)
+* mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
+* `insert_cmplib` option of mcf_xxx macros will be deprecated (the option is now checked automatically with value inserted only if needed)
+* mcf_xxx macros to have `wrap=` option defaulted to YES for convenience.  Set this option explicitly to avoid issues.
+* mp_getddl.sas to be renamed to mp_ds2ddl.sas (consistent with other ds2xxx macros).  A wrapper macro is already in place, and you are able to use this immediately.  The default for SHOWLOG will also be YES instead of NO.
+* mp_coretable.sas will be replaced by the standalone macros in the `ddl` folder (which are already available)
+
 ## Star Gazing
 
 If you find this library useful, please leave a [star](https://github.com/sasjs/core/stargazers) and help us grow our star graph!
@@ -195,10 +218,9 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 ![](https://starchart.cc/sasjs/core.svg)
 
 
-
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-10-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-11-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -219,6 +241,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
     <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
     <td align="center"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
+    <td align="center"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
   </tr>
 </table>
 

base/mf_abort.sas

@@ -11,7 +11,7 @@
   @cond
 **/
 
-%macro mf_abort(mac=mf_abort.sas, type=deprecated, msg=, iftrue=%str(1=1)
+%macro mf_abort(mac=mf_abort.sas, msg=, iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
   %if not(%eval(%unquote(&iftrue))) %then %return;

base/mf_dedup.sas

@@ -23,7 +23,6 @@
 
   <h4> Related Macros </h4>
   @li mf_trimstr.sas
-  @li mf_wordsinstr1butnotstr2.sas
 
   @version 9.2
   @author Allan Bowe

base/mf_existvar.sas

@@ -1,11 +1,11 @@
 /**
   @file
   @brief Checks if a variable exists in a data set.
-  @details Returns 0 if the variable does NOT exist, and return the position of
-    the var if it does.
-    Usage:
+  @details Returns 0 if the variable does NOT exist, and the position of the var
+  if it does.
+  Usage:
 
-        %put %mf_existvar(work.someds, somevar)
+      %put %mf_existvar(work.someds, somevar)
 
   @param [in] libds 2 part dataset or view reference
   @param [in] var variable name

base/mf_getapploc.sas

@@ -15,6 +15,7 @@
   @li /data
   @li /jobs
   @li /services
+  @li /tests
   @li /tests/jobs
   @li /tests/services
   @li /tests/macros
@@ -46,9 +47,13 @@
 
 /**
   * First check we are not in the tests/macros folder (which has no subfolders)
+  * or specifically in the testsetup or testteardown services
   */
-%if %index(&pgm,/tests/macros/) %then %do;
-  %let root=%substr(&pgm,1,%index(&pgm,/tests/macros)-1);
+%if %index(&pgm,/tests/macros/)
+or %index(&pgm,/tests/testsetup)
+or %index(&pgm,/tests/testteardown)
+%then %do;
+  %let root=%substr(&pgm,1,%index(&pgm,/tests)-1);
   &root
   %return;
 %end;

base/mf_getfilesize.sas

@@ -5,18 +5,19 @@
 
       %put %mf_getfilesize(fpath=C:\temp\myfile.txt);
 
-      or
+  or, provide a libds value as follows:
 
       data x;do x=1 to 100000;y=x;output;end;run;
       %put %mf_getfilesize(libds=work.x,format=yes);
 
-      gives:
+  Which gives:
 
-      2mb
+  > 2mb
+
+  @param [in] fpath= Full path and filename.  Provide this OR the libds value.
+  @param [in] libds= (0) Library.dataset value (assumes library is BASE engine)
+  @param [in] format= (NO) Set to yes to apply sizekmg. format
 
-  @param fpath= full path and filename.  Provide this OR the libds value.
-  @param libds= library.dataset value (assumes library is BASE engine)
-  @param format=  set to yes to apply sizekmg. format
   @returns bytes
 
   @version 9.2
@@ -26,16 +27,32 @@
 %macro mf_getfilesize(fpath=,libds=0,format=NO
 )/*/STORE SOURCE*/;
 
-  %if &libds ne 0 %then %do;
-    %let fpath=%sysfunc(pathname(%scan(&libds,1,.)))/%scan(&libds,2,.).sas7bdat;
-  %end;
+  %local rc fid fref bytes dsid lib vnum;
 
-  %local rc fid fref bytes;
-  %let rc=%sysfunc(filename(fref,&fpath));
-  %let fid=%sysfunc(fopen(&fref));
-  %let bytes=%sysfunc(finfo(&fid,File Size (bytes)));
-  %let rc=%sysfunc(fclose(&fid));
-  %let rc=%sysfunc(filename(fref));
+  %if &libds ne 0 %then %do;
+    %let libds=%upcase(&libds);
+    %if %index(&libds,.)=0 %then %let lib=WORK;
+    %else %let lib=%scan(&libds,1,.);
+    %let dsid=%sysfunc(open(
+      sashelp.vtable(where=(libname="&lib" and memname="%scan(&libds,-1,.)")
+        keep=libname memname filesize
+      )
+    ));
+    %if (&dsid ^= 0) %then %do;
+      %let vnum=%sysfunc(varnum(&dsid,FILESIZE));
+      %let rc=%sysfunc(fetch(&dsid));
+      %let bytes=%sysfunc(getvarn(&dsid,&vnum));
+      %let rc= %sysfunc(close(&dsid));
+    %end;
+    %else %put &sysmacroname: &libds could not be opened! %sysfunc(sysmsg());
+  %end;
+  %else %do;
+    %let rc=%sysfunc(filename(fref,&fpath));
+    %let fid=%sysfunc(fopen(&fref));
+    %let bytes=%sysfunc(finfo(&fid,File Size (bytes)));
+    %let rc=%sysfunc(fclose(&fid));
+    %let rc=%sysfunc(filename(fref));
+  %end;
 
   %if &format=NO %then %do;
     &bytes

base/mf_getplatform.sas

@@ -22,6 +22,12 @@
 )/*/STORE SOURCE*/;
 %local a b c;
 %if &switch.NONE=NONE %then %do;
+  %if %symexist(sasjsprocessmode) %then %do;
+    %if &sasjsprocessmode=Stored Program %then %do;
+      SASJS
+      %return;
+    %end;
+  %end;
   %if %symexist(sysprocessmode) %then %do;
     %if "&sysprocessmode"="SAS Object Server"
     or "&sysprocessmode"= "SAS Compute Server" %then %do;

base/mf_getuniquelibref.sas

@@ -14,27 +14,44 @@
 
 > mclib3
 
-  @param prefix= first part of libref.  Remember that librefs can only be 8 characters,
-    so a 7 letter prefix would mean that maxtries should be 10.
-  @param maxtries= the last part of the libref.  Provide an integer value.
+  A blank value is returned if no usable libname is determined.
+
+  @param [in] prefix= (mclib) first part of the returned libref. As librefs can
+    be as long as 8 characters, a maximum length of 7 characters is premitted
+    for this prefix.
+  @param [in] maxtries= Deprecated parameter. Remains here to ensure a
+    non-breaking change.  Will be removed in v5.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-
 %macro mf_getuniquelibref(prefix=mclib,maxtries=1000);
-  %local x libref;
-  %let x=0;
-  %do x=0 %to &maxtries;
-  %if %sysfunc(libref(&prefix&x)) ne 0 %then %do;
-    %let libref=&prefix&x;
-    %let rc=%sysfunc(libname(&libref,%sysfunc(pathname(work))));
-    %if &rc %then %put %sysfunc(sysmsg());
-    &prefix&x
-    %*put &sysmacroname: Libref &libref assigned as WORK and returned;
+  %local x;
+
+  %if ( %length(&prefix) gt 7 ) %then %do;
+    %put %str(ERR)OR: The prefix parameter cannot exceed 7 characters.;
+    0
     %return;
   %end;
+  %else %if (%sysfunc(NVALID(&prefix,v7))=0) %then %do;
+    %put %str(ERR)OR: Invalid prefix (&prefix);
+    0
+    %return;
   %end;
-  %put unable to find available libref in range &prefix.0-&maxtries;
+
+  /* Set maxtries equal to '10 to the power of [# unused characters] - 1' */
+  %let maxtries=%eval(10**(8-%length(&prefix))-1);
+
+  %do x = 0 %to &maxtries;
+    %if %sysfunc(libref(&prefix&x)) ne 0 %then %do;
+      &prefix&x
+      %return;
+    %end;
+    %let x = %eval(&x + 1);
+  %end;
+
+  %put %str(ERR)OR: No usable libref in range &prefix.0-&maxtries;
+  %put %str(ERR)OR- Try reducing the prefix or deleting some libraries!;
+  0
 %mend mf_getuniquelibref;

base/mf_verifymacvars.sas

@@ -1,6 +1,6 @@
 /**
   @file
-  @brief Checks if a set of macro variables exist / contain values.
+  @brief Checks if a set of macro variables exist AND contain values.
   @details Writes ERROR to log if abortType is SOFT, else will call %mf_abort.
   Usage:
 
@@ -14,10 +14,11 @@
   <h4> SAS Macros </h4>
   @li mf_abort.sas
 
-  @param verifyvars space separated list of macro variable names
-  @param makeupcase= set to YES to convert all variable VALUES to
+  @param [in] verifyvars Space separated list of macro variable names
+  @param [in] makeupcase= (NO) Set to YES to convert all variable VALUES to
     uppercase.
-  @param mAbort= Abort Type.  Default is SOFT (writes err to log).
+  @param [in] mAbort= (SOFT) Abort Type.  When SOFT, simply writes an err
+    message to the log.
     Set to any other value to call mf_abort (which can be configured to abort in
     various fashions according to context).
 
@@ -58,8 +59,14 @@
 
   %goto exit_success;
   %exit_err:
-    %if &mAbort=SOFT %then %put %str(ERR)OR: &abortmsg;
-    %else %mf_abort(mac=mf_verifymacvars,type=&mabort,msg=&abortmsg);
+    %put %str(ERR)OR: &abortmsg;
+    %mf_abort(iftrue=(&mabort ne SOFT),
+      mac=mf_verifymacvars,
+      msg=%str(&abortmsg)
+    )
+    0
+    %return;
   %exit_success:
+  1
 
 %mend mf_verifymacvars;

base/mf_wordsinstr1andstr2.sas

@@ -30,7 +30,6 @@
 
 %local count_base count_extr i i2 extr_word base_word match outvar;
 %if %length(&str1)=0 or %length(&str2)=0 %then %do;
-  %put %str(WARN)ING: empty string provided!;
   %put base string (str1)= &str1;
   %put compare string (str2) = &str2;
   %return;

base/mf_wordsinstr1butnotstr2.sas

@@ -3,6 +3,9 @@
   @brief Returns words that are in string 1 but not in string 2
   @details  Compares two space separated strings and returns the words that are
   in the first but not in the second.
+
+  Note - case sensitive!
+
   Usage:
 
       %let x= %mf_wordsInStr1ButNotStr2(
@@ -13,10 +16,8 @@
   returns:
   > sss bram boo
 
-  @param str1= string containing words to extract
-  @param str2= used to compare with the extract string
-
-  @warning CASE SENSITIVE!
+  @param [in] str1= string containing words to extract
+  @param [in] str2= used to compare with the extract string
 
   @version 9.2
   @author Allan Bowe
@@ -30,7 +31,6 @@
 
 %local count_base count_extr i i2 extr_word base_word match outvar;
 %if %length(&str1)=0 or %length(&str2)=0 %then %do;
-  %put %str(WARN)ING: empty string provided!;
   %put base string (str1)= &str1;
   %put compare string (str2) = &str2;
   %return;

base/mp_abort.sas

@@ -66,7 +66,10 @@
   %if %length(&mac)>0 %then %put NOTE- called by &mac;
   %put NOTE - &msg;
 
-  %if %symexist(_SYSINCLUDEFILEDEVICE) %then %do;
+  %if %symexist(_SYSINCLUDEFILEDEVICE)
+  /* abort cancel FILE does not restart outside the INCLUDE on Viya 3.5 */
+  and "&SYSPROCESSNAME " ne "Compute Server "
+  %then %do;
     %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do;
       data &errds;
         iftrue='1=1';
@@ -110,8 +113,8 @@
     %if %symexist(SYSPRINTTOLOG) %then %let logloc=&SYSPRINTTOLOG;
     %else %let logloc=%qsysfunc(getoption(LOG));
     proc printto log=log;run;
+    %let logline=0;
     %if %length(&logloc)>0 %then %do;
-      %let logline=0;
       data _null_;
         infile &logloc lrecl=5000;
         input; putlog _infile_;
@@ -160,7 +163,10 @@
       file _webout mod lrecl=32000 encoding='utf-8';
       length msg $32767 ;
       sasdatetime=datetime();
-      msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
+      msg=symget('msg');
+    %if &logline>0 %then %do;
+      msg=cats(msg,'\n\nLog Extract:\n',symget('logmsg'));
+    %end;
       /* escape the quotes */
       msg=tranwrd(msg,'"','\"');
       /* ditch the CRLFs as chrome complains */
@@ -170,7 +176,8 @@
       if symexist('_debug') then debug=quote(trim(symget('_debug')));
       else debug='""';
       put '>>weboutBEGIN<<';
-      put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
+      put '{"SYSDATE" : "' "&SYSDATE" '"';
+      put ',"SYSTIME" : "' "&SYSTIME" '"';
       put ',"sasjsAbort" : [{';
       put ' "MSG":' msg ;
       put ' ,"MAC": "' "&mac" '"}]';
@@ -199,7 +206,7 @@
       put ',"SYSVLONG" : ' sysvlong;
       syswarningtext=quote(trim(symget('syswarningtext')));
       put ",""SYSWARNINGTEXT"" : " syswarningtext;
-      put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+      put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
       put "}" @;
       put '>>weboutEND<<';
     run;

base/mp_applyformats.sas

@@ -1,9 +1,9 @@
 /**
   @file
   @brief Apply a set of formats to a table
-  @details Applies a set of formats to one or more SAS datasets.  Can be used
-    to migrate formats from one table to another.  The input table must contain
-    the following columns:
+  @details Applies a set of formats to the metadata of one or more SAS datasets.
+    Can be used to migrate formats from one table to another.  The input table
+    must contain the following columns:
 
     @li lib - the libref of the table to be updated
     @li ds - the dataset to be updated
@@ -142,16 +142,16 @@ run;
 proc sql noprint;
 select distinct lib into: liblist separated by ' ' from &inds;
 %put &=liblist;
-%do i=1 %to %sysfunc(countw(&liblist));
+%if %length(&liblist)>0 %then %do i=1 %to %sysfunc(countw(&liblist));
   %let lib=%scan(&liblist,1);
   %let engine=%mf_getengine(&lib);
   %if &engine ne V9 and &engine ne BASE %then %do;
     %let msg=&lib has &engine engine - formats cannot be applied;
-    proc sql;
     insert into &outds set lib="&lib",ds="_all_",var="_all", msg="&msg" ;
     %if &errds=0 %then %put %str(ERR)OR: &msg;
   %end;
 %end;
+quit;
 
 %if %mf_nobs(&outds)>0 %then %return;
 

base/mp_assertdsobs.sas

@@ -12,10 +12,6 @@
 
       %mp_assertdsobs(sashelp.class,test=ATMOST 20) %* pass if <21 obs present;
 
-  <h4> SAS Macros </h4>
-  @li mf_nobs.sas
-  @li mp_abort.sas
-
 
   @param [in] inds input dataset to test for presence of observations
   @param [in] desc= (Testing observations) The user provided test description
@@ -33,6 +29,11 @@
   |---|---|---|
   |User Provided description|PASS|Dataset &inds has XX obs|
 
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
   <h4> Related Macros </h4>
   @li mp_assertcolvals.sas
   @li mp_assert.sas
@@ -49,9 +50,10 @@
   outds=work.test_results
 )/*/STORE SOURCE*/;
 
-  %local nobs;
+  %local nobs ds;
   %let nobs=%mf_nobs(&inds);
   %let test=%upcase(&test);
+  %let ds=%mf_getuniquename(prefix=mp_assertdsobs);
 
   %if %substr(&test.xxxxx,1,6)=EQUALS %then %do;
     %let val=%scan(&test,2,%str( ));
@@ -84,7 +86,7 @@
     )
   %end;
 
-  data;
+  data &ds;
     length test_description $256 test_result $4 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
@@ -110,9 +112,6 @@
   %end;
   run;
 
-  %local ds;
-  %let ds=&syslast;
-
   proc append base=&outds data=&ds;
   run;
 

base/mp_assertscope.sas

@@ -0,0 +1,147 @@
+/**
+  @file
+  @brief Used to capture scope leakage of macro variables
+  @details
+
+  A common 'difficult to detect' bug in macros is where a nested macro
+  over-writes variables in a higher level macro.
+
+  This assertion takes a snapshot of the macro variables before and after
+  a macro invocation.  Differences are captured in the `&outds` table. This
+  makes it easy to detect whether any macro variables were modified or
+  changed.
+
+  The following variables are NOT tested (as they are known, global variables
+  used in SASjs):
+
+  @li &sasjs_prefix._FUNCTIONS
+
+  Global variables are initialised in mp_init.sas - which will also trigger
+  "strict mode" in your SAS session.  Whilst this is a default in SASjs
+  produced apps, if you prefer not to use this mode, simply instantiate the
+  following variable to prevent the macro from running:  `SASJS_PREFIX`
+
+  Example usage:
+
+      %mp_assertscope(SNAPSHOT)
+
+      %let oops=I did it again;
+
+      %mp_assertscope(COMPARE,
+        desc=Checking macro variables against previous snapshot
+      )
+
+  This macro is designed to work alongside `sasjs test` - for more information
+  about this facility, visit [cli.sasjs.io/test](https://cli.sasjs.io/test).
+
+  @param [in] action (SNAPSHOT) The action to take.  Valid values:
+    @li SNAPSHOT - take a copy of the current macro variables
+    @li COMPARE - compare the current macro variables against previous values
+  @param [in] scope= (GLOBAL) The scope of the variables to be checked.  This
+    corresponds to the values in the SCOPE column in `sashelp.vmacro`.
+  @param [in] desc= (Testing scope leakage) The user provided test description
+  @param [in] ignorelist= Provide a list of macro variable names to ignore from
+    the comparison
+  @param [in,out] scopeds= (work.mp_assertscope) The dataset to contain the
+    scope snapshot
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|No out of scope variables created or modified|
+
+  <h4> SAS Macros </h4>
+  @li mf_getquotedstr.sas
+  @li mp_init.sas
+
+  <h4> Related Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertcols.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertscope(action,
+  desc=Testing Scope Leakage,
+  scope=GLOBAL,
+  scopeds=work.mp_assertscope,
+  ignorelist=,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+%local ds test_result test_comments del add mod ilist;
+%let ilist=%upcase(&sasjs_prefix._FUNCTIONS SYS_PROCHTTP_STATUS_CODE
+  SYS_PROCHTTP_STATUS_CODE SYS_PROCHTTP_STATUS_PHRASE &ignorelist);
+
+/**
+  * this sets up the global vars, it will also enter STRICT mode.  If this
+  * behaviour is not desired, simply initiate the following global macro
+  * variable to prevent the macro from running: SASJS_PREFIX
+  */
+%mp_init()
+
+/* get current variables */
+%if &action=SNAPSHOT %then %do;
+  proc sql;
+  create table &scopeds as
+    select name,offset,value
+    from dictionary.macros
+    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
+    order by name,offset;
+%end;
+%else %if &action=COMPARE %then %do;
+
+  proc sql;
+  create table _data_ as
+    select name,offset,value
+    from dictionary.macros
+    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
+    order by name,offset;
+
+  %let ds=&syslast;
+
+  proc compare
+    base=&scopeds(where=(upcase(name) not in (%mf_getquotedstr(&ilist))))
+    compare=&ds;
+  run;
+
+  %if &sysinfo=0 %then %do;
+    %let test_result=PASS;
+    %let test_comments=&scope Variables Unmodified;
+  %end;
+  %else %do;
+    proc sql noprint undo_policy=none;
+    select distinct name into: del separated by ' '  from &scopeds
+      where name not in (select name from &ds);
+    select distinct name into: add separated by ' '  from &ds
+      where name not in (select name from &scopeds);
+    select distinct a.name into: mod separated by ' '
+      from &scopeds a
+      inner join &ds b
+      on a.name=b.name
+        and a.offset=b.offset
+      where a.value ne b.value;
+    %let test_result=FAIL;
+    %let test_comments=%str(Mod:(&mod) Add:(&add) Del:(&del));
+  %end;
+
+
+  data ;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_comments=symget('test_comments');
+    test_result=symget('test_result');
+  run;
+
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+%end;
+
+%mend mp_assertscope;

base/mp_cntlout.sas

@@ -0,0 +1,85 @@
+/**
+  @file mp_cntlout.sas
+  @brief Creates a cntlout dataset in a consistent format
+  @details The dataset produced by proc format in the cntlout option will vary
+  according to its contents.
+
+  When dealing with formats from an ETL perspective (eg in [Data Controller for
+  SAS](https://datacontroller.io)), it is important that the output dataset
+  has a consistent model (and compariable values).
+
+  This macro makes use of mddl_sas_cntlout.sas to provide the consistent model,
+  and will left-align the start and end values when dealing with numeric ranges
+  to enable consistency when checking for differences.
+
+  usage:
+
+      %mp_cntlout(libcat=yourlib.cat,cntlout=work.formatexport)
+
+  @param [in] libcat The library.catalog reference
+  @param [in] fmtlist= (0) provide a space separated list of specific formats to
+    extract
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [out] cntlout= (work.fmtextract) Libds reference for the output dataset
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mf_getvarformat.sas
+  @li mp_getformats.sas
+  @li mp_loadformat.sas
+  @li mp_ds2fmtds.sas
+
+  @version 9.2
+  @author Allan Bowe
+  @cond
+**/
+
+%macro mp_cntlout(
+  iftrue=(1=1)
+  ,libcat=
+  ,cntlout=work.fmtextract
+  ,fmtlist=0
+)/*/STORE SOURCE*/;
+%local ddlds cntlds i;
+
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+%let ddlds=%mf_getuniquename();
+%let cntlds=%mf_getuniquename();
+
+%mddl_sas_cntlout(libds=&ddlds)
+
+%if %index(&libcat,-)>0 and %scan(&libcat,2,-)=FC %then %do;
+  %let libcat=%scan(&libcat,1,-);
+%end;
+
+proc format lib=&libcat cntlout=&cntlds;
+%if "&fmtlist" ne "0" %then %do;
+  select
+  %do i=1 %to %sysfunc(countw(&fmtlist));
+    %scan(&fmtlist,&i,%str( ))
+  %end;
+  ;
+%end;
+run;
+
+data &cntlout;
+  if 0 then set &ddlds;
+  set &cntlds;
+  if type="N" then do;
+    start=cats(start);
+    end=cats(end);
+  end;
+run;
+proc sort;
+  by fmtname start;
+run;
+
+proc sql;
+drop table &ddlds,&cntlds;
+
+%mend mp_cntlout;
+/** @endcond */

base/mp_coretable.sas

@@ -0,0 +1,71 @@
+/**
+  @file
+  @brief Create the permanent Core tables
+  @details Several macros in the [core](https://github.com/sasjs/core) library
+    make use of permanent tables.  To avoid duplication in definitions, this
+    macro provides a central location for managing the corresponding DDL.
+
+  Note - this macro is likely to be deprecated in future in favour of a
+  dedicated "datamodel" folder (prefix mddl)
+
+  Any corresponding data would go in a seperate repo, to avoid this one
+  ballooning in size!
+
+  Example usage:
+
+      %mp_coretable(LOCKTABLE,libds=work.locktable)
+
+  @param [in] table_ref The type of table to create.  Example values:
+    @li DIFFTABLE
+    @li FILTER_DETAIL
+    @li FILTER_SUMMARY
+    @li LOCKANYTABLE
+    @li MAXKEYTABLE
+  @param [in] libds= (0) The library.dataset reference used to create the table.
+    If not provided, then the DDL is simply printed to the log.
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mddl_dc_filterdetail.sas
+  @li mddl_dc_filtersummary.sas
+  @li mddl_dc_locktable.sas
+  @li mddl_dc_maxkeytable.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filterstore.sas
+  @li mp_lockanytable.sas
+  @li mp_retainedkey.sas
+  @li mp_storediffs.sas
+  @li mp_stackdiffs.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_coretable(table_ref,libds=0
+)/*/STORE SOURCE*/;
+%local outds ;
+%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
+proc sql;
+%if &table_ref=DIFFTABLE %then %do;
+  %mddl_dc_difftable(libds=&outds)
+%end;
+%else %if &table_ref=LOCKTABLE %then %do;
+  %mddl_dc_locktable(libds=&outds)
+%end;
+%else %if &table_ref=FILTER_SUMMARY %then %do;
+  %mddl_dc_filtersummary(libds=&outds)
+%end;
+%else %if &table_ref=FILTER_DETAIL %then %do;
+  %mddl_dc_filterdetail(libds=&outds)
+%end;
+%else %if &table_ref=MAXKEYTABLE %then %do;
+  %mddl_dc_maxkeytable(libds=&outds)
+%end;
+
+%if &libds=0 %then %do;
+  describe table &syslast;
+  drop table &syslast;
+%end;
+%mend mp_coretable;

base/mp_createwebservice.sas

@@ -24,20 +24,22 @@ Usage:
         %webout(OBJ,example2) * Object format, easier to work with ;
         %webout(CLOSE)
     ;;;;
-    %mp_createwebservice(path=/Public/app/common,name=appInit,code=ft15f001,replace=YES)
+    %mp_createwebservice(path=/Public/app/common,name=appInit,replace=YES)
 
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
   @li mm_createwebservice.sas
   @li mv_createwebservice.sas
 
-  @param path= The full folder path where the service will be created
-  @param name= Service name.  Avoid spaces.
-  @param desc= The description of the service (optional)
-  @param precode= Space separated list of filerefs, pointing to the code that
-    needs to be attached to the beginning of the service (optional)
-  @param code= Space seperated fileref(s) of the actual code to be added
-  @param replace= select YES to replace any existing service in that location
+  @param [in,out] path= The full folder path where the service will be created
+  @param [in,out] name= Service name.  Avoid spaces.
+  @param [in] desc= The description of the service (optional)
+  @param [in] precode= Space separated list of filerefs, pointing to the code
+    that needs to be attached to the beginning of the service (optional)
+  @param [in] code= (ft15f001) Space seperated fileref(s) of the actual code to
+    be added
+  @param [in] replace= (YES) Select YES to replace any existing service in that
+    location
 
 
   @version 9.2

base/mp_csv2ds.sas

@@ -49,10 +49,6 @@
   ,mac=&sysmacroname
   ,msg=%str(the BASEDS variable must be provided)
 )
-%mp_abort(iftrue=( &baseds=0 )
-  ,mac=&sysmacroname
-  ,msg=%str(the BASEDS variable must be provided)
-)
 %mp_abort(iftrue=( %mf_existds(&baseds)=0 )
   ,mac=&sysmacroname
   ,msg=%str(the BASEDS dataset (&baseds) needs to be assigned, and to exist)

base/mp_deletefolder.sas

@@ -51,6 +51,7 @@
     data _null_;
       set work.&tempds end=last;
       length fref $8;
+      fref='';
       rc=filename(fref,filepath);
       rc=fdelete(fref);
       if rc then do;

base/mp_dirlist.sas

@@ -6,8 +6,7 @@
   Credit for the rename approach:
   https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003
 
-
-  usage:
+  Usage:
 
       %mp_dirlist(path=/some/location, outds=myTable, maxdepth=MAX)
 
@@ -23,12 +22,12 @@
   X CMD) do please raise an issue!
 
 
-  @param [in] path= for which to return contents
-  @param [in] fref= Provide a DISK engine fileref as an alternative to PATH
+  @param [in] path= (%sysfunc(pathname(work))) Path for which to return contents
+  @param [in] fref= (0) Provide a DISK engine fileref as an alternative to PATH
   @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
     subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
     recursion, set to MAX.
-  @param [out] outds= the output dataset to create
+  @param [out] outds= (work.mp_dirlist) The output dataset to create
   @param [out] getattrs= (NO)  If getattrs=YES then the doptname / foptname
     functions are used to scan all properties - any characters that are not
     valid in a SAS name (v7) are simply stripped, and the table is transposed
@@ -49,13 +48,15 @@
     - OS SPECIFIC variables, if <code>getattrs=</code> is used.
 
   <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_getvarlist.sas
+  @li mf_wordsinstr1butnotstr2.sas
   @li mp_dropmembers.sas
 
   <h4> Related Macros </h4>
   @li mp_dirlist.test.sas
 
   @version 9.2
-  @author Allan Bowe
 **/
 
 %macro mp_dirlist(path=%sysfunc(pathname(work))
@@ -81,7 +82,8 @@ data &out_ds(compress=no
     keep=file_or_folder filepath filename ext msg directory level
   );
   length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
-    ext $20 msg $200;
+    ext $20 msg $200 foption $16;
+  if _n_=1 then call missing(of _all_);
   retain level &level;
   %if &fref=0 %then %do;
     rc = filename(fref, "&path");
@@ -92,7 +94,13 @@ data &out_ds(compress=no
   %end;
   if rc = 0 then do;
     did = dopen(fref);
-    directory=dinfo(did,'Directory');
+    /* attribute is OS-dependent - could be "Directory" or "Directory Name" */
+    numopts=doptnum(did);
+    do i=1 to numopts;
+      foption=doptname(did,i);
+      if foption=:'Directory' then i=numopts;
+    end;
+    directory=dinfo(did,foption);
     if did=0 then do;
       putlog "NOTE: This directory is empty - " directory;
       msg=sysmsg();
@@ -193,9 +201,29 @@ data &out_ds;
   set &out_ds(where=(filepath ne ''));
 run;
 
-/* update main table */
-proc append base=&outds data=&out_ds;
-run;
+/**
+  * The above transpose can mean that some updates create additional columns.
+  * This necessitates the occasional use of datastep over proc append.
+  */
+%if %mf_existds(&outds) %then %do;
+  %local basevars appvars newvars;
+  %let basevars=%mf_getvarlist(&outds);
+  %let appvars=%mf_getvarlist(&out_ds);
+  %let newvars=%length(%mf_wordsinstr1butnotstr2(Str1=&appvars,Str2=&basevars));
+  %if &newvars>0 %then %do;
+    data &outds;
+      set &outds &out_ds;
+    run;
+  %end;
+  %else %do;
+    proc append base=&outds data=&out_ds force nowarn;
+    run;
+  %end;
+%end;
+%else %do;
+  proc append base=&outds data=&out_ds;
+  run;
+%end;
 
 /* recursive call */
 %if &maxdepth>&level or &maxdepth=MAX %then %do;

base/mp_ds2cards.sas

@@ -139,8 +139,9 @@ create table datalines1 as
 /**
   Due to long decimals cannot use best. format
   So - use bestd. format and then use character functions to strip trailing
-    zeros, if NOT an integer!!
-  resolved code = ifc(int(VARIABLE)=VARIABLE
+    zeros, if NOT an integer or missing!!  Cannot use int() as it upsets
+    note2err when there are missings.
+  resolved code = ifc( mod(coalesce(VARIABLE,0),1)=0
     ,put(VARIABLE,best32.)
     ,substrn(put(VARIABLE,bestd32.),1
     ,findc(put(VARIABLE,bestd32.),'0','TBK')));
@@ -151,7 +152,7 @@ data datalines_2;
   set datalines1 (where=(upcase(name) not in
     ('PROCESSED_DTTM','VALID_FROM_DTTM','VALID_TO_DTTM')));
   if type='num' then dataline=
-    cats('ifc(int(',name,')=',name,'
+    cats('ifc(mod(coalesce(',name,',0),1)=0
       ,put(',name,',best32.-l)
       ,substrn(put(',name,',bestd32.-l),1
       ,findc(put(',name,',bestd32.-l),"0","TBK")))');

base/mp_ds2csv.sas

@@ -1,23 +1,82 @@
 /**
   @file
-  @brief Export a dataset to a CSV file
-  @details Export to a file or a fileref
+  @brief Export a dataset to a CSV file WITH leading blanks
+  @details Export a dataset to a file or fileref, retaining leading blanks.
+
+  When using SASJS headerformat, the input statement is provided in the first
+  row of the CSV.
+
   Usage:
 
       %mp_ds2csv(sashelp.class,outref="%sysfunc(pathname(work))/file.csv")
 
-  @param ds The dataset to be exported
-  @param outfile= The output filename - should be quoted.
-  @param outref= The output fileref (takes precedence if provided)
-  @param outencoding= The output encoding to use (unquoted)
+      filename example temp;
+      %mp_ds2csv(sashelp.air,outref=example,headerformat=SASJS)
+      data; infile example; input;put _infile_; if _n_>5 then stop;run;
+
+      data _null_;
+        infile example;
+        input;
+        call symputx('stmnt',_infile_);
+        stop;
+      run;
+      data work.want;
+        infile example dsd firstobs=2;
+        input &stmnt;
+      run;
+
+  Why use mp_ds2csv over, say, proc export?
+
+  1. Ability to retain leading blanks (this is a major one)
+  2. Control the header format
+  3. Simple one-liner
+
+  @param [in] ds The dataset to be exported
+  @param [in] dlm= (COMMA) The delimeter to apply.  For SASJS, will always be
+    COMMA. Supported values:
+    @li COMMA
+    @li SEMICOLON
+  @param [in] headerformat= (LABEL) The format to use for the header section.
+    Valid values:
+    @li LABEL - Use the variable label (or name, if blank)
+    @li NAME - Use the variable name
+    @li SASJS - Used to create sasjs-formatted input CSVs, eg for use in
+      mp_testservice.sas.  This format will supply an input statement in the
+      first row, making ingestion by datastep a breeze.  Special misisng values
+      will be prefixed with a period (eg `.A`) to enable ingestion on both SAS 9
+      and Viya.  Dates / Datetimes etc are identified by the format type (lookup
+      with mcf_getfmttype.sas) and converted to human readable formats (not
+      numbers).
+  @param [out] outfile= The output filename - should be quoted.
+  @param [out] outref= (0) The output fileref (takes precedence if provided)
+  @param [in] outencoding= (0) The (quoted) output encoding to use, eg `"UTF-8"`
+  @param [in] termstr= (CRLF) The line seperator to use.  For SASJS, will
+    always be CRLF.  Valid values:
+    @li CRLF
+    @li LF
+
+  <h4> SAS Macros </h4>
+  @li mcf_getfmttype.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarformat.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
 
   @version 9.2
   @author Allan Bowe (credit mjsq)
 **/
 
-%macro mp_ds2csv(ds, outref=0, outfile=, outencoding=0
+%macro mp_ds2csv(ds
+  ,dlm=COMMA
+  ,outref=0
+  ,outfile=
+  ,outencoding=0
+  ,headerformat=LABEL
+  ,termstr=CRLF
 )/*/STORE SOURCE*/;
 
+%local outloc delim i varlist var vcnt vat dsv vcom vmiss fmttype vfmt;
+
 %if not %sysfunc(exist(&ds)) %then %do;
   %put %str(WARN)ING:  &ds does not exist;
   %return;
@@ -26,33 +85,128 @@
 %if %index(&ds,.)=0 %then %let ds=WORK.&ds;
 
 %if &outencoding=0 %then %let outencoding=;
-%else %let outencoding=encoding="&outencoding";
+%else %let outencoding=encoding=&outencoding;
 
-%local outloc;
 %if &outref=0 %then %let outloc=&outfile;
 %else %let outloc=&outref;
 
+%if &headerformat=SASJS %then %do;
+  %let delim=",";
+  %let termstr=CRLF;
+  %mcf_getfmttype(wrap=YES)
+%end;
+%else %if &dlm=COMMA %then %let delim=",";
+%else %let delim=";";
+
 /* credit to mjsq - https://stackoverflow.com/a/55642267 */
 
 /* first get headers */
 data _null_;
-  file &outloc dlm=',' dsd &outencoding lrecl=32767;
-  length header $ 2000;
+  file &outloc &outencoding lrecl=32767 termstr=&termstr;
+  length header $ 2000 varnm vfmt $32 dlm $1 fmttype $8;
+  call missing(of _all_);
   dsid=open("&ds.","i");
   num=attrn(dsid,"nvars");
+  dlm=&delim;
   do i=1 to num;
-    header = trim(left(coalescec(varlabel(dsid,i),varname(dsid,i))));
+    varnm=upcase(varname(dsid,i));
+    if i=num then dlm='';
+  %if &headerformat=NAME %then %do;
+    header=cats(varnm,dlm);
+  %end;
+  %else %if &headerformat=LABEL %then %do;
+    header = cats(coalescec(varlabel(dsid,i),varnm),dlm);
+  %end;
+  %else %if &headerformat=SASJS %then %do;
+    if vartype(dsid,i)='C' then header=cats(varnm,':$char',varlen(dsid,i),'.');
+    else do;
+      vfmt=coalescec(varfmt(dsid,i),'0');
+      fmttype=mcf_getfmttype(vfmt);
+      if fmttype='DATE' then header=cats(varnm,':date9.');
+      else if fmttype='DATETIME' then header=cats(varnm,':E8601DT26.6');
+      else if fmttype='TIME' then header=cats(varnm,':TIME12.');
+      else header=cats(varnm,':best.');
+    end;
+  %end;
+  %else %do;
+    %put &sysmacroname: Invalid headerformat value (&headerformat);
+    %return;
+  %end;
     put header @;
   end;
   rc=close(dsid);
 run;
 
+%let varlist=%mf_getvarlist(&ds);
+%let vcnt=%sysfunc(countw(&varlist));
+
+/**
+  * The $quote modifier (without a width) will take the length from the variable
+  * and increase by two.  However this will lead to truncation where the value
+  * contains double quotes (which are doubled up).  To get around this, scan the
+  * data to see the max number of double quotes, so that the appropriate width
+  * can be applied in the subsequent step.
+  */
+data _null_;
+  set &ds end=last;
+%do i=1 %to &vcnt;
+  %let var=%scan(&varlist,&i);
+  %if %mf_getvartype(&ds,&var)=C %then %do;
+    %let dsv1=%mf_getuniquename(prefix=csvcol1_);
+    %let dsv2=%mf_getuniquename(prefix=csvcol2_);
+    retain &dsv1 0;
+    &dsv2=length(&var)+countc(&var,'"');
+    if &dsv2>&dsv1 then &dsv1=&dsv2;
+    if last then call symputx(
+      "vlen&i"
+      /* should be no shorter than varlen, and no longer than 32767 */
+      ,cats('$quote',min(&dsv1+2,32767),'.')
+      ,'l'
+    );
+  %end;
+%end;
+
+%let vat=@;
+%let vcom=&delim;
+%let vmiss=%mf_getuniquename(prefix=csvcol3_);
 /* next, export data */
 data _null_;
   set &ds.;
-  file &outloc mod dlm=',' dsd &outencoding lrecl=32767;
-  put (_all_) (+0);
+  file &outloc mod dlm=&delim dsd &outencoding lrecl=32767 termstr=&termstr;
+  if _n_=1 then &vmiss='  ';
+  %do i=1 %to &vcnt;
+    %let var=%scan(&varlist,&i);
+    %if &i=&vcnt %then %do;
+      %let vat=;
+      %let vcom=;
+    %end;
+    %if %mf_getvartype(&ds,&var)=N %then %do;
+      %if &headerformat = SASJS %then %do;
+        %let vcom=&delim;
+        %let fmttype=%sysfunc(mcf_getfmttype(%mf_getvarformat(&ds,&var)0));
+        %if &fmttype=DATE %then %let vfmt=DATE9.;
+        %else %if &fmttype=DATETIME %then %let vfmt=E8601DT26.6;
+        %else %if &fmttype=TIME %then %let vfmt=TIME12.;
+        %else %do;
+          %let vfmt=;
+          %let vcom=;
+        %end;
+      %end;
+      %else %let vcom=;
+
+      /* must use period - in order to work in both 9.4 and Viya 3.5 */
+      if missing(&var) and &var ne %sysfunc(getoption(MISSING)) then do;
+        &vmiss=cats('.',&var);
+        put &vmiss &vat;
+      end;
+      else put &var &vfmt &vcom &vat;
+
+    %end;
+    %else %do;
+      %if &i ne &vcnt %then %let vcom=&delim;
+      put &var &&vlen&i &vcom &vat;
+    %end;
+  %end;
 run;
 
-
 %mend mp_ds2csv;

base/mp_ds2ddl.sas

@@ -0,0 +1,30 @@
+/**
+  @file
+  @brief A wrapper for mp_getddl.sas
+  @details In the next release, this will be the main version.
+
+
+  <h4> SAS Macros </h4>
+  @li mp_getddl.sas
+
+**/
+
+%macro mp_ds2ddl(libds,fref=getddl,flavour=SAS,showlog=YES,schema=
+  ,applydttm=NO
+)/*/STORE SOURCE*/;
+
+%local libref;
+%let libds=%upcase(&libds);
+%let libref=%scan(&libds,1,.);
+%if &libref=&libds %then %let libds=WORK.&libds;
+
+%mp_getddl(%scan(&libds,1,.)
+  ,%scan(&libds,2,.)
+  ,fref=&fref
+  ,flavour=SAS
+  ,showlog=&showlog
+  ,schema=&schema
+  ,applydttm=&applydttm
+)
+
+%mend mp_ds2ddl;

base/mp_ds2squeeze.sas

@@ -0,0 +1,120 @@
+/**
+  @file
+  @brief Create a smaller version of a dataset, without data loss
+  @details This macro will scan the input dataset and create a new one, that
+  has the minimum variable lengths needed to store the data without data loss.
+
+  Inspiration was taken from [How to Reduce the Disk Space Required by a
+  SAS® Data Set](https://www.lexjansen.com/nesug/nesug06/io/io18.pdf) by
+  Selvaratnam Sridharma.  The end of the referenced paper presents a macro named
+  "squeeze", hence the nomenclature.
+
+  Usage:
+
+      data big;
+        length my big $32000;
+        do i=1 to 1e4;
+          my=repeat('oh my',100);
+          big='dawg';
+          special=._;
+          output;
+        end;
+      run;
+
+      %mp_ds2squeeze(work.big,outds=work.smaller)
+
+  The following will also be printed to the log (exact values may differ
+  depending on your OS and COMPRESS settings):
+
+  > MP_DS2SQUEEZE: work.big was  625MB
+
+  > MP_DS2SQUEEZE: work.smaller is    5MB
+
+  @param [in] libds The library.dataset to be squeezed
+  @param [out] outds= (work.mp_ds2squeeze) The squeezed dataset to create
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> SAS Macros </h4>
+  @li mf_getfilesize.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_getmaxvarlengths.sas
+
+  <h4> Related Programs </h4>
+  @li mp_ds2squeeze.test.sas
+
+  @version 9.3
+  @author Allan Bowe
+**/
+
+%macro mp_ds2squeeze(
+  libds,
+  outds=work.mp_ds2squeeze,
+  mdebug=0
+)/*/STORE SOURCE*/;
+%local dbg source;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %do;
+  %let dbg=*;
+  %let source=/source2;
+%end;
+
+%local optval ds fref startsize;
+%let ds=%mf_getuniquename();
+%let fref=%mf_getuniquefileref();
+%let startsize=%mf_getfilesize(libds=&libds,format=yes);
+
+%mp_getmaxvarlengths(&libds,outds=&ds)
+
+data _null_;
+  set &ds end=last;
+  file &fref;
+  /* grab the types */
+  retain dsid;
+  if _n_=1 then dsid=open("&libds",'is');
+  if dsid le 0 then do;
+    msg=sysmsg();
+    put msg=;
+    stop;
+  end;
+  type=vartype(dsid,varnum(dsid, name));
+  if last then rc=close(dsid);
+  /* write out the length statement */
+  if _n_=1 then put 'length ';
+  length len $6;
+  if type='C' then do;
+    if maxlen=0 then len='$1';
+    else len=cats('$',maxlen);
+  end;
+  else do;
+    if maxlen=0 then len='3';
+    else len=cats(maxlen);
+  end;
+  put '  ' name ' ' len;
+  if last then put ';';
+run;
+
+/* configure varlenchk - as we are explicitly shortening the variables */
+%let optval=%sysfunc(getoption(varlenchk));
+options varlenchk=NOWARN;
+
+data &outds;
+  %inc &fref &source;
+  set &libds;
+run;
+
+options varlenchk=&optval;
+
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &ds;
+  filename &fref clear;
+%end;
+
+%put &sysmacroname: &libds was &startsize;
+%put &sysmacroname: &outds is %mf_getfilesize(libds=&outds,format=yes);
+
+%mend mp_ds2squeeze;

base/mp_filtercheck.sas

@@ -6,7 +6,8 @@
   SYSCC to 1008 if bad records are found, and call mp_abort.sas for a
   graceful service exit (configurable).
 
-  Used for dynamic filtering in [Data Controller for SAS®](https://datacontroller.io).
+  Used for dynamic filtering in [Data Controller for SAS®](
+  https://datacontroller.io).
 
   Usage:
 
@@ -109,7 +110,7 @@ data &outds;
     output;
   end;
   if mod(SUBGROUP_ID,1) ne 0 then do;
-    REASON_CD='SUBGROUP_ID should be integer, not '!!left(subgroup_id);
+    REASON_CD='SUBGROUP_ID should be integer, not '!!cats(subgroup_id);
     putlog REASON_CD= SUBGROUP_ID=;
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');
@@ -125,9 +126,9 @@ data &outds;
     output;
   end;
   if OPERATOR_NM not in
-  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
+  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE')
   then do;
-    REASON_CD='Invalid OPERATOR_NM: '!!left(OPERATOR_NM);
+    REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');

base/mp_filterstore.sas

@@ -0,0 +1,254 @@
+/**
+  @file
+  @brief Checks & Stores an input filter table and returns the Filter Key
+  @details Used to generate a FILTER_RK from an input query dataset.  This
+  process requires several permanent tables (names are configurable).  The
+  benefit of storing query values at backend is to enable stored 'views' of
+  filtered tables at frontend (ie, when building [SAS-Powered Apps](
+  https://sasapps.io)).  This macro is also used in [Data Controller for SAS](
+  https://datacontroller.io).
+
+  A more recent feature of this macro is the ability to support filter queries
+  on Format Catalogs.  This is achieved by adding a `-FC` suffix to the `libds`
+  parameter - where the "ds" in this case is the catalog name.
+
+
+  @param [in] libds= The target dataset to be filtered (lib should be assigned).
+    If filtering a format catalog, add the following suffix:  `-FC`.
+  @param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to
+    be validated.  Has the following format:
+|GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
+|---|---|---|---|---|---|
+|AND|AND|1|SOME_BESTNUM|>|1|
+|AND|AND|1|SOME_TIME|=|77333|
+  @param [in] filter_summary= (PERM.FILTER_SUMMARY) Permanent table containing
+    summary filter values.  The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(FILTER_SUMMARY)`. Example
+    values:
+|FILTER_RK:best.|FILTER_HASH:$32.|FILTER_TABLE:$41.|PROCESSED_DTTM:datetime19.|
+|---|---|---|---|
+|`1 `|`540E96F566D194AB58DD4C413C99C9DB `|`VIYA6014.MPE_TABLES `|`1956084246 `|
+|`2 `|`87737DB9EEE2650F5C89956CEAD0A14F `|`VIYA6014.MPE_X_TEST `|`1956084452.1`|
+|`3 `|`8048BD908DBBD83D013560734E90D394 `|`VIYA6014.MPE_TABLES `|`1956093620.6`|
+  @param [in] filter_detail= (PERM.FILTER_DETAIL) Permanent table containing
+    detailed (raw) filter values. The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(FILTER_DETAIL)`. Example
+    values:
+|FILTER_HASH:$32.|FILTER_LINE:best.|GROUP_LOGIC:$3.|SUBGROUP_LOGIC:$3.|SUBGROUP_ID:best.|VARIABLE_NM:$32.|OPERATOR_NM:$12.|RAW_VALUE:$4000.|PROCESSED_DTTM:datetime19.|
+|---|---|---|---|---|---|---|---|---|
+|`540E96F566D194AB58DD4C413C99C9DB `|`1 `|`AND `|`AND `|`1 `|`LIBREF `|`CONTAINS `|`DC`|`1956084245.8 `|
+|`540E96F566D194AB58DD4C413C99C9DB `|`2 `|`AND `|`OR `|`2 `|`DSN `|`= `|` MPE_LOCK_ANYTABLE `|`1956084245.8 `|
+|`87737DB9EEE2650F5C89956CEAD0A14F `|`1 `|`AND `|`AND `|`1 `|`PRIMARY_KEY_FIELD `|`IN `|`(1,2,3) `|`1956084451.9 `|
+  @param [in] lock_table= (PERM.LOCK_TABLE) Permanent locking table.  Used to
+    manage concurrent access.  The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(LOCKTABLE)`.
+  @param [in] maxkeytable= (0) Optional permanent reference table used for
+    retained key tracking.  Described in mp_retainedkey.sas.
+  @param [in] mdebug= set to 1 to enable DEBUG messages
+  @param [out] outresult= The result table with the FILTER_RK
+  @param [out] outquery= The original query, taken as extract after table load
+
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+  @li mf_getvalue.sas
+  @li mf_islibds.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_filtercheck.sas
+  @li mp_hashdataset.sas
+  @li mp_retainedkey.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filtercheck.sas
+  @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
+  @li mp_filterstore.test.sas
+
+  @version 9.2
+  @author [Allan Bowe](https://www.linkedin.com/in/allanbowe)
+
+**/
+
+%macro mp_filterstore(libds=,
+  queryds=work.filterquery,
+  filter_summary=PERM.FILTER_SUMMARY,
+  filter_detail=PERM.FILTER_DETAIL,
+  lock_table=PERM.LOCK_TABLE,
+  maxkeytable=PERM.MAXKEYTABLE,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+);
+%put &sysmacroname entry vars:;
+%put _local_;
+
+%local ds0 ds1 ds2 ds3 ds4 filter_hash orig_libds;
+%let libds=%upcase(&libds);
+%let orig_libds=&libds;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+%mp_abort(iftrue= (%mf_islibds(&filter_summary)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid filter_summary value: &filter_summary)
+)
+%mp_abort(iftrue= (%mf_islibds(&filter_detail)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid filter_detail value: &filter_detail)
+)
+%mp_abort(iftrue= (%mf_islibds(&lock_table)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid lock_table value: &lock_table)
+)
+
+/**
+  * validate query
+  * use format catalog export, if a format
+  */
+%if "%substr(&libds,%length(&libds)-2,3)"="-FC" %then %do;
+  %let libds=%scan(&libds,1,-); /* chop off -FC extension */
+  %let ds0=%mf_getuniquename(prefix=fmtds_);
+  %let libds=&ds0;
+  /*
+    There is no need to export the entire format catalog here - the validations
+    are done against the data model, not the data values.  So we can simply
+    hardcode the structure based on the cntlout dataset.
+  */
+  %mddl_sas_cntlout(libds=&ds0)
+
+%end;
+%mp_filtercheck(&queryds,targetds=&libds,abort=YES)
+
+/* hash the result */
+%let ds1=%mf_getuniquename(prefix=hashds);
+%mp_hashdataset(&queryds,outds=&ds1,salt=&orig_libds)
+%let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
+%if &mdebug=1 %then %do;
+  data _null_;
+    putlog "filter_hash=&filter_hash";
+    set &ds1;
+    putlog (_all_)(=);
+  run;
+%end;
+
+/* check if data already exists for this hash */
+data &outresult;
+  set &filter_summary;
+  where filter_hash="&filter_hash";
+run;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc after hash check)
+)
+%mp_abort(iftrue= ("&filter_hash "=" ")
+  ,mac=mp_filterstore
+  ,msg=%str(problem with filter_hash generation)
+)
+
+%if %mf_nobs(&outresult)=0 %then %do;
+
+  /* first update summary table */
+  %let ds3=%mf_getuniquename(prefix=filtersum);
+  data work.&ds3;
+    if 0 then set &filter_summary;
+    filter_table="&orig_libds";
+    filter_hash="&filter_hash";
+    PROCESSED_DTTM=%sysfunc(datetime());
+    output;
+    stop;
+  run;
+
+  %mp_lockanytable(LOCK,
+    lib=%scan(&filter_summary,1,.)
+    ,ds=%scan(&filter_summary,2,.)
+    ,ref=MP_FILTERSTORE summary update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  %let ds4=%mf_getuniquename(prefix=filtersumappend);
+  %mp_retainedkey(
+    base_lib=%scan(&filter_summary,1,.)
+    ,base_dsn=%scan(&filter_summary,2,.)
+    ,append_lib=work
+    ,append_dsn=&ds3
+    ,retained_key=filter_rk
+    ,business_key=filter_hash
+    ,maxkeytable=&maxkeytable
+    ,locktable=&lock_table
+    ,outds=work.&ds4
+  )
+  proc append base=&filter_summary data=&ds4;
+  run;
+
+  %mp_lockanytable(UNLOCK,
+    lib=%scan(&filter_summary,1,.)
+    ,ds=%scan(&filter_summary,2,.)
+    ,ref=MP_FILTERSTORE summary update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  %if &syscc ne 0 %then %do;
+    data _null_;
+      set &ds4;
+      putlog (_all_)(=);
+    run;
+    %goto err;
+  %end;
+
+  data &outresult;
+    set &filter_summary;
+    where filter_hash="&filter_hash";
+  run;
+
+  /* Next, update detail table */
+  %let ds2=%mf_getuniquename(prefix=filterdetail);
+  data &ds2;
+    if 0 then set &filter_detail;
+    set &queryds;
+    format filter_hash $hex32. filter_line 8.;
+    filter_hash="&filter_hash";
+    filter_line=_n_;
+    PROCESSED_DTTM=%sysfunc(datetime());
+  run;
+  %mp_lockanytable(LOCK,
+    lib=%scan(&filter_detail,1,.)
+    ,ds=%scan(&filter_detail,2,.)
+    ,ref=MP_FILTERSTORE update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+  proc append base=&filter_detail data=&ds2;
+  run;
+
+  %mp_lockanytable(UNLOCK,
+    lib=%scan(&filter_detail,1,.)
+    ,ds=%scan(&filter_detail,2,.)
+    ,ref=MP_FILTERSTORE detail update &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  %if &syscc ne 0 %then %do;
+    data _null_;
+      set &ds2;
+      putlog (_all_)(=);
+    run;
+    %goto err;
+  %end;
+
+%end;
+
+proc sort data=&filter_detail(where=(filter_hash="&filter_hash")) out=&outquery;
+  by filter_line;
+run;
+
+%err:
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc on macro exit)
+)
+
+%mend mp_filterstore;

base/mp_filtervalidate.sas

@@ -96,8 +96,7 @@ filename &fref1 clear;
     run;
     %mp_abort(
       mac=&sysmacroname,
-      msg=%str(Filter validation issues. ERR=%superq(SYSERRORTEXT)
-        , WARN=%superq(SYSWARNINGTEXT) )
+      msg=%str(Filter validation issues.)
     )
   %end;
   %let syscc=1008;

base/mp_getcols.sas

@@ -14,11 +14,11 @@
 
   @param ds The dataset from which to obtain column metadata
   @param outds= (work.cols) The output dataset to create. Sample data:
-  |NAME $|LENGTH 8|VARNUM 8|LABEL $|FORMAT $49|TYPE $1 |DDTYPE $|
-  |---|---|---|---|---|---|---|
-  |AIR|8|2|international airline travel (thousands)|8.|N|NUMERIC|
-  |DATE|8|1|DATE|MONYY.|N|DATE|
-  |REGION|3|3|REGION|$3.|C|CHARACTER|
+|NAME:$32.|LENGTH:best.|VARNUM:best.|LABEL:$256.|FMTNAME:$32.|FORMAT:$49.|TYPE:$1.|DDTYPE:$9.|
+|---|---|---|---|---|---|---|---|
+|`AIR `|`8 `|`2 `|`international airline travel (thousands) `|` `|`8. `|`N `|`NUMERIC `|
+|`DATE `|`8 `|`1 `|`DATE `|`MONYY `|`MONYY. `|`N `|`DATE `|
+|`REGION `|`3 `|`3 `|`REGION `|` `|`$3. `|`C `|`CHARACTER `|
 
   <h4> Related Macros </h4>
   @li mf_getvarlist.sas
@@ -30,26 +30,27 @@
 **/
 
 %macro mp_getcols(ds, outds=work.cols);
-
+%local dropds;
 proc contents noprint data=&ds
   out=_data_ (keep=name type length label varnum format:);
 run;
-data &outds(keep=name type length varnum format label ddtype);
-  set &syslast(rename=(format=format2 type=type2));
+%let dropds=&syslast;
+data &outds(keep=name type length varnum format label ddtype fmtname);
+  set &dropds(rename=(format=fmtname type=type2));
   name=upcase(name);
   if type2=2 then do;
     length format $49.;
-    if format2='' then format=cats('$',length,'.');
-    else if formatl=0 then format=cats(format2,'.');
-    else format=cats(format2,formatl,'.');
+    if fmtname='' then format=cats('$',length,'.');
+    else if formatl=0 then format=cats(fmtname,'.');
+    else format=cats(fmtname,formatl,'.');
     type='C';
     ddtype='CHARACTER';
   end;
   else do;
-    if format2='' then format=cats(length,'.');
-    else if formatl=0 then format=cats(format2,'.');
-    else if formatd=0 then format=cats(format2,formatl,'.');
-    else format=cats(format2,formatl,'.',formatd);
+    if fmtname='' then format=cats(length,'.');
+    else if formatl=0 then format=cats(fmtname,'.');
+    else if formatd=0 then format=cats(fmtname,formatl,'.');
+    else format=cats(fmtname,formatl,'.',formatd);
     type='N';
     if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
     else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
@@ -61,5 +62,6 @@ data &outds(keep=name type length varnum format label ddtype);
   end;
   if label='' then label=name;
 run;
-
+proc sql;
+drop table &dropds;
 %mend mp_getcols;

base/mp_getddl.sas

@@ -158,7 +158,7 @@ run;
         lab=" label="!!cats("'",tranwrd(label,"'","''"),"'");
       if notnull='yes' then notnul=' not null';
       if type='char' then typ=cats('char(',length,')');
-      else if length ne 8 then typ='num length='!!left(length);
+      else if length ne 8 then typ='num length='!!cats(length);
       else typ='num';
       put name typ fmt notnul lab;
     run;

base/mp_getformats.sas

@@ -40,6 +40,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
   |`WHICHPATH `|`**OTHER** `|`**OTHER** `|`big fat problem if not path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|`O `|` `|` `|` `|` `|
 
   <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
   @li mf_dedup.sas
   @li mf_getfmtlist.sas
   @li mf_getfmtname.sas
@@ -94,30 +95,7 @@ create table &outsummary as
 
 %if "&outdetail" ne "0" %then %do;
   /* ensure base table always exists */
-  proc sql;
-  create table &outdetail(
-      FMTNAME char(32)     label='Format name'
-      ,START char(16)     label='Starting value for format'
-      ,END char(16)     label='Ending value for format'
-      ,LABEL char(256)     label='Format value label'
-      ,MIN num length=3     label='Minimum length'
-      ,MAX num length=3     label='Maximum length'
-      ,DEFAULT num length=3     label='Default length'
-      ,LENGTH num length=3     label='Format length'
-      ,FUZZ num     label='Fuzz value'
-      ,PREFIX char(2)     label='Prefix characters'
-      ,MULT num     label='Multiplier'
-      ,FILL char(1)     label='Fill character'
-      ,NOEDIT num length=3     label='Is picture string noedit?'
-      ,TYPE char(1)     label='Type of format'
-      ,SEXCL char(1)     label='Start exclusion'
-      ,EEXCL char(1)     label='End exclusion'
-      ,HLO char(13)     label='Additional information'
-      ,DECSEP char(1)     label='Decimal separator'
-      ,DIG3SEP char(1)     label='Three-digit separator'
-      ,DATATYPE char(8)     label='Date/time/datetime?'
-      ,LANGUAGE char(8)     label='Language for date strings'
-  );
+  %mddl_sas_cntlout(libds=&outdetail)
   /* grab the location of each format */
   %let fmtcnt=0;
   data _null_;
@@ -134,7 +112,11 @@ create table &outsummary as
     proc format library=&&fmtloc&i CNTLOUT=&tempds;
       select &&fmtname&i;
     run;
-    proc append base=&outdetail data=&tempds;
+    data &tempds;
+      if 0 then set &outdetail;
+      set &tempds;
+    run;
+    proc append base=&outdetail data=&tempds ;
     run;
   %end;
 %end;

base/mp_getmaxvarlengths.sas

@@ -1,28 +1,46 @@
 /**
-  @file mp_getmaxvarlengths.sas
+  @file
   @brief Scans a dataset to find the max length of the variable values
   @details
   This macro will scan a base dataset and produce an output dataset with two
   columns:
 
   - NAME    Name of the base dataset column
-  - MAXLEN Maximum length of the data contained therein.
+  - MAXLEN  Maximum length of the data contained therein.
 
-  Character fields may be allocated very large widths (eg 32000) of which the
-  maximum  value is likely to be much narrower.  This macro was designed to
-  enable a HTML table to be appropriately sized however this could be used as
-  part of a data audit to ensure we aren't over-sizing our tables in relation to
-  the data therein.
+  Character fields are often allocated very large widths (eg 32000) of which the
+  maximum  value is likely to be much narrower.  Identifying such cases can be
+  helpful in the following scenarios:
+
+  @li Enabling a HTML table to be appropriately sized (`num2char=YES`)
+  @li Reducing the size of a dataset to save on storage (mp_ds2squeeze.sas)
+  @li Identifying columns containing nothing but missing values (`MAXLEN=0` in
+    the output table)
+
+  If the entire column is made up of (non-special) missing values then a value
+  of 0 is returned.
 
-  Numeric fields are converted using the relevant format to determine the width.
   Usage:
 
       %mp_getmaxvarlengths(sashelp.class,outds=work.myds)
 
-  @param libds Two part dataset (or view) reference.
-  @param outds= The output dataset to create
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] num2char= (NO) When set to NO, numeric fields are sized according
+    to the number of bytes used (or set to zero in the case of non-special
+    missings). When YES, the numeric field is converted to character (using the
+    format, if available), and that is sized instead, using `lengthn()`.
+  @param [out] outds= The output dataset to create, eg:
+  |NAME:$8.|MAXLEN:best.|
+  |---|---|
+  |`Name `|`7 `|
+  |`Sex `|`1 `|
+  |`Age `|`3 `|
+  |`Height `|`8 `|
+  |`Weight `|`3 `|
 
   <h4> SAS Macros </h4>
+  @li mcf_length.sas
+  @li mf_getuniquename.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
   @li mf_getvarformat.sas
@@ -30,20 +48,32 @@
   @version 9.2
   @author Allan Bowe
 
+  <h4> Related Macros </h4>
+  @li mp_ds2squeeze.sas
+  @li mp_getmaxvarlengths.test.sas
+
 **/
 
 %macro mp_getmaxvarlengths(
-    libds      /* libref.dataset to analyse */
-    ,outds=work.mp_getmaxvarlengths /* name of output dataset to create */
+  libds
+  ,num2char=NO
+  ,outds=work.mp_getmaxvarlengths
 )/*/STORE SOURCE*/;
 
-%local vars x var fmt;
+%local vars prefix x var fmt;
 %let vars=%mf_getvarlist(libds=&libds);
+%let prefix=%substr(%mf_getuniquename(),1,25);
+%let num2char=%upcase(&num2char);
+
+%if &num2char=NO %then %do;
+  /* compile length function for numeric fields */
+  %mcf_length(wrap=YES, insert_cmplib=YES)
+%end;
 
 proc sql;
 create table &outds (rename=(
     %do x=1 %to %sysfunc(countw(&vars,%str( )));
-      ________&x=%scan(&vars,&x)
+      &prefix.&x=%scan(&vars,&x)
     %end;
     ))
   as select
@@ -51,18 +81,21 @@ create table &outds (rename=(
       %let var=%scan(&vars,&x);
       %if &x>1 %then ,;
       %if %mf_getvartype(&libds,&var)=C %then %do;
-        max(length(&var)) as ________&x
+        max(lengthn(&var)) as &prefix.&x
       %end;
-      %else %do;
+      %else %if &num2char=YES %then %do;
         %let fmt=%mf_getvarformat(&libds,&var);
         %put fmt=&fmt;
         %if %str(&fmt)=%str() %then %do;
-          max(length(cats(&var))) as ________&x
+          max(lengthn(cats(&var))) as &prefix.&x
         %end;
         %else %do;
-          max(length(put(&var,&fmt))) as ________&x
+          max(lengthn(put(&var,&fmt))) as &prefix.&x
         %end;
       %end;
+      %else %do;
+        max(mcf_length(&var)) as &prefix.&x
+      %end;
     %end;
   from &libds;
 

base/mp_guesspk.sas

@@ -17,16 +17,21 @@
       %inc mc;
       %mp_guesspk(sashelp.class,outds=classpks)
 
-  @param baseds The dataset to analyse
-  @param outds= The output dataset to contain the possible PKs
-  @param max_guesses= (3) The total number of possible primary keys to generate.
-    A table may have multiple unlikely PKs, so no need to list them all.
-  @param min_rows= (5) The minimum number of rows a table should have in order
-    to try and guess the PK.
+  @param [in] baseds The dataset to analyse
+  @param [out] outds= The output dataset to contain the possible PKs
+  @param [in] max_guesses= (3) The total number of possible primary keys to
+    generate. A table may have multiple (unlikely) PKs, so no need to list them
+    all.
+  @param [in] min_rows= (5) The minimum number of rows a table should have in
+    order to try and guess the PK.
+  @param [in] ignore_cols (0) Space seperated list of columns which you are
+    sure are not part of the primary key (helps to avoid false positives)
+  @param [in] mdebug= Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
   @li mf_getvarlist.sas
   @li mf_getuniquename.sas
+  @li mf_wordsInstr1butnotstr2.sas
   @li mf_nobs.sas
 
   <h4> Related Macros </h4>
@@ -38,179 +43,226 @@
 **/
 
 %macro mp_guesspk(baseds
-      ,outds=mp_guesspk
-      ,max_guesses=3
-      ,min_rows=5
+  ,outds=mp_guesspk
+  ,max_guesses=3
+  ,min_rows=5
+  ,ignore_cols=0
+  ,mdebug=0
 )/*/STORE SOURCE*/;
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
 
-  /* declare local vars */
-  %local var vars vcnt i j k l tmpvar tmpds rows posspks ppkcnt;
-  %let vars=%mf_getvarlist(&baseds);
-  %let vcnt=%sysfunc(countw(&vars));
+/* declare local vars */
+%local var vars vcnt i j k l tmpvar tmpds rows posspks ppkcnt;
+%let vars=%upcase(%mf_getvarlist(&baseds));
+%let vars=%mf_wordsInStr1ButNotStr2(str1=&vars,str2=%upcase(&ignore_cols));
+%let vcnt=%sysfunc(countw(&vars));
 
-  %if &vcnt=0 %then %do;
-    %put &sysmacroname: &baseds has no variables!  Exiting.;
-    %return;
+%if &vcnt=0 %then %do;
+  %put &sysmacroname: &baseds has no variables!  Exiting.;
+  %return;
+%end;
+
+/* get null count and row count */
+%let tmpvar=%mf_getuniquename();
+proc sql noprint;
+create table _data_ as select
+  count(*) as &tmpvar
+%do i=1 %to &vcnt;
+  %let var=%scan(&vars,&i);
+  ,sum(case when &var is missing then 1 else 0 end) as &var
+%end;
+  from &baseds;
+
+/* transpose table and scan for not null cols */
+proc transpose;
+data _null_;
+  set &syslast end=last;
+  length vars $32767;
+  retain vars ;
+  if _name_="&tmpvar" then call symputx('rows',col1,'l');
+  else if col1=0 then vars=catx(' ',vars,_name_);
+  if last then call symputx('posspks',vars,'l');
+run;
+
+%let ppkcnt=%sysfunc(countw(&posspks));
+%if &ppkcnt=0 %then %do;
+  %put &sysmacroname: &baseds has no non-missing variables!  Exiting.;
+  %return;
+%end;
+
+proc sort data=&baseds(keep=&posspks) out=_data_ noduprec;
+  by _all_;
+run;
+%local pkds; %let pkds=&syslast;
+
+%if &rows > %mf_nobs(&pkds) %then %do;
+  %put &sysmacroname: &baseds has no combination of unique records! Exiting.;
+  %return;
+%end;
+
+/* now check cardinality */
+proc sql noprint;
+create table _data_ as select
+%do i=1 %to &ppkcnt;
+  %let var=%scan(&posspks,&i);
+  count(distinct &var) as &var
+  %if &i<&ppkcnt %then ,;
+%end;
+  from &pkds;
+
+/* transpose and sort by cardinality */
+proc transpose;
+proc sort; by descending col1;
+run;
+
+/* create initial PK list and re-order posspks list */
+data &outds(keep=pkguesses);
+  length pkguesses $5000 vars $5000;
+  set &syslast end=last;
+  retain vars ;
+  vars=catx(' ',vars,_name_);
+  if col1=&rows then do;
+    pkguesses=_name_;
+    output;
+  end;
+  if last then call symputx('posspks',vars,'l');
+run;
+
+%if %mf_nobs(&outds) ge &max_guesses %then %do;
+  %put &sysmacroname: %mf_nobs(&outds) possible primary key values found;
+  %return;
+%end;
+
+%if &ppkcnt=1 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %return;
+%end;
+
+/* begin scanning for uniques on pairs of PKs */
+%let tmpds=%mf_getuniquename();
+%local lev1 lev2;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do;
+      /* check for two level uniqueness */
+      proc sort data=&pkds(keep=&lev1 &lev2) out=&tmpds noduprec;
+        by _all_;
+      run;
+      %if %mf_nobs(&tmpds)=&rows %then %do;
+        proc sql;
+        insert into &outds values("&lev1 &lev2");
+        %if %mf_nobs(&outds) ge &max_guesses %then %do;
+          %put &sysmacroname: Max PKs reached at Level 2 for &baseds;
+          %goto exit;
+        %end;
+      %end;
+    %end;
   %end;
+%end;
 
-  /* get null count and row count */
-  %let tmpvar=%mf_getuniquename();
-  proc sql noprint;
-  create table _data_ as select
-    count(*) as &tmpvar
-  %do i=1 %to &vcnt;
-    %let var=%scan(&vars,&i);
-    ,sum(case when &var is missing then 1 else 0 end) as &var
-  %end;
-    from &baseds;
+%if &ppkcnt=2 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* transpose table and scan for not null cols */
-  proc transpose;
-  data _null_;
-    set &syslast end=last;
-    length vars $32767;
-    retain vars ;
-    if _name_="&tmpvar" then call symputx('rows',col1,'l');
-    else if col1=0 then vars=catx(' ',vars,_name_);
-    if last then call symputx('posspks',vars,'l');
-  run;
-
-  %let ppkcnt=%sysfunc(countw(&posspks));
-  %if &ppkcnt=0 %then %do;
-    %put &sysmacroname: &baseds has no non-missing variables!  Exiting.;
-    %return;
-  %end;
-
-  proc sort data=&baseds(keep=&posspks) out=_data_ noduprec;
-    by _all_;
-  run;
-  %local pkds; %let pkds=&syslast;
-
-  %if &rows > %mf_nobs(&pkds) %then %do;
-    %put &sysmacroname: &baseds has no combination of unique records! Exiting.;
-    %return;
-  %end;
-
-  /* now check cardinality */
-  proc sql noprint;
-  create table _data_ as select
-  %do i=1 %to &ppkcnt;
-    %let var=%scan(&posspks,&i);
-    count(distinct &var) as &var
-    %if &i<&ppkcnt %then ,;
-  %end;
-    from &pkds;
-
-  /* transpose and sort by cardinality */
-  proc transpose;
-  proc sort; by descending col1;
-  run;
-
-  /* create initial PK list and re-order posspks list */
-  data &outds(keep=pkguesses);
-    length pkguesses $5000 vars $5000;
-    set &syslast end=last;
-    retain vars ;
-    vars=catx(' ',vars,_name_);
-    if col1=&rows then do;
-      pkguesses=_name_;
-      output;
-    end;
-    if last then call symputx('posspks',vars,'l');
-  run;
-
-  %if %mf_nobs(&outds) ge &max_guesses %then %do;
-    %put &sysmacroname: %mf_nobs(&outds) possible primary key values found;
-    %return;
-  %end;
-
-  %if &ppkcnt=1 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
-
-  /* begin scanning for uniques on pairs of PKs */
-  %let tmpds=%mf_getuniquename();
-  %local lev1 lev2;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do;
-        /* check for two level uniqueness */
-        proc sort data=&pkds(keep=&lev1 &lev2) out=&tmpds noduprec;
+/* begin scanning for uniques on PK triplets */
+%local lev3;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
+      %let lev3=%scan(&posspks,&k);
+      %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do;
+        /* check for three level uniqueness */
+        proc sort data=&pkds(keep=&lev1 &lev2 &lev3) out=&tmpds noduprec;
           by _all_;
         run;
         %if %mf_nobs(&tmpds)=&rows %then %do;
           proc sql;
-          insert into &outds values("&lev1 &lev2");
+          insert into &outds values("&lev1 &lev2 &lev3");
           %if %mf_nobs(&outds) ge &max_guesses %then %do;
-            %put &sysmacroname: Max PKs reached at Level 2 for &baseds;
-            %return;
+            %put &sysmacroname: Max PKs reached at Level 3 for &baseds;
+            %goto exit;
           %end;
         %end;
       %end;
     %end;
   %end;
+%end;
 
-  %if &ppkcnt=2 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%if &ppkcnt=3 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* begin scanning for uniques on PK triplets */
-  %local lev3;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
-        %let lev3=%scan(&posspks,&k);
-        %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do;
-          /* check for three level uniqueness */
-          proc sort data=&pkds(keep=&lev1 &lev2 &lev3) out=&tmpds noduprec;
+/* scan for uniques on up to 4 PK fields */
+%local lev4;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
+      %let lev3=%scan(&posspks,&k);
+      %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
+        %let lev4=%scan(&posspks,&l);
+        %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then %do;
+          /* check for four level uniqueness */
+          proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4)
+              out=&tmpds noduprec;
             by _all_;
           run;
           %if %mf_nobs(&tmpds)=&rows %then %do;
             proc sql;
-            insert into &outds values("&lev1 &lev2 &lev3");
+            insert into &outds values("&lev1 &lev2 &lev3 &lev4");
             %if %mf_nobs(&outds) ge &max_guesses %then %do;
-              %put &sysmacroname: Max PKs reached at Level 3 for &baseds;
-              %return;
+              %put &sysmacroname: Max PKs reached at Level 4 for &baseds;
+              %goto exit;
             %end;
           %end;
         %end;
       %end;
     %end;
   %end;
+%end;
 
-  %if &ppkcnt=3 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%if &ppkcnt=4 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* scan for uniques on up to 4 PK fields */
-  %local lev4;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
-        %let lev3=%scan(&posspks,&k);
-        %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
-          %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then %do;
+/* scan for uniques on up to 4 PK fields */
+%local lev5 m;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
+      %let lev3=%scan(&posspks,&k);
+      %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
+        %let lev4=%scan(&posspks,&l);
+        %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
+        %do m=5 %to &ppkcnt;
+          %let lev5=%scan(&posspks,&m);
+          %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then %do;
             /* check for four level uniqueness */
-            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4)
+            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5)
                 out=&tmpds noduprec;
               by _all_;
             run;
             %if %mf_nobs(&tmpds)=&rows %then %do;
               proc sql;
-              insert into &outds values("&lev1 &lev2 &lev3 &lev4");
+              insert into &outds values("&lev1 &lev2 &lev3 &lev4 &lev5");
               %if %mf_nobs(&outds) ge &max_guesses %then %do;
-                %put &sysmacroname: Max PKs reached at Level 4 for &baseds;
-                %return;
+                %put &sysmacroname: Max PKs reached at Level 5 for &baseds;
+                %goto exit;
               %end;
             %end;
           %end;
@@ -218,37 +270,44 @@
       %end;
     %end;
   %end;
+%end;
 
-  %if &ppkcnt=4 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%if &ppkcnt=5 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* scan for uniques on up to 4 PK fields */
-  %local lev5 m;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
-        %let lev3=%scan(&posspks,&k);
-        %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
-          %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
-          %do m=5 %to &ppkcnt;
-            %let lev5=%scan(&posspks,&m);
-            %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then %do;
+/* scan for uniques on up to 4 PK fields */
+%local lev6 n;
+%do i=1 %to &ppkcnt;
+  %let lev1=%scan(&posspks,&i);
+  %do j=2 %to &ppkcnt;
+    %let lev2=%scan(&posspks,&j);
+    %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
+      %let lev3=%scan(&posspks,&k);
+      %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
+        %let lev4=%scan(&posspks,&l);
+        %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
+        %do m=5 %to &ppkcnt;
+          %let lev5=%scan(&posspks,&m);
+          %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5
+          %then %do n=6 %to &ppkcnt;
+            %let lev6=%scan(&posspks,&n);
+            %if &lev1 ne &lev6 & &lev2 ne &lev6 & &lev3 ne &lev6
+            & &lev4 ne &lev6 & &lev5 ne &lev6 %then
+            %do;
               /* check for four level uniqueness */
-              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5)
-                  out=&tmpds noduprec;
+              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5 &lev6)
+                out=&tmpds noduprec;
                 by _all_;
               run;
               %if %mf_nobs(&tmpds)=&rows %then %do;
                 proc sql;
-                insert into &outds values("&lev1 &lev2 &lev3 &lev4 &lev5");
+                insert into &outds
+                  values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
                 %if %mf_nobs(&outds) ge &max_guesses %then %do;
-                  %put &sysmacroname: Max PKs reached at Level 5 for &baseds;
-                  %return;
+                  %put &sysmacroname: Max PKs reached at Level 6 for &baseds;
+                  %goto exit;
                 %end;
               %end;
             %end;
@@ -257,56 +316,17 @@
       %end;
     %end;
   %end;
+%end;
 
-  %if &ppkcnt=5 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%if &ppkcnt=6 %then %do;
+  %put &sysmacroname: No more PK guess possible;
+  %goto exit;
+%end;
 
-  /* scan for uniques on up to 4 PK fields */
-  %local lev6 n;
-  %do i=1 %to &ppkcnt;
-    %let lev1=%scan(&posspks,&i);
-    %do j=2 %to &ppkcnt;
-      %let lev2=%scan(&posspks,&j);
-      %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt;
-        %let lev3=%scan(&posspks,&k);
-        %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt;
-          %let lev4=%scan(&posspks,&l);
-          %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then
-          %do m=5 %to &ppkcnt;
-            %let lev5=%scan(&posspks,&m);
-            %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then
-            %do n=6 %to &ppkcnt;
-              %let lev6=%scan(&posspks,&n);
-              %if &lev1 ne &lev6 & &lev2 ne &lev6 & &lev3 ne &lev6
-              & &lev4 ne &lev6 & &lev5 ne &lev6 %then
-              %do;
-                /* check for four level uniqueness */
-                proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5 &lev6)
-                  out=&tmpds noduprec;
-                  by _all_;
-                run;
-                %if %mf_nobs(&tmpds)=&rows %then %do;
-                  proc sql;
-                  insert into &outds
-                    values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
-                  %if %mf_nobs(&outds) ge &max_guesses %then %do;
-                    %put &sysmacroname: Max PKs reached at Level 6 for &baseds;
-                    %return;
-                  %end;
-                %end;
-              %end;
-            %end;
-          %end;
-        %end;
-      %end;
-    %end;
-  %end;
-
-  %if &ppkcnt=6 %then %do;
-    %put &sysmacroname: No more PK guess possible;
-    %return;
-  %end;
+%exit:
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &tmpds;
+%end;
 
 %mend mp_guesspk;

base/mp_include.sas

@@ -51,9 +51,10 @@ https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/mcrolref/n1j5tcc0n2xczyn1
     this dataset.
     It will then run an abort cancel FILE to stop the include running, and pass
     the dataset back.
-    NOTE - it is NOT possible to read this dataset as part of _this_ macro -
-    when running abort cancel FILE, ALL macros are closed, so instead it is
-    necessary to invoke "%mp_abort(mode=INCLUDE)" OUTSIDE of any macro wrappers.
+
+    IMPORTANT NOTE - it is NOT possible to read this dataset as part of _this_
+    macro! When running abort cancel FILE, ALL macros are closed, so instead it
+    is necessary to invoke "%mp_abort(mode=INCLUDE)" OUTSIDE of macro wrappers.
 
 
   @version 9.4

base/mp_init.sas

@@ -33,36 +33,44 @@
 %macro mp_init(prefix=SASJS
 )/*/STORE SOURCE*/;
 
-  %global
-    &prefix._INIT_NUM   /* initialisation time as numeric                   */
-    &prefix._INIT_DTTM  /* initialisation time in E8601DT26.6 format        */
-    &prefix.WORK        /* avoid typing %sysfunc(pathname(work)) every time */
-  ;
-  %if %eval(&&&prefix._INIT_NUM>0) %then %return;  /* only run once */
+%if %symexist(SASJS_PREFIX) %then %return;  /* only run once */
 
-  data _null_;
-    dttm=datetime();
-    call symputx("&prefix._init_num",dttm,'g');
-    call symputx("&prefix._init_dttm",put(dttm,E8601DT26.6),'g');
-    call symputx("&prefix.work",pathname('WORK'),'g');
-  run;
+%global
+  SASJS_PREFIX       /* the ONLY hard-coded global macro variable in SASjs    */
+  &prefix._FUNCTIONS /* used in mcf_init() to track core function compilation */
+  &prefix._INIT_NUM  /* initialisation time as numeric                        */
+  &prefix._INIT_DTTM /* initialisation time in E8601DT26.6 format             */
+  &prefix.WORK       /* avoid typing %sysfunc(pathname(work)) every time      */
+;
 
-  options
-    noautocorrect           /* disallow misspelled procedure names            */
-    compress=CHAR           /* default is none so ensure we have something!   */
-    datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
-    %str(err)orcheck=STRICT /* catch errs in libname/filename statements      */
-    fmterr                  /* ensure err when a format cannot be found       */
-    mergenoby=%str(ERR)OR   /* throw err when a merge has no BY variables     */
-    missing=.               /* changing this can cause hard to detect errs    */
-    noquotelenmax           /* avoid warnings for long strings                */
-    noreplace               /* avoid overwriting permanent datasets           */
-    ps=max                  /* reduce log size slightly                       */
-    ls=max                  /* reduce log even more and avoid word truncation */
-    validmemname=COMPATIBLE /* avoid special characters etc in table names    */
-    validvarname=V7         /* avoid special characters etc in variable names */
-    varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
-    varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
-  ;
+%let sasjs_prefix=&prefix;
+
+data _null_;
+  dttm=datetime();
+  call symputx("&sasjs_prefix._init_num",dttm,'g');
+  call symputx("&sasjs_prefix._init_dttm",put(dttm,E8601DT26.6),'g');
+  call symputx("&sasjs_prefix.work",pathname('WORK'),'g');
+run;
+
+options
+  compress=CHAR           /* default is none so ensure we have something!   */
+  datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
+  errorcheck=STRICT       /* catch errs in libname/filename statements      */
+  fmterr                  /* ensure err when a format cannot be found       */
+  mergenoby=%str(ERR)OR   /* throw err when a merge has no BY variables     */
+  missing=.               /* changing this can cause hard to detect errs    */
+  noquotelenmax           /* avoid warnings for long strings                */
+  noreplace               /* avoid overwriting permanent datasets           */
+  ps=max                  /* reduce log size slightly                       */
+  ls=max                  /* reduce log even more and avoid word truncation */
+  validmemname=COMPATIBLE /* avoid special characters etc in table names    */
+  validvarname=V7         /* avoid special characters etc in variable names */
+  varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
+  varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
+%if %substr(&sysver,1,1) ne 4 %then %do;
+  noautocorrect           /* disallow misspelled procedure names            */
+  dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
+%end;
+;
 
 %mend mp_init;

base/mp_jsonout.sas

@@ -19,11 +19,12 @@
 
         %mp_jsonout(OPEN,jref=tmp)
         %mp_jsonout(OBJ,class,jref=tmp)
+        %mp_jsonout(OBJ,class,dslabel=class2,jref=tmp,showmeta=YES)
         %mp_jsonout(CLOSE,jref=tmp)
 
         data _null_;
         infile tmp;
-        input;list;
+        input;putlog _infile_;
         run;
 
   If you are building web apps with SAS then you are strongly encouraged to use
@@ -31,22 +32,24 @@
   [sasjs adapter](https://github.com/sasjs/adapter).
   For more information see https://sasjs.io
 
-  @param action Valid values:
+  @param [in] action Valid values:
     @li OPEN - opens the JSON
     @li OBJ - sends a table with each row as an object
     @li ARR - sends a table with each row in an array
     @li CLOSE - closes the JSON
-
-  @param ds the dataset to send.  Must be a work table.
-  @param jref= the fileref to which to send the JSON
-  @param dslabel= the name to give the table in the exported JSON
-  @param fmt= Whether to keep or strip formats from the table
-  @param engine= Which engine to use to send the JSON, valid options are:
+  @param [in] ds The dataset to send.  Must be a work table.
+  @param [out] jref= (_webout) The fileref to which to send the JSON
+  @param [out] dslabel= The name to give the table in the exported JSON
+  @param [in] fmt= (Y) Whether to keep (Y) or strip (N) formats from the table
+  @param [in] engine= (DATASTEP) Which engine to use to send the JSON. Options:
     @li PROCJSON (default)
     @li DATASTEP (more reliable when data has non standard characters)
-
-  @param dbg= DEPRECATED - was used to conditionally add PRETTY to
-    proc json but this can cause line truncation in large files.
+  @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
+    (eg `null`) or as STRING values (eg `".a"` or `".b"`)
+  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+    such as the column formats and types.  The metadata is contained inside an
+    object with the same name as the table but prefixed with a dollar sign - ie,
+    `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
 
   <h4> Related Macros <h4>
   @li mp_ds2fmtds.sas
@@ -57,9 +60,14 @@
 
 **/
 
-%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0
+%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y
+  ,engine=DATASTEP
+  ,missing=NULL
+  ,showmeta=NO
 )/*/STORE SOURCE*/;
-%put output location=&jref;
+%local tempds colinfo fmtds i numcols;
+%let numcols=0;
+
 %if &action=OPEN %then %do;
   options nobomfile;
   data _null_;file &jref encoding='utf-8' ;
@@ -68,118 +76,129 @@
 %end;
 %else %if (&action=ARR or &action=OBJ) %then %do;
   options validvarname=upcase;
-  data _null_;file &jref mod encoding='utf-8' ;
+  data _null_; file &jref encoding='utf-8' mod;
     put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":";
 
+  /* grab col defs */
+  proc contents noprint data=&ds
+    out=_data_(keep=name type length format formatl formatd varnum label);
+  run;
+  %let colinfo=%scan(&syslast,2,.);
+  proc sort data=&colinfo;
+    by varnum;
+  run;
+  /* move meta to mac vars */
+  data _null_;
+    if _n_=1 then call symputx('numcols',nobs,'l');
+    set &colinfo end=last nobs=nobs;
+    name=upcase(name);
+    /* fix formats */
+    if type=2 or type=6 then do;
+      typelong='char';
+      length fmt $49.;
+      if format='' then fmt=cats('$',length,'.');
+      else if formatl=0 then fmt=cats(format,'.');
+      else fmt=cats(format,formatl,'.');
+      newlen=max(formatl,length);
+    end;
+    else do;
+      typelong='num';
+      if format='' then fmt='best.';
+      else if formatl=0 then fmt=cats(format,'.');
+      else if formatd=0 then fmt=cats(format,formatl,'.');
+      else fmt=cats(format,formatl,'.',formatd);
+      /* needs to be wide, for datetimes etc */
+      newlen=max(length,formatl,24);
+    end;
+    /* 32 char unique name */
+    newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
+
+    call symputx(cats('name',_n_),name,'l');
+    call symputx(cats('newname',_n_),newname,'l');
+    call symputx(cats('len',_n_),newlen,'l');
+    call symputx(cats('length',_n_),length,'l');
+    call symputx(cats('fmt',_n_),fmt,'l');
+    call symputx(cats('type',_n_),type,'l');
+    call symputx(cats('typelong',_n_),typelong,'l');
+    call symputx(cats('label',_n_),coalescec(label,name),'l');
+  run;
+
+  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+
   %if &engine=PROCJSON %then %do;
-    data;run;%let tempds=&syslast;
-    proc sql;drop table &tempds;
-    data &tempds /view=&tempds;set &ds;
+    %if &missing=STRING %then %do;
+      %put &sysmacroname: Special Missings not supported in proc json.;
+      %put &sysmacroname: Switching to DATASTEP engine;
+      %goto datastep;
+    %end;
+    data &tempds;set &ds;
     %if &fmt=N %then format _numeric_ best32.;;
+    /* PRETTY is necessary to avoid line truncation in large files */
     proc json out=&jref pretty
         %if &action=ARR %then nokeys ;
         ;export &tempds / nosastags fmtnumeric;
     run;
-    proc sql;drop view &tempds;
   %end;
   %else %if &engine=DATASTEP %then %do;
-    %local cols i tempds;
-    %let cols=0;
-    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 %then %do;
+    %datastep:
+    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1
+    %then %do;
       %put &sysmacroname:  &ds NOT FOUND!!!;
       %return;
     %end;
-    %if &fmt=Y %then %do;
-      %put converting every variable to a formatted variable;
-      /* see mp_ds2fmtds.sas for source */
-      proc contents noprint data=&ds
-        out=_data_(keep=name type length format formatl formatd varnum);
-      run;
-      proc sort;
-        by varnum;
-      run;
-      %local fmtds;
-      %let fmtds=%scan(&syslast,2,.);
-      /* prepare formats and varnames */
-      data _null_;
-        if _n_=1 then call symputx('nobs',nobs,'l');
-        set &fmtds end=last nobs=nobs;
-        name=upcase(name);
-        /* fix formats */
-        if type=2 or type=6 then do;
-          length fmt $49.;
-          if format='' then fmt=cats('$',length,'.');
-          else if formatl=0 then fmt=cats(format,'.');
-          else fmt=cats(format,formatl,'.');
-          newlen=max(formatl,length);
-        end;
-        else do;
-          if format='' then fmt='best.';
-          else if formatl=0 then fmt=cats(format,'.');
-          else if formatd=0 then fmt=cats(format,formatl,'.');
-          else fmt=cats(format,formatl,'.',formatd);
-          /* needs to be wide, for datetimes etc */
-          newlen=max(length,formatl,24);
-        end;
-        /* 32 char unique name */
-        newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
 
-        call symputx(cats('name',_n_),name,'l');
-        call symputx(cats('newname',_n_),newname,'l');
-        call symputx(cats('len',_n_),newlen,'l');
-        call symputx(cats('fmt',_n_),fmt,'l');
-        call symputx(cats('type',_n_),type,'l');
-      run;
-      data &fmtds;
+    %if &fmt=Y %then %do;
+      data _data_;
         /* rename on entry */
         set &ds(rename=(
-      %local i;
-      %do i=1 %to &nobs;
+      %do i=1 %to &numcols;
         &&name&i=&&newname&i
       %end;
         ));
-      %do i=1 %to &nobs;
+      %do i=1 %to &numcols;
         length &&name&i $&&len&i;
-        &&name&i=left(put(&&newname&i,&&fmt&i));
+        %if &&typelong&i=num %then %do;
+          &&name&i=left(put(&&newname&i,&&fmt&i));
+        %end;
+        %else %do;
+          &&name&i=put(&&newname&i,&&fmt&i);
+        %end;
         drop &&newname&i;
       %end;
         if _error_ then call symputx('syscc',1012);
       run;
-      %let ds=&fmtds;
-    %end; /* &fmt=Y */
-    data _null_;file &jref mod encoding='utf-8' ;
-      put "["; call symputx('cols',0,'l');
-    proc sort
-      data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
-      out=_data_;
-      by varnum;
-
-    data _null_;
-      set _last_ end=last;
-      call symputx(cats('name',_n_),name,'l');
-      call symputx(cats('type',_n_),type,'l');
-      call symputx(cats('len',_n_),length,'l');
-      if last then call symputx('cols',_n_,'l');
-    run;
+      %let fmtds=&syslast;
+    %end;
 
     proc format; /* credit yabwon for special null removal */
-      value bart ._ - .z = null
+    value bart (default=40)
+    %if &missing=NULL %then %do;
+      ._ - .z = null
+    %end;
+    %else %do;
+      ._ = [quote()]
+      . = null
+      .a - .z = [quote()]
+    %end;
       other = [best.];
 
-    data;run; %let tempds=&syslast; /* temp table for spesh char management */
-    proc sql; drop table &tempds;
-    data &tempds/view=&tempds;
+    data &tempds;
       attrib _all_ label='';
-      %do i=1 %to &cols;
-        %if &&type&i=char %then %do;
+      %do i=1 %to &numcols;
+        %if &&typelong&i=char or &fmt=Y %then %do;
           length &&name&i $32767;
           format &&name&i $32767.;
         %end;
       %end;
-      set &ds;
+      %if &fmt=Y %then %do;
+        set &fmtds;
+      %end;
+      %else %do;
+        set &ds;
+      %end;
       format _numeric_ bart.;
-    %do i=1 %to &cols;
-      %if &&type&i=char %then %do;
+    %do i=1 %to &numcols;
+      %if &&typelong&i=char or &fmt=Y %then %do;
         &&name&i='"'!!trim(prxchange('s/"/\"/',-1,
                     prxchange('s/'!!'0A'x!!'/\n/',-1,
                     prxchange('s/'!!'0D'x!!'/\r/',-1,
@@ -189,44 +208,64 @@
       %end;
     %end;
     run;
+
     /* write to temp loc to avoid _webout truncation
       - https://support.sas.com/kb/49/325.html */
     filename _sjs temp lrecl=131068 encoding='utf-8';
     data _null_; file _sjs lrecl=131068 encoding='utf-8' mod ;
+      if _n_=1 then put "[";
       set &tempds;
       if _n_>1 then put "," @; put
       %if &action=ARR %then "[" ; %else "{" ;
-      %do i=1 %to &cols;
+      %do i=1 %to &numcols;
         %if &i>1 %then  "," ;
         %if &action=OBJ %then """&&name&i"":" ;
         &&name&i
       %end;
       %if &action=ARR %then "]" ; %else "}" ; ;
-    proc sql;
-    drop view &tempds;
     /* now write the long strings to _webout 1 byte at a time */
     data _null_;
       length filein 8 fileid 8;
-      filein = fopen("_sjs",'I',1,'B');
-      fileid = fopen("&jref",'A',1,'B');
-      rec = '20'x;
+      filein=fopen("_sjs",'I',1,'B');
+      fileid=fopen("&jref",'A',1,'B');
+      rec='20'x;
       do while(fread(filein)=0);
-        rc = fget(filein,rec,1);
-        rc = fput(fileid, rec);
-        rc =fwrite(fileid);
+        rc=fget(filein,rec,1);
+        rc=fput(fileid, rec);
+        rc=fwrite(fileid);
       end;
-      rc = fclose(filein);
-      rc = fclose(fileid);
+      /* close out the table */
+      rc=fput(fileid, "]");
+      rc=fwrite(fileid);
+      rc=fclose(filein);
+      rc=fclose(fileid);
     run;
     filename _sjs clear;
-    data _null_; file &jref mod encoding='utf-8' ;
-      put "]";
+  %end;
+
+  proc sql;
+  drop table &colinfo, &tempds;
+
+  %if &showmeta=YES %then %do;
+    data _null_; file &jref encoding='utf-8' mod;
+      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{";
+      do i=1 to &numcols;
+        name=quote(trim(symget(cats('name',i))));
+        format=quote(trim(symget(cats('fmt',i))));
+        label=quote(trim(symget(cats('label',i))));
+        length=quote(trim(symget(cats('length',i))));
+        type=quote(trim(symget(cats('typelong',i))));
+        if i>1 then put "," @@;
+        put name ':{"format":' format ',"label":' label
+          ',"length":' length ',"type":' type '}';
+      end;
+      put '}}';
     run;
   %end;
 %end;
 
 %else %if &action=CLOSE %then %do;
-  data _null_;file &jref encoding='utf-8' mod ;
+  data _null_; file &jref encoding='utf-8' mod ;
     put "}";
   run;
 %end;

base/mp_loadformat.sas

@@ -0,0 +1,298 @@
+/**
+  @file
+  @brief Loads a format catalog from a staging dataset
+  @details When loading staged data, it is common to receive only the records
+  that have actually changed.  However, when loading a format catalog, if
+  records are missing they are presumed to be no longer required.
+
+  This macro will augment a staging dataset with other records from the same
+  format, to prevent loss of data - UNLESS the input dataset contains a marker
+  column, specifying that a particular row needs to be deleted (`delete_col=`).
+
+  This macro can also be used to identify which records would be (or were)
+  considered new, modified or deleted (`loadtarget=`) by creating the following
+  tables:
+
+  @li work.outds_add
+  @li work.outds_del
+  @li work.outds_mod
+
+  For example usage, see mp_loadformat.test.sas
+
+  @param [in] libcat The format catalog to be loaded
+  @param [in] libds The staging table to load
+  @param [in] loadtarget= (NO) Set to YES to actually load the target catalog
+  @param [in] delete_col= (_____DELETE__THIS__RECORD_____) The column used to
+    mark a record for deletion.  Values should be "Yes" or "No".
+  @param [out] auditlibds= (0) For change tracking, set to the libds of an audit
+    table as defined in mddl_dc_difftable.sas
+  @param [in] locklibds= (0) For multi-user (parallel) situations, set to the
+    libds of the DC lock table as defined in the mddl_dc_locktable.sas macro.
+  @param [out] outds_add= (0) Set a libds here to see the new records added
+  @param [out] outds_del= (0) Set a libds here to see the records deleted
+  @param [out] outds_mod= (0) Set a libds here to see the modified records
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_cntlout.sas
+  @li mp_lockanytable.sas
+
+  <h4> Related Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mddl_dc_locktable.sas
+  @li mp_loadformat.test.sas
+  @li mp_lockanytable.sas
+  @li mp_storediffs.sas
+  @li mp_stackdiffs.sas
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_loadformat(libcat,libds
+  ,loadtarget=NO
+  ,auditlibds=0
+  ,locklibds=0
+  ,delete_col=_____DELETE__THIS__RECORD_____
+  ,outds_add=0
+  ,outds_del=0
+  ,outds_mod=0
+  ,mdebug=0
+);
+/* set up local macro variables and temporary tables (with a prefix) */
+%local err msg prefix dslist i var fmtlist ibufsize;
+%let dslist=base_fmts template inlibds ds1 stagedata storediffs;
+%if &outds_add=0 %then %let dslist=&dslist outds_add;
+%if &outds_del=0 %then %let dslist=&dslist outds_del;
+%if &outds_mod=0 %then %let dslist=&dslist outds_mod;
+%let prefix=%substr(%mf_getuniquename(),1,21);
+%do i=1 %to %sysfunc(countw(&dslist));
+  %let var=%scan(&dslist,&i);
+  %local &var;
+  %let &var=%upcase(&prefix._&var);
+%end;
+
+/*
+format values can be up to 32767 wide.  SQL joins on such a wide column can
+cause buffer issues.  Update ibufsize and reset at the end.
+*/
+%let ibufsize=%sysfunc(getoption(ibufsize));
+options ibufsize=32767 ;
+
+/* in DC, format catalogs maybe specified in the libds with a -FC extension */
+%let libcat=%scan(&libcat,1,-);
+
+/* perform input validations */
+%let err=0;
+%let msg=0;
+data _null_;
+  if _n_=1 then putlog "&sysmacroname entry vars:";
+  set sashelp.vmacro;
+  where scope="&sysmacroname";
+  value=upcase(value);
+  if &mdebug=0 then put name '=' value;
+  if name=:'LOAD' and value not in ('YES','NO') then do;
+    call symputx('msg',"invalid value for "!!name!!":"!!value);
+    call symputx('err',1);
+    stop;
+  end;
+  else if name='LIBCAT' then do;
+    if exist(value,'CATALOG') le 0 then do;
+      call symputx('msg',"Unable to open catalog: "!!value);
+      call symputx('err',1);
+      stop;
+    end;
+  end;
+  else if name='LIBDS' then do;
+    if exist(value) le 0 then do;
+      call symputx('msg',"Unable to open staging table: "!!value);
+      call symputx('err',1);
+      stop;
+    end;
+  end;
+  else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
+  and missing(value) then do;
+    call symputx('msg',"missing value in var: "!!name);
+    call symputx('err',1);
+    stop;
+  end;
+run;
+
+%mp_abort(
+  iftrue=(&err ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(&msg)
+)
+
+/**
+  * First, extract only relevant formats from the catalog
+  */
+proc sql noprint;
+select distinct fmtname into: fmtlist separated by ' ' from &libds;
+
+%mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
+
+
+/**
+  * Ensure input table and base_formats have consistent lengths and types
+  */
+%mddl_sas_cntlout(libds=&template)
+data &inlibds;
+  if 0 then set &template;
+  set &libds;
+  if missing(type) then do;
+    if substr(fmtname,1,1)='$' then type='C';
+    else type='N';
+  end;
+  if type='N' then do;
+    start=cats(start);
+    end=cats(end);
+  end;
+run;
+
+
+/**
+  * Identify new records
+  */
+proc sql;
+create table &outds_add(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  left join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where b.fmtname is null
+    and upcase(a.&delete_col) ne "YES"
+  order by fmtname, start;;
+
+/**
+  * Identify deleted records
+  */
+create table &outds_del(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where upcase(a.&delete_col)="YES"
+  order by fmtname, start;
+
+/**
+  * Identify modified records
+  */
+create table &outds_mod (drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where upcase(a.&delete_col) ne "YES"
+  order by fmtname, start;
+
+options ibufsize=&ibufsize;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc prior to load prep)
+)
+
+%if &loadtarget=YES %then %do;
+  data &ds1;
+    merge &base_fmts(in=base)
+      &outds_mod(in=mod)
+      &outds_add(in=add)
+      &outds_del(in=del);
+    if not del and not mod;
+    by fmtname start;
+  run;
+  data &stagedata;
+    set &ds1 &outds_mod;
+  run;
+  proc sort;
+    by fmtname start;
+  run;
+%end;
+/* mp abort needs to run outside of conditional blocks */
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc prior to actual load)
+)
+%if &loadtarget=YES %then %do;
+  %if %mf_nobs(&stagedata)=0 %then %do;
+    %put There are no changes to load in &libcat!;
+    %return;
+  %end;
+  %if &locklibds ne 0 %then %do;
+    /* prevent parallel updates */
+    %mp_lockanytable(LOCK
+      ,lib=%scan(&libcat,1,.)
+      ,ds=%scan(&libcat,2,.)-FC
+      ,ref=MP_LOADFORMAT commencing format load
+      ,ctl_ds=&locklibds
+    )
+  %end;
+  /* do the actual load */
+  proc format lib=&libcat cntlin=&stagedata;
+  run;
+  %if &locklibds ne 0 %then %do;
+    /* unlock the table */
+    %mp_lockanytable(UNLOCK
+      ,lib=%scan(&libcat,1,.)
+      ,ds=%scan(&libcat,2,.)-FC
+      ,ref=MP_LOADFORMAT completed format load
+      ,ctl_ds=&locklibds
+    )
+  %end;
+  /* track the changes */
+  %if &auditlibds ne 0 %then %do;
+    %if &locklibds ne 0 %then %do;
+      %mp_lockanytable(LOCK
+        ,lib=%scan(&auditlibds,1,.)
+        ,ds=%scan(&auditlibds,2,.)
+        ,ref=MP_LOADFORMAT commencing audit table load
+        ,ctl_ds=&locklibds
+      )
+    %end;
+
+    %mp_storediffs(&libcat-FC
+      ,&inlibds
+      ,FMTNAME START
+      ,delds=&outds_del
+      ,modds=&outds_mod
+      ,appds=&outds_add
+      ,outds=&storediffs
+      ,mdebug=&mdebug
+    )
+
+    %if &locklibds ne 0 %then %do;
+      %mp_lockanytable(UNLOCK
+        ,lib=%scan(&auditlibds,1,.)
+        ,ds=%scan(&auditlibds,2,.)
+        ,ref=MP_LOADFORMAT commencing audit table load
+        ,ctl_ds=&locklibds
+      )
+    %end;
+  %end;
+%end;
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc after load)
+)
+
+%if &mdebug=0 %then %do;
+  proc datasets lib=work;
+    delete &prefix:;
+  run;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%mend mp_loadformat;

base/mp_lockanytable.sas

@@ -1,23 +1,23 @@
 /**
   @file
   @brief Mechanism for locking tables to prevent parallel modifications
-  @details Uses a control table to enable ANY table to be locked for updates.
+  @details Uses a control table to enable ANY table to be locked for updates
+  (not just SAS datasets).
   Only useful if every update uses the macro!   Used heavily within
   [Data Controller for SAS](https://datacontroller.io).
 
-  The underlying table is structured as per the MAKETABLE action.
-
   @param [in] action The action to be performed.  Valid values:
     @li LOCK - Sets the lock flag, also confirms if a SAS lock is available
     @li UNLOCK - Unlocks the table
-    @li MAKETABLE - creates the control table (ctl_ds)
   @param [in] lib= (WORK) The libref of the table to lock.  Should already be
     assigned.
   @param [in] ds= The dataset to lock
   @param [in] ref= A meaningful reference to enable the lock to be traced. Max
     length is 200 characters.
   @param [out] ctl_ds= (0) The control table which controls the actual locking.
-    Should already be assigned and available.
+    Should already be assigned and available.  The definition is available by
+    running the mddl_dc_locktable.sas macro.
+
   @param [in] loops= (25) Number of times to check for a lock.
   @param [in] loop_secs= (1) Seconds to wait between each lock attempt
 
@@ -49,7 +49,7 @@ data _null_;
   put name '=' value;
 run;
 
-%mp_abort(iftrue= (&ds=0 and &action ne MAKETABLE)
+%mp_abort(iftrue= ("&ds"="0" and &action ne MAKETABLE)
   ,mac=&sysmacroname
   ,msg=%str(dataset was not provided)
 )
@@ -86,7 +86,7 @@ run;
 /* do not proceed if no observations can be processed */
 %mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)
   ,mac=&sysmacroname
-  ,msg=%str(options obs = 0. syserrortext=&syserrortext)
+  ,msg=%str(cannot continue when options obs = 0)
 )
 
 %if &ACTION=LOCK %then %do;
@@ -221,19 +221,6 @@ run;
     %let abortme=1;
   %end;
 %end;
-%else %if &action=MAKETABLE %then %do;
-  proc sql;
-  create table &ctl_ds(
-      lock_lib char(8),
-      lock_ds char(32),
-      lock_status_cd char(10) not null,
-      lock_user_nm char(100) not null ,
-      lock_ref char(200),
-      lock_pid char(10),
-      lock_start_dttm num format=E8601DT26.6,
-      lock_end_dttm num format=E8601DT26.6,
-    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
-%end;
 %else %do;
   %let msg=lock_anytable given unsupported action (&action);
   %let abortme=1;

base/mp_lockfilecheck.sas

@@ -37,7 +37,7 @@ run;
   ,mac=checklock.sas
   ,msg=Aborting with syscc=&syscc on entry.
 )
-%mp_abort(iftrue= (&libds=0)
+%mp_abort(iftrue= ("&libds"="0")
   ,mac=&sysmacroname
   ,msg=%str(libds not provided)
 )
@@ -46,6 +46,12 @@ run;
 %let lib=%upcase(%scan(&libds,1,.));
 %let ds=%upcase(%scan(&libds,2,.));
 
+/* in DC, format catalogs are passed with a -FC suffix. No saslock here! */
+%if %scan(&libds,2,-)=FC %then %do;
+  %put &sysmacroname: Format Catalog detected, no lockfile applied to &libds;
+  %return;
+%end;
+
 /* do not proceed if no observations can be processed */
 %let msg=options obs = 0. syserrortext=%superq(syserrortext);
 %mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)

base/mp_md5.sas

@@ -0,0 +1,58 @@
+/**
+  @file
+  @brief Generates an md5 expression for hashing a set of variables
+  @details This is the same algorithm used to hash records in
+  [Data Controller for SAS](https://datacontroller.io) (free for up
+  to 5 users).
+
+  It is not designed to be efficient - it is designed to be effective,
+  given the range of edge cases (large floating points, special missing
+  numerics, thousands of columns, very wide columns).
+
+  It can be used only in data step, eg as follows:
+
+      data _null_;
+        set sashelp.class;
+        hashvar=%mp_md5(cvars=name sex, nvars=age height weight);
+        put hashvar=;
+      run;
+
+  Unfortunately it will not run in SQL - it fails with the following message:
+
+  > The width value for HEX is out of bounds. It should be between 1 and 16
+
+  The macro will also cause errors if the data contains (non-special) missings
+  and the (undocumented) `options dsoptions=nonote2err;` is in effect.
+
+  This can be avoided in two ways:
+
+  @li Global option:  `options dsoptions=nonote2err;`
+  @li Data step option: `data YOURLIB.YOURDATASET /nonote2err;`
+
+  @param cvars= Space seperated list of character variables
+  @param nvars= Space seperated list of numeric variables
+
+  <h4> Related Programs </h4>
+  @li mp_init.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_md5(cvars=,nvars=);
+%local i var sep;
+put(md5(
+  %do i=1 %to %sysfunc(countw(&cvars));
+    %let var=%scan(&cvars,&i,%str( ));
+    &sep put(md5(trim(&var)),$hex32.)
+    %let sep=!!;
+  %end;
+  %do i=1 %to %sysfunc(countw(&nvars));
+    %let var=%scan(&nvars,&i,%str( ));
+    /* multiply by 1 to strip precision errors (eg 0 != 0) */
+    /* but ONLY if not missing, else will lose any special missing values */
+    &sep put(md5(trim(put(ifn(missing(&var),&var,&var*1),binary64.))),$hex32.)
+    %let sep=!!;
+  %end;
+),hex32.)
+%mend mp_md5;

base/mp_replace.sas

@@ -0,0 +1,149 @@
+/**
+  @file
+  @brief Performs a text substitution on a file
+  @details Performs a find and replace on a file, either in place or to a new
+  file. Can be used on files where lines are longer than 32767.
+
+  Works by reading in the file byte by byte, then marking the beginning and end
+  of each matched string, before finally doing the replace.
+
+  Full credit for this highly efficient and syntactically satisfying SAS logic
+  goes to [Bartosz Jabłoński](https://www.linkedin.com/in/yabwon), founder of
+  the [SAS Packages](https://github.com/yabwon/SAS_PACKAGES) framework.
+
+  Usage:
+
+      %let file="%sysfunc(pathname(work))/file.txt";
+      %let str=replace/me;
+      %let rep=with/this;
+      data _null_;
+        file &file;
+        put 'blahblah';
+        put "blahblah&str.blah";
+        put 'blahblahblah';
+      run;
+      %mp_replace(&file, findvar=str, replacevar=rep)
+      data _null_;
+        infile &file;
+        input;
+        list;
+      run;
+
+  Note - if you are running a version of SAS that will allow the io package in
+  LUA, you can also use this macro: mp_gsubfile.sas
+
+  @param infile The QUOTED path to the file on which to perform the substitution
+  @param findvar= Macro variable NAME containing the string to search for
+  @param replacevar= Macro variable NAME containing the replacement string
+  @param outfile= (0) Optional QUOTED path to an the adjusted output file (to
+    avoid overwriting the first file).
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mp_gsubfile.test.sas
+
+  @version 9.4
+  @author Bartosz Jabłoński
+  @author Allan Bowe
+**/
+
+%macro mp_replace(infile,
+  findvar=,
+  replacevar=,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+%local inref dttm ds1;
+%let inref=%mf_getuniquefileref();
+%let outref=%mf_getuniquefileref();
+%if &outfile=0 %then %let outfile=&infile;
+%let ds1=%mf_getuniquename(prefix=allchars);
+%let ds2=%mf_getuniquename(prefix=startmark);
+
+/* START */
+%let dttm=%sysfunc(datetime());
+
+filename &inref &infile lrecl=1 recfm=n;
+
+data &ds1;
+  infile &inref;
+  input sourcechar $ 1. @@;
+  format sourcechar hex2.;
+run;
+
+data &ds2;
+  /* set find string to length in bytes to cover trailing spaces */
+  length string $ %length(%superq(&findvar));
+  string =symget("&findvar");
+  drop string;
+
+  firstchar=char(string,1);
+  findlen=lengthm(string); /* <- for trailing bytes */
+
+  do _N_=1 to nobs;
+    set &ds1 nobs=nobs point=_N_;
+    if sourcechar=firstchar then do;
+      pos=1;
+      s=0;
+      do point=_N_ to min(_N_ + findlen -1,nobs);
+        set &ds1 point=point;
+        if sourcechar=char(string, pos) then s + 1;
+        else goto _leave_;
+        pos+1;
+      end;
+      _leave_:
+      if s=findlen then do;
+        START =_N_;
+        _N_ =_N_+ s - 1;
+        STOP =_N_;
+        output;
+      end;
+    end;
+  end;
+  stop;
+  keep START STOP;
+run;
+
+data &ds1;
+  declare hash HS(dataset:"&ds2(keep=start)");
+  HS.defineKey("start");
+  HS.defineDone();
+  declare hash HE(dataset:"&ds2(keep=stop)");
+  HE.defineKey("stop");
+  HE.defineDone();
+  do until(eof);
+    set &ds1 end=eof curobs =n;
+    start = ^HS.check(key:n);
+    stop  = ^HE.check(key:n);
+    length strt $ 1;
+    strt =put(start,best. -L);
+    retain out 1;
+    if out   then output;
+    if start then out=0;
+    if stop  then out=1;
+  end;
+  stop;
+  keep sourcechar strt;
+run;
+
+filename &outref &outfile recfm=n;
+
+data _null_;
+  length replace $ %length(%superq(&replacevar));
+  replace=symget("&replacevar");
+  file &outref;
+  do until(eof);
+    set &ds1 end=eof;
+    if strt ="1" then put replace char.;
+    else put sourcechar char1.;
+  end;
+  stop;
+run;
+
+/* END */
+%put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) seconds to run;
+
+%mend mp_replace;

base/mp_retainedkey.sas

@@ -0,0 +1,246 @@
+/**
+  @file
+  @brief Generate and apply retained key values to a staging table
+  @details This macro will populate a staging table with a Retained Key based on
+  a business key and a base (target) table.
+
+  Definition of retained key ([source](
+    http://bukhantsov.org/2012/04/what-is-data-vault/)):
+
+  > The retained key is a key which is mapped to business key one-to-one. In
+  > comparison,  the surrogate key includes time and there can be many surrogate
+  > keys corresponding to one business key. This explains the name of the key,
+  > it is retained with insertion of a new version of a row while surrogate key
+  > is increasing.
+
+  This macro is designed to be used as part of a wider load / ETL process (such
+  as the one in [Data Controller for SAS](https://datacontroller.io)).
+
+  Specifically, the macro assumes that the base table has already been 'locked'
+  (eg with the mp_lockanytable.sas macro) prior to invocation.  Also, several
+  tables are assumed to exist (names are configurable):
+
+  @li work.staging_table - the staged data, minus the retained key element
+  @li permlib.base_table - the target table to be loaded (**not** loaded by this
+    macro)
+  @li permlib.maxkeytable - optional, used to store load metaadata.
+    The definition is available by running mp_coretable.sas as follows:
+    `mp_coretable(MAXKEYTABLE)`.
+  @li permlib.locktable - Necessary if maxkeytable is being populated. The
+    definition is available by running mp_coretable.sas as follows:
+    `mp_coretable(LOCKTABLE)`.
+
+
+  @param [in] base_lib= (WORK) Libref of the base (target) table.
+  @param [in] base_dsn= (BASETABLE) Name of the base (target) table.
+  @param [in] append_lib= (WORK) Libref of the staging table
+  @param [in] append_dsn= (APPENDTABLE) Name of the staging table
+  @param [in] retained_key= (DEFAULT_RK) Name of RK to generate (should exist on
+    base table)
+  @param [in] business_key= (PK1 PK2) Business key against which to generate
+    RK values.  Should be unique and not null on the staging table.
+  @param [in] check_uniqueness=(NO) Set to yes to perform a uniqueness check.
+    Recommended if there is a chance that the staging data is not unique on the
+    business key.
+  @param [in] maxkeytable= (0) Provide a maxkeytable libds reference here, to
+    store load metadata (maxkey val, load time).  Set to zero if metadata is not
+    required, eg, when preparing a 'dummy' load. Structure is described above.
+    See below for sample data.
+    |KEYTABLE:$32.|KEYCOLUMN:$32.|MAX_KEY:best.|PROCESSED_DTTM:E8601DT26.6|
+    |---|---|---|---|
+    |`DC487173.MPE_SELECTBOX `|`SELECTBOX_RK `|`55 `|`1950427787.8 `|
+    |`DC487173.MPE_FILTERANYTABLE `|`filter_rk `|`14 `|`1951053886.8 `|
+  @param [in] locktable= (0) If updating the maxkeytable, provide the libds
+    reference to the lock table (per mp_lockanytable.sas macro)
+  @param [in] filter_str= Apply a filter - useful for SCD2 or BITEMPORAL loads.
+    Example: `filter_str=%str( (where=( &now < &tech_to)) )`
+  @param [out] outds= (WORK.APPEND) Output table (staging table + retained key)
+
+  <h4> SAS Macros </h4>
+  @li mf_existvar.sas
+  @li mf_getquotedstr.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_lockanytable.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filterstore.sas
+  @li mp_retainedkey.test.sas
+
+  @version 9.2
+
+**/
+
+%macro mp_retainedkey(
+  base_lib=WORK
+  ,base_dsn=BASETABLE
+  ,append_lib=WORK
+  ,append_dsn=APPENDTABLE
+  ,retained_key=DEFAULT_RK
+  ,business_key= PK1 PK2
+  ,check_uniqueness=NO
+  ,maxkeytable=0
+  ,locktable=0
+  ,outds=WORK.APPEND
+  ,filter_str=
+);
+%put &sysmacroname entry vars:;
+%put _local_;
+
+%local base_libds app_libds key_field check maxkey idx_pk newkey_cnt iserr
+  msg x tempds1 tempds2 comma_pk appnobs checknobs dropvar tempvar idx_val;
+%let base_libds=%upcase(&base_lib..&base_dsn);
+%let app_libds=%upcase(&append_lib..&append_dsn);
+%let tempds1=%mf_getuniquename();
+%let tempds2=%mf_getuniquename();
+%let comma_pk=%mf_getquotedstr(in_str=%str(&business_key),dlm=%str(,),quote=);
+%let outds=%sysfunc(ifc(%index(&outds,.)=0,work.&outds,&outds));
+/* validation checks */
+%let iserr=0;
+%if &syscc>0 %then %do;
+  %let iserr=1;
+  %let msg=%str(SYSCC=&syscc on macro entry);
+%end;
+%else %if %sysfunc(exist(&base_libds))=0 %then %do;
+  %let iserr=1;
+  %let msg=%str(Base LIBDS (&base_libds) expected but NOT FOUND);
+%end;
+%else %if %sysfunc(exist(&app_libds))=0 %then %do;
+  %let iserr=1;
+  %let msg=%str(Append LIBDS (&app_libds) expected but NOT FOUND);
+%end;
+%else %if &maxkeytable ne 0 and %sysfunc(exist(&maxkeytable))=0  %then %do;
+  %let iserr=1;
+  %let msg=%str(Maxkeytable (&maxkeytable) expected but NOT FOUND);
+%end;
+%else %if &maxkeytable ne 0 and %sysfunc(exist(&locktable))=0  %then %do;
+  %let iserr=1;
+  %let msg=%str(Locktable (&locktable) expected but NOT FOUND);
+%end;
+%else %if %length(&business_key)=0 %then %do;
+  %let iserr=1;
+  %let msg=%str(Business key (&business_key) expected but NOT FOUND);
+%end;
+
+%do x=1 %to %sysfunc(countw(&business_key));
+  /* check business key values exist */
+  %let key_field=%scan(&business_key,&x,%str( ));
+  %if not %mf_existvar(&app_libds,&key_field) %then %do;
+    %let iserr=1;
+    %let msg=Business key (&key_field) not found on &app_libds!;
+    %goto err;
+  %end;
+  %else %if not %mf_existvar(&base_libds,&key_field) %then %do;
+    %let iserr=1;
+    %let msg=Business key (&key_field) not found on &base_libds!;
+    %goto err;
+  %end;
+%end;
+%err:
+%if &iserr=1 %then %do;
+  /* err case so first perform an unlock of the base table before exiting */
+  %mp_lockanytable(
+    UNLOCK,lib=&base_lib,ds=&base_dsn,ref=%superq(msg),ctl_ds=&locktable
+  )
+%end;
+%mp_abort(iftrue=(&iserr=1),mac=mp_retainedkey,msg=%superq(msg))
+
+proc sql noprint;
+select sum(max(&retained_key),0) into: maxkey from &base_libds;
+
+/**
+  * get base table RK and bus field values for lookup
+  */
+proc sql noprint;
+create table &tempds1 as
+  select distinct &comma_pk,&retained_key
+  from &base_libds &filter_str
+  order by &comma_pk,&retained_key;
+
+%if &check_uniqueness=YES %then %do;
+  select count(*) into:checknobs
+    from (select distinct &comma_pk from &app_libds);
+  select count(*) into: appnobs from &app_libds; /* might be view */
+  %if &checknobs ne &appnobs %then %do;
+    %let msg=Source table &app_libds is not unique on (&business_key);
+    %let iserr=1;
+  %end;
+%end;
+%if &iserr=1 %then %do;
+  /* err case so first perform an unlock of the base table before exiting */
+  %mp_lockanytable(
+    UNLOCK,lib=&base_lib,ds=&base_dsn,ref=%superq(msg),ctl_ds=&locktable
+  )
+%end;
+%mp_abort(iftrue= (&iserr=1),mac=mp_retainedkey,msg=%superq(msg))
+
+%if %mf_existvar(&app_libds,&retained_key)
+%then %let dropvar=(drop=&retained_key);
+
+/* prepare interim table with retained key populated for matching keys */
+proc sql noprint;
+create table &tempds2 as
+  select b.&retained_key, a.*
+  from &app_libds &dropvar a
+  left join &tempds1 b
+  on 1
+  %do idx_pk=1 %to %sysfunc(countw(&business_key));
+    %let idx_val=%scan(&business_key,&idx_pk);
+    and a.&idx_val=b.&idx_val
+  %end;
+  order by &retained_key;
+
+/* identify the number of entries without retained keys (new records) */
+select count(*) into: newkey_cnt
+  from &tempds2
+  where missing(&retained_key);
+quit;
+
+/**
+  * Update maxkey table if link provided
+  */
+%if &maxkeytable ne 0 %then %do;
+  proc sql noprint;
+  select count(*) into: check from &maxkeytable
+    where upcase(keytable)="&base_libds";
+
+  %mp_lockanytable(LOCK
+    ,lib=%scan(&maxkeytable,1,.)
+    ,ds=%scan(&maxkeytable,2,.)
+    ,ref=Updating maxkeyvalues with mp_retainedkey
+    ,ctl_ds=&locktable
+  )
+  proc sql;
+  %if &check=0 %then %do;
+  insert into &maxkeytable
+    set keytable="&base_libds"
+      ,keycolumn="&retained_key"
+      ,max_key=%eval(&maxkey+&newkey_cnt)
+      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt;
+  %end;
+  %else %do;
+  update &maxkeytable
+    set max_key=%eval(&maxkey+&newkey_cnt)
+      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt
+    where keytable="&base_libds";
+  %end;
+  %mp_lockanytable(UNLOCK
+    ,lib=%scan(&maxkeytable,1,.)
+    ,ds=%scan(&maxkeytable,2,.)
+    ,ref=Updating maxkeyvalues with maxkey=%eval(&maxkey+&newkey_cnt)
+    ,ctl_ds=&locktable
+  )
+%end;
+
+/* fill in the missing retained key values */
+%let tempvar=%mf_getuniquename();
+data &outds(drop=&tempvar);
+  retain &tempvar %eval(&maxkey+1);
+  set &tempds2;
+  if &retained_key =. then &retained_key=&tempvar;
+  &tempvar=&tempvar+1;
+run;
+
+%mend mp_retainedkey;
+

base/mp_stackdiffs.sas

@@ -0,0 +1,592 @@
+/**
+  @file
+  @brief Prepares an audit table for stacking (re-applying) the changes.
+  @details When the underlying data from a Base Table is refreshed, it can be
+    helpful to have any previously-applied changes, re-applied.
+
+    Such situation might arise if you are applying those changes using a tool
+    like [Data Controller for SAS®](https://datacontroller.io) - which records
+    all such changes in an audit table.
+    It may also apply if you are preparing a series of specific cell-level
+    transactions, that you would like to apply to multiple sets of (similarly
+    structured) Base Tables.
+
+    In both cases, it is necessary that the transactions are stored using
+    the mp_storediffs.sas macro, or at least that the underlying table is
+    structured as per the definition in mp_coretable.sas (DIFFTABLE entry)
+
+    <b>This</b> macro is used to convert the stored changes (tall format) into
+    staged changes (wide format), with base table values incorporated (in the
+    case of modified rows), ready for the subsequent load process.
+
+    Essentially then, what this macro does, is turn a table like this:
+
+|KEY_HASH:$32.|MOVE_TYPE:$1.|TGTVAR_NM:$32.|IS_PK:best.|IS_DIFF:best.|TGTVAR_TYPE:$1.|OLDVAL_NUM:best32.|NEWVAL_NUM:best32.|OLDVAL_CHAR:$32765.|NEWVAL_CHAR:$32765.|
+|---|---|---|---|---|---|---|---|---|---|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`NAME `|`1 `|`-1 `|`C `|`. `|`. `|` `|`Newbie `|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`AGE `|`0 `|`-1 `|`N `|`. `|`13 `|` `|` `|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`HEIGHT `|`0 `|`-1 `|`N `|`. `|`65.3 `|` `|` `|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`SEX `|`0 `|`-1 `|`C `|`. `|`. `|` `|`F `|
+|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`WEIGHT `|`0 `|`-1 `|`N `|`. `|`98 `|` `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`NAME `|`1 `|`-1 `|`C `|`. `|`. `|`Alfred `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`AGE `|`0 `|`-1 `|`N `|`14 `|`. `|` `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`HEIGHT `|`0 `|`-1 `|`N `|`69 `|`. `|` `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`SEX `|`0 `|`-1 `|`C `|`. `|`. `|`M `|` `|
+|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`WEIGHT `|`0 `|`-1 `|`N `|`112.5 `|`. `|` `|` `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`NAME `|`1 `|`0 `|`C `|`. `|`. `|`Alice `|`Alice `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`AGE `|`0 `|`1 `|`N `|`13 `|`99 `|` `|` `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`HEIGHT `|`0 `|`0 `|`N `|`56.5 `|`56.5 `|` `|` `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`SEX `|`0 `|`0 `|`C `|`. `|`. `|`F `|`F `|
+|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`WEIGHT `|`0 `|`0 `|`N `|`84 `|`84 `|` `|` `|
+
+    Into three tables like this:
+
+  <b> `work.outmod`: </b>
+  |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.|
+  |---|---|---|---|---|
+  |`Alice `|`F `|`99 `|`56.5 `|`84 `|
+
+  <b> `work.outadd`: </b>
+  |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.|
+  |---|---|---|---|---|
+  |`Newbie `|`F `|`13 `|`65.3 `|`98 `|
+
+  <b> `work.outdel`: </b>
+  |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.|
+  |---|---|---|---|---|
+  |`Alfred `|`M `|`14 `|`69 `|`112.5 `|
+
+    As you might expect, there are a bunch of extra features and checks.
+
+    The macro supports both SCD2 (TXTEMPORAL) and UPDATE loadtypes. If the
+    base table contains a PROCESSED_DTTM column (or similar), this can be
+    ignored by declaring it in the `processed_dttm_var` parameter.
+
+    The macro is also flexible where columns have been added or removed from
+    the base table UNLESS there is a change to the primary key.
+
+    Changes to the primary key fields are NOT supported, and are likely to cause
+    unexpected results.
+
+    The following pre-flight checks are made:
+
+    @li All primary key columns exist on the base table
+    @li There is no change in variable TYPE for any of the columns
+    @li There is no reduction in variable LENGTH below the max-length of the
+      supplied values
+
+    Rules for stacking changes are as follows:
+
+    <table>
+    <tr>
+      <th>Transaction Type</th><th>Key Behaviour</th><th>Column Behaviour</th>
+    </tr>
+    <tr>
+      <td>Deletes</td>
+      <td>
+        The row is added to `&outDEL.` UNLESS it no longer exists
+        in the base table, in which case it is added to `&errDS.` instead.
+      </td>
+      <td>
+        Deletes are unaffected by the addition or removal of non Primary-Key
+        columns.
+      </td>
+    </tr>
+    <tr>
+      <td>Inserts</td>
+      <td>
+        Previously newly added rows are added to the `outADD` table UNLESS they
+        are present in the Base table.<br>In this case they are added to the
+        `&errDS.` table instead.
+      </td>
+      <td>
+        Inserts are unaffected by the addition of columns in the Base Table
+        (they are padded with blanks).  Deleted columns are only a problem if
+        they appear on the previous insert - in which case the record is added
+        to `&errDS.`.
+      </td>
+    </tr>
+    <tr>
+      <td>Updates</td>
+      <td>
+        Previously modified rows are merged with base table values such that
+        only the individual cells that were _previously_ changed are re-applied.
+        Where the row contains cells that were not marked as having changed in
+        the prior transaction, the 'blanks' are filled with base table values in
+        the `outMOD` table.<br>
+        If the row no longer exists on the base table, then the row is added to
+        the `errDS` table instead.
+      </td>
+      <td>
+        Updates are unaffected by the addition of columns in the Base Table -
+        the new cells are simply populated with Base Table values.  Deleted
+        columns are only a problem if they relate to a modified cell
+        (`is_diff=1`) - in which case the record is added to `&errDS.`.
+      </td>
+    </tr>
+    </table>
+
+    To illustrate the above with a diagram:
+
+  @dot
+    digraph {
+      rankdir="TB"
+      start[label="Transaction Type?" shape=Mdiamond]
+      del[label="Does Base Row exist?" shape=rectangle]
+      add [label="Does Base Row exist?" shape=rectangle]
+      mod [label="Does Base Row exist?" shape=rectangle]
+      chkmod [label="Do all modified\n(is_diff=1) cells exist?" shape=rectangle]
+      chkadd [label="Do all inserted cells exist?" shape=rectangle]
+      outmod [label="outMOD\nTable" shape=Msquare style=filled]
+      outadd [label="outADD\nTable" shape=Msquare style=filled]
+      outdel [label="outDEL\nTable" shape=Msquare style=filled]
+      outerr [label="ErrDS Table" shape=Msquare fillcolor=Orange style=filled]
+      start -> del [label="Delete"]
+      start -> add [label="Insert"]
+      start -> mod [label="Update"]
+
+      del -> outdel [label="Yes"]
+      del -> outerr [label="No" color="Red" fontcolor="Red"]
+      add -> chkadd [label="No"]
+      add -> outerr [label="Yes" color="Red" fontcolor="Red"]
+      mod -> outerr [label="No" color="Red" fontcolor="Red"]
+      mod -> chkmod [label="Yes"]
+      chkmod -> outerr [label="No" color="Red" fontcolor="Red"]
+      chkmod -> outmod [label="Yes"]
+      chkadd -> outerr [label="No" color="Red" fontcolor="Red"]
+      chkadd -> outadd [label="Yes"]
+
+    }
+  @enddot
+
+    For examples of usage, check out the mp_stackdiffs.test.sas program.
+
+
+  @param [in] baselibds Base Table against which the changes will be applied,
+    in libref.dataset format.
+  @param [in] auditlibds Dataset with previously applied transactions, to be
+    re-applied. Use libref.dataset format.
+    DDL as follows:  %mp_coretable(DIFFTABLE)
+  @param [in] key Space seperated list of key variables
+  @param [in] mdebug= Set to 1 to enable DEBUG messages and preserve outputs
+  @param [in] processed_dttm_var= (0) If a variable is being used to mark
+    the processed datetime, put the name of the variable here.  It will NOT
+    be included in the staged dataset (the load process is expected to
+    provide this)
+  @param [out] errds= (work.errds) Output table containing problematic records.
+    The columns of this table are:
+    @li PK_VARS - Space separated list of primary key variable names
+    @li PK_VALS - Slash separted list of PK variable values
+    @li ERR_MSG - Explanation of why this record is problematic
+  @param [out] outmod= (work.outmod) Output table containing modified records
+  @param [out] outadd= (work.outadd) Output table containing additional records
+  @param [out] outdel= (work.outdel) Output table containing deleted records
+
+  <h4> SAS Macros </h4>
+  @li mf_existvarlist.sas
+  @li mf_getquotedstr.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_islibds.sas
+  @li mf_nobs.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_abort.sas
+  @li mp_ds2squeeze.sas
+
+  <h4> Related Macros </h4>
+  @li mp_coretable.sas
+  @li mp_stackdiffs.test.sas
+  @li mp_storediffs.sas
+
+  @todo The current approach assumes that a variable called KEY_HASH is not on
+    the base table.  This part will need to be refactored (eg using
+    mf_getuniquename.sas) when such a use case arises.
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mp_stackdiffs(baselibds
+  ,auditlibds
+  ,key
+  ,mdebug=0
+  ,processed_dttm_var=0
+  ,errds=work.errds
+  ,outmod=work.outmod
+  ,outadd=work.outadd
+  ,outdel=work.outdel
+)/*/STORE SOURCE*/;
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+/* input parameter validations */
+%mp_abort(iftrue= (%mf_islibds(&baselibds) ne 1)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid baselibds: &baselibds)
+)
+%mp_abort(iftrue= (%mf_islibds(&auditlibds) ne 1)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid auditlibds: &auditlibds)
+)
+%mp_abort(iftrue= (%length(&key)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Missing key variables!)
+)
+%mp_abort(iftrue= (
+    %mf_existVarList(&auditlibds,LIBREF DSN MOVE_TYPE KEY_HASH TGTVAR_NM IS_PK
+      IS_DIFF TGTVAR_TYPE OLDVAL_NUM NEWVAL_NUM OLDVAL_CHAR NEWVAL_CHAR)=0
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Input &auditlibds is missing required columns!)
+)
+
+
+/* set up macro vars */
+%local prefix dslist x var keyjoin commakey keepvars missvars fref;
+%let prefix=%substr(%mf_getuniquename(),1,25);
+%let dslist=ds1d ds2d ds3d ds1a ds2a ds3a ds1m ds2m ds3m pks dups base
+  delrec delerr addrec adderr modrec moderr;
+%do x=1 %to %sysfunc(countw(&dslist));
+  %let var=%scan(&dslist,&x);
+  %local &var;
+  %let &var=%upcase(&prefix._&var);
+%end;
+
+%let key=%upcase(&key);
+%let commakey=%mf_getquotedstr(&key,quote=N);
+
+%let keyjoin=1=1;
+%do x=1 %to %sysfunc(countw(&key));
+  %let var=%scan(&key,&x);
+  %let keyjoin=&keyjoin and a.&var=b.&var;
+%end;
+
+data &errds;
+  length pk_vars $256 pk_vals $4098 err_msg $512;
+  call missing (of _all_);
+  stop;
+run;
+
+/**
+  * Prepare raw DELETE table
+  * Records are in the OLDVAL_xxx columns
+  */
+%let keepvars=MOVE_TYPE KEY_HASH TGTVAR_NM TGTVAR_TYPE IS_PK
+              OLDVAL_NUM OLDVAL_CHAR
+              NEWVAL_NUM NEWVAL_CHAR;
+proc sort data=&auditlibds(where=(move_type='D') keep=&keepvars)
+  out=&ds1d(drop=move_type);
+by KEY_HASH TGTVAR_NM;
+run;
+proc transpose data=&ds1d(where=(tgtvar_type='N'))
+    out=&ds2d(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var OLDVAL_NUM;
+run;
+proc transpose data=&ds1d(where=(tgtvar_type='C'))
+    out=&ds3d(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var OLDVAL_CHAR;
+run;
+%mp_ds2squeeze(&ds2d,outds=&ds2d)
+%mp_ds2squeeze(&ds3d,outds=&ds3d)
+data &outdel;
+  if 0 then set &baselibds;
+  set &ds2d;
+  set &ds3d;
+  drop key_hash;
+  if not missing(%scan(&key,1));
+run;
+proc sort;
+  by &key;
+run;
+
+/**
+  * Prepare raw APPEND table
+  * Records are in the NEWVAL_xxx columns
+  */
+proc sort data=&auditlibds(where=(move_type='A') keep=&keepvars)
+    out=&ds1a(drop=move_type);
+  by KEY_HASH TGTVAR_NM;
+run;
+proc transpose data=&ds1a(where=(tgtvar_type='N'))
+    out=&ds2a(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var NEWVAL_NUM;
+run;
+proc transpose data=&ds1a(where=(tgtvar_type='C'))
+    out=&ds3a(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var NEWVAL_CHAR;
+run;
+%mp_ds2squeeze(&ds2a,outds=&ds2a)
+%mp_ds2squeeze(&ds3a,outds=&ds3a)
+data &outadd;
+  if 0 then set &baselibds;
+  set &ds2a;
+  set &ds3a;
+  drop key_hash;
+  if not missing(%scan(&key,1));
+run;
+proc sort;
+  by &key;
+run;
+
+/**
+  * Prepare raw MODIFY table
+  * Keep only primary key - will add modified values later
+  */
+proc sort data=&auditlibds(
+      where=(move_type='M' and is_pk=1) keep=&keepvars
+    ) out=&ds1m(drop=move_type);
+  by KEY_HASH TGTVAR_NM;
+run;
+proc transpose data=&ds1m(where=(tgtvar_type='N'))
+    out=&ds2m(drop=_name_);
+  by KEY_HASH ;
+  id TGTVAR_NM;
+  var NEWVAL_NUM;
+run;
+proc transpose data=&ds1m(where=(tgtvar_type='C'))
+    out=&ds3m(drop=_name_);
+  by KEY_HASH;
+  id TGTVAR_NM;
+  var NEWVAL_CHAR;
+run;
+%mp_ds2squeeze(&ds2m,outds=&ds2m)
+%mp_ds2squeeze(&ds3m,outds=&ds3m)
+data &outmod;
+  if 0 then set &baselibds;
+  set &ds2m;
+  set &ds3m;
+  if not missing(%scan(&key,1));
+run;
+proc sort;
+  by &key;
+run;
+
+/**
+  * Extract matching records from the base table
+  * Do this in one join for efficiency.
+  * At a later date, this should be optimised for large database tables by using
+  * passthrough and a temporary table.
+  */
+data &pks;
+  if 0 then set &baselibds;
+  set &outadd &outmod &outdel;
+  keep &key;
+run;
+
+proc sort noduprec dupout=&dups;
+by &key;
+run;
+data _null_;
+  set &dups;
+  putlog (_all_)(=);
+run;
+%mp_abort(iftrue= (%mf_nobs(&dups) ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(duplicates (%mf_nobs(&dups)) found on &auditlibds!)
+)
+
+proc sql;
+create table &base as
+  select a.*
+  from &baselibds a, &pks b
+  where &keyjoin;
+
+/**
+  * delete check
+  * This is straightforward as it relates to records only
+  */
+proc sql;
+create table &delrec as
+  select a.*
+  from &outdel a
+  left join &base b
+  on &keyjoin
+  where b.%scan(&key,1) is null
+  order by &commakey;
+
+data &delerr;
+  if 0 then set &errds;
+  set &delrec;
+  PK_VARS="&key";
+  PK_VALS=catx('/',&commakey);
+  ERR_MSG="Rows cannot be deleted as they do not exist on the Base dataset";
+  keep PK_VARS PK_VALS ERR_MSG;
+run;
+proc append base=&errds data=&delerr;
+run;
+
+data &outdel;
+  merge &outdel (in=a) &delrec (in=b);
+  by &key;
+  if not b;
+run;
+
+/**
+  * add check
+  * Problems - where record already exists, or base table has columns missing
+  */
+%let missvars=%mf_wordsinstr1butnotstr2(
+  Str1=%upcase(%mf_getvarlist(&outadd)),
+  Str2=%upcase(%mf_getvarlist(&baselibds))
+);
+%if %length(&missvars)>0 %then %do;
+    /* add them to the err table */
+  data &adderr;
+    if 0 then set &errds;
+    set &outadd;
+    PK_VARS="&key";
+    PK_VALS=catx('/',&commakey);
+    ERR_MSG="Rows cannot be added due to missing base vars: &missvars";
+    keep PK_VARS PK_VALS ERR_MSG;
+  run;
+  proc append base=&errds data=&adderr;
+  run;
+  proc sql;
+  delete * from &outadd;
+%end;
+%else %do;
+  proc sql;
+  /* find records that already exist on base table */
+  create table &addrec as
+    select a.*
+    from &outadd a
+    inner join &base b
+    on &keyjoin
+    order by &commakey;
+
+  /* add them to the err table */
+  data &adderr;
+    if 0 then set &errds;
+    set &addrec;
+    PK_VARS="&key";
+    PK_VALS=catx('/',&commakey);
+    ERR_MSG="Rows cannot be added as they already exist on the Base dataset";
+    keep PK_VARS PK_VALS ERR_MSG;
+  run;
+  proc append base=&errds data=&adderr;
+  run;
+
+  /* remove invalid rows from the outadd table */
+  data &outadd;
+    merge &outadd (in=a) &addrec (in=b);
+    by &key;
+    if not b;
+  run;
+%end;
+
+/**
+  * mod check
+  * Problems - where record does not exist or baseds has modified cols missing
+  */
+proc sql noprint;
+select distinct tgtvar_nm into: missvars separated by ' '
+  from &auditlibds
+  where move_type='M' and is_diff=1;
+%let missvars=%mf_wordsinstr1butnotstr2(
+  Str1=&missvars,
+  Str2=%upcase(%mf_getvarlist(&baselibds))
+);
+%if %length(&missvars)>0 %then %do;
+    /* add them to the err table */
+  data &moderr;
+    if 0 then set &errds;
+    set &outmod;
+    PK_VARS="&key";
+    PK_VALS=catx('/',&commakey);
+    ERR_MSG="Rows cannot be modified due to missing base vars: &missvars";
+    keep PK_VARS PK_VALS ERR_MSG;
+  run;
+  proc append base=&errds data=&moderr;
+  run;
+  proc sql;
+  delete * from &outmod;
+%end;
+%else %do;
+  /* now check for records that do not exist (therefore cannot be modified) */
+  proc sql;
+  create table &modrec as
+    select a.*
+    from &outmod a
+    left join &base b
+    on &keyjoin
+    where b.%scan(&key,1) is null
+    order by &commakey;
+  data &moderr;
+    if 0 then set &errds;
+    set &modrec;
+    PK_VARS="&key";
+    PK_VALS=catx('/',&commakey);
+    ERR_MSG="Rows cannot be modified as they do not exist on the Base dataset";
+    keep PK_VARS PK_VALS ERR_MSG;
+  run;
+  proc append base=&errds data=&moderr;
+  run;
+  /* delete the above records from the outmod table */
+  data &outmod;
+    merge &outmod (in=a) &modrec (in=b);
+    by &key;
+    if not b;
+  run;
+  /* now - we can prepare the final MOD table (which is currently PK only) */
+  proc sql undo_policy=none;
+  create table &outmod as
+    select a.key_hash
+      ,b.*
+    from &outmod a
+    inner join &base b
+    on &keyjoin
+    order by &commakey;
+  /* now - to update outmod with modified (is_diff=1) values */
+  %let fref=%mf_getuniquefileref();
+  data _null_;
+    file &fref;
+    set &auditlibds(where=(move_type='M')) end=lastobs;
+    by key_hash;
+    retain comma 'N';
+    if _n_=1 then put 'proc sql;';
+    if first.key_hash then do;
+      comma='N';
+      put "update &outmod set " @@;
+    end;
+    if is_diff=1 then do;
+      if comma='N' then do;
+        put '  '@@;
+        comma='Y';
+      end;
+      else put '  ,'@@;
+      if tgtvar_type='C' then do;
+        length qstr $32767;
+        qstr=quote(trim(NEWVAL_CHAR));
+        put tgtvar_nm '=' qstr;
+      end;
+      else put tgtvar_nm '=' newval_num;
+      if comma='  ' then comma='  ,';
+    end;
+    if last.key_hash then put '  where key_hash=trim("' key_hash '");';
+    if lastobs then put "alter table &outmod drop key_hash;";
+  run;
+  %inc &fref/source2;
+%end;
+
+%if &mdebug=0 %then %do;
+  proc datasets lib=work;
+    delete &prefix:;
+  run;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%mend mp_stackdiffs;
+/** @endcond */

base/mp_storediffs.sas

@@ -49,41 +49,23 @@
   @param [in] appds= (0) Dataset with appended records
   @param [in] modds= (0) Dataset with modified records
   @param [out] outds= (work.mp_storediffs) Output table containing stored data.
-    Has the following format:
+    DDL as follows:  %mp_coretable(DIFFTABLE)
 
-        proc sql;
-        create table &outds(
-          load_ref char(36) label='unique load reference',
-          processed_dttm num format=E8601DT26.6 label='Processed at timestamp',
-          libref char(8) label='Library Reference (8 chars)',
-          dsn char(32) label='Dataset Name (32 chars)',
-          key_hash char(32) label=
-            'MD5 Hash of primary key values (pipe seperated)',
-          move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified',
-          is_pk num label='Is Primary Key Field? (1/0)',
-          is_diff num label=
-            'Did value change? (1/0/-1).  Always -1 for appends and deletes.',
-          tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
-          tgtvar_nm char(32) label='Target variable name (32 chars)',
-          oldval_num num format=best32. label='Old (numeric) value',
-          newval_num num format=best32. label='New (numeric) value',
-          oldval_char char(32765) label='Old (character) value',
-          newval_char char(32765) label='New (character) value',
-          constraint pk_mpe_audit
-            primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
-        );
-
-    @param [in] processed_dttm= (0) Provide a datetime constant in relation to
-      the actual load time.  If not provided, current timestamp is used.
-    @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
-    @param [out] loadref= (0) Provide a unique key to reference the load,
-      otherwise a UUID will be generated.
+  @param [in] processed_dttm= (0) Provide a datetime constant in relation to
+    the actual load time.  If not provided, current timestamp is used.
+  @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
+  @param [out] loadref= (0) Provide a unique key to reference the load,
+    otherwise a UUID will be generated.
 
   <h4> SAS Macros </h4>
   @li mf_getquotedstr.sas
   @li mf_getuniquename.sas
   @li mf_getvarlist.sas
 
+  <h4> Related Macros </h4>
+  @li mp_stackdiffs.sas
+  @li mp_storediffs.test.sas
+
   @version 9.2
   @author Allan Bowe
 **/
@@ -108,7 +90,7 @@
 %else %let dbg=*;
 
 /* set up unique and temporary vars */
-%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist;
+%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist vlist;
 %let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_ds1));
 %let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_ds2));
 %let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_ds3));
@@ -162,12 +144,21 @@ proc transpose data=&ds1
   by &inds_keep &hashkey;
   var _character_;
 run;
+
+%if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do;
+  /* this is a format catalog - cannot query cols directly */
+  %let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH"
+    ,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO"
+    ,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
+%end;
+%else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
+
 data &ds4;
   length &inds_keep $41 tgtvar_nm $32;
   set &ds2 &ds3 indsname=&inds_auto;
 
   tgtvar_nm=upcase(tgtvar_nm);
-  if tgtvar_nm in (%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE)));
+  if tgtvar_nm in (%upcase(&vlist));
 
   if &inds_auto="&ds2" then tgtvar_type='N';
   else if &inds_auto="&ds3" then tgtvar_type='C';

base/mp_streamfile.sas

@@ -12,17 +12,20 @@
 
       %mp_streamfile(contenttype=csv,inloc=/some/where.txt,outname=myfile.txt)
 
+  @param [in] contenttype= (TEXT) Either TEXT, ZIP, CSV, EXCEL
+  @param [in] inloc= /path/to/file.ext to be sent
+  @param [in] inref= fileref of file to be sent (if provided, overrides `inloc`)
+  @param [in] iftrue= (1=1) Provide a condition under which to execute.
+  @param [out] outname= the name of the file, as downloaded by the browser
+  @param [out] outref= (_webout) The destination where the file should be
+    streamed.
+
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
+  @li mfs_httpheader.sas
   @li mp_binarycopy.sas
 
-  @param contenttype= Either TEXT, ZIP, CSV, EXCEL (default TEXT)
-  @param inloc= /path/to/file.ext to be sent
-  @param inref= fileref of file to be sent (if provided, overrides `inloc`)
-  @param outname= the name of the file, as downloaded by the browser
-
   @author Allan Bowe
-  @source https://github.com/sasjs/core
 
 **/
 
@@ -30,12 +33,16 @@
   contenttype=TEXT
   ,inloc=
   ,inref=0
+  ,iftrue=%str(1=1)
   ,outname=
+  ,outref=_webout
 )/*/STORE SOURCE*/;
 
-%let contentype=%upcase(&contenttype);
-%local platform; %let platform=%mf_getplatform();
+%if not(%eval(%unquote(&iftrue))) %then %return;
 
+%let contentype=%upcase(&contenttype);
+%let outref=%upcase(&outref);
+%local platform; %let platform=%mf_getplatform();
 
 /**
   * check engine type to avoid the below err message:
@@ -44,39 +51,101 @@
 %local streamweb;
 %let streamweb=0;
 data _null_;
-  set sashelp.vextfl(where=(upcase(fileref)="_WEBOUT"));
+  set sashelp.vextfl(where=(upcase(fileref)="&outref"));
   if xengine='STREAM' then call symputx('streamweb',1,'l');
 run;
 
-%if &contentype=ZIP %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+%if &contentype=CSV %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/zip');
+      rc=stpsrv_header('Content-type','application/csv');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.zip'
-      contenttype='application/zip'
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
+      contenttype='application/csv'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,application/csv)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
 %else %if &contentype=EXCEL %then %do;
   /* suitable for XLS format */
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/vnd.ms-excel');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
       contenttype='application/vnd.ms-excel'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,application/vnd.ms-excel)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
+%end;
+%else %if &contentype=GIF or &contentype=JPEG or &contentype=PNG %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type',"image/%lowcase(&contenttype)");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI"
+      contenttype="image/%lowcase(&contenttype)";
+  %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,image/%lowcase(&contenttype))
+  %end;
+%end;
+%else %if &contentype=HTML %then %do;
+  %if &platform=SASVIYA %then %do;
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
+      contenttype="text/html";
+  %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,text/html)
+  %end;
+%end;
+%else %if &contentype=TEXT %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type','application/text');
+      rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
+      contenttype='application/text'
+      contentdisp="attachment; filename=&outname";
+  %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,application/text)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
+%end;
+%else %if &contentype=WOFF or &contentype=WOFF2 or &contentype=TTF %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type',"font/%lowcase(&contenttype)");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI"
+      contenttype="font/%lowcase(&contenttype)";
+  %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,font/%lowcase(&contenttype))
+  %end;
 %end;
 %else %if &contentype=XLSX %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
       rc=stpsrv_header('Content-type',
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
@@ -84,54 +153,44 @@ run;
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
       contenttype=
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type
+      ,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+    )
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
-%else %if &contentype=TEXT %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+%else %if &contentype=ZIP %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/text');
+      rc=stpsrv_header('Content-type','application/zip');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
-      contenttype='application/text'
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.zip'
+      contenttype='application/zip'
       contentdisp="attachment; filename=&outname";
   %end;
-%end;
-%else %if &contentype=CSV %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
-    data _null_;
-      rc=stpsrv_header('Content-type','application/csv');
-      rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
-    run;
-  %end;
-  %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
-      contenttype='application/csv'
-      contentdisp="attachment; filename=&outname";
-  %end;
-%end;
-%else %if &contentype=HTML %then %do;
-  %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
-      contenttype="text/html";
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,application/zip)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
   %end;
 %end;
 %else %do;
   %put %str(ERR)OR: Content Type &contenttype NOT SUPPORTED by &sysmacroname!;
-  %return;
 %end;
 
 %if &inref ne 0 %then %do;
-  %mp_binarycopy(inref=&inref,outref=_webout)
+  %mp_binarycopy(inref=&inref,outref=&outref)
 %end;
 %else %do;
-  %mp_binarycopy(inloc="&inloc",outref=_webout)
+  %mp_binarycopy(inloc="&inloc",outref=&outref)
 %end;
 
 %mend mp_streamfile;

base/mp_testservice.sas

@@ -1,12 +1,10 @@
 /**
-  @file mp_testservice.sas
-  @brief Will execute a test against a SASjs web service on SAS 9 or Viya
+  @file
+  @brief Will execute a SASjs web service on SAS 9 or Viya
   @details Prepares the input files and retrieves the resulting datasets from
   the response JSON.
 
-      %mp_testjob(
-        duration=60*5
-      )
+
 
   Note - the _webout fileref should NOT be assigned prior to running this macro.
 
@@ -14,6 +12,10 @@
   @param [in] inputfiles=(0) A list of space seperated fileref:filename pairs as
     follows:
         inputfiles=inref:filename inref2:filename2
+  @param [in] inputdatasets= (0) All datasets in this space seperated list are
+    converted into SASJS-formatted CSVs (see mp_ds2csv.sas) files and added to
+    the list of `inputfiles` for ingestion.  The dataset will be sent with the
+    same name (no need for a colon modifier).
   @param [in] inputparams=(0) A dataset containing name/value pairs in the
     following format:
     |name:$32|value:$1000|
@@ -38,9 +40,13 @@
   @li mf_getuniquename.sas
   @li mp_abort.sas
   @li mp_binarycopy.sas
+  @li mp_ds2csv.sas
   @li mv_getjobresult.sas
   @li mv_jobflow.sas
 
+  <h4> Related Programs </h4>
+  @li mp_testservice.test.sas
+
   @version 9.4
   @author Allan Bowe
 
@@ -48,6 +54,7 @@
 
 %macro mp_testservice(program,
   inputfiles=0,
+  inputdatasets=0,
   inputparams=0,
   debug=log,
   mdebug=0,
@@ -56,7 +63,7 @@
   viyaresult=WEBOUT_JSON,
   viyacontext=SAS Job Execution compute context
 )/*/STORE SOURCE*/;
-%local dbg;
+%local dbg pcnt fref1 webref i webcount var platform;
 %if &mdebug=1 %then %do;
   %put &sysmacroname entry vars:;
   %put _local_;
@@ -64,7 +71,6 @@
 %else %let dbg=*;
 
 /* sanitise inputparams */
-%local pcnt;
 %let pcnt=0;
 %if &inputparams ne 0 %then %do;
   data _null_;
@@ -76,7 +82,7 @@
     else do;
       x+1;
       call symputx(name,quote(cats(value)),'l');
-      call symputx('pval'!!left(x),name,'l');
+      call symputx(cats('pval',x),name,'l');
       call symputx('pcnt',x,'l');
     end;
   run;
@@ -86,17 +92,25 @@
   )
 %end;
 
+/* convert inputdatasets to filerefs */
+%if "&inputdatasets" ne "0" %then %do;
+  %if %quote(&inputfiles)=0 %then %let inputfiles=;
+  %do i=1 %to %sysfunc(countw(&inputdatasets,%str( )));
+    %let var=%scan(&inputdatasets,&i,%str( ));
+    %local dsref&i;
+    %let dsref&i=%mf_getuniquefileref();
+    %mp_ds2csv(&var,outref=&&dsref&i,headerformat=SASJS)
+    %let inputfiles=&inputfiles &&dsref&i:%scan(&var,-1,.);
+  %end;
+%end;
+
 
-%local fref1 webref;
 %let fref1=%mf_getuniquefileref();
 %let webref=%mf_getuniquefileref();
-
-%local platform;
 %let platform=%mf_getplatform();
 %if &platform=SASMETA %then %do;
 
   /* parse the input files */
-  %local webcount i var;
   %if %quote(&inputfiles) ne 0 %then %do;
     %let webcount=%sysfunc(countw(&inputfiles));
     %put &=webcount;

base/mp_validatecol.sas

@@ -20,10 +20,13 @@
       ;;;;
       run;
 
+  For more examples, see mp_validatecol.test.sas
+
   Tip - when contributing, use https://regex101.com to test the regex validity!
 
   @param [in] incol The column to be validated
   @param [in] rule The rule to apply.  Current rules:
+    @li ISINT - checks if the variable is an integer
     @li ISNUM - checks if the variable is numeric
     @li LIBDS - matches LIBREF.DATASET format
     @li FORMAT - checks if the provided format is syntactically valid
@@ -32,6 +35,9 @@
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas
 
+  <h4> Related Macros </h4>
+  @li mp_validatecol.test.sas
+
   @version 9.3
 **/
 
@@ -41,7 +47,15 @@
 %local tempcol;
 %let tempcol=%mf_getuniquename();
 
-%if &rule=ISNUM %then %do;
+%if &rule=ISINT %then %do;
+  &outcol=0;
+  if not missing(&incol) then do;
+    &tempcol=input(&incol,?? best32.);
+    if not missing(&tempcol) then if mod(&tempcol,1)=0 then &outcol=1;
+  end;
+  drop &tempcol;
+%end;
+%else %if &rule=ISNUM %then %do;
   /*
     credit SØREN LASSEN
     https://sasmacro.blogspot.com/2009/06/welcome-isnum-macro.html

base/mp_wait4file.sas

@@ -4,9 +4,7 @@
   @details Loops with a `sleep()` command until a file arrives or the max wait
   period expires.
 
-  @example
-
-  Wait 3 minutes OR for /tmp/flag.txt to appear
+  Example: Wait 3 minutes OR for /tmp/flag.txt to appear
 
     %mp_wait4file(/tmp/flag.txt , maxwait=60*3)
 

build.py

@@ -84,7 +84,7 @@ options noquotelenmax;
 """
 f = open('all.sas', "w")             # r / r+ / rb / rb+ / w / wb
 f.write(header)
-folders=['base','meta','metax','server','viya','lua','fcmp']
+folders=['base','ddl','meta','metax','server','viya','lua','fcmp']
 for folder in folders:
     filenames = [fn for fn in Path('./' + folder).iterdir() if fn.match("*.sas")]
     filenames.sort()

ddl/mddl_dc_difftable.sas

@@ -0,0 +1,34 @@
+/**
+  @file
+  @brief Difftable DDL
+  @details Used to store changes to tables.  Used by mp_storediffs.sas
+      and mp_stackdiffs.sas
+
+**/
+
+
+%macro mddl_dc_difftable(libds=WORK.DIFFTABLE);
+
+  proc sql;
+  create table &libds(
+      load_ref char(36) label='unique load reference',
+      processed_dttm num format=E8601DT26.6 label='Processed at timestamp',
+      libref char(8) label='Library Reference (8 chars)',
+      dsn char(32) label='Dataset Name (32 chars)',
+      key_hash char(32) label=
+        'MD5 Hash of primary key values (pipe seperated)',
+      move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified',
+      is_pk num label='Is Primary Key Field? (1/0)',
+      is_diff num label=
+        'Did value change? (1/0/-1).  Always -1 for appends and deletes.',
+      tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
+      tgtvar_nm char(32) label='Target variable name (32 chars)',
+      oldval_num num format=best32. label='Old (numeric) value',
+      newval_num num format=best32. label='New (numeric) value',
+      oldval_char char(32765) label='Old (character) value',
+      newval_char char(32765) label='New (character) value',
+    constraint pk_mpe_audit
+      primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
+  );
+
+%mend mddl_dc_difftable;

ddl/mddl_dc_filterdetail.sas

@@ -0,0 +1,27 @@
+/**
+  @file
+  @brief Filtertable DDL
+  @details For storing detailed filter values.  Used by
+      mp_filterstore.sas.
+
+**/
+
+
+%macro mddl_dc_filterdetail(libds=WORK.FILTER_DETAIL);
+
+  proc sql;
+  create table &libds(
+      filter_hash char(32) not null,
+      filter_line num not null,
+      group_logic char(3) not null,
+      subgroup_logic char(3) not null,
+      subgroup_id num not null,
+      variable_nm varchar(32) not null,
+      operator_nm varchar(12) not null,
+      raw_value varchar(4000) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_hash,filter_line)
+  );
+
+%mend mddl_dc_filterdetail;

ddl/mddl_dc_filtersummary.sas

@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Filtersummary DDL
+  @details For storing summary filter values.  Used by
+      mp_filterstore.sas.
+
+**/
+
+
+%macro mddl_dc_filtersummary(libds=WORK.FILTER_SUMMARY);
+
+  proc sql;
+  create table &libds(
+      filter_rk num not null,
+      filter_hash char(32) not null,
+      filter_table char(41) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_rk)
+  );
+
+%mend mddl_dc_filtersummary;

ddl/mddl_dc_locktable.sas

@@ -0,0 +1,25 @@
+/**
+  @file
+  @brief Locktable DDL
+  @details For "locking" tables prior to multipass loads. Used by
+      mp_lockanytable.sas
+
+**/
+
+
+%macro mddl_dc_locktable(libds=WORK.LOCKTABLE);
+
+  proc sql;
+  create table &libds(
+      lock_lib char(8),
+      lock_ds char(32),
+      lock_status_cd char(10) not null,
+      lock_user_nm char(100) not null ,
+      lock_ref char(200),
+      lock_pid char(10),
+      lock_start_dttm num format=E8601DT26.6,
+      lock_end_dttm num format=E8601DT26.6,
+    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds)
+  );
+
+%mend mddl_dc_locktable;

ddl/mddl_dc_maxkeytable.sas

@@ -0,0 +1,24 @@
+/**
+  @file
+  @brief Maxkeytable DDL
+  @details For storing the maximum retained key information.  Used
+      by mp_retainedkey.sas
+
+**/
+
+
+%macro mddl_dc_maxkeytable(libds=WORK.MAXKEYTABLE);
+
+  proc sql;
+  create table &libds(
+      keytable varchar(41) label='Base table in libref.dataset format',
+      keycolumn char(32) format=$32.
+        label='The Retained key field containing the key values.',
+      max_key num label=
+        'Integer representing current max RK or SK value in the KEYTABLE',
+      processed_dttm num format=E8601DT26.6
+        label='Datetime this value was last updated',
+    constraint pk_mpe_maxkeyvalues
+        primary key(keytable));
+
+%mend mddl_dc_maxkeytable;

ddl/mddl_sas_cntlout.sas

@@ -0,0 +1,43 @@
+/**
+  @file
+  @brief The CNTLOUT table generated by proc format
+  @details This table will actually change format depending on the data values,
+  therefore the max possible lengths are described here to enable consistency
+  when dealing with format data.
+
+**/
+
+
+%macro mddl_sas_cntlout(libds=WORK.CNTLOUT);
+
+proc sql;
+create table &libds(
+    FMTNAME char(32)      label='Format name'
+    /*
+      to accomodate larger START values, mp_loadformat.sas will need the
+      SQL dependency removed (proc sql needs to accomodate 3 index values in
+      a 32767 ibufsize limit)
+    */
+    ,START char(10000)    label='Starting value for format'
+    ,END char(32767)      label='Ending value for format'
+    ,LABEL char(32767)    label='Format value label'
+    ,MIN num length=3     label='Minimum length'
+    ,MAX num length=3     label='Maximum length'
+    ,DEFAULT num length=3 label='Default length'
+    ,LENGTH num length=3  label='Format length'
+    ,FUZZ num             label='Fuzz value'
+    ,PREFIX char(2)       label='Prefix characters'
+    ,MULT num             label='Multiplier'
+    ,FILL char(1)         label='Fill character'
+    ,NOEDIT num length=3  label='Is picture string noedit?'
+    ,TYPE char(1)         label='Type of format'
+    ,SEXCL char(1)        label='Start exclusion'
+    ,EEXCL char(1)        label='End exclusion'
+    ,HLO char(13)         label='Additional information'
+    ,DECSEP char(1)       label='Decimal separator'
+    ,DIG3SEP char(1)      label='Three-digit separator'
+    ,DATATYPE char(8)     label='Date/time/datetime?'
+    ,LANGUAGE char(8)     label='Language for date strings'
+);
+
+%mend mddl_sas_cntlout;

fcmp/mcf_getfmttype.sas

@@ -0,0 +1,118 @@
+/**
+  @file
+  @brief Returns the type of the format
+  @details
+  Returns the type, eg DATE / DATETIME / TIME (based on hard-coded lookup)
+  else CHAR / NUM.
+
+  This macro may be extended in the future to support custom formats - this
+  would necessitate a call to `dosubl()` for running a proc format with cntlout.
+
+  The function itself takes the following (positional) parameters:
+
+  | PARAMETER | DESCRIPTION |
+  |---|---|
+  |fmtnm| Format name to be tested.  Can be with or without the w.d extension.|
+
+  Usage:
+
+      %mcf_getfmttype(wrap=YES, insert_cmplib=YES)
+
+      data _null_;
+        fmt1=mcf_getfmttype('DATE9.');
+        fmt2=mcf_getfmttype('DATETIME');
+        put (fmt:)(=);
+      run;
+      %put fmt3=%sysfunc(mcf_getfmttype(TIME9.));
+
+  Returns:
+
+  > fmt1=DATE fmt2=DATETIME
+  > fmt3=TIME
+
+  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
+  @param [out] lib= (work) The output library in which to create the catalog.
+  @param [out] cat= (sasjs) The output catalog in which to create the package.
+  @param [out] pkg= (utils) The output package in which to create the function.
+    Uses a 3 part format:  libref.catalog.package
+  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
+    values inserted only if needed.
+
+  <h4> SAS Macros </h4>
+  @li mcf_init.sas
+
+  <h4> Related Programs </h4>
+  @li mcf_getfmttype.test.sas
+  @li mp_init.sas
+
+  @todo "Custom Format Lookups" To enable site-specific formats, make
+  use of a set of SASJS_FMTLIST_(DATATYPE) global variables.
+
+**/
+
+%macro mcf_getfmttype(wrap=NO
+  ,insert_cmplib=DEPRECATED
+  ,lib=WORK
+  ,cat=SASJS
+  ,pkg=UTILS
+)/*/STORE SOURCE*/;
+%local i var cmpval found;
+
+%if %mcf_init(mcf_getfmttype)=1 %then %return;
+
+%if &wrap=YES  %then %do;
+  proc fcmp outlib=&lib..&cat..&pkg;
+%end;
+
+function mcf_getfmttype(fmtnm $) $8;
+  if substr(fmtnm,1,1)='$' then return('CHAR');
+  else do;
+    /* extract NAME */
+    length fmt $32;
+    fmt=scan(fmtnm,1,'.');
+    do while (
+      substr(fmt,length(fmt),1) in ('1','2','3','4','5','6','7','8','9','0')
+      );
+      if length(fmt)=1 then fmt='W';
+      else fmt=substr(fmt,1,length(fmt)-1);
+    end;
+
+    /* apply lookups */
+    if cats(fmt) in ('DATETIME','B8601DN','B8601DN','B8601DT','B8601DT'
+      ,'B8601DZ','B8601DZ','DATEAMPM','DTDATE','DTMONYY','DTWKDATX','DTYEAR'
+      ,'DTYYQC','E8601DN','E8601DN','E8601DT','E8601DT','E8601DZ','E8601DZ')
+      then return('DATETIME');
+    else if fmt in ('DATE','YYMMDD','B8601DA','B8601DA','DAY','DDMMYY'
+      ,'DDMMYYB','DDMMYYC','DDMMYYD','DDMMYYN','DDMMYYP','DDMMYYS','DDMMYYx'
+      ,'DOWNAME','E8601DA','E8601DA','JULDAY','JULIAN','MMDDYY','MMDDYYB'
+      ,'MMDDYYC','MMDDYYD','MMDDYYN','MMDDYYP','MMDDYYS','MMDDYYx','MMYY'
+      ,'MMYYC','MMYYD','MMYYN','MMYYP','MMYYS','MMYYx','MONNAME','MONTH'
+      ,'MONYY','PDJULG','PDJULI','QTR','QTRR','WEEKDATE','WEEKDATX','WEEKDAY'
+      ,'WEEKU','WEEKV','WEEKW','WORDDATE','WORDDATX','YEAR','YYMM','YYMMC'
+      ,'YYMMD','YYMMDDB','YYMMDDC','YYMMDDD','YYMMDDN','YYMMDDP','YYMMDDS'
+      ,'YYMMDDx','YYMMN','YYMMP','YYMMS','YYMMx','YYMON','YYQ','YYQC','YYQD'
+      ,'YYQN','YYQP','YYQR','YYQRC','YYQRD','YYQRN','YYQRP','YYQRS','YYQRx'
+      ,'YYQS','YYQx','YYQZ') then return('DATE');
+    else if fmt in ('TIME','B8601LZ','B8601LZ','B8601TM','B8601TM','B8601TZ'
+      ,'B8601TZ','E8601LZ','E8601LZ','E8601TM','E8601TM','E8601TZ','E8601TZ'
+      ,'HHMM','HOUR','MMSS','TIMEAMPM','TOD') then return('TIME');
+    else return('NUM');
+  end;
+endsub;
+
+%if &wrap=YES %then %do;
+  quit;
+%end;
+
+/* insert the CMPLIB if not already there */
+%let cmpval=%sysfunc(getoption(cmplib));
+%let found=0;
+%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
+  %let var=%scan(&cmpval,&i,%str( %(%)));
+  %if &var=&lib..&cat %then %let found=1;
+%end;
+%if &found=0 %then %do;
+  options insert=(CMPLIB=(&lib..&cat));
+%end;
+
+%mend mcf_getfmttype;

fcmp/mcf_init.sas

@@ -0,0 +1,44 @@
+/**
+  @file
+  @brief Sets up the mcf_xx functions
+  @details
+  There is no (efficient) way to determine if an mcf_xx macro has already been
+  invoked.  So, we make use of a global macro variable list to keep track.
+
+  Usage:
+
+      %mcf_init(MCF_LENGTH)
+
+  Returns:
+
+  > 1 (if already initialised) else 0
+
+  @param [in] func The function to be initialised
+
+  <h4> Related Macros </h4>
+  @li mcf_init.test.sas
+
+**/
+
+%macro mcf_init(func
+)/*/STORE SOURCE*/;
+
+%if not (%symexist(SASJS_PREFIX)) %then %do;
+  %global SASJS_PREFIX;
+  %let SASJS_PREFIX=SASJS;
+%end;
+
+%let func=%upcase(&func);
+
+/* the / character is just a seperator */
+%global &sasjs_prefix._FUNCTIONS;
+%if %index(&&&sasjs_prefix._FUNCTIONS,&func/)>0 %then %do;
+  1
+  %return;
+%end;
+%else %do;
+  %let &sasjs_prefix._FUNCTIONS=&&&sasjs_prefix._FUNCTIONS &func/;
+  0
+%end;
+
+%mend mcf_init;

fcmp/mcf_length.sas

@@ -0,0 +1,90 @@
+/**
+  @file
+  @brief Returns the length of a numeric value
+  @details
+  Returns the length, in bytes, of a numeric value.  If the value is
+  missing, then 0 is returned.
+
+  The function itself takes the following (positional) parameters:
+
+  | PARAMETER | DESCRIPTION |
+  |---|---|
+  | var | variable (or value) to be tested|
+
+  Usage:
+
+      %mcf_length(wrap=YES, insert_cmplib=YES)
+
+      data _null_;
+        ina=1;
+        inb=10000000;
+        inc=12345678;
+        ind=.;
+        outa=mcf_length(ina);
+        outb=mcf_length(inb);
+        outc=mcf_length(inc);
+        outd=mcf_length(ind);
+        put (out:)(=);
+      run;
+
+  Returns:
+
+  > outa=3 outb=4 outc=5 outd=0
+
+  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
+  @param [out] lib= (work) The output library in which to create the catalog.
+  @param [out] cat= (sasjs) The output catalog in which to create the package.
+  @param [out] pkg= (utils) The output package in which to create the function.
+    Uses a 3 part format:  libref.catalog.package
+  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
+    values inserted only if needed.
+
+  <h4> SAS Macros </h4>
+  @li mcf_init.sas
+
+  <h4> Related Programs </h4>
+  @li mcf_length.test.sas
+  @li mp_init.sas
+
+**/
+
+%macro mcf_length(wrap=NO
+  ,insert_cmplib=DEPRECATED
+  ,lib=WORK
+  ,cat=SASJS
+  ,pkg=UTILS
+)/*/STORE SOURCE*/;
+%local i var cmpval found;
+%if %mcf_init(mcf_length)=1 %then %return;
+
+%if &wrap=YES  %then %do;
+  proc fcmp outlib=&lib..&cat..&pkg;
+%end;
+
+function mcf_length(var);
+  if var=. then len=0;
+  else if missing(var) or trunc(var,3)=var then len=3;
+  else if trunc(var,4)=var then len=4;
+  else if trunc(var,5)=var then len=5;
+  else if trunc(var,6)=var then len=6;
+  else if trunc(var,7)=var then len=7;
+  else len=8;
+  return(len);
+endsub;
+
+%if &wrap=YES %then %do;
+  quit;
+%end;
+
+/* insert the CMPLIB if not already there */
+%let cmpval=%sysfunc(getoption(cmplib));
+%let found=0;
+%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
+  %let var=%scan(&cmpval,&i,%str( %(%)));
+  %if &var=&lib..&cat %then %let found=1;
+%end;
+%if &found=0 %then %do;
+  options insert=(CMPLIB=(&lib..&cat));
+%end;
+
+%mend mcf_length;

fcmp/mcf_stpsrv_header.sas

@@ -47,29 +47,33 @@
 
 
   @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
-  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
-    CMPLIB reference.
   @param [out] lib= (work) The output library in which to create the catalog.
   @param [out] cat= (sasjs) The output catalog in which to create the package.
   @param [out] pkg= (utils) The output package in which to create the function.
     Uses a 3 part format:  libref.catalog.package
+  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
+    values inserted only if needed.
 
   <h4> SAS Macros </h4>
-  @li mf_existfunction.sas
+  @li mcf_init.sas
+
+  <h4> Related Programs </h4>
+  @li mcf_stpsrv_header.test.sas
+  @li mp_init.sas
 
 **/
 
 %macro mcf_stpsrv_header(wrap=NO
-  ,insert_cmplib=NO
+  ,insert_cmplib=DEPRECATED
   ,lib=WORK
   ,cat=SASJS
   ,pkg=UTILS
 )/*/STORE SOURCE*/;
-
-%if %mf_existfunction(stpsrv_header)=1 %then %return;
+%local i var cmpval found;
+%if %mcf_init(stpsrv_header)=1 %then %return;
 
 %if &wrap=YES  %then %do;
-  proc fcmp outcat=&lib..&cat..&pkg;
+  proc fcmp outlib=&lib..&cat..&pkg;
 %end;
 
 function stpsrv_header(name $, value $);
@@ -92,7 +96,14 @@ endsub;
   quit;
 %end;
 
-%if &insert_cmplib=YES %then %do;
+/* insert the CMPLIB if not already there */
+%let cmpval=%sysfunc(getoption(cmplib));
+%let found=0;
+%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
+  %let var=%scan(&cmpval,&i,%str( %(%)));
+  %if &var=&lib..&cat %then %let found=1;
+%end;
+%if &found=0 %then %do;
   options insert=(CMPLIB=(&lib..&cat));
 %end;
 

fcmp/mcf_string2file.sas

@@ -32,24 +32,33 @@
       run;
 
   @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
-  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
-    CMPLIB reference.
   @param [out] lib= (work) The output library in which to create the catalog.
   @param [out] cat= (sasjs) The output catalog in which to create the package.
   @param [out] pkg= (utils) The output package in which to create the function.
     Uses a 3 part format:  libref.catalog.package
+  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
+    values inserted only if needed.
+
+  <h4> SAS Macros </h4>
+  @li mcf_init.sas
+
+  <h4> Related Programs </h4>
+  @li mcf_stpsrv_header.test.sas
+  @li mp_init.sas
 
 **/
 
 %macro mcf_string2file(wrap=NO
-  ,insert_cmplib=NO
+  ,insert_cmplib=DEPRECATED
   ,lib=WORK
   ,cat=SASJS
   ,pkg=UTILS
 )/*/STORE SOURCE*/;
+%local i var cmpval found;
+%if %mcf_init(mcf_string2file)=1 %then %return;
 
 %if &wrap=YES  %then %do;
-  proc fcmp outcat=&lib..&cat..&pkg;
+  proc fcmp outlib=&lib..&cat..&pkg;
 %end;
 
 function mcf_string2file(filepath $, string $, mode $);
@@ -72,7 +81,14 @@ endsub;
   quit;
 %end;
 
-%if &insert_cmplib=YES %then %do;
+/* insert the CMPLIB if not already there */
+%let cmpval=%sysfunc(getoption(cmplib));
+%let found=0;
+%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
+  %let var=%scan(&cmpval,&i,%str( %(%)));
+  %if &var=&lib..&cat %then %let found=1;
+%end;
+%if &found=0 %then %do;
   options insert=(CMPLIB=(&lib..&cat));
 %end;
 

meta/mm_assignlib.sas

@@ -40,6 +40,12 @@
       /* now try and assign it */
       if libname("&libref",,'meta',cats('liburi="',liburi,'";')) ne 0 then do;
         putlog "&libref could not be assigned";
+        putlog liburi=;
+        /**
+          * Fetch the system message for display in the abort modal.  This is
+          * not always helpful though.  One example, previously received:
+          * NOTE: Libref XX refers to the same library metadata as libref XX.
+          */
         call symputx('msg',sysmsg(),'l');
         if "&mabort"='HARD' then call symputx('mp_abort',1,'l');
       end;
@@ -61,7 +67,7 @@
 
   %if &mp_abort=1 %then %do;
     %mp_abort(iftrue= (&mp_abort=1)
-      ,mac=&sysmacroname
+      ,mac=mm_assignlib.sas
       ,msg=&msg
     )
     %return;

meta/mm_createapplication.sas

@@ -59,7 +59,10 @@
 %&mD.put Executing &sysmacroname..sas;
 %&mD.put _local_;
 
-%mf_verifymacvars(tree name)
+%mp_abort(iftrue= (%mf_verifymacvars(tree name)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Empty inputs: tree name)
+)
 
 /**
   * check tree exists

meta/mm_createdocument.sas

@@ -47,7 +47,10 @@
 %&mD.put Executing &sysmacroname..sas;
 %&mD.put _local_;
 
-%mf_verifymacvars(tree name)
+%mp_abort(iftrue= (%mf_verifymacvars(tree name)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Empty inputs: tree name)
+)
 
 /**
   * check tree exists

meta/mm_createfolder.sas

@@ -12,13 +12,14 @@
     The macro is idempotent - if you run it twice, it will only create a folder
     once.
 
-  usage:
+  Usage:
 
       %mm_createfolder(path=/some/meta/folder)
 
   @param [in] path= Name of the folder to create.
   @param [in] mdebug= set DBG to 1 to disable DEBUG messages
 
+
   @version 9.4
   @author Allan Bowe
 

meta/mm_createlibrary.sas

@@ -133,12 +133,14 @@ run;
 filename &frefin temp;
 filename &frefout temp;
 
+%mp_abort(iftrue= (
+    &engine=BASE & %mf_verifymacvars(libname libref engine servercontext tree)=0
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Empty inputs: libname libref engine servercontext tree)
+)
+
 %if &engine=BASE %then %do;
-
-  %mf_verifymacvars(libname libref engine servercontext tree)
-
-
-
   /**
     * Check that the ServerContext exists
     */

meta/mm_createstp.sas

@@ -12,7 +12,7 @@
     This macro is idempotent - if you run it twice, it will only create an STP
     once.
 
-  usage (type 1 STP):
+  Usage (type 1 STP):
 
       %mm_createstp(stpname=MyNewSTP
         ,filename=mySpecialProgram.sas
@@ -31,7 +31,8 @@
         putlog (_all_)(=);
       run;
 
-  usage (type 2 STP):
+  Usage (type 2 STP):
+
       %mm_createstp(stpname=MyNewType2STP
         ,filename=mySpecialProgram.sas
         ,directory=SASEnvironment/SASCode/STPs
@@ -39,14 +40,6 @@
         ,Server=SASApp
         ,stptype=2)
 
-  <h4> SAS Macros </h4>
-  @li mf_nobs.sas
-  @li mf_verifymacvars.sas
-  @li mm_getdirectories.sas
-  @li mm_updatestpsourcecode.sas
-  @li mp_dropmembers.sas
-  @li mm_getservercontexts.sas
-
   @param stpname= Stored Process name.  Avoid spaces - testing has shown that
     the check to avoid creating multiple STPs in the same folder with the same
     name does not work when the name contains spaces.
@@ -77,6 +70,18 @@
     - fileuri
     - texturi
 
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mf_verifymacvars.sas
+  @li mm_getdirectories.sas
+  @li mm_updatestpsourcecode.sas
+  @li mm_getservercontexts.sas
+  @li mp_abort.sas
+  @li mp_dropmembers.sas
+
+  <h4> Related Macros </h4>
+  @li mm_createwebservice.sas
+
   @version 9.2
   @author Allan Bowe
 
@@ -105,7 +110,12 @@
 %&mD.put Executing mm_CreateSTP.sas;
 %&mD.put _local_;
 
-%mf_verifymacvars(stpname filename directory tree)
+%mp_abort(
+  iftrue=(%mf_verifymacvars(stpname filename directory tree)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Empty inputs: stpname filename directory tree)
+)
+
 %mp_dropmembers(%scan(&outds,2,.))
 
 /**
@@ -174,6 +184,7 @@ run;
 
   data &outds (keep=stpuri prompturi fileuri texturi);
     length stpuri prompturi fileuri texturi serveruri $256 ;
+    if _n_=1 then call missing (of _all_);
     set &outds;
 
     /* final checks on uris */

meta/mm_createwebservice.sas

@@ -12,6 +12,7 @@ Usage:
     %* parmcards lets us write to a text file from open code ;
     filename ft15f001 temp;
     parmcards4;
+        %webout(FETCH)
         %* do some sas, any inputs are now already WORK tables;
         data example1 example2;
           set sashelp.class;
@@ -24,11 +25,8 @@ Usage:
     ;;;;
     %mm_createwebservice(path=/Public/app/common,name=appInit,code=ft15f001)
 
-  <h4> SAS Macros </h4>
-  @li mm_createstp.sas
-  @li mf_getuser.sas
-  @li mm_createfolder.sas
-  @li mm_deletestp.sas
+  For more examples of using these web services with the SASjs Adapter, see:
+  https://github.com/sasjs/adapter#readme
 
   @param path= The full path (in SAS Metadata) where the service will be created
   @param name= Stored Process name.  Avoid spaces - testing has shown that
@@ -37,16 +35,22 @@ Usage:
   @param desc= The description of the service (optional)
   @param precode= Space separated list of filerefs, pointing to the code that
     needs to be attached to the beginning of the service (optional)
-  @param code=(ft15f001) Space seperated fileref(s) of the actual code to be
+  @param code= (ft15f001) Space seperated fileref(s) of the actual code to be
     added
-  @param server=(SASApp) The server which will run the STP.  Server name or uri
+  @param server= (SASApp) The server which will run the STP.  Server name or uri
     is fine.
-  @param mDebug=(0) set to 1 to show debug messages in the log
-  @param replace=(YES) select NO to avoid replacing an existing service in that
+  @param mDebug= (0) set to 1 to show debug messages in the log
+  @param replace= (YES) select NO to avoid replacing an existing service in that
     location
-  @param adapter=(sasjs) the macro uses the sasjs adapter by default.  To use
+  @param adapter= (sasjs) the macro uses the sasjs adapter by default.  To use
     another adapter, add a (different) fileref here.
 
+  <h4> SAS Macros </h4>
+  @li mm_createstp.sas
+  @li mf_getuser.sas
+  @li mm_createfolder.sas
+  @li mm_deletestp.sas
+
   @version 9.2
   @author Allan Bowe
 
@@ -89,9 +93,14 @@ data _null_;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
   put ' ';
-  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0 ';
+  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
+  put '  ,engine=DATASTEP ';
+  put '  ,missing=NULL ';
+  put '  ,showmeta=NO ';
   put ')/*/STORE SOURCE*/; ';
-  put '%put output location=&jref; ';
+  put '%local tempds colinfo fmtds i numcols; ';
+  put '%let numcols=0; ';
+  put ' ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
   put '  data _null_;file &jref encoding=''utf-8'' ; ';
@@ -100,118 +109,129 @@ data _null_;
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
   put '  options validvarname=upcase; ';
-  put '  data _null_;file &jref mod encoding=''utf-8'' ; ';
+  put '  data _null_; file &jref encoding=''utf-8'' mod; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
   put ' ';
+  put '  /* grab col defs */ ';
+  put '  proc contents noprint data=&ds ';
+  put '    out=_data_(keep=name type length format formatl formatd varnum label); ';
+  put '  run; ';
+  put '  %let colinfo=%scan(&syslast,2,.); ';
+  put '  proc sort data=&colinfo; ';
+  put '    by varnum; ';
+  put '  run; ';
+  put '  /* move meta to mac vars */ ';
+  put '  data _null_; ';
+  put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
+  put '    set &colinfo end=last nobs=nobs; ';
+  put '    name=upcase(name); ';
+  put '    /* fix formats */ ';
+  put '    if type=2 or type=6 then do; ';
+  put '      typelong=''char''; ';
+  put '      length fmt $49.; ';
+  put '      if format='''' then fmt=cats(''$'',length,''.''); ';
+  put '      else if formatl=0 then fmt=cats(format,''.''); ';
+  put '      else fmt=cats(format,formatl,''.''); ';
+  put '      newlen=max(formatl,length); ';
+  put '    end; ';
+  put '    else do; ';
+  put '      typelong=''num''; ';
+  put '      if format='''' then fmt=''best.''; ';
+  put '      else if formatl=0 then fmt=cats(format,''.''); ';
+  put '      else if formatd=0 then fmt=cats(format,formatl,''.''); ';
+  put '      else fmt=cats(format,formatl,''.'',formatd); ';
+  put '      /* needs to be wide, for datetimes etc */ ';
+  put '      newlen=max(length,formatl,24); ';
+  put '    end; ';
+  put '    /* 32 char unique name */ ';
+  put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put ' ';
+  put '    call symputx(cats(''name'',_n_),name,''l''); ';
+  put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '    call symputx(cats(''len'',_n_),newlen,''l''); ';
+  put '    call symputx(cats(''length'',_n_),length,''l''); ';
+  put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
+  put '    call symputx(cats(''type'',_n_),type,''l''); ';
+  put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
+  put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
+  put '  run; ';
+  put ' ';
+  put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
-  put '    data;run;%let tempds=&syslast; ';
-  put '    proc sql;drop table &tempds; ';
-  put '    data &tempds /view=&tempds;set &ds; ';
+  put '    %if &missing=STRING %then %do; ';
+  put '      %put &sysmacroname: Special Missings not supported in proc json.; ';
+  put '      %put &sysmacroname: Switching to DATASTEP engine; ';
+  put '      %goto datastep; ';
+  put '    %end; ';
+  put '    data &tempds;set &ds; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
+  put '    /* PRETTY is necessary to avoid line truncation in large files */ ';
   put '    proc json out=&jref pretty ';
   put '        %if &action=ARR %then nokeys ; ';
   put '        ;export &tempds / nosastags fmtnumeric; ';
   put '    run; ';
-  put '    proc sql;drop view &tempds; ';
   put '  %end; ';
   put '  %else %if &engine=DATASTEP %then %do; ';
-  put '    %local cols i tempds; ';
-  put '    %let cols=0; ';
-  put '    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 %then %do; ';
+  put '    %datastep: ';
+  put '    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 ';
+  put '    %then %do; ';
   put '      %put &sysmacroname:  &ds NOT FOUND!!!; ';
   put '      %return; ';
   put '    %end; ';
-  put '    %if &fmt=Y %then %do; ';
-  put '      %put converting every variable to a formatted variable; ';
-  put '      /* see mp_ds2fmtds.sas for source */ ';
-  put '      proc contents noprint data=&ds ';
-  put '        out=_data_(keep=name type length format formatl formatd varnum); ';
-  put '      run; ';
-  put '      proc sort; ';
-  put '        by varnum; ';
-  put '      run; ';
-  put '      %local fmtds; ';
-  put '      %let fmtds=%scan(&syslast,2,.); ';
-  put '      /* prepare formats and varnames */ ';
-  put '      data _null_; ';
-  put '        if _n_=1 then call symputx(''nobs'',nobs,''l''); ';
-  put '        set &fmtds end=last nobs=nobs; ';
-  put '        name=upcase(name); ';
-  put '        /* fix formats */ ';
-  put '        if type=2 or type=6 then do; ';
-  put '          length fmt $49.; ';
-  put '          if format='''' then fmt=cats(''$'',length,''.''); ';
-  put '          else if formatl=0 then fmt=cats(format,''.''); ';
-  put '          else fmt=cats(format,formatl,''.''); ';
-  put '          newlen=max(formatl,length); ';
-  put '        end; ';
-  put '        else do; ';
-  put '          if format='''' then fmt=''best.''; ';
-  put '          else if formatl=0 then fmt=cats(format,''.''); ';
-  put '          else if formatd=0 then fmt=cats(format,formatl,''.''); ';
-  put '          else fmt=cats(format,formatl,''.'',formatd); ';
-  put '          /* needs to be wide, for datetimes etc */ ';
-  put '          newlen=max(length,formatl,24); ';
-  put '        end; ';
-  put '        /* 32 char unique name */ ';
-  put '        newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
   put ' ';
-  put '        call symputx(cats(''name'',_n_),name,''l''); ';
-  put '        call symputx(cats(''newname'',_n_),newname,''l''); ';
-  put '        call symputx(cats(''len'',_n_),newlen,''l''); ';
-  put '        call symputx(cats(''fmt'',_n_),fmt,''l''); ';
-  put '        call symputx(cats(''type'',_n_),type,''l''); ';
-  put '      run; ';
-  put '      data &fmtds; ';
+  put '    %if &fmt=Y %then %do; ';
+  put '      data _data_; ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
-  put '      %local i; ';
-  put '      %do i=1 %to &nobs; ';
+  put '      %do i=1 %to &numcols; ';
   put '        &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
-  put '      %do i=1 %to &nobs; ';
+  put '      %do i=1 %to &numcols; ';
   put '        length &&name&i $&&len&i; ';
-  put '        &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '        %if &&typelong&i=num %then %do; ';
+  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '        %end; ';
+  put '        %else %do; ';
+  put '          &&name&i=put(&&newname&i,&&fmt&i); ';
+  put '        %end; ';
   put '        drop &&newname&i; ';
   put '      %end; ';
   put '        if _error_ then call symputx(''syscc'',1012); ';
   put '      run; ';
-  put '      %let ds=&fmtds; ';
-  put '    %end; /* &fmt=Y */ ';
-  put '    data _null_;file &jref mod encoding=''utf-8'' ; ';
-  put '      put "["; call symputx(''cols'',0,''l''); ';
-  put '    proc sort ';
-  put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
-  put '      out=_data_; ';
-  put '      by varnum; ';
-  put ' ';
-  put '    data _null_; ';
-  put '      set _last_ end=last; ';
-  put '      call symputx(cats(''name'',_n_),name,''l''); ';
-  put '      call symputx(cats(''type'',_n_),type,''l''); ';
-  put '      call symputx(cats(''len'',_n_),length,''l''); ';
-  put '      if last then call symputx(''cols'',_n_,''l''); ';
-  put '    run; ';
+  put '      %let fmtds=&syslast; ';
+  put '    %end; ';
   put ' ';
   put '    proc format; /* credit yabwon for special null removal */ ';
-  put '      value bart ._ - .z = null ';
+  put '    value bart (default=40) ';
+  put '    %if &missing=NULL %then %do; ';
+  put '      ._ - .z = null ';
+  put '    %end; ';
+  put '    %else %do; ';
+  put '      ._ = [quote()] ';
+  put '      . = null ';
+  put '      .a - .z = [quote()] ';
+  put '    %end; ';
   put '      other = [best.]; ';
   put ' ';
-  put '    data;run; %let tempds=&syslast; /* temp table for spesh char management */ ';
-  put '    proc sql; drop table &tempds; ';
-  put '    data &tempds/view=&tempds; ';
+  put '    data &tempds; ';
   put '      attrib _all_ label=''''; ';
-  put '      %do i=1 %to &cols; ';
-  put '        %if &&type&i=char %then %do; ';
+  put '      %do i=1 %to &numcols; ';
+  put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
   put '          length &&name&i $32767; ';
   put '          format &&name&i $32767.; ';
   put '        %end; ';
   put '      %end; ';
-  put '      set &ds; ';
+  put '      %if &fmt=Y %then %do; ';
+  put '        set &fmtds; ';
+  put '      %end; ';
+  put '      %else %do; ';
+  put '        set &ds; ';
+  put '      %end; ';
   put '      format _numeric_ bart.; ';
-  put '    %do i=1 %to &cols; ';
-  put '      %if &&type&i=char %then %do; ';
+  put '    %do i=1 %to &numcols; ';
+  put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
   put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
   put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
   put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
@@ -221,49 +241,71 @@ data _null_;
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
+  put ' ';
   put '    /* write to temp loc to avoid _webout truncation ';
   put '      - https://support.sas.com/kb/49/325.html */ ';
   put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
   put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
+  put '      if _n_=1 then put "["; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
   put '      %if &action=ARR %then "[" ; %else "{" ; ';
-  put '      %do i=1 %to &cols; ';
+  put '      %do i=1 %to &numcols; ';
   put '        %if &i>1 %then  "," ; ';
   put '        %if &action=OBJ %then """&&name&i"":" ; ';
   put '        &&name&i ';
   put '      %end; ';
   put '      %if &action=ARR %then "]" ; %else "}" ; ; ';
-  put '    proc sql; ';
-  put '    drop view &tempds; ';
   put '    /* now write the long strings to _webout 1 byte at a time */ ';
   put '    data _null_; ';
   put '      length filein 8 fileid 8; ';
-  put '      filein = fopen("_sjs",''I'',1,''B''); ';
-  put '      fileid = fopen("&jref",''A'',1,''B''); ';
-  put '      rec = ''20''x; ';
+  put '      filein=fopen("_sjs",''I'',1,''B''); ';
+  put '      fileid=fopen("&jref",''A'',1,''B''); ';
+  put '      rec=''20''x; ';
   put '      do while(fread(filein)=0); ';
-  put '        rc = fget(filein,rec,1); ';
-  put '        rc = fput(fileid, rec); ';
-  put '        rc =fwrite(fileid); ';
+  put '        rc=fget(filein,rec,1); ';
+  put '        rc=fput(fileid, rec); ';
+  put '        rc=fwrite(fileid); ';
   put '      end; ';
-  put '      rc = fclose(filein); ';
-  put '      rc = fclose(fileid); ';
+  put '      /* close out the table */ ';
+  put '      rc=fput(fileid, "]"); ';
+  put '      rc=fwrite(fileid); ';
+  put '      rc=fclose(filein); ';
+  put '      rc=fclose(fileid); ';
   put '    run; ';
   put '    filename _sjs clear; ';
-  put '    data _null_; file &jref mod encoding=''utf-8'' ; ';
-  put '      put "]"; ';
+  put '  %end; ';
+  put ' ';
+  put '  proc sql; ';
+  put '  drop table &colinfo, &tempds; ';
+  put ' ';
+  put '  %if &showmeta=YES %then %do; ';
+  put '    data _null_; file &jref encoding=''utf-8'' mod; ';
+  put '      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; ';
+  put '      do i=1 to &numcols; ';
+  put '        name=quote(trim(symget(cats(''name'',i)))); ';
+  put '        format=quote(trim(symget(cats(''fmt'',i)))); ';
+  put '        label=quote(trim(symget(cats(''label'',i)))); ';
+  put '        length=quote(trim(symget(cats(''length'',i)))); ';
+  put '        type=quote(trim(symget(cats(''typelong'',i)))); ';
+  put '        if i>1 then put "," @@; ';
+  put '        put name '':{"format":'' format '',"label":'' label ';
+  put '          '',"length":'' length '',"type":'' type ''}''; ';
+  put '      end; ';
+  put '      put ''}}''; ';
   put '    run; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8'' mod ; ';
+  put '  data _null_; file &jref encoding=''utf-8'' mod ; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
   put '%mend mp_jsonout; ';
-  put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y); ';
+  put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
+  put '  ,showmeta=NO ';
+  put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
   put '%local i tempds jsonengine; ';
@@ -321,14 +363,15 @@ data _null_;
   put '  %if %str(&_debug) ge 131 %then %do; ';
   put '    put ''>>weboutBEGIN<<''; ';
   put '  %end; ';
-  put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
+  put '    put ''{"SYSDATE" : "'' "&SYSDATE" ''"''; ';
+  put '    put '',"SYSTIME" : "'' "&SYSTIME" ''"''; ';
   put '  run; ';
   put ' ';
   put '%end; ';
   put ' ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '    ,engine=&jsonengine,dbg=%str(&_debug) ';
+  put '    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta ';
   put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
@@ -343,15 +386,12 @@ data _null_;
   put '      set &tempds; ';
   put '      if not (upcase(name) =:"DATA"); /* ignore temp datasets */ ';
   put '      i+1; ';
-  put '      call symputx(''wt''!!left(i),name,''l''); ';
+  put '      call symputx(cats(''wt'',i),name,''l''); ';
   put '      call symputx(''wtcnt'',i,''l''); ';
   put '    data _null_; file &fref mod encoding=''utf-8''; ';
   put '      put ",""WORK"":{"; ';
   put '    %do i=1 %to &wtcnt; ';
   put '      %let wt=&&wt&i; ';
-  put '      proc contents noprint data=&wt ';
-  put '        out=_data_ (keep=name type length format:); ';
-  put '      run;%let tempds=%scan(&syslast,2,.); ';
   put '      data _null_; file &fref mod encoding=''utf-8''; ';
   put '        dsid=open("WORK.&wt",''is''); ';
   put '        nlobs=attrn(dsid,''NLOBS''); ';
@@ -361,8 +401,7 @@ data _null_;
   put '        put " ""&wt"" : {"; ';
   put '        put ''"nlobs":'' nlobs; ';
   put '        put '',"nvars":'' nvars; ';
-  put '      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=&jsonengine) ';
-  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=&jsonengine) ';
+  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) ';
   put '      data _null_; file &fref mod encoding=''utf-8''; ';
   put '        put "}"; ';
   put '    %end; ';
@@ -390,7 +429,11 @@ data _null_;
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
   put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
-  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''" ''; ';
+  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
+  put '    length memsize $32; ';
+  put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
+  put '    memsize=quote(cats(memsize)); ';
+  put '    put '',"MEMSIZE" : '' memsize; ';
   put '    put "}" @; ';
   put '  %if %str(&_debug) ge 131 %then %do; ';
   put '    put ''>>weboutEND<<''; ';
@@ -418,8 +461,10 @@ data _null_;
   put ' ';
   put '%mend mf_getuser; ';
 /* WEBOUT END */
-  put '%macro webout(action,ds,dslabel=,fmt=);';
-  put '  %mm_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt)';
+  put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO);';
+  put '  %mm_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt,missing=&missing';
+  put '    ,showmeta=&showmeta';
+  put '  )';
   put '%mend;';
 run;
 

meta/mm_getauthinfo.sas

@@ -1,16 +1,21 @@
 /**
   @file mm_getauthinfo.sas
-  @brief extracts authentication info
-  @details usage:
+  @brief Extracts authentication info for each user in metadata
+  @details
+  Usage:
 
-    %mm_getauthinfo(outds=auths)
+      %mm_getauthinfo(outds=auths)
 
-  @param outds= the ONE LEVEL work dataset to create
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
+  @param [out] outds= (mm_getauthinfo) The output dataset to create
 
   <h4> SAS Macros </h4>
-  @li mm_getobjects.sas
   @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
   @li mm_getdetails.sas
+  @li mm_getobjects.sas
+
 
   @version 9.4
   @author Allan Bowe
@@ -18,67 +23,69 @@
 **/
 
 %macro mm_getauthinfo(outds=mm_getauthinfo
+  ,mdebug=0
 )/*/STORE SOURCE*/;
+%local prefix fileref;
+%let prefix=%substr(%mf_getuniquename(),1,25);
 
-%if %length(&outds)>30 %then %do;
-  %put %str(ERR)OR: Temp tables are created with the &outds prefix, which
-    therefore needs to be 30 characters or less;
-  %return;
-%end;
-%if %index(&outds,'.')>0 %then %do;
-  %put %str(ERR)OR: Table &outds should be ONE LEVEL (no library);
-  %return;
-%end;
-
-%mm_getobjects(type=Login,outds=&outds.0)
+%mm_getobjects(type=Login,outds=&prefix.0)
 
 %local fileref;
 %let fileref=%mf_getuniquefileref();
 
 data _null_;
   file &fileref;
-  set &outds.0 end=last;
+  set &prefix.0 end=last;
   /* run macro */
-  str=cats('%mm_getdetails(uri=',id,",outattrs=&outds.d",_n_
-    ,",outassocs=&outds.a",_n_,")");
+  str=cats('%mm_getdetails(uri=',id,",outattrs=&prefix.d",_n_
+    ,",outassocs=&prefix.a",_n_,")");
   put str;
   /* transpose attributes */
-  str=cats("proc transpose data=&outds.d",_n_,"(drop=type) out=&outds.da"
+  str=cats("proc transpose data=&prefix.d",_n_,"(drop=type) out=&prefix.da"
     ,_n_,"(drop=_name_);var value;id name;run;");
   put str;
   /* add extra info to attributes */
-  str=cats("data &outds.da",_n_,";length login_id login_name $256; login_id="
-    ,quote(trim(id)),";set &outds.da",_n_
+  str=cats("data &prefix.da",_n_,";length login_id login_name $256; login_id="
+    ,quote(trim(id)),";set &prefix.da",_n_
     ,";login_name=trim(subpad(name,1,256));drop name;run;");
   put str;
   /* add extra info to associations */
-  str=cats("data &outds.a",_n_,";length login_id login_name $256; login_id="
+  str=cats("data &prefix.a",_n_,";length login_id login_name $256; login_id="
     ,quote(trim(id)),";login_name=",quote(trim(name))
-    ,";set &outds.a",_n_,";run;");
+    ,";set &prefix.a",_n_,";run;");
   put str;
   if last then do;
     /* collate attributes */
-    str=cats("data &outds._logat; set &outds.da1-&outds.da",_n_,";run;");
+    str=cats("data &prefix._logat; set &prefix.da1-&prefix.da",_n_,";run;");
     put str;
     /* collate associations */
-    str=cats("data &outds._logas; set &outds.a1-&outds.a",_n_,";run;");
+    str=cats("data &prefix._logas; set &prefix.a1-&prefix.a",_n_,";run;");
     put str;
     /* tidy up */
-    str=cats("proc delete data=&outds.da1-&outds.da",_n_,";run;");
+    str=cats("proc delete data=&prefix.da1-&prefix.da",_n_,";run;");
     put str;
-    str=cats("proc delete data=&outds.d1-&outds.d",_n_,";run;");
+    str=cats("proc delete data=&prefix.d1-&prefix.d",_n_,";run;");
     put str;
-    str=cats("proc delete data=&outds.a1-&outds.a",_n_,";run;");
+    str=cats("proc delete data=&prefix.a1-&prefix.a",_n_,";run;");
     put str;
   end;
 run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fileref;
+    if _n_=1 then putlog // "Now executing the following code:" //;
+    input; putlog _infile_;
+  run;
+%end;
 %inc &fileref;
+filename &fileref clear;
 
 /* get libraries */
-proc sort data=&outds._logas(where=(assoc='Libraries')) out=&outds._temp;
+proc sort data=&prefix._logas(where=(assoc='Libraries')) out=&prefix._temp;
   by login_id;
-data &outds._temp;
-  set &outds._temp;
+data &prefix._temp;
+  set &prefix._temp;
   by login_id;
   length library_list $32767;
   retain library_list;
@@ -86,31 +93,27 @@ data &outds._temp;
   else library_list=catx(' !! ',library_list,name);
 proc sql;
 /* get auth domain */
-create table &outds._dom as
+create table &prefix._dom as
   select login_id,name as domain
-  from &outds._logas
+  from &prefix._logas
   where assoc='Domain';
-create unique index login_id on &outds._dom(login_id);
+create unique index login_id on &prefix._dom(login_id);
 /* join it all together */
-create table &outds._logins as
+create table &outds as
   select a.*
     ,c.domain
     ,b.library_list
-  from &outds._logat (drop=ishidden lockedby usageversion publictype) a
-  left join &outds._temp b
+  from &prefix._logat (drop=ishidden lockedby usageversion publictype) a
+  left join &prefix._temp b
   on a.login_id=b.login_id
-  left join &outds._dom c
+  left join &prefix._dom c
   on a.login_id=c.login_id;
-drop table &outds._temp;
-drop table &outds._logat;
-drop table &outds._logas;
 
-data _null_;
-  infile &fileref;
-  if _n_=1 then putlog // "Now executing the following code:" //;
-  input; putlog _infile_;
-run;
+%if &mdebug=0 %then %do;
+  proc datasets lib=work;
+    delete &prefix:;
+  run;
+%end;
 
-filename &fileref clear;
 
 %mend mm_getauthinfo;

meta/mm_getgroupmembers.sas

@@ -1,21 +1,28 @@
 /**
   @file
   @brief Creates dataset with all members of a metadata group
-  @details
+  @details This macro will query SAS metadata and return all the members
+    of a particular group.
 
-  usage:
+  Usage:
 
-    %mm_getgroupmembers(someGroupName
-      ,outds=work.mm_getgroupmembers
-      ,emails=YES)
+      %mm_getgroupmembers(someGroupName
+        ,outds=work.mm_getgroupmembers
+        ,emails=YES
+      )
 
   @param group metadata group for which to bring back members
-  @param outds= the dataset to create that contains the list of members
-  @param emails= set to YES to bring back email addresses
-  @param id= set to yes if passing an ID rather than a group name
+  @param outds= (work.mm_getgroupmembers) The dataset to create that contains
+    the list of members
+  @param emails= (NO) Set to YES to bring back email addresses
+  @param id= (NO) Set to yes if passing an ID rather than a group name
 
   @returns outds  dataset containing all members of the metadata group
 
+  <h4> Related Macros </h4>
+  @li mm_getgorups.sas
+  @li mm_adduser2group.sas
+
   @version 9.2
   @author Allan Bowe
 

meta/mm_getservercontexts.sas

@@ -35,13 +35,17 @@ data _null_;
   set repos;
   where repositorytype in('CUSTOM','FOUNDATION');
   keep id name ;
-  call symputx('repo'!!left(_n_),name,'l');
+  call symputx(cats('repo',_n_),name,'l');
   call symputx('repocnt',_n_,'l');
 run;
 
 filename __mc1 temp;
 filename __mc2 temp;
-data &outds; length serveruri servername $200; stop;run;
+data &outds;
+  length serveruri servername $200;
+  call missing (of _all_);
+  stop;
+run;
 %do x=1 %to &repocnt;
   options metarepository=&&repo&x;
   proc metadata in=
@@ -60,13 +64,16 @@ data &outds; length serveruri servername $200; stop;run;
   data _null_;
     file __mc2;
     put '<SXLEMAP version="1.2" name="SASContexts"><TABLE name="SASContexts">';
-    put "<TABLE-PATH syntax='XPath'>/GetMetadataObjects/Objects/ServerContext</TABLE-PATH>";
+    put "<TABLE-PATH syntax='XPath'>/GetMetadataObjects/Objects/ServerContext";
+    put "</TABLE-PATH>";
     put '<COLUMN name="serveruri">';
-    put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/ServerContext/@Id</PATH>";
+    put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/ServerContext/@Id";
+    put "</PATH>";
     put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>200</LENGTH>";
     put '</COLUMN>';
     put '<COLUMN name="servername">';
-    put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/ServerContext/@Name</PATH>";
+    put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/ServerContext/@Name";
+    put "</PATH>";
     put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>200</LENGTH>";
     put '</COLUMN>';
     put '</TABLE></SXLEMAP>';

meta/mm_updatestpsourcecode.sas

@@ -13,9 +13,6 @@
   @param [in] stpcode= the source file (or fileref) containing the SAS code to load
     into the stp.  For multiple files, they should simply be concatenated first.
   @param [in] minify= set to YES in order to strip comments, blank lines, and CRLFs.
-
-  @param frefin= deprecated - a unique fileref is now always used
-  @param frefout= deprecated - a unique fileref is now always used
   @param mDebug= set to 1 to show debug messages in the log
 
   @version 9.3
@@ -30,16 +27,8 @@
   ,stpcode=
   ,minify=NO
   ,mdebug=0
-  /* deprecated */
-  ,frefin=inmeta
-  ,frefout=outmeta
 );
 
-%if &frefin ne inmeta or &frefout ne outmeta %then %do;
-  %put %str(WARN)ING: the frefin and frefout parameters will be deprecated in
-    an upcoming release.;
-%end;
-
 /* first, check if STP exists */
 %local tsuri;
 %let tsuri=stopifempty ;

meta/mm_webout.sas

@@ -23,17 +23,25 @@
         %mm_webout(CLOSE)
 
 
-  @param action Either FETCH, OPEN, ARR, OBJ or CLOSE
-  @param ds The dataset to send back to the frontend
-  @param dslabel= value to use instead of the real name for sending to JSON
-  @param fmt=(Y) Set to N to send back unformatted values
-  @param fref=(_webout) The fileref to which to write the JSON
+  @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE
+  @param [in] ds The dataset to send back to the frontend
+  @param [out] dslabel= Value to use instead of table name for sending to JSON
+  @param [in] fmt=(Y) Set to N to send back unformatted values
+  @param [out] fref= (_webout) The fileref to which to write the JSON
+  @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
+    (eg `null`) or as STRING values (eg `".a"` or `".b"`)
+  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+    such as the column formats and types.  The metadata is contained inside an
+    object with the same name as the table but prefixed with a dollar sign - ie,
+    `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
 
   @version 9.3
   @author Allan Bowe
 
 **/
-%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y);
+%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
+  ,showmeta=NO
+);
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
 %local i tempds jsonengine;
@@ -91,14 +99,15 @@
   %if %str(&_debug) ge 131 %then %do;
     put '>>weboutBEGIN<<';
   %end;
-    put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
+    put '{"SYSDATE" : "' "&SYSDATE" '"';
+    put ',"SYSTIME" : "' "&SYSTIME" '"';
   run;
 
 %end;
 
 %else %if &action=ARR or &action=OBJ %then %do;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=&jsonengine,dbg=%str(&_debug)
+    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta
   )
 %end;
 %else %if &action=CLOSE %then %do;
@@ -113,15 +122,12 @@
       set &tempds;
       if not (upcase(name) =:"DATA"); /* ignore temp datasets */
       i+1;
-      call symputx('wt'!!left(i),name,'l');
+      call symputx(cats('wt',i),name,'l');
       call symputx('wtcnt',i,'l');
     data _null_; file &fref mod encoding='utf-8';
       put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
-      proc contents noprint data=&wt
-        out=_data_ (keep=name type length format:);
-      run;%let tempds=%scan(&syslast,2,.);
       data _null_; file &fref mod encoding='utf-8';
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
@@ -131,8 +137,7 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=&jsonengine)
-      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=&jsonengine)
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES)
       data _null_; file &fref mod encoding='utf-8';
         put "}";
     %end;
@@ -160,7 +165,11 @@
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
-    put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
+    length memsize $32;
+    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
+    memsize=quote(cats(memsize));
+    put ',"MEMSIZE" : ' memsize;
     put "}" @;
   %if %str(&_debug) ge 131 %then %do;
     put '>>weboutEND<<';

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sasjs/core",
-  "description": "Production Ready Macros for SAS Application Developers",
+  "description": "Macros for SAS Application Developers",
   "license": "MIT",
   "keywords": [
     "SAS",
@@ -33,9 +33,7 @@
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "^2.39.0"
-  },
-  "dependencies": {
-    "ts-loader": "^9.2.6"
+    "@sasjs/cli": "3.6.0",
+    "@sasjs/core": "4.4.7"
   }
 }

sasjs/doxy/Doxyfile

@@ -8,6 +8,7 @@ FILE_PATTERNS          = *.sas \
                          *.dox
 GENERATE_LATEX         = NO
 GENERATE_TREEVIEW      = YES
+HAVE_DOT               = YES
 HIDE_FRIEND_COMPOUNDS  = YES
 HIDE_IN_BODY_DOCS      = YES
 HIDE_SCOPE_NAMES       = YES

sasjs/doxy/new_header.html

@@ -1,67 +1,71 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!DOCTYPE html
+  PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 <!-- HTML header for doxygen 1.8.17-->
 <html xmlns="https://www.w3.org/1999/xhtml" lang="en">
-  <head>
-    <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8" />
-    <meta http-equiv="X-UA-Compatible" content="IE=9" />
-    <meta property="og:type" content="website">
-    <meta name="author" content="Allan Bowe">
-    <meta name="generator" content="Doxygen $doxygenversion" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <!--BEGIN PROJECT_NAME-->
-    <meta name="description" content="$projectbrief" />
-    <!--END PROJECT_NAME-->
-    <!--BEGIN !PROJECT_NAME-->
-    <title>$title</title>
-    <!--END !PROJECT_NAME-->
-    <link href="$relpath^tabs.css" rel="stylesheet" type="text/css" />
-    <script type="text/javascript" src="$relpath^jquery.js"></script>
-    <script type="text/javascript" src="$relpath^dynsections.js"></script>
-    $treeview $search $mathjax
-    <link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
-    <link rel="shortcut icon" href="$relpath^favicon.ico" type="image/x-icon" />
-    $extrastylesheet
-  </head>
-  <body>
-    <div id="top">
-      <!-- do not remove this div, it is closed by doxygen! -->
 
-      <!--BEGIN TITLEAREA-->
-      <div id="titlearea">
-        <table cellspacing="0" cellpadding="0">
-          <tbody>
-            <tr style="height: 56px">
-              <!--BEGIN PROJECT_LOGO-->
-              <td id="projectlogo">
-                <a href="$relpath^"
-                  ><img alt="Logo" src="$relpath^$projectlogo"
-                /></a>
-              </td>
-              <!--END PROJECT_LOGO-->
-              <td id="projectalign" style="padding-left: 0.5em">
-                <div id="projectbrief">
-                  Production Ready Macros for SAS Application Developers<br />
-                  <a href="https://github.com/sasjs/core">
-                    https://github.com/sasjs/core
-                  </a>
-                </div>
-              </td>
-                  </tr>
-                </table>
-              </td>
+<head>
+  <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8" />
+  <meta http-equiv="X-UA-Compatible" content="IE=9" />
+  <meta property="og:type" content="website">
+  <meta property="og:title" content="MacroCore" />
+  <meta property="og:url" content="https://core.sasjs.io" />
+  <meta property="og:image" content="https://core.sasjs.io/Macro_core_website_1.png" />
+  <meta name="author" content="Allan Bowe">
+  <meta name="generator" content="Doxygen $doxygenversion" />
+  <meta name="viewport" content="width=device-width, initial-scale=1" />
+  <!--BEGIN PROJECT_NAME-->
+  <meta name="description" content="$projectbrief" />
+  <meta name="og:description" content="$projectbrief" />
+  <!--END PROJECT_NAME-->
+  <title>$title</title>
+  <link href="$relpath^tabs.css" rel="stylesheet" type="text/css" />
+  <script type="text/javascript" src="$relpath^jquery.js"></script>
+  <script type="text/javascript" src="$relpath^dynsections.js"></script>
+  $treeview $search $mathjax
+  <link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
+  <link rel="shortcut icon" href="$relpath^favicon.ico" type="image/x-icon" />
+  $extrastylesheet
+</head>
+
+<body>
+  <div id="top">
+    <!-- do not remove this div, it is closed by doxygen! -->
+
+    <!--BEGIN TITLEAREA-->
+    <div id="titlearea">
+      <table cellspacing="0" cellpadding="0">
+        <tbody>
+          <tr style="height: 56px">
+            <!--BEGIN PROJECT_LOGO-->
+            <td id="projectlogo">
+              <a href="$relpath^"><img alt="Logo" src="$relpath^$projectlogo" /></a>
+            </td>
+            <!--END PROJECT_LOGO-->
+            <td id="projectalign" style="padding-left: 0.5em">
+              <div id="projectbrief">
+                Macros for SAS Application Developers<br />
+                <a href="https://github.com/sasjs/core">
+                  https://github.com/sasjs/core
+                </a>
+              </div>
+            </td>
+          </tr>
+      </table>
+      </td>
 
 
-              <!--BEGIN DISABLE_INDEX-->
-              <!--BEGIN SEARCHENGINE-->
-              <td>$searchbox</td>
-              <!--END SEARCHENGINE-->
-              <!--END DISABLE_INDEX-->
-            </tr>
-          </tbody>
-        </table>
-      </div>
-      <!--END TITLEAREA-->
-      <!-- end header part -->
+      <!--BEGIN DISABLE_INDEX-->
+      <!--BEGIN SEARCHENGINE-->
+      <td>$searchbox</td>
+      <!--END SEARCHENGINE-->
+      <!--END DISABLE_INDEX-->
+      </tr>
+      </tbody>
+      </table>
     </div>
-  </body>
-</html>
+    <!--END TITLEAREA-->
+    <!-- end header part -->
+  </div>
+</body>
+
+</html>

sasjs/sasjsconfig.json

@@ -1,20 +1,22 @@
 {
-  "$schema": "https://cli.sasjs.io/sasjsconfig-schema.json",
+  "$schema": "https://raw.githubusercontent.com/sasjs/utils/main/src/types/sasjsconfig-schema.json",
   "macroFolders": [
     "base",
+    "ddl",
     "fcmp",
     "meta",
     "metax",
     "server",
     "viya",
     "lua",
-    "tests/crossplatform"
+    "tests/crossplatform",
+    "tests/ddl"
   ],
   "docConfig": {
     "displayMacroCore": false,
     "enableLineage": false,
     "doxyContent": {
-      "favIcon": "runningman.jpg",
+      "favIcon": "favicon.ico",
       "logo": "Macro_core_website_1.png",
       "readMe": "../../README.md"
     }
@@ -40,6 +42,7 @@
         "tests/viyaonly"
       ],
       "programFolders": [],
+      "binaryFolders": [],
       "deployConfig": {
         "deployServicePack": true,
         "deployScripts": []
@@ -58,6 +61,7 @@
         "tests/sas9only"
       ],
       "programFolders": [],
+      "binaryFolders": [],
       "deployConfig": {
         "deployServicePack": true,
         "deployScripts": []
@@ -67,19 +71,26 @@
     },
     {
       "name": "server",
-      "serverUrl": "https://sas.analytium.co.uk:5001",
+      "serverUrl": "https://sas.analytium.co.uk:5000",
       "serverType": "SASJS",
-      "appLoc": "/Shared Data/temp/macrocore",
+      "httpsAgentOptions": {
+        "allowInsecureRequests": false
+      },
+      "appLoc": "/sasjs/core",
       "macroFolders": [
         "tests/serveronly"
       ],
+      "programFolders": [],
+      "binaryFolders": [],
       "deployConfig": {
-        "deployServicePack": true
+        "deployServicePack": true,
+        "deployScripts": []
       }
     },
     {
       "name": "docsonly",
       "serverType": "SAS9",
+      "appLoc": "dummy",
       "macroFolders": [
         "tests/sas9only",
         "tests/viyaonly"

server/mfs_httpheader.sas

@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief Sets the http headers in the SASjs/server response
+  @details For GET requests, SASjs server will use the file generated by this
+  macro for setting the appropriate http headers in the response.
+
+  It works by writing a file to the session directory, that is then ingested by
+  the server.
+
+  The location of this file is driven by the global variable
+  `sasjs_stpsrv_header_loc` which is made available in the autoexec.
+
+  Usage:
+
+      %mfs_httpheader(Content-type,application/csv)
+
+  @param [in] header_name Name of the http header to set
+  @param [in] header_value Value of the http header to set
+
+  <h4> Related Macros </h4>
+  @li mcf_stpsrv_header.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mfs_httpheader(header_name
+  ,header_value
+)/*/STORE SOURCE*/;
+%local fref fid i;
+
+%if %sysfunc(filename(fref,&sasjs_stpsrv_header_loc)) ne 0 %then %do;
+  %put &=fref &=sasjs_stpsrv_header_loc;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let fid=%sysfunc(fopen(&fref,A));
+
+%if &fid=0 %then %do;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let rc=%sysfunc(fput(&fid,%str(&header_name): %str(&header_value)));
+%let rc=%sysfunc(fwrite(&fid));
+
+%let rc=%sysfunc(fclose(&fid));
+%let rc=%sysfunc(filename(&fref));
+
+%mend mfs_httpheader;

server/ms_createfile.sas

@@ -0,0 +1,89 @@
+/**
+  @file
+  @brief Creates a file on SASjs Drive
+  @details Creates a file on SASjs Drive. To use the file as a Stored Program,
+  it must have a ".sas" extension.
+
+  Example:
+
+      filename stpcode temp;
+      data _null_;
+        file stpcode;
+        put '%put hello world;';
+      run;
+      %ms_createfile(/some/stored/program.sas, inref=stpcode)
+
+  @param [in] driveloc The full path to the file in SASjs Drive
+  @param [in] inref= (0) The fileref containing the file to create.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+
+**/
+
+%macro ms_createfile(driveloc
+    ,inref=0
+    ,mdebug=0
+  );
+
+%local fname0 fname1 boundary fname statcd msg;
+%let fname0=%mf_getuniquefileref();
+%let fname1=%mf_getuniquefileref();
+%let boundary=%mf_getuniquename();
+
+data _null_;
+  file &fname0 termstr=crlf;
+  infile &inref end=eof;
+  if _n_ = 1 then do;
+    put "--&boundary.";
+    put 'Content-Disposition: form-data; name="filePath"';
+    put ;
+    put "&driveloc";
+    put "--&boundary";
+    put 'Content-Disposition: form-data; name="file"; filename="ignore.sas"';
+    put "Content-Type: text/plain";
+    put ;
+  end;
+  input;
+  put _infile_; /* add the actual file to be sent */
+  if eof then do;
+    put ;
+    put "--&boundary--";
+  end;
+run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fname0;
+    input;
+    put _infile_;
+  run;
+%end;
+
+proc http method='POST' in=&fname0 out=&fname1
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file";
+  headers "Content-Type"="multipart/form-data; boundary=&boundary";
+%if &mdebug=1 %then %do;
+  debug level=1;
+%end;
+run;
+
+%let statcd=0;
+data _null_;
+  infile &fname1;
+  input;
+  putlog _infile_;
+  if _infile_='{"status":"success"}' then call symputx('statcd',1,'l');
+  else call symputx('msg',_infile_,'l');
+run;
+
+%mp_abort(
+  iftrue=(&statcd=0)
+  ,mac=ms_createfile.sas
+  ,msg=%superq(msg)
+)
+
+%mend ms_createfile;

server/ms_runstp.sas

@@ -0,0 +1,77 @@
+/**
+  @file
+  @brief Executes a SASjs Server Stored Program
+  @details Runs a Stored Program (using POST method) and extracts the webout and
+  log from the response JSON.
+
+  Example:
+
+      %ms_runstp(/some/stored/program
+        ,debug=131
+        ,outref=weboot
+      )
+
+  @param [in] pgm The full path to the Stored Program in SASjs Drive (_program
+    parameter)
+  @param [in] debug= (131) The value to supply to the _debug URL parameter
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outref= (outweb) The output fileref to contain the response JSON
+    (will be created using temp engine)
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_abort.sas
+
+**/
+
+%macro ms_runstp(pgm
+    ,debug=131
+    ,outref=outweb
+    ,mdebug=0
+  );
+%local dbg fname1;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+%let fname1=%mf_getuniquefileref();
+
+%mp_abort(iftrue=("&pgm"="")
+  ,mac=&sysmacroname
+  ,msg=%str(Program not provided)
+)
+
+data _null_;
+  file &fname1;
+  infile "&_sasjs_tokenfile";
+  input;
+  put 'Authorization: Bearer' _infile_;
+run;
+
+filename &outref temp;
+
+/* prepare request*/
+proc http method='POST' headerin=&fname1 out=&outref
+  url="&_sasjs_apiserverurl.&_sasjs_apipath?_program=&pgm%str(&)_debug=131";
+run;
+%if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  or &mdebug=1 %then %do;
+  data _null_;infile &outref;input;putlog _infile_;run;
+%end;
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  ,mac=&sysmacroname
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+
+%if &mdebug=1 %then %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%else %do;
+  /* clear refs */
+  filename &fname1 clear;
+%end;
+%mend ms_runstp;

server/ms_webout.sas

@@ -20,15 +20,22 @@
         %ms_webout(CLOSE)
 
 
-  @param action Either FETCH, OPEN, ARR, OBJ or CLOSE
-  @param ds The dataset to send back to the frontend
-  @param dslabel= value to use instead of the real name for sending to JSON
-  @param fmt=(Y) Set to N to send back unformatted values
-  @param fref=(_webout) The fileref to which to write the JSON
+  @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE
+  @param [in] ds The dataset to send back to the frontend
+  @param [out] dslabel= value to use instead of table name for sending to JSON
+  @param [in] fmt= (Y) Set to N to send back unformatted values
+  @param [out] fref= (_webout) The fileref to which to write the JSON
+  @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
+    (eg `null`) or as STRING values (eg `".a"` or `".b"`)
+  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+    such as the column formats and types.  The metadata is contained inside an
+    object with the same name as the table but prefixed with a dollar sign - ie,
+    `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
 
   <h4> SAS Macros </h4>
-  @li mp_jsonout.sas
   @li mf_getuser.sas
+  @li mp_jsonout.sas
+  @li mfs_httpheader.sas
 
   <h4> Related Macros </h4>
   @li mv_webout.sas
@@ -39,7 +46,9 @@
 
 **/
 
-%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y);
+%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
+  ,showmeta=NO
+);
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
 
@@ -78,11 +87,11 @@
   /* fix encoding */
   OPTIONS NOBOMFILE;
 
+  /* set the header */
+  %mfs_httpheader(Content-type,application/json)
+
   /* setup json */
   data _null_;file &fref encoding='utf-8' termstr=lf;
-  %if %str(&_debug) ge 131 %then %do;
-    put '>>weboutBEGIN<<';
-  %end;
     put '{"SYSDATE" : "' "&SYSDATE" '"';
     put ',"SYSTIME" : "' "&SYSTIME" '"';
   run;
@@ -91,7 +100,7 @@
 
 %else %if &action=ARR or &action=OBJ %then %do;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=DATASTEP,dbg=%str(&_debug)
+    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta
   )
 %end;
 %else %if &action=CLOSE %then %do;
@@ -106,15 +115,12 @@
       set &tempds;
       if not (upcase(name) =:"DATA"); /* ignore temp datasets */
       i+1;
-      call symputx('wt'!!left(i),name,'l');
+      call symputx(cats('wt',i),name,'l');
       call symputx('wtcnt',i,'l');
     data _null_; file &fref mod encoding='utf-8' termstr=lf;
       put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
-      proc contents noprint data=&wt
-        out=_data_ (keep=name type length format:);
-      run;%let tempds=%scan(&syslast,2,.);
       data _null_; file &fref mod encoding='utf-8' termstr=lf;
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
@@ -124,8 +130,7 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
-      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES)
       data _null_; file &fref mod encoding='utf-8' termstr=lf;
         put "}";
     %end;
@@ -157,17 +162,15 @@
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
-    put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length autoexec $512;
     autoexec=quote(urlencode(trim(getoption('autoexec'))));
     put ',"AUTOEXEC" : ' autoexec;
+    length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
     memsize=quote(cats(memsize));
     put ',"MEMSIZE" : ' memsize;
     put "}" @;
-  %if %str(&_debug) ge 131 %then %do;
-    put '>>weboutEND<<';
-  %end;
   run;
 %end;
 

tests/crossplatform/mcf_getfmttype.test.sas

@@ -0,0 +1,39 @@
+/**
+  @file
+  @brief Testing mcf_getfmttype.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mcf_getfmttype.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%mp_assertscope(SNAPSHOT)
+%mcf_getfmttype(wrap=YES, insert_cmplib=YES)
+%mp_assertscope(COMPARE,ignorelist=SASJS_FUNCTIONS)
+
+%mp_assert(
+  iftrue=(%sysfunc(mcf_getfmttype(DATE9.))=DATE),
+  desc=Check DATE format
+)
+%mp_assert(
+  iftrue=(%sysfunc(mcf_getfmttype($6))=CHAR),
+  desc=Check CHAR format
+)
+%mp_assert(
+  iftrue=(%sysfunc(mcf_getfmttype(8.))=NUM),
+  desc=Check NUM format
+)
+%mp_assert(
+  iftrue=(%sysfunc(mcf_getfmttype(E8601DT))=DATETIME),
+  desc=Check DATETIME format
+)
+
+/* test 2 - compile again test for warnings */
+%mcf_getfmttype(wrap=YES, insert_cmplib=YES)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check syscc=0 after re-initialisation
+)

tests/crossplatform/mcf_init.test.sas

@@ -0,0 +1,46 @@
+/**
+  @file
+  @brief Testing mcf_init.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mcf_init.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(%mcf_init(test)=0),
+  desc=Check if new func returns 0
+)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errs on basic invocation
+)
+%mp_assert(
+  iftrue=(%mcf_init(test)=1),
+  desc=Check if second invocation returns 1
+)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errs on second invocation
+)
+%mp_assert(
+  iftrue=(%mcf_init(test2)=0),
+  desc=Check if new invocation returns 0
+)
+%mp_assert(
+  iftrue=(%mcf_init(test2)=1),
+  desc=Check if second new invocation returns 1
+)
+%mp_assert(
+  iftrue=(%mcf_init(test)=1),
+  desc=Check original returns 1
+)
+%mp_assert(
+  iftrue=(%mcf_init(t)=1),
+  desc=Check subset returns 1
+)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errs at end
+)

tests/crossplatform/mcf_length.test.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Testing mcf_length.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mcf_length.sas
+  @li mp_assert.sas
+
+**/
+
+%mcf_length(wrap=YES, insert_cmplib=YES)
+
+data test;
+  call symputx('null',mcf_length(.));
+  call symputx('special',mcf_length(._));
+  call symputx('three',mcf_length(1));
+  call symputx('four',mcf_length(10000000));
+  call symputx('five',mcf_length(12345678));
+  call symputx('six',mcf_length(1234567890));
+  call symputx('seven',mcf_length(12345678901234));
+  call symputx('eight',mcf_length(12345678901234567));
+run;
+
+%mp_assert(
+  iftrue=(%str(&null)=%str(0)),
+  desc=Check if NULL returns 0
+)
+%mp_assert(
+  iftrue=(%str(&special)=%str(3)),
+  desc=Check if special missing ._ returns 3
+)
+%mp_assert(
+  iftrue=(%str(&three)=%str(3)),
+  desc=Check for length 3
+)
+%mp_assert(
+  iftrue=(%str(&four)=%str(4)),
+  desc=Check for length 4
+)
+%mp_assert(
+  iftrue=(%str(&five)=%str(5)),
+  desc=Check for length 5
+)
+%mp_assert(
+  iftrue=(%str(&six)=%str(6)),
+  desc=Check for length 6
+)
+%mp_assert(
+  iftrue=(%str(&seven)=%str(7)),
+  desc=Check for length 3
+)
+%mp_assert(
+  iftrue=(%str(&eight)=%str(8)),
+  desc=Check for length 8
+)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check syscc=0 before re-initialisation
+)
+
+/* test 2 - compile again test for warnings */
+%mcf_length(wrap=YES, insert_cmplib=YES)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check syscc=0 after re-initialisation
+)

tests/crossplatform/mf_existvar.test.sas

@@ -10,13 +10,11 @@
 
 
 %mp_assert(
-  iftrue=(%mf_existvar(sashelp.class,age)=1),
-  desc=Checking existing var exists,
-  outds=work.test_results
+  iftrue=(%mf_existvar(sashelp.class,age)>0),
+  desc=Checking existing var exists
 )
 
 %mp_assert(
   iftrue=(%mf_existvar(sashelp.class,isjustanumber)=0),
-  desc=Checking non existing var does not exist,
-  outds=work.test_results
+  desc=Checking non existing var does not exist
 )

tests/crossplatform/mf_getapploc.test.sas

@@ -46,4 +46,20 @@
   ),
   desc=Checking tests/macros appLoc matches (which has no subfolder),
   outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/tests/testsetup)"="/some/area"
+  ),
+  desc=Checking tests/testsetup operation,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/tests/testteardown)"="/some/area"
+  ),
+  desc=Checking tests/teardown operation,
+  outds=work.test_results
 )

tests/crossplatform/mf_getfilesize.test.sas

@@ -0,0 +1,30 @@
+/**
+  @file
+  @brief Testing mf_getfilesize macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getfilesize.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+data test;
+  x=1;
+run;
+
+%mp_assertscope(SNAPSHOT)
+%put %mf_getfilesize(libds=work.test)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking syscc
+)
+
+%put %mf_getfilesize(libds=test);
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking syscc with one level name
+)

tests/crossplatform/mf_getuniquelibref.test.sas

@@ -0,0 +1,54 @@
+/**
+  @file
+  @brief Testing mf_getuniquelibref macro
+  @details To test performance you can also use the following macro:
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquelibref.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+/* check valid libs */
+%mp_assertscope(SNAPSHOT)
+%let libshort=%mf_getuniquelibref(prefix=lib);
+%mp_assertscope(COMPARE,ignorelist=LIBSHORT)
+libname &libshort (work);
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking for valid libref &libshort,
+  outds=work.test_results
+)
+
+%let lib7=%mf_getuniquelibref(prefix=libref7);
+libname &lib7 (work);
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking for valid libref &lib7,
+  outds=work.test_results
+)
+
+
+/* check for invalid libs */
+
+%let lib8=%mf_getuniquelibref(prefix=lib8char);
+%mp_assert(
+  iftrue=(&lib8=0),
+  desc=Invalid prefix (8 chars),
+  outds=work.test_results
+)
+
+%let liblong=%mf_getuniquelibref(prefix=invalidlib);
+%mp_assert(
+  iftrue=(&liblong=0),
+  desc=Checking for invalid libref (long),
+  outds=work.test_results
+)
+
+%let badlib=%mf_getuniquelibref(prefix=8adlib);
+%mp_assert(
+  iftrue=(&badlib=0),
+  desc=Checking for invalid libref (8adlib),
+  outds=work.test_results
+)

tests/crossplatform/mf_verifymacvars.test.sas

@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Testing mf_verifymacvars macro
+
+  <h4> SAS Macros </h4>
+  @li mf_verifymacvars.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%let var1=x;
+%let var2=y;
+
+%mp_assertscope(SNAPSHOT)
+%mp_assert(
+  iftrue=(%mf_verifymacvars(var1 var2)=1),
+  desc=Checking macvars exist,
+  outds=work.test_results
+)
+%mp_assertscope(COMPARE)
+

tests/crossplatform/mp_applyformats.test.sas

@@ -34,7 +34,7 @@ run;
 %mp_applyformats(work.cols2)
 
 %mp_assert(
-  iftrue=("&orig_fmt"=""),
+  iftrue=("&origfmt"=""),
   desc=Check that formats were cleared,
   outds=work.test_results
 )

tests/crossplatform/mp_assert.test.sas

@@ -0,0 +1,15 @@
+/**
+  @file
+  @brief Testing mp_assert macro
+  @details This is quite "meta".. it's just testing itself
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(1=1),
+  desc=Checking result was created,
+  outds=work.test_results
+)

tests/crossplatform/mp_assertscope.test.sas

@@ -0,0 +1,80 @@
+/**
+  @file
+  @brief Testing mp_assertscope macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getvalue.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+
+**/
+
+%macro dostuff(action);
+  %if &action=ADD %then %do;
+    %global NEWVAR1 NEWVAR2;
+  %end;
+  %else %if &action=DEL %then %do;
+    %symdel NEWVAR1 NEWVAR2;
+  %end;
+  %else %if &action=MOD %then %do;
+    %let NEWVAR1=Let us pray..;
+  %end;
+  %else %if &action=NOTHING %then %do;
+    %local a b c d e;
+  %end;
+%mend dostuff;
+
+
+/* check for adding variables */
+%mp_assertscope(SNAPSHOT)
+%dostuff(ADD)
+%mp_assertscope(COMPARE,outds=work.testing_the_tester1)
+%mp_assert(
+  iftrue=(
+    "%mf_getvalue(work.testing_the_tester1,test_comments)"
+      ="Mod:() Add:(NEWVAR1 NEWVAR2) Del:()"
+  ),
+  desc=Checking result when vars added,
+  outds=work.test_results
+)
+
+
+/* check for modifying variables */
+%mp_assertscope(SNAPSHOT)
+%dostuff(MOD)
+%mp_assertscope(COMPARE,outds=work.testing_the_tester2)
+%mp_assert(
+  iftrue=(
+    "%mf_getvalue(work.testing_the_tester2,test_comments)"
+      ="Mod:(NEWVAR1) Add:() Del:()"
+  ),
+  desc=Checking result when vars modified,
+  outds=work.test_results
+)
+
+/* check for deleting variables */
+%mp_assertscope(SNAPSHOT)
+%dostuff(DEL)
+%mp_assertscope(COMPARE,outds=work.testing_the_tester3)
+%mp_assert(
+  iftrue=(
+    "%mf_getvalue(work.testing_the_tester3,test_comments)"
+      ="Mod:() Add:() Del:(NEWVAR1 NEWVAR2)"
+  ),
+  desc=Checking result when vars deleted,
+  outds=work.test_results
+)
+
+/* check for doing nothing */
+%mp_assertscope(SNAPSHOT)
+%dostuff(NOTHING)
+%mp_assertscope(COMPARE,outds=work.testing_the_tester4)
+%mp_assert(
+  iftrue=(
+    "%mf_getvalue(work.testing_the_tester4,test_comments)"
+      ="GLOBAL Variables Unmodified"
+  ),
+  desc=Checking results when nothing created,
+  outds=work.test_results
+)

tests/crossplatform/mp_cntlout.test.sas

@@ -0,0 +1,39 @@
+/**
+  @file
+  @brief Testing mp_cntlout.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_cntlout.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+libname perm (work);
+data work.loadfmts;
+  length fmtname $32;
+  eexcl='Y';
+  type='N';
+  do i=1 to 100;
+    fmtname=cats('SASJS_',i,'X');
+    do j=1 to 100;
+      start=cats(j);
+      end=cats(j+1);
+      label= cats('Dummy ',start);
+      output;
+    end;
+  end;
+run;
+proc format cntlin=work.loadfmts library=perm.testcat;
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mp_cntlout(libcat=perm.testcat,cntlout=work.cntlout)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.cntlout)=10000),
+  desc=Checking first hash diff,
+  outds=work.test_results
+)