Merge pull request #205 from sasjs/issue204

fix: updating mp_hashdataset to cope with STRICT mode.  Adding test a…

.all-contributorsrc

@@ -117,6 +117,15 @@
       "contributions": [
         "bug"
       ]
+    },
+    {
+      "login": "yabwon",
+      "name": "Bart Jablonski",
+      "avatar_url": "https://avatars.githubusercontent.com/u/9314894?v=4",
+      "profile": "https://github.com/yabwon",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.git-hooks/pre-commit

@@ -1,5 +1,12 @@
 #!/bin/bash
-sasjs lint
+
+# Ensure lint is passing
+LINT=`sasjs lint`
+if [[ "$LINT" != "✔ All matched files use @sasjs/lint code style!" ]]; then
+  echo "$LINT"
+  echo "To commit in spite of these warnings, use the -n parameter."
+  exit 1
+fi
 
 # Avoid commits to the master branch
 BRANCH=`git rev-parse --abbrev-ref HEAD`

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [sasjs]

.gitpod.yml

@@ -1,7 +1,7 @@
 tasks:
-  - init: |
-      nvm install --lts
-      npm i -g @sasjs/cli
+  - init: npm install -g npm
+  - command: npm i
+  - command: npm i -g @sasjs/cli
 
 image:
   file: .gitpod.dockerfile
@@ -24,4 +24,4 @@ github:
     # add a "Review in Gitpod" button to pull requests (defaults to false)
     addBadge: false
     # add a label once the prebuild is ready to pull requests (defaults to false)
-    addLabel: prebuilt-in-gitpod
+    addLabel: prebuilt-in-gitpod

README.md

@@ -31,19 +31,28 @@ Documentation: https://core.sasjs.io
 
 ## Components
 
-### BASE library (SAS9/Viya)
+### BASE library (All Platforms)
 
 - OS independent
-- Not metadata aware
+- Works on all SAS Platforms
 - No X command
 - Prefixes: _mf_, _mp_
 
-#### FCMP library (SAS9/Viya)
+### DDL library (All Platforms)
+
+- OS independent
+- Works on all SAS Platforms
+- No X command
+- Prefixes: _mddl_(lib)_ -> where lib can be "SAS" (in relation to a SAS component) or "DC" (in relation to a Data Controller component)
+
+This library will not be used for storing data entries (such as formats or datalines).  Where this becomes necessary in the future, a new repo will be created, in order to keep the NPM bundle size down (for the benefit of those looking to embed purely macros in their applications).
+
+### FCMP library (All Platforms)
+
 - Function and macro names are identical, except for special cases
 - Prefixes: _mcf_
 
-The fcmp macros are used to generate fcmp functions, and can be used with or
-without the `proc fcmp` wrapper.
+The fcmp macros are used to generate fcmp functions, and can be used with or without the `proc fcmp` wrapper.
 
 ### META library (SAS9 only)
 
@@ -81,7 +90,7 @@ Macros used for interfacing with SAS Viya.
 
 Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
 
-To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert your LUA into a data step with put statements, and create the macro wrapper with a `ml_` prefix.  You can then use your module in any program by running:
+To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert all files with a ".lua" extension into a macro wrapper with an `ml_` prefix (embedding the necessary data step put statements).  You can then use your module in any program by running:
 
 ```sas
 /* compile the lua module */
@@ -95,8 +104,7 @@ endsubmit;
 run;
 ```
 
-- X command enabled
-- Prefixes: _mmw_,_mmu_,_mmx_
+- Prefixes: _ml_
 
 ## Installation
 
@@ -126,14 +134,15 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 - one macro per file
 - prefixes:
   - _mcf_ for macro compiled functions (proc fcmp)
+  - _mddl_ for macros containing DDL (Data Definition Language)
   - _mf_ for macro functions (can be used in open code).
   - _ml_ for macros that are used to compile LUA modules
   - _mm_ for metadata macros (interface with the metadata server).
-  - _mmx_ for macros that use metadata and are XCMD enabled
+  - _mmx_ for macros that use metadata and are XCMD enabled (working on both windows and unix)
   - _mp_ for macro procedures (which generate sas code)
   - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
   - _mv_ for macro procedures that will only work in Viya
-  - _mx_ for macros that are XCMD enabled
+  - _mx_ for macros that are XCMD enabled (working on both windows and unix)
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
@@ -193,11 +202,14 @@ When contributing to this library, it is therefore important to ensure that all
 
 ## Breaking Changes
 
-We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major (breaking) release becomes necessary:
+We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major/breaking release (v5) becomes necessary:
 
+* mf_getuniquelibref.sas to have the deprecated maxtried parameter removed (no longer needed)
 * mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
 * `insert_cmplib` option of mcf_xxx macros will be deprecated (the option is now checked automatically with value inserted only if needed)
-* mcf_xxx macros to have `wrap=` option defaulted to YES for convenience
+* mcf_xxx macros to have `wrap=` option defaulted to YES for convenience.  Set this option explicitly to avoid issues.
+* mp_getddl.sas to be renamed to mp_ds2ddl.sas (consistent with other ds2xxx macros).  A wrapper macro is already in place, and you are able to use this immediately.  The default for SHOWLOG will also be YES instead of NO.
+* mp_coretable.sas will be replaced by the standalone macros in the `ddl` folder (which are already available)
 
 ## Star Gazing
 
@@ -205,11 +217,19 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 ![](https://starchart.cc/sasjs/core.svg)
 
+## Other SAS Repositories
 
+The following repositories are also worth checking out:
+
+* [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
+* [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)
+* [KatjaGlassConsulting/SMILE-SmartSASMacros](https://github.com/KatjaGlassConsulting/SMILE-SmartSASMacros)
+* [scottbass/sas](https://github.com/scottbass/SAS)
+* [yabwon/sas_packages](https://github.com/yabwon/SAS_PACKAGES)
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-10-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-11-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -230,6 +250,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
     <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
     <td align="center"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
+    <td align="center"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
   </tr>
 </table>
 

base/mf_deletefile.sas

@@ -0,0 +1,31 @@
+/**
+  @file
+  @brief Deletes a physical file, if it exists
+  @details Usage:
+
+      %mf_writefile(&sasjswork/myfile.txt,l1=some content)
+
+      %mf_deletefile(&sasjswork/myfile.txt)
+
+      %mf_deletefile(&sasjswork/myfile.txt)
+
+
+  @param filepath Full path to the target file
+
+  @returns The return code from the fdelete() invocation
+
+  <h4> Related Macros </h4>
+  @li mf_deletefile.test.sas
+  @li mf_writefile.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_deletefile(file
+)/*/STORE SOURCE*/;
+  %local rc fref;
+  %let rc= %sysfunc(filename(fref,&file));
+  %if %sysfunc(fdelete(&fref)) ne 0 %then %put %sysfunc(sysmsg());
+  %let rc= %sysfunc(filename(fref));
+%mend mf_deletefile;

base/mf_existfeature.sas

@@ -9,19 +9,17 @@
 
         %put %mf_existfeature(PROCLUA);
 
-  @param feature the feature to detect.  Leave blank to list all in log.
+  @param [in] feature The feature to detect.
 
   @return output returns 1 or 0 (or -1 if not found)
 
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
 
-
   @version 8
   @author Allan Bowe
 **/
 /** @cond */
-
 %macro mf_existfeature(feature
 )/*/STORE SOURCE*/;
   %let feature=%upcase(&feature);
@@ -29,7 +27,11 @@
   %let platform=%mf_getplatform();
 
   %if &feature= %then %do;
-    %put Supported features:  PROCLUA;
+    %put No feature was requested for detection;
+  %end;
+  %else %if &feature=COLCONSTRAINTS %then %do;
+    %if %substr(&sysver,1,1)=4 %then 0;
+    %else 1;
   %end;
   %else %if &feature=PROCLUA %then %do;
     /* https://blogs.sas.com/content/sasdummy/2015/08/03/using-lua-within-your-sas-programs */
@@ -43,5 +45,4 @@
     %put &sysmacroname: &feature not found;
   %end;
 %mend mf_existfeature;
-
-/** @endcond */
+/** @endcond */

base/mf_getapploc.sas

@@ -15,6 +15,7 @@
   @li /data
   @li /jobs
   @li /services
+  @li /tests
   @li /tests/jobs
   @li /tests/services
   @li /tests/macros
@@ -46,9 +47,13 @@
 
 /**
   * First check we are not in the tests/macros folder (which has no subfolders)
+  * or specifically in the testsetup or testteardown services
   */
-%if %index(&pgm,/tests/macros/) %then %do;
-  %let root=%substr(&pgm,1,%index(&pgm,/tests/macros)-1);
+%if %index(&pgm,/tests/macros/)
+or %index(&pgm,/tests/testsetup)
+or %index(&pgm,/tests/testteardown)
+%then %do;
+  %let root=%substr(&pgm,1,%index(&pgm,/tests)-1);
   &root
   %return;
 %end;

base/mf_getplatform.sas

@@ -22,6 +22,12 @@
 )/*/STORE SOURCE*/;
 %local a b c;
 %if &switch.NONE=NONE %then %do;
+  %if %symexist(sasjsprocessmode) %then %do;
+    %if &sasjsprocessmode=Stored Program %then %do;
+      SASJS
+      %return;
+    %end;
+  %end;
   %if %symexist(sysprocessmode) %then %do;
     %if "&sysprocessmode"="SAS Object Server"
     or "&sysprocessmode"= "SAS Compute Server" %then %do;

base/mf_getuniquelibref.sas

@@ -14,27 +14,44 @@
 
 > mclib3
 
-  @param prefix= first part of libref.  Remember that librefs can only be 8 characters,
-    so a 7 letter prefix would mean that maxtries should be 10.
-  @param maxtries= the last part of the libref.  Provide an integer value.
+  A blank value is returned if no usable libname is determined.
+
+  @param [in] prefix= (mclib) first part of the returned libref. As librefs can
+    be as long as 8 characters, a maximum length of 7 characters is premitted
+    for this prefix.
+  @param [in] maxtries= Deprecated parameter. Remains here to ensure a
+    non-breaking change.  Will be removed in v5.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-
 %macro mf_getuniquelibref(prefix=mclib,maxtries=1000);
-  %local x libref;
-  %let x=0;
-  %do x=0 %to &maxtries;
-  %if %sysfunc(libref(&prefix&x)) ne 0 %then %do;
-    %let libref=&prefix&x;
-    %let rc=%sysfunc(libname(&libref,%sysfunc(pathname(work))));
-    %if &rc %then %put %sysfunc(sysmsg());
-    &prefix&x
-    %*put &sysmacroname: Libref &libref assigned as WORK and returned;
+  %local x;
+
+  %if ( %length(&prefix) gt 7 ) %then %do;
+    %put %str(ERR)OR: The prefix parameter cannot exceed 7 characters.;
+    0
     %return;
   %end;
+  %else %if (%sysfunc(NVALID(&prefix,v7))=0) %then %do;
+    %put %str(ERR)OR: Invalid prefix (&prefix);
+    0
+    %return;
   %end;
-  %put unable to find available libref in range &prefix.0-&maxtries;
+
+  /* Set maxtries equal to '10 to the power of [# unused characters] - 1' */
+  %let maxtries=%eval(10**(8-%length(&prefix))-1);
+
+  %do x = 0 %to &maxtries;
+    %if %sysfunc(libref(&prefix&x)) ne 0 %then %do;
+      &prefix&x
+      %return;
+    %end;
+    %let x = %eval(&x + 1);
+  %end;
+
+  %put %str(ERR)OR: No usable libref in range &prefix.0-&maxtries;
+  %put %str(ERR)OR- Try reducing the prefix or deleting some libraries!;
+  0
 %mend mf_getuniquelibref;

base/mf_getuser.sas

@@ -33,7 +33,8 @@
   %else %if %symexist(&metavar) %then %do;
     %if %length(&&&metavar)=0 %then %let user=&sysuserid;
     /* sometimes SAS will add @domain extension - remove for consistency */
-    %else %let user=%scan(&&&metavar,1,@);
+    /* but be sure to quote in case of usernames with commas */
+    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@));
   %end;
   %else %let user=&sysuserid;
 

base/mf_verifymacvars.sas

@@ -1,6 +1,6 @@
 /**
   @file
-  @brief Checks if a set of macro variables exist / contain values.
+  @brief Checks if a set of macro variables exist AND contain values.
   @details Writes ERROR to log if abortType is SOFT, else will call %mf_abort.
   Usage:
 
@@ -14,10 +14,11 @@
   <h4> SAS Macros </h4>
   @li mf_abort.sas
 
-  @param verifyvars space separated list of macro variable names
-  @param makeupcase= set to YES to convert all variable VALUES to
+  @param [in] verifyvars Space separated list of macro variable names
+  @param [in] makeupcase= (NO) Set to YES to convert all variable VALUES to
     uppercase.
-  @param mAbort= Abort Type.  Default is SOFT (writes err to log).
+  @param [in] mAbort= (SOFT) Abort Type.  When SOFT, simply writes an err
+    message to the log.
     Set to any other value to call mf_abort (which can be configured to abort in
     various fashions according to context).
 
@@ -58,8 +59,14 @@
 
   %goto exit_success;
   %exit_err:
-    %if &mAbort=SOFT %then %put %str(ERR)OR: &abortmsg;
-    %else %mf_abort(mac=mf_verifymacvars,type=&mabort,msg=&abortmsg);
+    %put &abortmsg;
+    %mf_abort(iftrue=(&mabort ne SOFT),
+      mac=mf_verifymacvars,
+      msg=%str(&abortmsg)
+    )
+    0
+    %return;
   %exit_success:
+  1
 
 %mend mf_verifymacvars;

base/mp_abort.sas

@@ -85,7 +85,7 @@
   %end;
 
   /* Stored Process Server web app context */
-  %if %symexist(_metaperson)
+  %if %symexist(_METAFOLDER)
     or "&SYSPROCESSNAME "="Compute Server "
     or &mode=INCLUDE
   %then %do;
@@ -161,12 +161,14 @@
     /* send response in SASjs JSON format */
     data _null_;
       file _webout mod lrecl=32000 encoding='utf-8';
-      length msg $32767 ;
+      length msg syswarningtext syserrortext $32767 ;
       sasdatetime=datetime();
       msg=symget('msg');
     %if &logline>0 %then %do;
       msg=cats(msg,'\n\nLog Extract:\n',symget('logmsg'));
     %end;
+      /* escape the escapes */
+      msg=tranwrd(msg,'\','\\');
       /* escape the quotes */
       msg=tranwrd(msg,'"','\"');
       /* ditch the CRLFs as chrome complains */
@@ -260,4 +262,4 @@
   %end;
 %mend mp_abort;
 
-/** @endcond */
+/** @endcond */

base/mp_assertscope.sas

@@ -74,7 +74,8 @@
   outds=work.test_results
 )/*/STORE SOURCE*/;
 %local ds test_result test_comments del add mod ilist;
-%let ilist=%upcase(&sasjs_prefix._FUNCTIONS &ignorelist);
+%let ilist=%upcase(&sasjs_prefix._FUNCTIONS SYS_PROCHTTP_STATUS_CODE
+  SYS_PROCHTTP_STATUS_CODE SYS_PROCHTTP_STATUS_PHRASE &ignorelist);
 
 /**
   * this sets up the global vars, it will also enter STRICT mode.  If this
@@ -89,7 +90,7 @@
   create table &scopeds as
     select name,offset,value
     from dictionary.macros
-    where scope="&scope" and name not in (%mf_getquotedstr(&ilist))
+    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
     order by name,offset;
 %end;
 %else %if &action=COMPARE %then %do;
@@ -98,12 +99,14 @@
   create table _data_ as
     select name,offset,value
     from dictionary.macros
-    where scope="&scope" and name not in (%mf_getquotedstr(&ilist))
+    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
     order by name,offset;
 
   %let ds=&syslast;
 
-  proc compare base=&scopeds compare=&ds;
+  proc compare
+    base=&scopeds(where=(upcase(name) not in (%mf_getquotedstr(&ilist))))
+    compare=&ds;
   run;
 
   %if &sysinfo=0 %then %do;
@@ -141,4 +144,4 @@
   drop table &ds;
 %end;
 
-%mend mp_assertscope;
+%mend mp_assertscope;

base/mp_cntlout.sas

@@ -0,0 +1,85 @@
+/**
+  @file mp_cntlout.sas
+  @brief Creates a cntlout dataset in a consistent format
+  @details The dataset produced by proc format in the cntlout option will vary
+  according to its contents.
+
+  When dealing with formats from an ETL perspective (eg in [Data Controller for
+  SAS](https://datacontroller.io)), it is important that the output dataset
+  has a consistent model (and compariable values).
+
+  This macro makes use of mddl_sas_cntlout.sas to provide the consistent model,
+  and will left-align the start and end values when dealing with numeric ranges
+  to enable consistency when checking for differences.
+
+  usage:
+
+      %mp_cntlout(libcat=yourlib.cat,cntlout=work.formatexport)
+
+  @param [in] libcat The library.catalog reference
+  @param [in] fmtlist= (0) provide a space separated list of specific formats to
+    extract
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [out] cntlout= (work.fmtextract) Libds reference for the output dataset
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mf_getvarformat.sas
+  @li mp_getformats.sas
+  @li mp_loadformat.sas
+  @li mp_ds2fmtds.sas
+
+  @version 9.2
+  @author Allan Bowe
+  @cond
+**/
+
+%macro mp_cntlout(
+  iftrue=(1=1)
+  ,libcat=
+  ,cntlout=work.fmtextract
+  ,fmtlist=0
+)/*/STORE SOURCE*/;
+%local ddlds cntlds i;
+
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+%let ddlds=%mf_getuniquename();
+%let cntlds=%mf_getuniquename();
+
+%mddl_sas_cntlout(libds=&ddlds)
+
+%if %index(&libcat,-)>0 and %scan(&libcat,2,-)=FC %then %do;
+  %let libcat=%scan(&libcat,1,-);
+%end;
+
+proc format lib=&libcat cntlout=&cntlds;
+%if "&fmtlist" ne "0" %then %do;
+  select
+  %do i=1 %to %sysfunc(countw(&fmtlist));
+    %scan(&fmtlist,&i,%str( ))
+  %end;
+  ;
+%end;
+run;
+
+data &cntlout;
+  if 0 then set &ddlds;
+  set &cntlds;
+  if type="N" then do;
+    start=cats(start);
+    end=cats(end);
+  end;
+run;
+proc sort;
+  by fmtname start;
+run;
+
+proc sql;
+drop table &ddlds,&cntlds;
+
+%mend mp_cntlout;
+/** @endcond */

base/mp_coretable.sas

@@ -5,24 +5,32 @@
     make use of permanent tables.  To avoid duplication in definitions, this
     macro provides a central location for managing the corresponding DDL.
 
+  Note - this macro is likely to be deprecated in future in favour of a
+  dedicated "datamodel" folder (prefix mddl)
+
+  Any corresponding data would go in a seperate repo, to avoid this one
+  ballooning in size!
+
   Example usage:
 
       %mp_coretable(LOCKTABLE,libds=work.locktable)
 
   @param [in] table_ref The type of table to create.  Example values:
-    @li DIFFTABLE - Used to store changes to tables.  Used by mp_storediffs.sas
-      and mp_stackdiffs.sas
-    @li FILTER_DETAIL - For storing detailed filter values.  Used by
-      mp_filterstore.sas.
-    @li FILTER_SUMMARY - For storing summary filter values.  Used by
-      mp_filterstore.sas.
-    @li LOCKANYTABLE - For "locking" tables prior to multipass loads. Used by
-      mp_lockanytable.sas
-    @li MAXKEYTABLE - For storing the maximum retained key information.  Used
-      by mp_retainedkey.sas
+    @li DIFFTABLE
+    @li FILTER_DETAIL
+    @li FILTER_SUMMARY
+    @li LOCKANYTABLE
+    @li MAXKEYTABLE
   @param [in] libds= (0) The library.dataset reference used to create the table.
     If not provided, then the DDL is simply printed to the log.
 
+  <h4> SAS Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mddl_dc_filterdetail.sas
+  @li mddl_dc_filtersummary.sas
+  @li mddl_dc_locktable.sas
+  @li mddl_dc_maxkeytable.sas
+
   <h4> Related Macros </h4>
   @li mp_filterstore.sas
   @li mp_lockanytable.sas
@@ -41,76 +49,21 @@
 %let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
 proc sql;
 %if &table_ref=DIFFTABLE %then %do;
-  create table &outds(
-      load_ref char(36) label='unique load reference',
-      processed_dttm num format=E8601DT26.6 label='Processed at timestamp',
-      libref char(8) label='Library Reference (8 chars)',
-      dsn char(32) label='Dataset Name (32 chars)',
-      key_hash char(32) label=
-        'MD5 Hash of primary key values (pipe seperated)',
-      move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified',
-      is_pk num label='Is Primary Key Field? (1/0)',
-      is_diff num label=
-        'Did value change? (1/0/-1).  Always -1 for appends and deletes.',
-      tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
-      tgtvar_nm char(32) label='Target variable name (32 chars)',
-      oldval_num num format=best32. label='Old (numeric) value',
-      newval_num num format=best32. label='New (numeric) value',
-      oldval_char char(32765) label='Old (character) value',
-      newval_char char(32765) label='New (character) value',
-    constraint pk_mpe_audit
-      primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
-  );
+  %mddl_dc_difftable(libds=&outds)
 %end;
 %else %if &table_ref=LOCKTABLE %then %do;
-  create table &outds(
-      lock_lib char(8),
-      lock_ds char(32),
-      lock_status_cd char(10) not null,
-      lock_user_nm char(100) not null ,
-      lock_ref char(200),
-      lock_pid char(10),
-      lock_start_dttm num format=E8601DT26.6,
-      lock_end_dttm num format=E8601DT26.6,
-    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+  %mddl_dc_locktable(libds=&outds)
 %end;
 %else %if &table_ref=FILTER_SUMMARY %then %do;
-  create table &outds(
-      filter_rk num not null,
-      filter_hash char(32) not null,
-      filter_table char(41) not null,
-      processed_dttm num not null format=E8601DT26.6,
-    constraint pk_mpe_filteranytable
-      primary key(filter_rk));
+  %mddl_dc_filtersummary(libds=&outds)
 %end;
 %else %if &table_ref=FILTER_DETAIL %then %do;
-  create table &outds(
-      filter_hash char(32) not null,
-      filter_line num not null,
-      group_logic char(3) not null,
-      subgroup_logic char(3) not null,
-      subgroup_id num not null,
-      variable_nm varchar(32) not null,
-      operator_nm varchar(12) not null,
-      raw_value varchar(4000) not null,
-      processed_dttm num not null format=E8601DT26.6,
-    constraint pk_mpe_filteranytable
-      primary key(filter_hash,filter_line));
+  %mddl_dc_filterdetail(libds=&outds)
 %end;
 %else %if &table_ref=MAXKEYTABLE %then %do;
-  create table &outds(
-      keytable varchar(41) label='Base table in libref.dataset format',
-      keycolumn char(32) format=$32.
-        label='The Retained key field containing the key values.',
-      max_key num label=
-        'Integer representing current max RK or SK value in the KEYTABLE',
-      processed_dttm num format=E8601DT26.6
-        label='Datetime this value was last updated',
-    constraint pk_mpe_maxkeyvalues
-        primary key(keytable));
+  %mddl_dc_maxkeytable(libds=&outds)
 %end;
 
-
 %if &libds=0 %then %do;
   describe table &syslast;
   drop table &syslast;

base/mp_dirlist.sas

@@ -94,6 +94,13 @@ data &out_ds(compress=no
   %end;
   if rc = 0 then do;
     did = dopen(fref);
+    if did=0 then do;
+      putlog "NOTE: This directory is empty, or does not exist - &path";
+      msg=sysmsg();
+      put msg;
+      put _all_;
+      stop;
+    end;
     /* attribute is OS-dependent - could be "Directory" or "Directory Name" */
     numopts=doptnum(did);
     do i=1 to numopts;
@@ -101,12 +108,6 @@ data &out_ds(compress=no
       if foption=:'Directory' then i=numopts;
     end;
     directory=dinfo(did,foption);
-    if did=0 then do;
-      putlog "NOTE: This directory is empty - " directory;
-      msg=sysmsg();
-      put _all_;
-      stop;
-    end;
     rc = filename(fref);
   end;
   else do;
@@ -242,4 +243,4 @@ run;
 proc sql;
 drop table &out_ds;
 
-%mend mp_dirlist;
+%mend mp_dirlist;

base/mp_ds2ddl.sas

@@ -0,0 +1,30 @@
+/**
+  @file
+  @brief A wrapper for mp_getddl.sas
+  @details In the next release, this will be the main version.
+
+
+  <h4> SAS Macros </h4>
+  @li mp_getddl.sas
+
+**/
+
+%macro mp_ds2ddl(libds,fref=getddl,flavour=SAS,showlog=YES,schema=
+  ,applydttm=NO
+)/*/STORE SOURCE*/;
+
+%local libref;
+%let libds=%upcase(&libds);
+%let libref=%scan(&libds,1,.);
+%if &libref=&libds %then %let libds=WORK.&libds;
+
+%mp_getddl(%scan(&libds,1,.)
+  ,%scan(&libds,2,.)
+  ,fref=&fref
+  ,flavour=SAS
+  ,showlog=&showlog
+  ,schema=&schema
+  ,applydttm=&applydttm
+)
+
+%mend mp_ds2ddl;

base/mp_filtercheck.sas

@@ -6,7 +6,8 @@
   SYSCC to 1008 if bad records are found, and call mp_abort.sas for a
   graceful service exit (configurable).
 
-  Used for dynamic filtering in [Data Controller for SAS®](https://datacontroller.io).
+  Used for dynamic filtering in [Data Controller for SAS®](
+  https://datacontroller.io).
 
   Usage:
 
@@ -125,7 +126,7 @@ data &outds;
     output;
   end;
   if OPERATOR_NM not in
-  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
+  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE')
   then do;
     REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;

base/mp_filterstore.sas

@@ -8,8 +8,13 @@
   https://sasapps.io)).  This macro is also used in [Data Controller for SAS](
   https://datacontroller.io).
 
+  A more recent feature of this macro is the ability to support filter queries
+  on Format Catalogs.  This is achieved by adding a `-FC` suffix to the `libds`
+  parameter - where the "ds" in this case is the catalog name.
 
-  @param [in] libds= The target dataset to be filtered (lib should be assigned)
+
+  @param [in] libds= The target dataset to be filtered (lib should be assigned).
+    If filtering a format catalog, add the following suffix:  `-FC`.
   @param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to
     be validated.  Has the following format:
 |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
@@ -45,6 +50,7 @@
 
 
   <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
   @li mf_getuniquename.sas
   @li mf_getvalue.sas
   @li mf_islibds.sas
@@ -78,7 +84,10 @@
 %put &sysmacroname entry vars:;
 %put _local_;
 
-%local ds1 ds2 ds3 ds4 filter_hash;
+%local ds0 ds1 ds2 ds3 ds4 filter_hash orig_libds;
+%let libds=%upcase(&libds);
+%let orig_libds=&libds;
+
 %mp_abort(iftrue= (&syscc ne 0)
   ,mac=mp_filterstore
   ,msg=%str(syscc=&syscc on macro entry)
@@ -96,12 +105,27 @@
   ,msg=%str(Invalid lock_table value: &lock_table)
 )
 
-/* validate query */
+/**
+  * validate query
+  * use format catalog export, if a format
+  */
+%if "%substr(&libds,%length(&libds)-2,3)"="-FC" %then %do;
+  %let libds=%scan(&libds,1,-); /* chop off -FC extension */
+  %let ds0=%mf_getuniquename(prefix=fmtds_);
+  %let libds=&ds0;
+  /*
+    There is no need to export the entire format catalog here - the validations
+    are done against the data model, not the data values.  So we can simply
+    hardcode the structure based on the cntlout dataset.
+  */
+  %mddl_sas_cntlout(libds=&ds0)
+
+%end;
 %mp_filtercheck(&queryds,targetds=&libds,abort=YES)
 
 /* hash the result */
 %let ds1=%mf_getuniquename(prefix=hashds);
-%mp_hashdataset(&queryds,outds=&ds1,salt=&libds)
+%mp_hashdataset(&queryds,outds=&ds1,salt=&orig_libds)
 %let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
 %if &mdebug=1 %then %do;
   data _null_;
@@ -132,7 +156,7 @@ run;
   %let ds3=%mf_getuniquename(prefix=filtersum);
   data work.&ds3;
     if 0 then set &filter_summary;
-    filter_table=symget('libds');
+    filter_table="&orig_libds";
     filter_hash="&filter_hash";
     PROCESSED_DTTM=%sysfunc(datetime());
     output;

base/mp_getformats.sas

@@ -40,6 +40,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
   |`WHICHPATH `|`**OTHER** `|`**OTHER** `|`big fat problem if not path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|`O `|` `|` `|` `|` `|
 
   <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
   @li mf_dedup.sas
   @li mf_getfmtlist.sas
   @li mf_getfmtname.sas
@@ -94,30 +95,7 @@ create table &outsummary as
 
 %if "&outdetail" ne "0" %then %do;
   /* ensure base table always exists */
-  proc sql;
-  create table &outdetail(
-      FMTNAME char(32)     label='Format name'
-      ,START char(16)     label='Starting value for format'
-      ,END char(16)     label='Ending value for format'
-      ,LABEL char(256)     label='Format value label'
-      ,MIN num length=3     label='Minimum length'
-      ,MAX num length=3     label='Maximum length'
-      ,DEFAULT num length=3     label='Default length'
-      ,LENGTH num length=3     label='Format length'
-      ,FUZZ num     label='Fuzz value'
-      ,PREFIX char(2)     label='Prefix characters'
-      ,MULT num     label='Multiplier'
-      ,FILL char(1)     label='Fill character'
-      ,NOEDIT num length=3     label='Is picture string noedit?'
-      ,TYPE char(1)     label='Type of format'
-      ,SEXCL char(1)     label='Start exclusion'
-      ,EEXCL char(1)     label='End exclusion'
-      ,HLO char(13)     label='Additional information'
-      ,DECSEP char(1)     label='Decimal separator'
-      ,DIG3SEP char(1)     label='Three-digit separator'
-      ,DATATYPE char(8)     label='Date/time/datetime?'
-      ,LANGUAGE char(8)     label='Language for date strings'
-  );
+  %mddl_sas_cntlout(libds=&outdetail)
   /* grab the location of each format */
   %let fmtcnt=0;
   data _null_;
@@ -134,7 +112,11 @@ create table &outsummary as
     proc format library=&&fmtloc&i CNTLOUT=&tempds;
       select &&fmtname&i;
     run;
-    proc append base=&outdetail data=&tempds;
+    data &tempds;
+      if 0 then set &outdetail;
+      set &tempds;
+    run;
+    proc append base=&outdetail data=&tempds ;
     run;
   %end;
 %end;

base/mp_gsubfile.sas

@@ -48,6 +48,11 @@
   outfile=0
 )/*/STORE SOURCE*/;
 
+  %if "%substr(&sysver,1,4)"="V.04" %then %do;
+    %put %str(ERR)OR: Viya 4 does not support the IO library in lua;
+    %return;
+  %end;
+
   %ml_gsubfile()
 
 %mend mp_gsubfile;

base/mp_hashdataset.sas

@@ -1,8 +1,8 @@
 /**
   @file
   @brief Returns a unique hash for a dataset
-  @details Ignores metadata attributes, used only to hash values. Compared
-  datasets must be in the same order.
+  @details Ignores metadata attributes, used only to hash values. If used to
+  compare datasets, they must have their columns and rows in the same order.
 
       %mp_hashdataset(sashelp.class,outds=myhash)
 
@@ -17,13 +17,16 @@
   @li mf_getattrn.sas
   @li mf_getuniquename.sas
   @li mf_getvarlist.sas
-  @li mf_getvartype.sas
+  @li mp_md5.sas
+
+  <h4> Related Files </h4>
+  @li mp_hashdataset.test.sas
 
   @param [in] libds dataset to hash
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
   @param [in] iftrue= A condition under which the macro should be executed.
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
-  will contain one column (hashkey) with one observation (a hex32.
+  will contain one column (hashkey) with one observation (a $hex32.
   representation of the input hash)
   |hashkey:$32.|
   |---|
@@ -35,48 +38,48 @@
 
 %macro mp_hashdataset(
   libds,
-  outds=,
+  outds=work._data_,
   salt=,
   iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
-  %if not(%eval(%unquote(&iftrue))) %then %return;
+%local keyvar /* roll up the md5 */
+  prevkeyvar /* retain prev record md5 */
+  lastvar /* last var in input ds */
+  cvars nvars;
 
-  %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
-    %put %str(WARN)ING: Dataset &libds is empty, or is not a dataset;
-  %end;
-  %else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
-    %put %str(ERR)OR: Dataset &libds is not a dataset;
-  %end;
-  %else %do;
-    %local keyvar /* roll up the md5 */
-      prevkeyvar /* retain prev record md5 */
-      lastvar /* last var in input ds */
-      varlist var i;
-    /* avoid naming conflict for hash key vars */
-    %let keyvar=%mf_getuniquename();
-    %let prevkeyvar=%mf_getuniquename();
-    %let lastvar=%mf_getuniquename();
-    %let varlist=%mf_getvarlist(&libds);
-    data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
-      length &prevkeyvar &keyvar $32;
-      retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
-      set &libds end=&lastvar;
-      /* hash should include previous row */
-      &keyvar=put(md5(&prevkeyvar
-      /* loop every column, hashing every individual value */
-    %do i=1 %to %sysfunc(countw(&varlist));
-      %let var=%scan(&varlist,&i,%str( ));
-      %if %mf_getvartype(&libds,&var)=C %then %do;
-          !!put(md5(trim(&var)),$hex32.)
-      %end;
-      %else %do;
-          !!put(md5(trim(put(&var*1,binary64.))),$hex32.)
-      %end;
-    %end;
-      ),$hex32.);
-      &prevkeyvar=&keyvar;
-      if &lastvar then output;
-    run;
-  %end;
-%mend mp_hashdataset;
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+/* avoid naming conflict for hash key vars */
+%let keyvar=%mf_getuniquename();
+%let prevkeyvar=%mf_getuniquename();
+%let lastvar=%mf_getuniquename();
+
+%if %mf_getattrn(&libds,NLOBS)=0 %then %do;
+  data &outds;
+    length hashkey $32;
+    retain hashkey "%sysfunc(md5(%str(&salt)),$hex32.)";
+    output;
+    stop;
+  run;
+  %put &sysmacroname: Dataset &libds is empty, or is not a dataset;
+  %put &sysmacroname: hashkey of &outds is based on salt (&salt) only;
+%end;
+%else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
+  %put %str(ERR)OR: Dataset &libds is not a dataset;
+%end;
+%else %do;
+  data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
+    length &prevkeyvar &keyvar $32;
+    retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
+    set &libds end=&lastvar;
+    /* hash should include previous row */
+    &keyvar=%mp_md5(
+      cvars=%mf_getvarlist(&libds,typefilter=C) &prevkeyvar,
+      nvars=%mf_getvarlist(&libds,typefilter=N)
+    );
+    &prevkeyvar=&keyvar;
+    if &lastvar then output;
+  run;
+%end;
+%mend mp_hashdataset;

base/mp_jsonout.sas

@@ -203,8 +203,15 @@
                     prxchange('s/'!!'0A'x!!'/\n/',-1,
                     prxchange('s/'!!'0D'x!!'/\r/',-1,
                     prxchange('s/'!!'09'x!!'/\t/',-1,
+                    prxchange('s/'!!'00'x!!'/\\u0000/',-1, /* NUL */
+                    prxchange('s/'!!'0E'x!!'/\\u000E/',-1, /* SS  */
+                    prxchange('s/'!!'0F'x!!'/\\u000F/',-1, /* SF  */
+                    prxchange('s/'!!'01'x!!'/\\u0001/',-1, /* SOH */
+                    prxchange('s/'!!'02'x!!'/\\u0002/',-1, /* STX */
+                    prxchange('s/'!!'02'x!!'/\\u0010/',-1, /* DLE */
+                    prxchange('s/'!!'11'x!!'/\\u0011/',-1, /* DC1 */
                     prxchange('s/\\/\\\\/',-1,&&name&i)
-        )))))!!'"';
+        ))))))))))))!!'"';
       %end;
     %end;
     run;

base/mp_loadformat.sas

@@ -0,0 +1,303 @@
+/**
+  @file
+  @brief Loads a format catalog from a staging dataset
+  @details When loading staged data, it is common to receive only the records
+  that have actually changed.  However, when loading a format catalog, if
+  records are missing they are presumed to be no longer required.
+
+  This macro will augment a staging dataset with other records from the same
+  format, to prevent loss of data - UNLESS the input dataset contains a marker
+  column, specifying that a particular row needs to be deleted (`delete_col=`).
+
+  This macro can also be used to identify which records would be (or were)
+  considered new, modified or deleted (`loadtarget=`) by creating the following
+  tables:
+
+  @li work.outds_add
+  @li work.outds_del
+  @li work.outds_mod
+
+  For example usage, see mp_loadformat.test.sas
+
+  @param [in] libcat The format catalog to be loaded
+  @param [in] libds The staging table to load
+  @param [in] loadtarget= (NO) Set to YES to actually load the target catalog
+  @param [in] delete_col= (_____DELETE__THIS__RECORD_____) The column used to
+    mark a record for deletion.  Values should be "Yes" or "No".
+  @param [out] auditlibds= (0) For change tracking, set to the libds of an audit
+    table as defined in mddl_dc_difftable.sas
+  @param [in] locklibds= (0) For multi-user (parallel) situations, set to the
+    libds of the DC lock table as defined in the mddl_dc_locktable.sas macro.
+  @param [out] outds_add= (0) Set a libds here to see the new records added
+  @param [out] outds_del= (0) Set a libds here to see the records deleted
+  @param [out] outds_mod= (0) Set a libds here to see the modified records
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_cntlout.sas
+  @li mp_lockanytable.sas
+  @li mp_storediffs.sas
+
+  <h4> Related Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mddl_dc_locktable.sas
+  @li mp_loadformat.test.sas
+  @li mp_lockanytable.sas
+  @li mp_stackdiffs.sas
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_loadformat(libcat,libds
+  ,loadtarget=NO
+  ,auditlibds=0
+  ,locklibds=0
+  ,delete_col=_____DELETE__THIS__RECORD_____
+  ,outds_add=0
+  ,outds_del=0
+  ,outds_mod=0
+  ,mdebug=0
+);
+/* set up local macro variables and temporary tables (with a prefix) */
+%local err msg prefix dslist i var fmtlist ibufsize;
+%let dslist=base_fmts template inlibds ds1 stagedata storediffs;
+%if &outds_add=0 %then %let dslist=&dslist outds_add;
+%if &outds_del=0 %then %let dslist=&dslist outds_del;
+%if &outds_mod=0 %then %let dslist=&dslist outds_mod;
+%let prefix=%substr(%mf_getuniquename(),1,21);
+%do i=1 %to %sysfunc(countw(&dslist));
+  %let var=%scan(&dslist,&i);
+  %local &var;
+  %let &var=%upcase(&prefix._&var);
+%end;
+
+/*
+format values can be up to 32767 wide.  SQL joins on such a wide column can
+cause buffer issues.  Update ibufsize and reset at the end.
+*/
+%let ibufsize=%sysfunc(getoption(ibufsize));
+options ibufsize=32767 ;
+
+/* in DC, format catalogs maybe specified in the libds with a -FC extension */
+%let libcat=%scan(&libcat,1,-);
+
+/* perform input validations */
+%let err=0;
+%let msg=0;
+data _null_;
+  if _n_=1 then putlog "&sysmacroname entry vars:";
+  set sashelp.vmacro;
+  where scope="&sysmacroname";
+  value=upcase(value);
+  if &mdebug=0 then put name '=' value;
+  if name=:'LOAD' and value not in ('YES','NO') then do;
+    call symputx('msg',"invalid value for "!!name!!":"!!value);
+    call symputx('err',1);
+    stop;
+  end;
+  else if name='LIBCAT' then do;
+    if exist(value,'CATALOG') le 0 then do;
+      call symputx('msg',"Unable to open catalog: "!!value);
+      call symputx('err',1);
+      stop;
+    end;
+  end;
+  else if name='LIBDS' then do;
+    if exist(value) le 0 then do;
+      call symputx('msg',"Unable to open staging table: "!!value);
+      call symputx('err',1);
+      stop;
+    end;
+  end;
+  else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
+  and missing(value) then do;
+    call symputx('msg',"missing value in var: "!!name);
+    call symputx('err',1);
+    stop;
+  end;
+run;
+
+%mp_abort(
+  iftrue=(&err ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(&msg)
+)
+
+/**
+  * First, extract only relevant formats from the catalog
+  */
+proc sql noprint;
+select distinct upcase(fmtname) into: fmtlist separated by ' ' from &libds;
+
+%mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
+
+
+/**
+  * Ensure input table and base_formats have consistent lengths and types
+  */
+%mddl_sas_cntlout(libds=&template)
+data &inlibds;
+  length &delete_col $3;
+  if 0 then set &template;
+  set &libds;
+  if &delete_col='' then &delete_col='No';
+  fmtname=upcase(fmtname);
+  if missing(type) then do;
+    if substr(fmtname,1,1)='$' then type='C';
+    else type='N';
+  end;
+  if type='N' then do;
+    start=cats(start);
+    end=cats(end);
+  end;
+run;
+
+/**
+  * Identify new records
+  */
+proc sql;
+create table &outds_add(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  left join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where b.fmtname is null
+    and upcase(a.&delete_col) ne "YES"
+  order by fmtname, start;;
+
+/**
+  * Identify deleted records
+  */
+create table &outds_del(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where upcase(a.&delete_col)="YES"
+  order by fmtname, start;
+
+/**
+  * Identify modified records
+  */
+create table &outds_mod (drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where upcase(a.&delete_col) ne "YES"
+  order by fmtname, start;
+
+options ibufsize=&ibufsize;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc prior to load prep)
+)
+
+%if &loadtarget=YES %then %do;
+  data &ds1;
+    merge &base_fmts(in=base)
+      &outds_mod(in=mod)
+      &outds_add(in=add)
+      &outds_del(in=del);
+    if not del and not mod;
+    by fmtname start;
+  run;
+  data &stagedata;
+    set &ds1 &outds_mod;
+  run;
+  proc sort;
+    by fmtname start;
+  run;
+%end;
+/* mp abort needs to run outside of conditional blocks */
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc prior to actual load)
+)
+%if &loadtarget=YES %then %do;
+  %if %mf_nobs(&stagedata)=0 %then %do;
+    %put There are no changes to load in &libcat!;
+    %return;
+  %end;
+  %if &locklibds ne 0 %then %do;
+    /* prevent parallel updates */
+    %mp_lockanytable(LOCK
+      ,lib=%scan(&libcat,1,.)
+      ,ds=%scan(&libcat,2,.)-FC
+      ,ref=MP_LOADFORMAT commencing format load
+      ,ctl_ds=&locklibds
+    )
+  %end;
+  /* do the actual load */
+  proc format lib=&libcat cntlin=&stagedata;
+  run;
+  %if &locklibds ne 0 %then %do;
+    /* unlock the table */
+    %mp_lockanytable(UNLOCK
+      ,lib=%scan(&libcat,1,.)
+      ,ds=%scan(&libcat,2,.)-FC
+      ,ref=MP_LOADFORMAT completed format load
+      ,ctl_ds=&locklibds
+    )
+  %end;
+  /* track the changes */
+  %if &auditlibds ne 0 %then %do;
+    %if &locklibds ne 0 %then %do;
+      %mp_lockanytable(LOCK
+        ,lib=%scan(&auditlibds,1,.)
+        ,ds=%scan(&auditlibds,2,.)
+        ,ref=MP_LOADFORMAT commencing audit table load
+        ,ctl_ds=&locklibds
+      )
+    %end;
+
+    %mp_storediffs(&libcat-FC
+      ,&base_fmts
+      ,FMTNAME START
+      ,delds=&outds_del
+      ,modds=&outds_mod
+      ,appds=&outds_add
+      ,outds=&storediffs
+      ,mdebug=&mdebug
+    )
+
+    proc append base=&auditlibds data=&storediffs;
+    run;
+
+    %if &locklibds ne 0 %then %do;
+      %mp_lockanytable(UNLOCK
+        ,lib=%scan(&auditlibds,1,.)
+        ,ds=%scan(&auditlibds,2,.)
+        ,ref=MP_LOADFORMAT commencing audit table load
+        ,ctl_ds=&locklibds
+      )
+    %end;
+  %end;
+%end;
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc after load)
+)
+
+%if &mdebug=0 %then %do;
+  proc datasets lib=work;
+    delete &prefix:;
+  run;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%mend mp_loadformat;

base/mp_lockanytable.sas

@@ -1,7 +1,8 @@
 /**
   @file
   @brief Mechanism for locking tables to prevent parallel modifications
-  @details Uses a control table to enable ANY table to be locked for updates.
+  @details Uses a control table to enable ANY table to be locked for updates
+  (not just SAS datasets).
   Only useful if every update uses the macro!   Used heavily within
   [Data Controller for SAS](https://datacontroller.io).
 
@@ -15,7 +16,7 @@
     length is 200 characters.
   @param [out] ctl_ds= (0) The control table which controls the actual locking.
     Should already be assigned and available.  The definition is available by
-    running mp_coretable.sas as follows:  `%mp_coretable(LOCKTABLE)`.
+    running the mddl_dc_locktable.sas macro.
 
   @param [in] loops= (25) Number of times to check for a lock.
   @param [in] loop_secs= (1) Seconds to wait between each lock attempt
@@ -48,7 +49,7 @@ data _null_;
   put name '=' value;
 run;
 
-%mp_abort(iftrue= (&ds=0 and &action ne MAKETABLE)
+%mp_abort(iftrue= ("&ds"="0" and &action ne MAKETABLE)
   ,mac=&sysmacroname
   ,msg=%str(dataset was not provided)
 )

base/mp_lockfilecheck.sas

@@ -37,7 +37,7 @@ run;
   ,mac=checklock.sas
   ,msg=Aborting with syscc=&syscc on entry.
 )
-%mp_abort(iftrue= (&libds=0)
+%mp_abort(iftrue= ("&libds"="0")
   ,mac=&sysmacroname
   ,msg=%str(libds not provided)
 )
@@ -46,6 +46,12 @@ run;
 %let lib=%upcase(%scan(&libds,1,.));
 %let ds=%upcase(%scan(&libds,2,.));
 
+/* in DC, format catalogs are passed with a -FC suffix. No saslock here! */
+%if %scan(&libds,2,-)=FC %then %do;
+  %put &sysmacroname: Format Catalog detected, no lockfile applied to &libds;
+  %return;
+%end;
+
 /* do not proceed if no observations can be processed */
 %let msg=options obs = 0. syserrortext=%superq(syserrortext);
 %mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)

base/mp_md5.sas

@@ -0,0 +1,58 @@
+/**
+  @file
+  @brief Generates an md5 expression for hashing a set of variables
+  @details This is the same algorithm used to hash records in
+  [Data Controller for SAS](https://datacontroller.io) (free for up
+  to 5 users).
+
+  It is not designed to be efficient - it is designed to be effective,
+  given the range of edge cases (large floating points, special missing
+  numerics, thousands of columns, very wide columns).
+
+  It can be used only in data step, eg as follows:
+
+      data _null_;
+        set sashelp.class;
+        hashvar=%mp_md5(cvars=name sex, nvars=age height weight);
+        put hashvar=;
+      run;
+
+  Unfortunately it will not run in SQL - it fails with the following message:
+
+  > The width value for HEX is out of bounds. It should be between 1 and 16
+
+  The macro will also cause errors if the data contains (non-special) missings
+  and the (undocumented) `options dsoptions=nonote2err;` is in effect.
+
+  This can be avoided in two ways:
+
+  @li Global option:  `options dsoptions=nonote2err;`
+  @li Data step option: `data YOURLIB.YOURDATASET /nonote2err;`
+
+  @param cvars= Space seperated list of character variables
+  @param nvars= Space seperated list of numeric variables
+
+  <h4> Related Programs </h4>
+  @li mp_init.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_md5(cvars=,nvars=);
+%local i var sep;
+put(md5(
+  %do i=1 %to %sysfunc(countw(&cvars));
+    %let var=%scan(&cvars,&i,%str( ));
+    &sep put(md5(trim(&var)),$hex32.)
+    %let sep=!!;
+  %end;
+  %do i=1 %to %sysfunc(countw(&nvars));
+    %let var=%scan(&nvars,&i,%str( ));
+    /* multiply by 1 to strip precision errors (eg 0 != 0) */
+    /* but ONLY if not missing, else will lose any special missing values */
+    &sep put(md5(trim(put(ifn(missing(&var),&var,&var*1),binary64.))),$hex32.)
+    %let sep=!!;
+  %end;
+),$hex32.)
+%mend mp_md5;

base/mp_replace.sas

@@ -0,0 +1,149 @@
+/**
+  @file
+  @brief Performs a text substitution on a file
+  @details Performs a find and replace on a file, either in place or to a new
+  file. Can be used on files where lines are longer than 32767.
+
+  Works by reading in the file byte by byte, then marking the beginning and end
+  of each matched string, before finally doing the replace.
+
+  Full credit for this highly efficient and syntactically satisfying SAS logic
+  goes to [Bartosz Jabłoński](https://www.linkedin.com/in/yabwon), founder of
+  the [SAS Packages](https://github.com/yabwon/SAS_PACKAGES) framework.
+
+  Usage:
+
+      %let file="%sysfunc(pathname(work))/file.txt";
+      %let str=replace/me;
+      %let rep=with/this;
+      data _null_;
+        file &file;
+        put 'blahblah';
+        put "blahblah&str.blah";
+        put 'blahblahblah';
+      run;
+      %mp_replace(&file, findvar=str, replacevar=rep)
+      data _null_;
+        infile &file;
+        input;
+        list;
+      run;
+
+  Note - if you are running a version of SAS that will allow the io package in
+  LUA, you can also use this macro: mp_gsubfile.sas
+
+  @param infile The QUOTED path to the file on which to perform the substitution
+  @param findvar= Macro variable NAME containing the string to search for
+  @param replacevar= Macro variable NAME containing the replacement string
+  @param outfile= (0) Optional QUOTED path to an the adjusted output file (to
+    avoid overwriting the first file).
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mp_gsubfile.test.sas
+
+  @version 9.4
+  @author Bartosz Jabłoński
+  @author Allan Bowe
+**/
+
+%macro mp_replace(infile,
+  findvar=,
+  replacevar=,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+%local inref dttm ds1;
+%let inref=%mf_getuniquefileref();
+%let outref=%mf_getuniquefileref();
+%if &outfile=0 %then %let outfile=&infile;
+%let ds1=%mf_getuniquename(prefix=allchars);
+%let ds2=%mf_getuniquename(prefix=startmark);
+
+/* START */
+%let dttm=%sysfunc(datetime());
+
+filename &inref &infile lrecl=1 recfm=n;
+
+data &ds1;
+  infile &inref;
+  input sourcechar $char1. @@;
+  format sourcechar hex2.;
+run;
+
+data &ds2;
+  /* set find string to length in bytes to cover trailing spaces */
+  length string $ %length(%superq(&findvar));
+  string =symget("&findvar");
+  drop string;
+
+  firstchar=char(string,1);
+  findlen=lengthm(string); /* <- for trailing bytes */
+
+  do _N_=1 to nobs;
+    set &ds1 nobs=nobs point=_N_;
+    if sourcechar=firstchar then do;
+      pos=1;
+      s=0;
+      do point=_N_ to min(_N_ + findlen -1,nobs);
+        set &ds1 point=point;
+        if sourcechar=char(string, pos) then s + 1;
+        else goto _leave_;
+        pos+1;
+      end;
+      _leave_:
+      if s=findlen then do;
+        START =_N_;
+        _N_ =_N_+ s - 1;
+        STOP =_N_;
+        output;
+      end;
+    end;
+  end;
+  stop;
+  keep START STOP;
+run;
+
+data &ds1;
+  declare hash HS(dataset:"&ds2(keep=start)");
+  HS.defineKey("start");
+  HS.defineDone();
+  declare hash HE(dataset:"&ds2(keep=stop)");
+  HE.defineKey("stop");
+  HE.defineDone();
+  do until(eof);
+    set &ds1 end=eof curobs =n;
+    start = ^HS.check(key:n);
+    stop  = ^HE.check(key:n);
+    length strt $ 1;
+    strt =put(start,best. -L);
+    retain out 1;
+    if out   then output;
+    if start then out=0;
+    if stop  then out=1;
+  end;
+  stop;
+  keep sourcechar strt;
+run;
+
+filename &outref &outfile recfm=n;
+
+data _null_;
+  length replace $ %length(%superq(&replacevar));
+  replace=symget("&replacevar");
+  file &outref;
+  do until(eof);
+    set &ds1 end=eof;
+    if strt ="1" then put replace char.;
+    else put sourcechar char1.;
+  end;
+  stop;
+run;
+
+/* END */
+%put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) seconds to run;
+
+%mend mp_replace;

base/mp_storediffs.sas

@@ -90,7 +90,7 @@
 %else %let dbg=*;
 
 /* set up unique and temporary vars */
-%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist;
+%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist vlist;
 %let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_ds1));
 %let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_ds2));
 %let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_ds3));
@@ -144,12 +144,21 @@ proc transpose data=&ds1
   by &inds_keep &hashkey;
   var _character_;
 run;
+
+%if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do;
+  /* this is a format catalog - cannot query cols directly */
+  %let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH"
+    ,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO"
+    ,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
+%end;
+%else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
+
 data &ds4;
   length &inds_keep $41 tgtvar_nm $32;
   set &ds2 &ds3 indsname=&inds_auto;
 
   tgtvar_nm=upcase(tgtvar_nm);
-  if tgtvar_nm in (%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE)));
+  if tgtvar_nm in (%upcase(&vlist));
 
   if &inds_auto="&ds2" then tgtvar_type='N';
   else if &inds_auto="&ds3" then tgtvar_type='C';

base/mp_streamfile.sas

@@ -12,7 +12,7 @@
 
       %mp_streamfile(contenttype=csv,inloc=/some/where.txt,outname=myfile.txt)
 
-  @param [in] contenttype= (TEXTS) Either TEXT, ZIP, CSV, EXCEL
+  @param [in] contenttype= (TEXT) Either TEXT, ZIP, CSV, EXCEL
   @param [in] inloc= /path/to/file.ext to be sent
   @param [in] inref= fileref of file to be sent (if provided, overrides `inloc`)
   @param [in] iftrue= (1=1) Provide a condition under which to execute.
@@ -22,6 +22,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
+  @li mfs_httpheader.sas
   @li mp_binarycopy.sas
 
   @author Allan Bowe
@@ -55,7 +56,7 @@ data _null_;
 run;
 
 %if &contentype=CSV %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/csv');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -66,10 +67,14 @@ run;
       contenttype='application/csv'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,application/csv)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
 %else %if &contentype=EXCEL %then %do;
   /* suitable for XLS format */
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/vnd.ms-excel');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -80,15 +85,36 @@ run;
       contenttype='application/vnd.ms-excel'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,application/vnd.ms-excel)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
+%end;
+%else %if &contentype=GIF or &contentype=JPEG or &contentype=PNG %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type',"image/%lowcase(&contenttype)");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI"
+      contenttype="image/%lowcase(&contenttype)";
+  %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,image/%lowcase(&contenttype))
+  %end;
 %end;
 %else %if &contentype=HTML %then %do;
   %if &platform=SASVIYA %then %do;
     filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
       contenttype="text/html";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,text/html)
+  %end;
 %end;
 %else %if &contentype=TEXT %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/text');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -99,9 +125,13 @@ run;
       contenttype='application/text'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,application/text)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
 %else %if &contentype=WOFF or &contentype=WOFF2 or &contentype=TTF %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
       rc=stpsrv_header('Content-type',"font/%lowcase(&contenttype)");
     run;
@@ -110,9 +140,12 @@ run;
     filename &outref filesrvc parenturi="&SYS_JES_JOB_URI"
       contenttype="font/%lowcase(&contenttype)";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,font/%lowcase(&contenttype))
+  %end;
 %end;
 %else %if &contentype=XLSX %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
       rc=stpsrv_header('Content-type',
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
@@ -125,9 +158,15 @@ run;
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type
+      ,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+    )
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
 %else %if &contentype=ZIP %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/zip');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -138,6 +177,10 @@ run;
       contenttype='application/zip'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,application/zip)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
 %else %do;
   %put %str(ERR)OR: Content Type &contenttype NOT SUPPORTED by &sysmacroname!;

base/mp_wait4file.sas

@@ -4,9 +4,7 @@
   @details Loops with a `sleep()` command until a file arrives or the max wait
   period expires.
 
-  @example
-
-  Wait 3 minutes OR for /tmp/flag.txt to appear
+  Example: Wait 3 minutes OR for /tmp/flag.txt to appear
 
     %mp_wait4file(/tmp/flag.txt , maxwait=60*3)
 

build.py

@@ -84,7 +84,7 @@ options noquotelenmax;
 """
 f = open('all.sas', "w")             # r / r+ / rb / rb+ / w / wb
 f.write(header)
-folders=['base','meta','metax','server','viya','lua','fcmp']
+folders=['base','ddl','meta','metax','server','viya','lua','fcmp']
 for folder in folders:
     filenames = [fn for fn in Path('./' + folder).iterdir() if fn.match("*.sas")]
     filenames.sort()

ddl/mddl_dc_difftable.sas

@@ -0,0 +1,34 @@
+/**
+  @file
+  @brief Difftable DDL
+  @details Used to store changes to tables.  Used by mp_storediffs.sas
+      and mp_stackdiffs.sas
+
+**/
+
+
+%macro mddl_dc_difftable(libds=WORK.DIFFTABLE);
+
+  proc sql;
+  create table &libds(
+      load_ref char(36) label='unique load reference',
+      processed_dttm num format=E8601DT26.6 label='Processed at timestamp',
+      libref char(8) label='Library Reference (8 chars)',
+      dsn char(32) label='Dataset Name (32 chars)',
+      key_hash char(32) label=
+        'MD5 Hash of primary key values (pipe seperated)',
+      move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified',
+      is_pk num label='Is Primary Key Field? (1/0)',
+      is_diff num label=
+        'Did value change? (1/0/-1).  Always -1 for appends and deletes.',
+      tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
+      tgtvar_nm char(32) label='Target variable name (32 chars)',
+      oldval_num num format=best32. label='Old (numeric) value',
+      newval_num num format=best32. label='New (numeric) value',
+      oldval_char char(32765) label='Old (character) value',
+      newval_char char(32765) label='New (character) value',
+    constraint pk_mpe_audit
+      primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
+  );
+
+%mend mddl_dc_difftable;

ddl/mddl_dc_filterdetail.sas

@@ -0,0 +1,27 @@
+/**
+  @file
+  @brief Filtertable DDL
+  @details For storing detailed filter values.  Used by
+      mp_filterstore.sas.
+
+**/
+
+
+%macro mddl_dc_filterdetail(libds=WORK.FILTER_DETAIL);
+
+  proc sql;
+  create table &libds(
+      filter_hash char(32) not null,
+      filter_line num not null,
+      group_logic char(3) not null,
+      subgroup_logic char(3) not null,
+      subgroup_id num not null,
+      variable_nm varchar(32) not null,
+      operator_nm varchar(12) not null,
+      raw_value varchar(4000) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_hash,filter_line)
+  );
+
+%mend mddl_dc_filterdetail;

ddl/mddl_dc_filtersummary.sas

@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Filtersummary DDL
+  @details For storing summary filter values.  Used by
+      mp_filterstore.sas.
+
+**/
+
+
+%macro mddl_dc_filtersummary(libds=WORK.FILTER_SUMMARY);
+
+  proc sql;
+  create table &libds(
+      filter_rk num not null,
+      filter_hash char(32) not null,
+      filter_table char(41) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_rk)
+  );
+
+%mend mddl_dc_filtersummary;

ddl/mddl_dc_locktable.sas

@@ -0,0 +1,25 @@
+/**
+  @file
+  @brief Locktable DDL
+  @details For "locking" tables prior to multipass loads. Used by
+      mp_lockanytable.sas
+
+**/
+
+
+%macro mddl_dc_locktable(libds=WORK.LOCKTABLE);
+
+  proc sql;
+  create table &libds(
+      lock_lib char(8),
+      lock_ds char(32),
+      lock_status_cd char(10) not null,
+      lock_user_nm char(100) not null ,
+      lock_ref char(200),
+      lock_pid char(10),
+      lock_start_dttm num format=E8601DT26.6,
+      lock_end_dttm num format=E8601DT26.6,
+    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds)
+  );
+
+%mend mddl_dc_locktable;

ddl/mddl_dc_maxkeytable.sas

@@ -0,0 +1,24 @@
+/**
+  @file
+  @brief Maxkeytable DDL
+  @details For storing the maximum retained key information.  Used
+      by mp_retainedkey.sas
+
+**/
+
+
+%macro mddl_dc_maxkeytable(libds=WORK.MAXKEYTABLE);
+
+  proc sql;
+  create table &libds(
+      keytable varchar(41) label='Base table in libref.dataset format',
+      keycolumn char(32) format=$32.
+        label='The Retained key field containing the key values.',
+      max_key num label=
+        'Integer representing current max RK or SK value in the KEYTABLE',
+      processed_dttm num format=E8601DT26.6
+        label='Datetime this value was last updated',
+    constraint pk_mpe_maxkeyvalues
+        primary key(keytable));
+
+%mend mddl_dc_maxkeytable;

ddl/mddl_sas_cntlout.sas

@@ -0,0 +1,43 @@
+/**
+  @file
+  @brief The CNTLOUT table generated by proc format
+  @details This table will actually change format depending on the data values,
+  therefore the max possible lengths are described here to enable consistency
+  when dealing with format data.
+
+**/
+
+
+%macro mddl_sas_cntlout(libds=WORK.CNTLOUT);
+
+proc sql;
+create table &libds(
+    FMTNAME char(32)      label='Format name'
+    /*
+      to accomodate larger START values, mp_loadformat.sas will need the
+      SQL dependency removed (proc sql needs to accomodate 3 index values in
+      a 32767 ibufsize limit)
+    */
+    ,START char(10000)    label='Starting value for format'
+    ,END char(32767)      label='Ending value for format'
+    ,LABEL char(32767)    label='Format value label'
+    ,MIN num length=3     label='Minimum length'
+    ,MAX num length=3     label='Maximum length'
+    ,DEFAULT num length=3 label='Default length'
+    ,LENGTH num length=3  label='Format length'
+    ,FUZZ num             label='Fuzz value'
+    ,PREFIX char(2)       label='Prefix characters'
+    ,MULT num             label='Multiplier'
+    ,FILL char(1)         label='Fill character'
+    ,NOEDIT num length=3  label='Is picture string noedit?'
+    ,TYPE char(1)         label='Type of format'
+    ,SEXCL char(1)        label='Start exclusion'
+    ,EEXCL char(1)        label='End exclusion'
+    ,HLO char(13)         label='Additional information'
+    ,DECSEP char(1)       label='Decimal separator'
+    ,DIG3SEP char(1)      label='Three-digit separator'
+    ,DATATYPE char(8)     label='Date/time/datetime?'
+    ,LANGUAGE char(8)     label='Language for date strings'
+);
+
+%mend mddl_sas_cntlout;

fcmp/mcf_stpsrv_header.sas

@@ -1,110 +0,0 @@
-/**
-  @file
-  @brief Provides a replacement for the stpsrv_header function
-  @details The stpsrv_header is normally a built-in function, used to set the
-  headers for SAS 9 Stored Processes as documented here:
-  https://go.documentation.sas.com/doc/en/itechcdc/9.4/stpug/srvhead.htm
-
-  The purpose of this custom function is to provide a replacement when running
-  similar code as a web service against
-  [sasjs/server](https://github.com/sasjs/server).  It operates by creating a
-  text file with the headers.  The location of this text file is determined by
-  a macro variable (`sasjs_stpsrv_header_loc`) which needs to be injected into
-  each service by the calling process, eg:
-
-      %let sasjs_stpsrv_header_loc = C:/temp/some_uuid/stpsrv_header.txt;
-
-  Note - the function works by appending headers to the file.  If multiple same-
-  named headers are provided, they will all be appended - the calling process
-  needs to pick up the last one.  This will mean removing the attribute if the
-  final record has an empty value.
-
-  The function takes the following (positional) parameters:
-
-  | PARAMETER | DESCRIPTION |
-  |------------|-------------|
-  | name $  | name of the header attribute to create|
-  | value  $  | value of the header attribute|
-
-  It returns 0 if successful, or -1 if an error occured.
-
-  Usage:
-
-      %let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/stpsrv_header.txt;
-
-      %mcf_stpsrv_header(wrap=YES, insert_cmplib=YES)
-
-      data _null_;
-        rc=stpsrv_header('Content-type','application/text');
-        rc=stpsrv_header('Content-disposition',"attachment; filename=file.txt");
-      run;
-
-      data _null_;
-        infile "&sasjs_stpsrv_header_loc";
-        input;
-        putlog _infile_;
-      run;
-
-
-  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
-  @param [out] lib= (work) The output library in which to create the catalog.
-  @param [out] cat= (sasjs) The output catalog in which to create the package.
-  @param [out] pkg= (utils) The output package in which to create the function.
-    Uses a 3 part format:  libref.catalog.package
-  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
-    values inserted only if needed.
-
-  <h4> SAS Macros </h4>
-  @li mcf_init.sas
-
-  <h4> Related Programs </h4>
-  @li mcf_stpsrv_header.test.sas
-  @li mp_init.sas
-
-**/
-
-%macro mcf_stpsrv_header(wrap=NO
-  ,insert_cmplib=DEPRECATED
-  ,lib=WORK
-  ,cat=SASJS
-  ,pkg=UTILS
-)/*/STORE SOURCE*/;
-%local i var cmpval found;
-%if %mcf_init(stpsrv_header)=1 %then %return;
-
-%if &wrap=YES  %then %do;
-  proc fcmp outlib=&lib..&cat..&pkg;
-%end;
-
-function stpsrv_header(name $, value $);
-  length loc $128 val $512;
-  loc=symget('sasjs_stpsrv_header_loc');
-  val=trim(name)!!': '!!value;
-  length fref $8;
-  rc=filename(fref,loc);
-  if (rc ne 0) then return( -1 );
-  fid = fopen(fref,'a');
-  if (fid = 0) then return( -1 );
-  rc=fput(fid, val);
-  rc=fwrite(fid);
-  rc=fclose(fid);
-  rc=filename(fref);
-  return(0);
-endsub;
-
-%if &wrap=YES %then %do;
-  quit;
-%end;
-
-/* insert the CMPLIB if not already there */
-%let cmpval=%sysfunc(getoption(cmplib));
-%let found=0;
-%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
-  %let var=%scan(&cmpval,&i,%str( %(%)));
-  %if &var=&lib..&cat %then %let found=1;
-%end;
-%if &found=0 %then %do;
-  options insert=(CMPLIB=(&lib..&cat));
-%end;
-
-%mend mcf_stpsrv_header;

meta/mm_assignlib.sas

@@ -13,8 +13,8 @@
   <h4> SAS Macros </h4>
   @li mp_abort.sas
 
-  @param libref the libref (not name) of the metadata library
-  @param mAbort= If not assigned, HARD will call %mp_abort(), SOFT will
+  @param [in] libref The libref (not name) of the metadata library
+  @param [in] mAbort= If not assigned, HARD will call %mp_abort(), SOFT will
     silently return
 
   @returns libname statement
@@ -28,11 +28,11 @@
     libref
     ,mAbort=HARD
 )/*/STORE SOURCE*/;
-
+%local mp_abort msg;
+%let mp_abort=0;
 %if %sysfunc(libref(&libref)) %then %do;
-  %local mp_abort msg; %let mp_abort=0;
   data _null_;
-    length liburi LibName $200;
+    length liburi LibName msg $200;
     call missing(of _all_);
     nobj=metadata_getnobj("omsobj:SASLibrary?@Libref='&libref'",1,liburi);
     if nobj=1 then do;
@@ -40,7 +40,30 @@
       /* now try and assign it */
       if libname("&libref",,'meta',cats('liburi="',liburi,'";')) ne 0 then do;
         putlog "&libref could not be assigned";
-        call symputx('msg',sysmsg(),'l');
+        putlog liburi=;
+        /**
+          * Fetch the system message for display in the abort modal.  This is
+          * not always helpful though.  One example, previously received:
+          * NOTE: Libref XX refers to the same library metadata as libref XX.
+          */
+        msg=sysmsg();
+        if msg=:'ERROR: Libref SAVE is not assigned.' then do;
+          msg=catx(" ",
+            "Could not assign %upcase(&libref).",
+            "Please check metadata permissions!  Libname:",libname,
+            "Liburi:",liburi
+          );
+        end;
+        else if msg="ERROR: User does not have appropriate authorization "!!
+          "level for library SAVE."
+        then do;
+          msg=catx(" ",
+            "ERROR: User does not have appropriate authorization level",
+            "for library %upcase(&libref), libname:",libname,
+            "Liburi:",liburi
+          );
+        end;
+        call symputx('msg',msg,'l');
         if "&mabort"='HARD' then call symputx('mp_abort',1,'l');
       end;
       else do;
@@ -59,20 +82,16 @@
     end;
   run;
 
-  %if &mp_abort=1 %then %do;
-    %mp_abort(iftrue= (&mp_abort=1)
-      ,mac=&sysmacroname
-      ,msg=&msg
-    )
-    %return;
-  %end;
-  %else %if %length(&msg)>2 %then %do;
-    %put NOTE: &msg;
-    %return;
-  %end;
+  %put NOTE: &msg;
 
 %end;
 %else %do;
   %put NOTE: Library &libref is already assigned;
 %end;
+
+%mp_abort(iftrue= (&mp_abort=1)
+  ,mac=mm_assignlib.sas
+  ,msg=%superq(msg)
+)
+
 %mend mm_assignlib;

meta/mm_createapplication.sas

@@ -59,7 +59,10 @@
 %&mD.put Executing &sysmacroname..sas;
 %&mD.put _local_;
 
-%mf_verifymacvars(tree name)
+%mp_abort(iftrue= (%mf_verifymacvars(tree name)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Empty inputs: tree name)
+)
 
 /**
   * check tree exists

meta/mm_createdocument.sas

@@ -47,7 +47,10 @@
 %&mD.put Executing &sysmacroname..sas;
 %&mD.put _local_;
 
-%mf_verifymacvars(tree name)
+%mp_abort(iftrue= (%mf_verifymacvars(tree name)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Empty inputs: tree name)
+)
 
 /**
   * check tree exists

meta/mm_createfolder.sas

@@ -12,13 +12,14 @@
     The macro is idempotent - if you run it twice, it will only create a folder
     once.
 
-  usage:
+  Usage:
 
       %mm_createfolder(path=/some/meta/folder)
 
   @param [in] path= Name of the folder to create.
   @param [in] mdebug= set DBG to 1 to disable DEBUG messages
 
+
   @version 9.4
   @author Allan Bowe
 

meta/mm_createlibrary.sas

@@ -133,12 +133,14 @@ run;
 filename &frefin temp;
 filename &frefout temp;
 
+%mp_abort(iftrue= (
+    &engine=BASE & %mf_verifymacvars(libname libref engine servercontext tree)=0
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Empty inputs: libname libref engine servercontext tree)
+)
+
 %if &engine=BASE %then %do;
-
-  %mf_verifymacvars(libname libref engine servercontext tree)
-
-
-
   /**
     * Check that the ServerContext exists
     */

meta/mm_createstp.sas

@@ -12,7 +12,7 @@
     This macro is idempotent - if you run it twice, it will only create an STP
     once.
 
-  usage (type 1 STP):
+  Usage (type 1 STP):
 
       %mm_createstp(stpname=MyNewSTP
         ,filename=mySpecialProgram.sas
@@ -31,7 +31,8 @@
         putlog (_all_)(=);
       run;
 
-  usage (type 2 STP):
+  Usage (type 2 STP):
+
       %mm_createstp(stpname=MyNewType2STP
         ,filename=mySpecialProgram.sas
         ,directory=SASEnvironment/SASCode/STPs
@@ -74,8 +75,9 @@
   @li mf_verifymacvars.sas
   @li mm_getdirectories.sas
   @li mm_updatestpsourcecode.sas
-  @li mp_dropmembers.sas
   @li mm_getservercontexts.sas
+  @li mp_abort.sas
+  @li mp_dropmembers.sas
 
   <h4> Related Macros </h4>
   @li mm_createwebservice.sas
@@ -108,7 +110,12 @@
 %&mD.put Executing mm_CreateSTP.sas;
 %&mD.put _local_;
 
-%mf_verifymacvars(stpname filename directory tree)
+%mp_abort(
+  iftrue=(%mf_verifymacvars(stpname filename directory tree)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Empty inputs: stpname filename directory tree)
+)
+
 %mp_dropmembers(%scan(&outds,2,.))
 
 /**

meta/mm_createwebservice.sas

@@ -236,8 +236,15 @@ data _null_;
   put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
   put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
   put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
+  put '                    prxchange(''s/''!!''00''x!!''/\\u0000/'',-1, /* NUL */ ';
+  put '                    prxchange(''s/''!!''0E''x!!''/\\u000E/'',-1, /* SS  */ ';
+  put '                    prxchange(''s/''!!''0F''x!!''/\\u000F/'',-1, /* SF  */ ';
+  put '                    prxchange(''s/''!!''01''x!!''/\\u0001/'',-1, /* SOH */ ';
+  put '                    prxchange(''s/''!!''02''x!!''/\\u0002/'',-1, /* STX */ ';
+  put '                    prxchange(''s/''!!''02''x!!''/\\u0010/'',-1, /* DLE */ ';
+  put '                    prxchange(''s/''!!''11''x!!''/\\u0011/'',-1, /* DC1 */ ';
   put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '        )))))!!''"''; ';
+  put '        ))))))))))))!!''"''; ';
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
@@ -453,7 +460,8 @@ data _null_;
   put '  %else %if %symexist(&metavar) %then %do; ';
   put '    %if %length(&&&metavar)=0 %then %let user=&sysuserid; ';
   put '    /* sometimes SAS will add @domain extension - remove for consistency */ ';
-  put '    %else %let user=%scan(&&&metavar,1,@); ';
+  put '    /* but be sure to quote in case of usernames with commas */ ';
+  put '    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@)); ';
   put '  %end; ';
   put '  %else %let user=&sysuserid; ';
   put ' ';

meta/mm_deletelibrary.sas

@@ -78,6 +78,7 @@ filename &fname2 clear;
 %local isgone;
 data _null_;
   length type uri $256;
+  call missing (of _all_);
   rc=metadata_resolve("omsobj:SASLibrary?@Id='&liburi'",type,uri);
   call symputx('isgone',type,'l');
 run;

meta/mm_getauthinfo.sas

@@ -1,16 +1,21 @@
 /**
   @file mm_getauthinfo.sas
-  @brief extracts authentication info
-  @details usage:
+  @brief Extracts authentication info for each user in metadata
+  @details
+  Usage:
 
-    %mm_getauthinfo(outds=auths)
+      %mm_getauthinfo(outds=auths)
 
-  @param outds= the ONE LEVEL work dataset to create
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
+  @param [out] outds= (mm_getauthinfo) The output dataset to create
 
   <h4> SAS Macros </h4>
-  @li mm_getobjects.sas
   @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
   @li mm_getdetails.sas
+  @li mm_getobjects.sas
+
 
   @version 9.4
   @author Allan Bowe
@@ -18,67 +23,69 @@
 **/
 
 %macro mm_getauthinfo(outds=mm_getauthinfo
+  ,mdebug=0
 )/*/STORE SOURCE*/;
+%local prefix fileref;
+%let prefix=%substr(%mf_getuniquename(),1,25);
 
-%if %length(&outds)>30 %then %do;
-  %put %str(ERR)OR: Temp tables are created with the &outds prefix, which
-    therefore needs to be 30 characters or less;
-  %return;
-%end;
-%if %index(&outds,'.')>0 %then %do;
-  %put %str(ERR)OR: Table &outds should be ONE LEVEL (no library);
-  %return;
-%end;
-
-%mm_getobjects(type=Login,outds=&outds.0)
+%mm_getobjects(type=Login,outds=&prefix.0)
 
 %local fileref;
 %let fileref=%mf_getuniquefileref();
 
 data _null_;
   file &fileref;
-  set &outds.0 end=last;
+  set &prefix.0 end=last;
   /* run macro */
-  str=cats('%mm_getdetails(uri=',id,",outattrs=&outds.d",_n_
-    ,",outassocs=&outds.a",_n_,")");
+  str=cats('%mm_getdetails(uri=',id,",outattrs=&prefix.d",_n_
+    ,",outassocs=&prefix.a",_n_,")");
   put str;
   /* transpose attributes */
-  str=cats("proc transpose data=&outds.d",_n_,"(drop=type) out=&outds.da"
+  str=cats("proc transpose data=&prefix.d",_n_,"(drop=type) out=&prefix.da"
     ,_n_,"(drop=_name_);var value;id name;run;");
   put str;
   /* add extra info to attributes */
-  str=cats("data &outds.da",_n_,";length login_id login_name $256; login_id="
-    ,quote(trim(id)),";set &outds.da",_n_
+  str=cats("data &prefix.da",_n_,";length login_id login_name $256; login_id="
+    ,quote(trim(id)),";set &prefix.da",_n_
     ,";login_name=trim(subpad(name,1,256));drop name;run;");
   put str;
   /* add extra info to associations */
-  str=cats("data &outds.a",_n_,";length login_id login_name $256; login_id="
+  str=cats("data &prefix.a",_n_,";length login_id login_name $256; login_id="
     ,quote(trim(id)),";login_name=",quote(trim(name))
-    ,";set &outds.a",_n_,";run;");
+    ,";set &prefix.a",_n_,";run;");
   put str;
   if last then do;
     /* collate attributes */
-    str=cats("data &outds._logat; set &outds.da1-&outds.da",_n_,";run;");
+    str=cats("data &prefix._logat; set &prefix.da1-&prefix.da",_n_,";run;");
     put str;
     /* collate associations */
-    str=cats("data &outds._logas; set &outds.a1-&outds.a",_n_,";run;");
+    str=cats("data &prefix._logas; set &prefix.a1-&prefix.a",_n_,";run;");
     put str;
     /* tidy up */
-    str=cats("proc delete data=&outds.da1-&outds.da",_n_,";run;");
+    str=cats("proc delete data=&prefix.da1-&prefix.da",_n_,";run;");
     put str;
-    str=cats("proc delete data=&outds.d1-&outds.d",_n_,";run;");
+    str=cats("proc delete data=&prefix.d1-&prefix.d",_n_,";run;");
     put str;
-    str=cats("proc delete data=&outds.a1-&outds.a",_n_,";run;");
+    str=cats("proc delete data=&prefix.a1-&prefix.a",_n_,";run;");
     put str;
   end;
 run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fileref;
+    if _n_=1 then putlog // "Now executing the following code:" //;
+    input; putlog _infile_;
+  run;
+%end;
 %inc &fileref;
+filename &fileref clear;
 
 /* get libraries */
-proc sort data=&outds._logas(where=(assoc='Libraries')) out=&outds._temp;
+proc sort data=&prefix._logas(where=(assoc='Libraries')) out=&prefix._temp;
   by login_id;
-data &outds._temp;
-  set &outds._temp;
+data &prefix._temp;
+  set &prefix._temp;
   by login_id;
   length library_list $32767;
   retain library_list;
@@ -86,31 +93,27 @@ data &outds._temp;
   else library_list=catx(' !! ',library_list,name);
 proc sql;
 /* get auth domain */
-create table &outds._dom as
+create table &prefix._dom as
   select login_id,name as domain
-  from &outds._logas
+  from &prefix._logas
   where assoc='Domain';
-create unique index login_id on &outds._dom(login_id);
+create unique index login_id on &prefix._dom(login_id);
 /* join it all together */
-create table &outds._logins as
+create table &outds as
   select a.*
     ,c.domain
     ,b.library_list
-  from &outds._logat (drop=ishidden lockedby usageversion publictype) a
-  left join &outds._temp b
+  from &prefix._logat (drop=ishidden lockedby usageversion publictype) a
+  left join &prefix._temp b
   on a.login_id=b.login_id
-  left join &outds._dom c
+  left join &prefix._dom c
   on a.login_id=c.login_id;
-drop table &outds._temp;
-drop table &outds._logat;
-drop table &outds._logas;
 
-data _null_;
-  infile &fileref;
-  if _n_=1 then putlog // "Now executing the following code:" //;
-  input; putlog _infile_;
-run;
+%if &mdebug=0 %then %do;
+  proc datasets lib=work;
+    delete &prefix:;
+  run;
+%end;
 
-filename &fileref clear;
 
 %mend mm_getauthinfo;

meta/mm_spkexport.sas

@@ -49,20 +49,25 @@
 
 
   <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_isblank.sas
   @li mf_loc.sas
   @li mm_tree.sas
-  @li mf_getuniquefileref.sas
-  @li mf_isblank.sas
   @li mp_abort.sas
 
-  @param metaloc= the metadata folder to export
-  @param secureref= fileref containing the username / password (should point to
-    a file in a secure location).  Leave blank to substitute $bash type vars.
-  @param outref= fileref to which to write the command
-  @param cmdoutloc= the directory to which the command will write the SPK
-    (default=WORK)
-  @param cmdoutname= the name of the spk / log files to create (will be
-    identical just with .spk or .log extension)
+
+  @param [in] metaloc= the metadata folder to export
+  @param [in] secureref= fileref containing the username / password (should
+    point to a file in a secure location). Leave blank to substitute $bash vars.
+  @param [in] excludevars= (0) A space seperated list of macro variable names,
+    each of which contains a value that should be used to filter the output
+    objects.
+  @param [out] outref= fileref to which to write the command
+  @param [out] cmdoutloc= (%sysfunc(pathname(work))) The directory to which the
+    command will write the SPK
+  @param [out] cmdoutname= (mmxport) The name of the spk / log files to create
+    (will be identical just with .spk or .log extension)
 
   @version 9.4
   @author Allan Bowe
@@ -71,6 +76,7 @@
 
 %macro mm_spkexport(metaloc=
   ,secureref=
+  ,excludevars=0
   ,outref=
   ,cmdoutloc=%sysfunc(pathname(work))
   ,cmdoutname=mmxport
@@ -82,7 +88,7 @@
 %end;
 
 /* set creds */
-%local mmxuser mmxpath;
+%local mmxuser mmxpath i var;
 %let mmxuser=$1;
 %let mmxpass=$2;
 %if %mf_isblank(&secureref)=0 %then %do;
@@ -90,35 +96,51 @@
 %end;
 
 /* setup metadata connection options */
-%local host port platform_object_path connx_string;
+%local host port platform_object_path ds;
 %let host=%sysfunc(getoption(metaserver));
 %let port=%sysfunc(getoption(metaport));
 %let platform_object_path=%mf_loc(POF);
+%let ds=%mf_getuniquename(prefix=spkexportable);
 
-%let connx_string=%str(-host &host -port &port -user &mmxuser %trim(
-  )-password &mmxpass);
-
-%mm_tree(root=%str(&metaloc) ,types=EXPORTABLE ,outds=exportable)
+%mm_tree(root=%str(&metaloc),types=EXPORTABLE ,outds=&ds)
 
 %if %mf_isblank(&outref)=1 %then %let outref=%mf_getuniquefileref();
 
 data _null_;
-  set exportable end=last;
+  set &ds end=last;
   file &outref lrecl=32767;
   length str $32767;
   if _n_=1 then do;
+    put "# Script generated by &sysuserid on %sysfunc(datetime(),datetime19.)";
     put "cd ""&platform_object_path"" \";
-    put "; ./ExportPackage &connx_string -disableX11 \";
-    put " -package ""&cmdoutloc/&cmdoutname..spk"" \";
+    put "; ./ExportPackage -host &host -port &port -user &mmxuser \";
+    put "  -disableX11 -password &mmxpass \";
+    put "  -package ""&cmdoutloc/&cmdoutname..spk"" \";
   end;
+/* exclude particular patterns from the exported SPK */
+%if "&excludevars" ne "0" %then %do;
+  %do i=1 %to %sysfunc(countw(&excludevars));
+    %let var=%scan(&excludevars,&i);
+    if _n_=1 then do;
+      length excludestr&i $1000;
+      retain excludestr&i;
+      excludestr&i=symget("&var");
+      putlog excludestr&i=;
+      putlog path=;
+    end;
+    if index(path,cats(excludestr&i))=0 and index(name,cats(excludestr&i))=0;
+  %end;
+  /* ignore top level folder else all subcontent will be exported regardless */
+  if _n_>1;
+%end;
   str=' -objects '!!cats('"',path,'/',name,"(",publictype,')" \');
   put str;
   if last then put " -log ""&cmdoutloc/&cmdoutname..log"" 2>&1 ";
 run;
 
 %mp_abort(iftrue= (&syscc ne 0)
-  ,mac=&sysmacroname
+  ,mac=mm_spkexport
   ,msg=%str(syscc=&syscc)
 )
 
-%mend mm_spkexport;
+%mend mm_spkexport;

package-lock.json

@@ -7,7 +7,8 @@
       "name": "@sasjs/core",
       "license": "MIT",
       "devDependencies": {
-        "@sasjs/cli": "3.6.0"
+        "@sasjs/cli": "3.6.0",
+        "@sasjs/core": "4.4.7"
       }
     },
     "node_modules/@sasjs/adapter": {
@@ -108,6 +109,12 @@
         "valid-url": "^1.0.9"
       }
     },
+    "node_modules/@sasjs/core": {
+      "version": "4.4.7",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.7.tgz",
+      "integrity": "sha512-9+HmwspvWu/gH9ElnmRaGdjOCspOidBRUYUfxLzgOy1Ya1JMZ2RErMklCAMg7XI0Us5jecd2FXdo8zlQDhRkWQ==",
+      "dev": true
+    },
     "node_modules/@sasjs/lint": {
       "version": "1.11.2",
       "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.2.tgz",
@@ -1143,9 +1150,9 @@
       }
     },
     "node_modules/follow-redirects": {
-      "version": "1.14.7",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz",
-      "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==",
+      "version": "1.14.8",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.8.tgz",
+      "integrity": "sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA==",
       "dev": true,
       "funding": [
         {
@@ -2822,6 +2829,12 @@
         }
       }
     },
+    "@sasjs/core": {
+      "version": "4.4.7",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.7.tgz",
+      "integrity": "sha512-9+HmwspvWu/gH9ElnmRaGdjOCspOidBRUYUfxLzgOy1Ya1JMZ2RErMklCAMg7XI0Us5jecd2FXdo8zlQDhRkWQ==",
+      "dev": true
+    },
     "@sasjs/lint": {
       "version": "1.11.2",
       "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.2.tgz",
@@ -3665,9 +3678,9 @@
       }
     },
     "follow-redirects": {
-      "version": "1.14.7",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz",
-      "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==",
+      "version": "1.14.8",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.8.tgz",
+      "integrity": "sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA==",
       "dev": true
     },
     "form-data": {

package.json

@@ -33,6 +33,7 @@
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "3.6.0"
+    "@sasjs/cli": "3.6.0",
+    "@sasjs/core": "4.4.7"
   }
 }

sasjs/sasjsconfig.json

@@ -2,13 +2,12 @@
   "$schema": "https://raw.githubusercontent.com/sasjs/utils/main/src/types/sasjsconfig-schema.json",
   "macroFolders": [
     "base",
+    "ddl",
     "fcmp",
-    "meta",
-    "metax",
-    "server",
-    "viya",
     "lua",
-    "tests/crossplatform"
+    "server",
+    "tests/crossplatform",
+    "tests/ddl"
   ],
   "docConfig": {
     "displayMacroCore": false,
@@ -37,6 +36,7 @@
       },
       "appLoc": "/Public/temp/macrocore",
       "macroFolders": [
+        "viya",
         "tests/viyaonly"
       ],
       "programFolders": [],
@@ -56,6 +56,8 @@
       },
       "appLoc": "/Shared Data/temp/macrocore",
       "macroFolders": [
+        "meta",
+        "metax",
         "tests/sas9only"
       ],
       "programFolders": [],
@@ -69,23 +71,49 @@
     },
     {
       "name": "server",
-      "serverUrl": "https://sas.analytium.co.uk:5000",
+      "serverUrl": " ",
       "serverType": "SASJS",
-      "appLoc": "/Shared Data/temp/macrocore",
+      "httpsAgentOptions": {
+        "allowInsecureRequests": false
+      },
+      "appLoc": "/sasjs/core",
       "macroFolders": [
+        "server",
         "tests/serveronly"
       ],
+      "programFolders": [],
+      "binaryFolders": [],
       "deployConfig": {
-        "deployServicePack": true
+        "deployServicePack": true,
+        "deployScripts": []
       }
     },
     {
       "name": "docsonly",
       "serverType": "SAS9",
+      "appLoc": "dummy",
       "macroFolders": [
+        "meta",
+        "metax",
+        "server",
+        "viya",
         "tests/sas9only",
         "tests/viyaonly"
       ]
+    },
+    {
+      "name": "viya4",
+      "serverUrl": "https://azureuse011059.my-trials.sas.com",
+      "serverType": "SASVIYA",
+      "appLoc": "/Public/temp/macrocore",
+      "macroFolders": [
+        "viya",
+        "tests/viyaonly"
+      ],
+      "deployConfig": {
+        "deployServicePack": true
+      },
+      "contextName": "SAS Job Execution compute context"
     }
   ]
-}
+}

server/mfs_httpheader.sas

@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief Sets the http headers in the SASjs/server response
+  @details For GET requests, SASjs server will use the file generated by this
+  macro for setting the appropriate http headers in the response.
+
+  It works by writing a file to the session directory, that is then ingested by
+  the server.
+
+  The location of this file is driven by the global variable
+  `sasjs_stpsrv_header_loc` which is made available in the autoexec.
+
+  Usage:
+
+      %mfs_httpheader(Content-type,application/csv)
+
+  @param [in] header_name Name of the http header to set
+  @param [in] header_value Value of the http header to set
+
+  <h4> Related Macros </h4>
+  @li mcf_stpsrv_header.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mfs_httpheader(header_name
+  ,header_value
+)/*/STORE SOURCE*/;
+%local fref fid i;
+
+%if %sysfunc(filename(fref,&sasjs_stpsrv_header_loc)) ne 0 %then %do;
+  %put &=fref &=sasjs_stpsrv_header_loc;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let fid=%sysfunc(fopen(&fref,A));
+
+%if &fid=0 %then %do;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let rc=%sysfunc(fput(&fid,%str(&header_name): %str(&header_value)));
+%let rc=%sysfunc(fwrite(&fid));
+
+%let rc=%sysfunc(fclose(&fid));
+%let rc=%sysfunc(filename(&fref));
+
+%mend mfs_httpheader;

server/ms_createfile.sas

@@ -0,0 +1,98 @@
+/**
+  @file
+  @brief Creates a file on SASjs Drive
+  @details Creates a file on SASjs Drive. To use the file as a Stored Program,
+  it must have a ".sas" extension.
+
+  Example:
+
+      filename stpcode temp;
+      data _null_;
+        file stpcode;
+        put '%put hello world;';
+      run;
+      %ms_createfile(/some/stored/program.sas, inref=stpcode)
+
+  @param [in] driveloc The full path to the file in SASjs Drive
+  @param [in] inref= (0) The fileref containing the file to create.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+
+**/
+
+%macro ms_createfile(driveloc
+    ,inref=0
+    ,mdebug=0
+  );
+
+%local fname0 fname1 fname2 boundary fname statcd msg;
+%let fname0=%mf_getuniquefileref();
+%let fname1=%mf_getuniquefileref();
+%let fname2=%mf_getuniquefileref();
+%let boundary=%mf_getuniquename();
+
+data _null_;
+  file &fname0 termstr=crlf;
+  infile &inref end=eof;
+  if _n_ = 1 then do;
+    put "--&boundary.";
+    put 'Content-Disposition: form-data; name="filePath"';
+    put ;
+    put "&driveloc";
+    put "--&boundary";
+    put 'Content-Disposition: form-data; name="file"; filename="ignore.sas"';
+    put "Content-Type: text/plain";
+    put ;
+  end;
+  input;
+  put _infile_; /* add the actual file to be sent */
+  if eof then do;
+    put ;
+    put "--&boundary--";
+  end;
+run;
+
+data _null_;
+  file &fname1;
+  put "Content-Type: multipart/form-data; boundary=&boundary";
+run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fname0;
+    input;
+    put _infile_;
+  data _null_;
+    infile &fname1;
+    input;
+    put _infile_;
+  run;
+%end;
+
+proc http method='POST' in=&fname0 headerin=&fname1 out=&fname2
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file";
+%if &mdebug=1 %then %do;
+  debug level=1;
+%end;
+run;
+
+%let statcd=0;
+data _null_;
+  infile &fname2;
+  input;
+  putlog _infile_;
+  if _infile_='{"status":"success"}' then call symputx('statcd',1,'l');
+  else call symputx('msg',_infile_,'l');
+run;
+
+%mp_abort(
+  iftrue=(&statcd=0)
+  ,mac=ms_createfile.sas
+  ,msg=%superq(msg)
+)
+
+%mend ms_createfile;

server/ms_deletefile.sas

@@ -0,0 +1,35 @@
+/**
+  @file
+  @brief Deletes a file from SASjs Drive
+  @details Deletes a file from SASjs Drive, if it exists.
+
+  Example:
+
+      filename stpcode temp;
+      data _null_;
+        file stpcode;
+        put '%put hello world;';
+      run;
+      %ms_createfile(/some/stored/program.sas, inref=stpcode)
+
+      %ms_deletefile(/some/stored/program.sas)
+
+  @param [in] driveloc The full path to the file in SASjs Drive
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+
+**/
+
+%macro ms_deletefile(driveloc
+    ,mdebug=0
+  );
+
+proc http method='DELETE'
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file?_filePath=&driveloc";
+%if &mdebug=1 %then %do;
+  debug level=2;
+%end;
+run;
+
+
+%mend ms_deletefile;

server/ms_getfile.sas

@@ -0,0 +1,32 @@
+/**
+  @file
+  @brief Gets a file from SASjs Drive
+  @details Fetches a file on SASjs Drive and stores it in the output fileref.
+
+  Example:
+
+      %ms_getfile(/some/stored/file.ext, outref=myfile)
+
+  @param [in] driveloc The full path to the file in SASjs Drive
+  @param [out] outref= (msgetfil) The fileref to contain the file.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+
+**/
+
+%macro ms_getfile(driveloc
+    ,outref=msgetfil
+    ,mdebug=0
+  );
+
+filename &outref temp;
+
+proc http method='GET' out=&outref
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file?_filePath=&driveloc";
+%if &mdebug=1 %then %do;
+  debug level=2;
+%end;
+run;
+
+
+%mend ms_getfile;

server/ms_runstp.sas

@@ -0,0 +1,77 @@
+/**
+  @file
+  @brief Executes a SASjs Server Stored Program
+  @details Runs a Stored Program (using POST method) and extracts the webout and
+  log from the response JSON.
+
+  Example:
+
+      %ms_runstp(/some/stored/program
+        ,debug=131
+        ,outref=weboot
+      )
+
+  @param [in] pgm The full path to the Stored Program in SASjs Drive (_program
+    parameter)
+  @param [in] debug= (131) The value to supply to the _debug URL parameter
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outref= (outweb) The output fileref to contain the response JSON
+    (will be created using temp engine)
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_abort.sas
+
+**/
+
+%macro ms_runstp(pgm
+    ,debug=131
+    ,outref=outweb
+    ,mdebug=0
+  );
+%local dbg fname1;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+%let fname1=%mf_getuniquefileref();
+
+%mp_abort(iftrue=("&pgm"="")
+  ,mac=&sysmacroname
+  ,msg=%str(Program not provided)
+)
+
+data _null_;
+  file &fname1;
+  infile "&_sasjs_tokenfile";
+  input;
+  put 'Authorization: Bearer' _infile_;
+run;
+
+filename &outref temp;
+
+/* prepare request*/
+proc http method='POST' headerin=&fname1 out=&outref
+  url="&_sasjs_apiserverurl.&_sasjs_apipath?_program=&pgm%str(&)_debug=131";
+run;
+%if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  or &mdebug=1 %then %do;
+  data _null_;infile &outref;input;putlog _infile_;run;
+%end;
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  ,mac=&sysmacroname
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+
+%if &mdebug=1 %then %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%else %do;
+  /* clear refs */
+  filename &fname1 clear;
+%end;
+%mend ms_runstp;

server/ms_webout.sas

@@ -33,8 +33,9 @@
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
 
   <h4> SAS Macros </h4>
-  @li mp_jsonout.sas
   @li mf_getuser.sas
+  @li mp_jsonout.sas
+  @li mfs_httpheader.sas
 
   <h4> Related Macros </h4>
   @li mv_webout.sas
@@ -86,11 +87,11 @@
   /* fix encoding */
   OPTIONS NOBOMFILE;
 
+  /* set the header */
+  %mfs_httpheader(Content-type,application/json)
+
   /* setup json */
   data _null_;file &fref encoding='utf-8' termstr=lf;
-  %if %str(&_debug) ge 131 %then %do;
-    put '>>weboutBEGIN<<';
-  %end;
     put '{"SYSDATE" : "' "&SYSDATE" '"';
     put ',"SYSTIME" : "' "&SYSTIME" '"';
   run;
@@ -113,6 +114,7 @@
     data _null_;
       set &tempds;
       if not (upcase(name) =:"DATA"); /* ignore temp datasets */
+      if not (upcase(name)=:"_DATA_");
       i+1;
       call symputx(cats('wt',i),name,'l');
       call symputx('wtcnt',i,'l');
@@ -170,9 +172,6 @@
     memsize=quote(cats(memsize));
     put ',"MEMSIZE" : ' memsize;
     put "}" @;
-  %if %str(&_debug) ge 131 %then %do;
-    put '>>weboutEND<<';
-  %end;
   run;
 %end;
 

tests/crossplatform/mcf_stpsrv_header.test.sas

@@ -1,39 +0,0 @@
-/**
-  @file
-  @brief Testing mcf_stpsrv_header macro
-
-  <h4> SAS Macros </h4>
-  @li mcf_stpsrv_header.sas
-  @li mp_assert.sas
-
-**/
-
-%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/stpsrv_header.txt;
-
-%mcf_stpsrv_header(wrap=YES, insert_cmplib=YES)
-
-data _null_;
-  rc=stpsrv_header('Content-type','application/text');
-  rc=stpsrv_header('Content-disposition',"attachment; filename=file.txt");
-run;
-
-%let test1=FAIL;
-%let test2=FAIL;
-
-data _null_;
-  infile "&sasjs_stpsrv_header_loc";
-  input;
-  if _n_=1 and _infile_='Content-type: application/text'
-  then call symputx('test1','PASS');
-  else if _n_=2 & _infile_='Content-disposition: attachment; filename=file.txt'
-  then call symputx('test2','PASS');
-run;
-
-%mp_assert(
-  iftrue=(%str(&test1)=%str(PASS)),
-  desc=Check first header line
-)
-%mp_assert(
-  iftrue=(%str(&test2)=%str(PASS)),
-  desc=Check second header line
-)

tests/crossplatform/mf_deletefile.test.sas

@@ -0,0 +1,29 @@
+/**
+  @file
+  @brief Testing mf_deletefile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_deletefile.sas
+  @li mf_writefile.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%let test1file=&sasjswork/myfile1.txt;
+
+%mf_writefile(&test1file,l1=some content)
+
+%mp_assert(
+  iftrue=(%sysfunc(fileexist(&test1file))=1),
+  desc=Check &test1file exists
+)
+
+%mp_assertscope(SNAPSHOT)
+%mf_deletefile(&test1file)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(%sysfunc(fileexist(&test1file))=0),
+  desc=Check &test1file no longer exists
+)

tests/crossplatform/mf_getapploc.test.sas

@@ -46,4 +46,20 @@
   ),
   desc=Checking tests/macros appLoc matches (which has no subfolder),
   outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/tests/testsetup)"="/some/area"
+  ),
+  desc=Checking tests/testsetup operation,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/tests/testteardown)"="/some/area"
+  ),
+  desc=Checking tests/teardown operation,
+  outds=work.test_results
 )

tests/crossplatform/mf_getuniquelibref.test.sas

@@ -0,0 +1,54 @@
+/**
+  @file
+  @brief Testing mf_getuniquelibref macro
+  @details To test performance you can also use the following macro:
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquelibref.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+/* check valid libs */
+%mp_assertscope(SNAPSHOT)
+%let libshort=%mf_getuniquelibref(prefix=lib);
+%mp_assertscope(COMPARE,ignorelist=LIBSHORT)
+libname &libshort (work);
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking for valid libref &libshort,
+  outds=work.test_results
+)
+
+%let lib7=%mf_getuniquelibref(prefix=libref7);
+libname &lib7 (work);
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking for valid libref &lib7,
+  outds=work.test_results
+)
+
+
+/* check for invalid libs */
+
+%let lib8=%mf_getuniquelibref(prefix=lib8char);
+%mp_assert(
+  iftrue=(&lib8=0),
+  desc=Invalid prefix (8 chars),
+  outds=work.test_results
+)
+
+%let liblong=%mf_getuniquelibref(prefix=invalidlib);
+%mp_assert(
+  iftrue=(&liblong=0),
+  desc=Checking for invalid libref (long),
+  outds=work.test_results
+)
+
+%let badlib=%mf_getuniquelibref(prefix=8adlib);
+%mp_assert(
+  iftrue=(&badlib=0),
+  desc=Checking for invalid libref (8adlib),
+  outds=work.test_results
+)

tests/crossplatform/mf_verifymacvars.test.sas

@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Testing mf_verifymacvars macro
+
+  <h4> SAS Macros </h4>
+  @li mf_verifymacvars.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%let var1=x;
+%let var2=y;
+
+%mp_assertscope(SNAPSHOT)
+%mp_assert(
+  iftrue=(%mf_verifymacvars(var1 var2)=1),
+  desc=Checking macvars exist,
+  outds=work.test_results
+)
+%mp_assertscope(COMPARE)
+

tests/crossplatform/mp_cntlout.test.sas

@@ -0,0 +1,39 @@
+/**
+  @file
+  @brief Testing mp_cntlout.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_cntlout.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+libname perm (work);
+data work.loadfmts;
+  length fmtname $32;
+  eexcl='Y';
+  type='N';
+  do i=1 to 100;
+    fmtname=cats('SASJS_',i,'X');
+    do j=1 to 100;
+      start=cats(j);
+      end=cats(j+1);
+      label= cats('Dummy ',start);
+      output;
+    end;
+  end;
+run;
+proc format cntlin=work.loadfmts library=perm.testcat;
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mp_cntlout(libcat=perm.testcat,cntlout=work.cntlout)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.cntlout)=10000),
+  desc=Checking first hash diff,
+  outds=work.test_results
+)

tests/crossplatform/mp_dirlist.test.sas

@@ -47,4 +47,18 @@ run;
   iftrue=(%mf_nobs(work.mytable3)=2),
   desc=Top level returned,
   outds=work.test_results
-)
+)
+
+%mp_dirlist(path=&root/b, outds=work.myTable4)
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable4)=0),
+  desc=Empty table for empty directory,
+  outds=work.test_results
+)
+
+%mp_dirlist(path=&root/notexisting, outds=work.myTable5)
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable5)=0),
+  desc=Empty table for non-existing directory,
+  outds=work.test_results
+)

tests/crossplatform/mp_filterstore.test.2.sas

@@ -0,0 +1,112 @@
+/**
+  @file
+  @brief Testing mp_filterstore macro with a format catalog
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
+  @li mp_coretable.sas
+  @li mp_filterstore.sas
+
+**/
+
+libname permlib (work);
+
+%mp_coretable(LOCKTABLE,libds=permlib.locktable)
+%mp_coretable(FILTER_SUMMARY,libds=permlib.filtsum)
+%mp_coretable(FILTER_DETAIL,libds=permlib.filtdet)
+%mp_coretable(MAXKEYTABLE,libds=permlib.maxkey)
+
+/* valid filter */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$12. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,Start,>,"'2'"
+AND,AND,1,Fmtname,=,"'MORDOR'"
+OR,OR,2,Label,IN,"('Dragon1','Dragon2')"
+OR,OR,2,End,=,"'6'"
+OR,OR,2,Start,GE,"'10'"
+;;;;
+run;
+
+/* make some formats */
+PROC FORMAT library=permlib.testfmts;
+  picture MyMSdt other='%0Y-%0m-%0dT%0H:%0M:%0S' (datatype=datetime);
+RUN;
+data work.fmts;
+  length fmtname $32;
+  do fmtname='SMAUG','MORDOR','GOLLUM';
+    do start=1 to 10;
+      label= cats('Dragon ',start);
+      output;
+    end;
+  end;
+run;
+proc sort data=work.fmts nodupkey;
+  by fmtname start;
+run;
+proc format cntlin=work.fmts library=permlib.testfmts;
+run;
+proc format library=permlib.testfmts;
+  invalue indays (default=13) other=42;
+run;
+
+
+%mp_assertscope(SNAPSHOT)
+%mp_filterstore(libds=permlib.testfmts-fc,
+  queryds=work.inds,
+  filter_summary=permlib.filtsum,
+  filter_detail=permlib.filtdet,
+  lock_table=permlib.locktable,
+  maxkeytable=permlib.maxkey,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+)
+%mp_assertscope(COMPARE)
+
+%mp_assert(iftrue=(&syscc=0),
+  desc=Ensure macro runs without errors,
+  outds=work.test_results
+)
+/* ensure only one record created */
+%mp_assertdsobs(permlib.filtsum,
+  desc=Initial query,
+  test=ATMOST 1,
+  outds=work.test_results
+)
+/* check RK is correct */
+proc sql noprint;
+select max(filter_rk) into: test1 from work.result;
+%mp_assert(iftrue=(&test1=1),
+  desc=Ensure filter rk is correct,
+  outds=work.test_results
+)
+
+/* Test 2 - load same table again and ensure we get the same RK */
+%mp_filterstore(libds=permlib.testfmts-fc,
+  queryds=work.inds,
+  filter_summary=permlib.filtsum,
+  filter_detail=permlib.filtdet,
+  lock_table=permlib.locktable,
+  maxkeytable=permlib.maxkey,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+)
+/* ensure only one record created */
+%mp_assertdsobs(permlib.filtsum,
+  desc=Initial query - same obs,
+  test=ATMOST 1,
+  outds=work.test_results
+)
+/* check RK is correct */
+proc sql noprint;
+select max(filter_rk) into: test2 from work.result;
+%mp_assert(iftrue=(&test2=1),
+  desc=Ensure filter rk is correct for second run,
+  outds=work.test_results
+)

tests/crossplatform/mp_gsubfile.test.sas

@@ -8,6 +8,13 @@
 
 **/
 
+
+%macro gsubtest();
+%if "%substr(&sysver,1,4)"="V.04" %then %do;
+  %put %str(ERR)OR: Viya 4 does not support the IO library in lua;
+  %return;
+%end;
+
 /**
   * test 1 - simple replace
   */
@@ -63,4 +70,8 @@ run;
   iftrue=("&strcheck2b"="&str2"),
   desc=Check that multi line replacement was successful (line3),
   outds=work.test_results
-)
+)
+
+%mend gsubtest;
+
+%gsubtest()

tests/crossplatform/mp_hashdataset.test.sas

@@ -0,0 +1,59 @@
+/**
+  @file
+  @brief Testing mp_hashdataset.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_hashdataset.sas
+  @li mp_assert.sas
+
+**/
+
+/* test 1 - regular DS */
+data work.test;
+  set sashelp.vextfl;
+  missval=.;
+  misscval='';
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mp_hashdataset(test)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Regular test works,
+  outds=work.test_results
+)
+
+%mp_hashdataset(test,outds=work.test2)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=hash with output runs without errors,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.test2)=1),
+  desc=output has 1 row,
+  outds=work.test_results
+)
+
+
+data work.test3a;
+  set work.test;
+  stop;
+run;
+%mp_hashdataset(test3a,outds=work.test3b)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=hash with zero-row input runs without errors,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.test3b)=1),
+  desc=test 3 output has 1 row,
+  outds=work.test_results
+)

tests/crossplatform/mp_loadformat.test.sas

@@ -0,0 +1,93 @@
+/**
+  @file
+  @brief Testing mp_loadformat.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mp_loadformat.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+/* prep format catalog */
+libname perm (work);
+
+%mddl_dc_difftable(libds=perm.audit)
+
+data work.loadfmts;
+  length fmtname $32;
+  eexcl='Y';
+  type='N';
+  do i=1 to 100;
+    fmtname=cats('SASJS_',i,'X');
+    do j=1 to 100;
+      start=cats(j);
+      end=cats(j+1);
+      label= cats('Dummy ',start);
+      output;
+    end;
+  end;
+run;
+proc format cntlin=work.loadfmts library=perm.testcat;
+run;
+
+/* make some test data */
+data work.stagedata;
+  set work.loadfmts;
+  type='N';
+  eexcl='Y';
+  if _n_<150 then deleteme='Yes';
+  else if _n_<250 then label='mod'!!cats(_n_);
+  else if _n_<350 then do;
+    start=cats(_n_);
+    end=cats(_n_+1);
+    label='newval'!!cats(_N_);
+  end;
+  else stop;
+run;
+
+/* load the above */
+%mp_assertscope(SNAPSHOT)
+%mp_loadformat(perm.testcat
+  ,work.stagedata
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_test1
+  ,outds_del=del_test1
+  ,outds_mod=mod_test1
+  ,mdebug=1
+)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(%mf_nobs(del_test1)=149),
+  desc=Test 1 - delete obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(add_test1)=100),
+  desc=Test 1 - add obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(mod_test1)=100),
+  desc=Test 1 - mod obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(perm.audit)=7329),
+  desc=Test 1 - audit table updated,
+  outds=work.test_results
+)
+data work.difftest;
+  set perm.audit;
+  where is_diff=1;
+run;
+%mp_assert(
+  iftrue=(%mf_nobs(work.difftest)>0),
+  desc=Test 1 - diffs were found,
+  outds=work.test_results
+)

tests/crossplatform/mp_md5.test.sas

@@ -0,0 +1,41 @@
+/**
+  @file
+  @brief Testing mp_md5.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_md5.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+%global hash1 hash2 hash3;
+
+%mp_assertscope(SNAPSHOT)
+data work.test1 /nonote2err;
+  c1='';
+  c2=repeat('x',32767);
+  c3='  f';
+  n1=.a;
+  n2=.;
+  n3=1.0000000001;
+  hash=%mp_md5(cvars=c1 c2 c3,nvars=n1 n2 n3);
+  call symputx('hash1',hash);
+  n1=.b;
+  hash=%mp_md5(cvars=c1 c2 c3,nvars=n1 n2 n3);
+  call symputx('hash2',hash);
+  c3='f';
+  hash=%mp_md5(cvars=c1 c2 c3,nvars=n1 n2 n3);
+  call symputx('hash3',hash);
+run;
+%mp_assertscope(COMPARE,ignorelist=HASH1 HASH2 HASH3)
+
+%mp_assert(
+  iftrue=("&hash1" ne "&hash2"),
+  desc=Checking first hash diff,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&hash2" ne "&hash3"),
+  desc=Checking first hash diff,
+  outds=work.test_results
+)

tests/crossplatform/mp_replace.test.sas

@@ -0,0 +1,95 @@
+/**
+  @file
+  @brief Testing mp_replace.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_replace.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+
+%let test1="&sasjswork/file.txt";
+%let str=replace/me;
+%let rep=with/this;
+data _null_;
+  file &test1;
+  put 'blahblah';
+  put "blahblah&str.blah";
+  put 'blahblahblah';
+run;
+%mp_assertscope(SNAPSHOT)
+%mp_replace(&test1, findvar=str, replacevar=rep)
+%mp_assertscope(COMPARE)
+data _null_;
+  infile &test1;
+  input;
+  if _n_=2 then call symputx('test1result',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&test1result" = "blahblah&rep.blah"),
+  desc=Checking first replace,
+  outds=work.test_results
+)
+
+
+%let test2="&sasjswork/file2.txt";
+%let str=%str(replacewith trailing spaces   );
+%let rep=%str( with more spaces  );
+data _null_;
+  file &test2;
+  put 'blahblah';
+  put "blahblah&str.blah&str. replace &str.X";
+  put "blahbreplacewith&str.spacesahblah";
+run;
+%mp_replace(&test2, findvar=str, replacevar=rep)
+
+data _null_;
+  infile &test2;
+  input;
+  if _n_=2 then call symputx('test2resulta',_infile_);
+  if _n_=3 then call symputx('test2resultb',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&test2resulta" = "blahblah&rep.blah&rep. replace &rep.X"),
+  desc=Checking second replace 2nd row,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test2resultb" = "blahbreplacewith&rep.spacesahblah"),
+  desc=Checking second replace 3rd row,
+  outds=work.test_results
+)
+
+
+%let test3="&sasjswork/file3.txt";
+%let str=%str(replace.string.with.dots   );
+%let rep=%str( more.dots);
+data _null_;
+  file &test3;
+  put 'blahblah';
+  put "blahblah&str.blah&str. replace &str.X";
+  put "blahbreplacewith&str.spacesahblah";
+run;
+%mp_replace(&test3, findvar=str, replacevar=rep)
+
+data _null_;
+  infile &test3;
+  input;
+  if _n_=2 then call symputx('test3resulta',_infile_);
+  if _n_=3 then call symputx('test3resultb',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&test3resulta" = "blahblah&rep.blah&rep. replace &rep.X"),
+  desc=Checking third replace 2nd row (dots),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test3resultb" = "blahbreplacewith&rep.spacesahblah"),
+  desc=Checking third replace 3rd row (dots),
+  outds=work.test_results
+)

tests/crossplatform/mp_stackdiffs.test.sas

@@ -42,6 +42,7 @@ run;
   ,mdebug=1
 )
 
+
 %mp_assertscope(SNAPSHOT)
 
 /**

tests/ddl/mddl_dc_difftable.test.sas

@@ -0,0 +1,19 @@
+/**
+  @file
+  @brief Difftable DDL test
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mf_existds.sas
+  @li mp_assert.sas
+
+**/
+
+
+%mddl_dc_difftable(libds=WORK.DIFFTABLE)
+
+%mp_assert(
+  iftrue=(%mf_existds(WORK.DIFFTABLE)=1),
+  desc=Checking table was created,
+  outds=work.test_results
+)

tests/ddl/mddl_dc_filterdetail.test.sas

@@ -0,0 +1,18 @@
+/**
+  @file
+  @brief Filtertable DDL test
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_filterdetail.sas
+  @li mf_existds.sas
+  @li mp_assert.sas
+
+**/
+
+%mddl_dc_filterdetail(libds=WORK.TEST)
+
+%mp_assert(
+  iftrue=(%mf_existds(WORK.TEST)=1),
+  desc=Checking table was created,
+  outds=work.test_results
+)

tests/ddl/mddl_dc_filtersummary.test.sas

@@ -0,0 +1,18 @@
+/**
+  @file
+  @brief Filtersummary DDL test
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_filtersummary.sas
+  @li mf_existds.sas
+  @li mp_assert.sas
+
+**/
+
+%mddl_dc_filtersummary(libds=WORK.TEST)
+
+%mp_assert(
+  iftrue=(%mf_existds(WORK.TEST)=1),
+  desc=Checking table was created,
+  outds=work.test_results
+)

tests/ddl/mddl_dc_locktable.test.sas

@@ -0,0 +1,18 @@
+/**
+  @file
+  @brief Locktable DDL test
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_locktable.sas
+  @li mf_existds.sas
+  @li mp_assert.sas
+
+**/
+
+%mddl_dc_locktable(libds=WORK.TEST)
+
+%mp_assert(
+  iftrue=(%mf_existds(WORK.TEST)=1),
+  desc=Checking table was created,
+  outds=work.test_results
+)

tests/ddl/mddl_dc_maxkeytable.test.sas

@@ -0,0 +1,18 @@
+/**
+  @file
+  @brief Maxkeytable DDL test
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_maxkeytable.sas
+  @li mf_existds.sas
+  @li mp_assert.sas
+
+**/
+
+%mddl_dc_maxkeytable(libds=WORK.TEST)
+
+%mp_assert(
+  iftrue=(%mf_existds(WORK.TEST)=1),
+  desc=Checking table was created,
+  outds=work.test_results
+)

tests/ddl/mddl_sas_cntlout.test.sas

@@ -0,0 +1,18 @@
+/**
+  @file
+  @brief mddl_sas_cntlout ddl test
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_existds.sas
+  @li mp_assert.sas
+
+**/
+
+%mddl_sas_cntlout(libds=WORK.TEST)
+
+%mp_assert(
+  iftrue=(%mf_existds(WORK.TEST)=1),
+  desc=Checking table was created,
+  outds=work.test_results
+)

tests/sas9only/mm_getauthinfo.test.sas

@@ -0,0 +1,21 @@
+/**
+  @file
+  @brief Testing mm_getauthinfo macro
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mm_getauthinfo.sas
+  @li mp_assertscope.sas
+
+**/
+
+
+%mp_assertscope(SNAPSHOT)
+%mm_getauthinfo(outds=auths)
+%mp_assertscope(COMPARE)
+
+
+%mp_assert(
+  iftrue=(%mf_existds(work.auths)=1),
+  desc=Check if the auths dataset was created
+)

tests/sas9only/mm_spkexport.test.sas

@@ -0,0 +1,41 @@
+/**
+  @file
+  @brief Testing mm_webout macro
+
+  <h4> SAS Macros </h4>
+  @li mm_spkexport.sas
+  @li mp_assert.sas
+  @li mp_as
+
+**/
+
+
+%* create sample text file as input to the macro;
+filename tmp temp;
+data _null_;
+  file tmp;
+  put '%let mmxuser="sasdemo";';
+  put '%let mmxpass="Mars321";';
+run;
+
+filename myref "%sysfunc(pathname(work))/mmxexport.sh"
+  permission='A::u::rwx,A::g::r-x,A::o::---';
+%mp_assertscope(SNAPSHOT)
+%mm_spkexport(metaloc=%str(/Shared Data)
+    ,outref=myref
+    ,secureref=tmp
+    ,cmdoutloc=%str(/tmp)
+)
+%mp_assertscope(COMPARE)
+
+data _null_;
+  infile tmp;
+  input;
+  putlog _infile_;
+  call symputx('nobs',_n_);
+run;
+
+%mp_assert(
+  iftrue=(&nobs>2),
+  desc=Check if content was created
+)

tests/serveronly/mfs_httpheader.test.sas

@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief Testing mfs_httpheader.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mfs_httpheader.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/header.txt;
+
+%mp_assertscope(SNAPSHOT)
+%mfs_httpheader(Content-type,application/csv)
+%mp_assertscope(COMPARE)
+
+data _null_;
+  infile "&sasjs_stpsrv_header_loc";
+  input;
+  if _n_=1 then call symputx('test1',_infile_);
+run;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check code ran without errors,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test1"="Content-type: application/csv"),
+  desc=Checking line was created,
+  outds=work.test_results
+)
+
+%mfs_httpheader(Content-type,application/text)
+%let test2=0;
+data _null_;
+  infile "&sasjs_stpsrv_header_loc";
+  input;
+  if _n_=2 then call symputx('test2',_infile_);
+run;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check code ran without errors for test2,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test2"="Content-type: application/text"),
+  desc=Checking line was created,
+  outds=work.test_results
+)

tests/serveronly/ms_createfile.test.sas

@@ -0,0 +1,30 @@
+/**
+  @file
+  @brief Testing ms_createfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li ms_createfile.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+
+filename stpcode temp;
+data _null_;
+  file stpcode;
+  put '%put hello world;';
+run;
+
+options mprint;
+%let fname=%mf_getuniquename();
+
+%mp_assertscope(SNAPSHOT)
+%ms_createfile(/sasjs/tests/&fname..sas
+  ,inref=stpcode
+  ,mdebug=1
+)
+%mp_assertscope(COMPARE)
+
+
+

tests/serveronly/ms_deletefile.test.sas

@@ -0,0 +1,61 @@
+/**
+  @file
+  @brief Testing ms_deletefile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li ms_createfile.sas
+  @li ms_deletefile.sas
+  @li ms_getfile.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+
+/* first make a remote file */
+filename stpcode temp;
+%let fname=%mf_getuniquename();
+data _null_;
+  file stpcode;
+  put "data &fname;run;";
+run;
+%ms_createfile(/sasjs/tests/&fname..sas
+  ,inref=stpcode
+  ,mdebug=1
+)
+
+%ms_getfile(/sasjs/tests/&fname..sas,outref=testref)
+
+%let test1=0;
+data _null_;
+  infile testref;
+  input;
+  call symputx('test1',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&test1"="data &fname;run;"),
+  desc=Make sure the file was created,
+  outds=work.test_results
+)
+
+%mp_assertscope(SNAPSHOT)
+%ms_deletefile(/sasjs/tests/&fname..sas,mdebug=1)
+%mp_assertscope(COMPARE)
+
+%ms_getfile(/sasjs/tests/&fname..sas,outref=testref2)
+
+%let test2=0;
+data _null_;
+  infile testref2;
+  input;
+  call symputx('test2',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&test2"="%str(Err)or: File does not exist."),
+  desc=Make sure the file was deleted,
+  outds=work.test_results
+)
+
+

tests/serveronly/ms_getfile.test.sas

@@ -0,0 +1,45 @@
+/**
+  @file
+  @brief Testing ms_getfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li ms_createfile.sas
+  @li ms_getfile.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+
+/* first make a remote file */
+filename stpcode temp;
+%let fname=%mf_getuniquename();
+data _null_;
+  file stpcode;
+  put "data &fname;run;";
+run;
+%ms_createfile(/sasjs/tests/&fname..sas
+  ,inref=stpcode
+  ,mdebug=1
+)
+
+%mp_assertscope(SNAPSHOT)
+%ms_getfile(/sasjs/tests/&fname..sas,outref=testref)
+%mp_assertscope(COMPARE)
+
+%let test1=0;
+data _null_;
+  infile testref;
+  input;
+  call symputx('test1',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&test1"="data &fname;run;"),
+  desc=Checking file was created with the same content,
+  outds=work.test_results
+)
+
+
+
+

tests/serveronly/ms_runstp.test.sas

@@ -0,0 +1,44 @@
+/**
+  @file
+  @brief Testing ms_runstp.sas macro
+
+  <h4> SAS Macros </h4>
+  @li ms_runstp.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%mp_assertscope(SNAPSHOT)
+%ms_runstp(/Public/app/frs/allan/services/common/appinit
+  ,debug=131
+  ,outref=weboot
+)
+%mp_assertscope(COMPARE)
+
+libname webeen json (weboot);
+
+data _null_;
+  infile weboot;
+  input;
+  putlog _infile_;
+run;
+
+data work.httpheaders;
+  set webeen.httpheaders;
+  call symputx('test1',content_type);
+run;
+
+data work.log;
+  set webeen.log;
+  put (_all_)(=);
+  if _n_>10 then stop;
+run;
+
+%mp_assert(
+  iftrue=("&test1"="application/json"),
+  desc=Checking line was created,
+  outds=work.test_results
+)
+
+

tests/testinit.sas

@@ -19,7 +19,7 @@
 %macro loglevel();
   %if "&_debug"="2477" or "&_debug"="fields,log,trace" %then %do;
     %put debug mode activated;
-    options mprint;
+    options mprint mprintnest;
   %end;
 %mend loglevel;
 

tests/viyaonly/mv_deleteviyafolder.test.sas

@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief Testing mv_deleteviyafolder macro function
+
+  <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mfv_existfolder.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mv_createfolder.sas
+  @li mv_deleteviyafolder.sas
+
+**/
+
+options mprint sgen;
+
+%let folder=%mf_uid();
+%let tgtfolder=&mcTestAppLoc/temp/&folder;
+
+/* create a folder */
+%mv_createfolder(path=&tgtfolder)
+
+
+%mp_assert(
+  iftrue=(%mfv_existfolder(&tgtfolder)=1),
+  desc=Check if created folder exists
+)
+
+%mp_assertscope(SNAPSHOT)
+%mv_deleteviyafolder(path=&tgtfolder)
+/* ignore proc json vars */
+%mp_assertscope(COMPARE
+  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADP2LEN MCLIB0_JADVLEN MCLIB2_JADP1LEN
+    MCLIB2_JADVLEN
+)
+
+%mp_assert(
+  iftrue=(%mfv_existfolder(&tgtfolder)=0),
+  desc=Check if deleted folder is gone
+)
+
+/* delete folder with content */
+%mv_createfolder(path=&tgtfolder/content/and/stuff)
+%mp_assert(
+  iftrue=(%mfv_existfolder(&tgtfolder/content/and/stuff)=1),
+  desc=Check if folder with content exists
+)
+%mv_deleteviyafolder(path=&tgtfolder)
+%mp_assert(
+  iftrue=(%mfv_existfolder(&tgtfolder)=0),
+  desc=Check if deleted folder with subfolders is gone
+)

tests/viyaonly/mv_getjobcode.test.sas

@@ -4,6 +4,7 @@
 
   <h4> SAS Macros </h4>
   @li mp_assert.sas
+  @li mp_assertscope.sas
   @li mv_createjob.sas
   @li mv_getjobcode.sas
 
@@ -27,11 +28,17 @@ run;
 )
 
 /* now get the code back */
+%mp_assertscope(SNAPSHOT)
 %mv_getjobcode(
   path=&mcTestAppLoc/services/temp,
   name=some_job,
   outref=mycode
 )
+/* exclude automatic proc json macro variables from scope check */
+%mp_assertscope(COMPARE,
+  ignorelist=MCLIB2_JADP1LEN MCLIB2_JADP2LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
+    MCLIB2_JADP3LEN
+)
 
 %let diditexist=NO;
 data work.test1;
@@ -46,4 +53,4 @@ run;
 %mp_assert(
   iftrue=(&diditexist=NO),
   desc=Check if the code that was sent was successfully retrieved
-)
+)

tests/viyaonly/mv_getjoblog.test.sas

@@ -4,6 +4,7 @@
 
   <h4> SAS Macros </h4>
   @li mp_assert.sas
+  @li mp_assertscope.sas
   @li mv_createjob.sas
   @li mv_jobexecute.sas
   @li mv_jobwaitfor.sas
@@ -49,8 +50,12 @@ data _null_;
 run;
 
 %* Finally, fetch the log;
-%mv_getjoblog(uri=%str(&uri),outref=mylog)
-
+%mp_assertscope(SNAPSHOT)
+%mv_getjoblog(uri=%str(&uri),outref=mylog,mdebug=1)
+/* ignore auto proc json vars */
+%mp_assertscope(COMPARE
+  ,ignorelist=MCLIB2_JADP2LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
+)
 
 data _null_;
   infile mylog end=eof;
@@ -67,4 +72,4 @@ run;
 %mp_assert(
   iftrue=(%str(&found)=1),
   desc=Check if the log was still fetched even though endsas was submitted
-)
+)

tests/viyaonly/mv_jobflow.test.1.sas

@@ -76,3 +76,6 @@ run;
   iftrue=(&syscc ne 0),
   desc=Check that non zero return code is returned if called job fails
 )
+
+/* set syscc to zero for final check in testterm */
+%let syscc=0;

viya/mv_createwebservice.sas

@@ -380,8 +380,15 @@ data _null_;
   put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
   put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
   put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
+  put '                    prxchange(''s/''!!''00''x!!''/\\u0000/'',-1, /* NUL */ ';
+  put '                    prxchange(''s/''!!''0E''x!!''/\\u000E/'',-1, /* SS  */ ';
+  put '                    prxchange(''s/''!!''0F''x!!''/\\u000F/'',-1, /* SF  */ ';
+  put '                    prxchange(''s/''!!''01''x!!''/\\u0001/'',-1, /* SOH */ ';
+  put '                    prxchange(''s/''!!''02''x!!''/\\u0002/'',-1, /* STX */ ';
+  put '                    prxchange(''s/''!!''02''x!!''/\\u0010/'',-1, /* DLE */ ';
+  put '                    prxchange(''s/''!!''11''x!!''/\\u0011/'',-1, /* DC1 */ ';
   put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '        )))))!!''"''; ';
+  put '        ))))))))))))!!''"''; ';
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
@@ -655,7 +662,8 @@ data _null_;
   put '  %else %if %symexist(&metavar) %then %do; ';
   put '    %if %length(&&&metavar)=0 %then %let user=&sysuserid; ';
   put '    /* sometimes SAS will add @domain extension - remove for consistency */ ';
-  put '    %else %let user=%scan(&&&metavar,1,@); ';
+  put '    /* but be sure to quote in case of usernames with commas */ ';
+  put '    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@)); ';
   put '  %end; ';
   put '  %else %let user=&sysuserid; ';
   put ' ';

viya/mv_deleteviyafolder.sas

@@ -8,10 +8,16 @@
       %mv_deleteviyafolder(path=/Public/test)
 
 
-  @param path= The full path of the folder to be deleted
-  @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values are "password" or "authorization_code" (unquoted).
-    The default is authorization_code.
+  @param [in] path= The full path of the folder to be deleted
+  @param [in] access_token_var= (ACCESS_TOKEN) The global macro variable to
+    contain the access token
+  @param [in] grant_type= (sas_services) Valid values are:
+    @li password
+    @li authorization_code
+    @li detect - will check if access_token exists, if not will use sas_services
+      if a SASStudioV session else authorization_code.  Default option.
+    @li sas_services - will use oauth_bearer=sas_services.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
 
 
   @version VIYA V.03.04
@@ -19,6 +25,7 @@
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas
+  @li mf_existds.sas
   @li mf_getplatform.sas
   @li mf_getuniquefileref.sas
   @li mf_getuniquelibref.sas
@@ -29,6 +36,7 @@
 %macro mv_deleteviyafolder(path=
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
+    ,mdebug=0
   );
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
@@ -105,14 +113,17 @@ run;
 %let libref1a=%mf_getuniquelibref();
 libname &libref1a JSON fileref=&fname1a;
 
-data _null_;
-  set &libref1a..items_links;
-  if href=:'/folders/folders' then return;
-  if rel='deleteResource' then
-    call execute('proc http method="DELETE" url='!!quote("&base_uri"!!trim(href))
-    !!'; headers "Authorization"="Bearer &&&access_token_var" '
-    !!' "Accept"="*/*";run; /**/');
-run;
+%if %mf_existds(&libref1a..items_links) %then %do;
+  data _null_;
+    set &libref1a..items_links;
+    if href=:'/folders/folders' then return;
+    if rel='deleteResource' then
+      call execute('proc http method="DELETE" url='
+      !!quote("&base_uri"!!trim(href))
+      !!'; headers "Authorization"="Bearer &&&access_token_var" '
+      !!' "Accept"="*/*";run; /**/');
+  run;
+%end;
 
 %put &sysmacroname: perform the delete operation ;
 %local fname2;
@@ -133,9 +144,11 @@ run;
 %end;
 %else %put &sysmacroname: &path successfully deleted;
 
-/* clear refs */
-filename &fname1 clear;
-filename &fname2 clear;
-libname &libref1 clear;
+%if &mdebug=0 %then %do;
+  /* clear refs */
+  filename &fname1 clear;
+  filename &fname2 clear;
+  libname &libref1 clear;
+%end;
 
-%mend mv_deleteviyafolder;
+%mend mv_deleteviyafolder;

viya/mv_getgroups.sas

@@ -32,7 +32,7 @@
     ,grant_type=sas_services
     ,outds=work.viyagroups
   );
-%local oauth_bearer;
+%local oauth_bearer base_uri fname1 libref1;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
   %else %let grant_type=sas_services;
@@ -50,11 +50,10 @@
 )
 
 options noquotelenmax;
-%local base_uri; /* location of rest apis */
+/* location of rest apis */
 %let base_uri=%mf_getplatform(VIYARESTAPI);
 
 /* fetching folder details for provided path */
-%local fname1;
 %let fname1=%mf_getuniquefileref();
 %let libref1=%mf_getuniquelibref();
 
@@ -78,9 +77,8 @@ data &outds;
 run;
 
 
-
 /* clear refs */
 filename &fname1 clear;
 libname &libref1 clear;
 
-%mend mv_getgroups;
+%mend mv_getgroups;

viya/mv_getjobcode.sas

@@ -33,7 +33,6 @@
   @li mf_getplatform.sas
   @li mf_getuniquefileref.sas
   @li mv_getfoldermembers.sas
-  @li ml_json.sas
 
 **/
 
@@ -44,9 +43,9 @@
     ,grant_type=sas_services
     ,mdebug=0
   );
-%local dbg;
+%local dbg bufsize varcnt fname1 fname2 errmsg;
 %if &mdebug=1 %then %do;
-  %put &sysmacroname entry vars:;
+  %put &sysmacroname local entry vars:;
   %put _local_;
 %end;
 %else %let dbg=*;
@@ -104,7 +103,6 @@ run;
 )
 
 /* prepare request*/
-%local  fname1;
 %let fname1=%mf_getuniquefileref();
 proc http method='GET' out=&fname1 &oauth_bearer
   url="&base_uri&joburi";
@@ -114,37 +112,95 @@ proc http method='GET' out=&fname1 &oauth_bearer
   %end;
   ;
 run;
-%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
-%do;
-  data _null_;infile &fname1;input;putlog _infile_;run;
-  %mp_abort(mac=&sysmacroname
-    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
-  )
-%end;
-%local  fname2 fname3 fpath1 fpath2 fpath3;
-%let fname2=%mf_getuniquefileref();
-%let fname3=%mf_getuniquefileref();
-%let fpath1=%sysfunc(pathname(&fname1));
-%let fpath2=%sysfunc(pathname(&fname2));
-%let fpath3=%sysfunc(pathname(&fname3));
 
-/* compile the lua JSON module */
-%ml_json()
-/* read using LUA - this allows the code to be of any length */
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fname1;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  ,mac=&sysmacroname
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+%let fname2=%mf_getuniquefileref();
+filename &fname2 temp ;
+
+/* cannot use lua IO package as not available in Viya 4 */
+/* so use data step to read the JSON until the string `"code":"` is found */
 data _null_;
-  file "&fpath3..lua";
-  put '
-    infile = io.open (sas.symget("fpath1"), "r")
-    outfile = io.open (sas.symget("fpath2"), "w")
-    io.input(infile)
-    local resp=json.decode(io.read())
-    local job=resp["code"]
-    outfile:write(job)
-    io.close(infile)
-    io.close(outfile)
-  ';
+  file &fname2 recfm=n;
+  infile &fname1 lrecl=1 recfm=n;
+  input sourcechar $char1. @@;
+  format sourcechar hex2.;
+  retain startwrite 0;
+  if startwrite=0 and sourcechar='"' then do;
+    reentry:
+    input sourcechar $ 1. @@;
+    if sourcechar='c' then do;
+      reentry2:
+      input sourcechar $ 1. @@;
+      if sourcechar='o' then do;
+        input sourcechar $ 1. @@;
+        if sourcechar='d' then do;
+          input sourcechar $ 1. @@;
+          if sourcechar='e' then do;
+            input sourcechar $ 1. @@;
+            if sourcechar='"' then do;
+              input sourcechar $ 1. @@;
+              if sourcechar=':' then do;
+                input sourcechar $ 1. @@;
+                if sourcechar='"' then do;
+                  putlog 'code found';
+                  startwrite=1;
+                  input sourcechar $ 1. @@;
+                end;
+              end;
+              else if sourcechar='c' then goto reentry2;
+            end;
+          end;
+          else if sourcechar='"' then goto reentry;
+        end;
+        else if sourcechar='"' then goto reentry;
+      end;
+      else if sourcechar='"' then goto reentry;
+    end;
+    else if sourcechar='"' then goto reentry;
+  end;
+  /* once the `"code":"` string is found, write until unescaped `"` is found */
+  if startwrite=1 then do;
+    if sourcechar='\' then do;
+      input sourcechar $ 1. @@;
+      if sourcechar in ('"','\') then put sourcechar char1.;
+      else if sourcechar='n' then put '0A'x;
+      else if sourcechar='r' then put '0D'x;
+      else if sourcechar='t' then put '09'x;
+      else if sourcechar='u' then do;
+        length uni $4;
+        input uni $ 4. @@;
+        sourcechar=unicode('\u'!!uni);
+        put sourcechar char1.;
+      end;
+      else do;
+        call symputx('errmsg',"Uncaught escape char: "!!sourcechar,'l');
+        call symputx('syscc',99);
+        stop;
+      end;
+    end;
+    else if sourcechar='"' then stop;
+    else put sourcechar char1.;
+  end;
 run;
-%inc "&fpath3..lua";
+
+%mp_abort(iftrue=("&syscc"="99")
+  ,mac=mv_getjobcode
+  ,msg=%str(&errmsg)
+)
+
 /* export to desired destination */
 %if "&outref"="0" %then %do;
   data _null_;
@@ -169,7 +225,6 @@ run;
   /* clear refs */
   filename &fname1 clear;
   filename &fname2 clear;
-  filename &fname3 clear;
 %end;
 
 %mend mv_getjobcode;

viya/mv_getjoblog.sas

@@ -86,7 +86,8 @@
   @li mp_abort.sas
   @li mf_getplatform.sas
   @li mf_existfileref.sas
-  @li ml_json.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
 
 **/
 
@@ -95,7 +96,7 @@
     ,grant_type=sas_services
     ,mdebug=0
   );
-%local dbg;
+%local dbg libref1 libref2 loglocation fname1 fname2;
 %if &mdebug=1 %then %do;
   %put &sysmacroname entry vars:;
   %put _local_;
@@ -154,8 +155,8 @@ options noquotelenmax;
 %let base_uri=%mf_getplatform(VIYARESTAPI);
 
 /* prepare request*/
-%local  fname1;
 %let fname1=%mf_getuniquefileref();
+%let fname2=%mf_getuniquefileref();
 proc http method='GET' out=&fname1 &oauth_bearer
   url="&base_uri&uri";
   headers
@@ -175,37 +176,19 @@ run;
     ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
   )
 %end;
-%local  fname2 fname3 fpath1 fpath2 fpath3;
-%let fname2=%mf_getuniquefileref();
-%let fname3=%mf_getuniquefileref();
-%let fpath1=%sysfunc(pathname(&fname1));
-%let fpath2=%sysfunc(pathname(&fname2));
-%let fpath3=%sysfunc(pathname(&fname3));
 
-/* compile the lua JSON module */
-%ml_json()
-/* read using LUA - this allows the code to be of any length */
+%let libref1=%mf_getuniquelibref();
+libname &libref1 JSON fileref=&fname1;
 data _null_;
-  file "&fpath3..lua";
-  put '
-    infile = io.open (sas.symget("fpath1"), "r")
-    outfile = io.open (sas.symget("fpath2"), "w")
-    io.input(infile)
-    local resp=json.decode(io.read())
-    local logloc=resp["logLocation"]
-    outfile:write(logloc)
-    io.close(infile)
-    io.close(outfile)
-  ';
+  set &libref1..root;
+  call symputx('loglocation',loglocation,'l');
 run;
-%inc "&fpath3..lua";
-/* get log path*/
+
+/* validate log path*/
 %let errflg=1;
-%let errmsg=No entry in &fname2 fileref;
+%let errmsg=No loglocation entry in &fname1 fileref;
 data _null_;
-  infile &fname2;
-  input;
-  uri=cats(_infile_);
+  uri=symget('loglocation');
   if length(uri)<12 then do;
     call symputx('errflg',1);
     call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
@@ -232,7 +215,7 @@ run;
 
 /* we have a log uri - now fetch the log */
 %&dbg.put &sysmacroname: querying &base_uri&logloc/content;
-proc http method='GET' out=&fname1 &oauth_bearer
+proc http method='GET' out=&fname2 &oauth_bearer
   url="&base_uri&logloc/content?limit=10000";
   headers
   %if &grant_type=authorization_code %then %do;
@@ -243,14 +226,14 @@ run;
 
 %if &mdebug=1 %then %do;
   %put &sysmacroname: fetching log content from &base_uri&logloc/content;
-  data _null_;infile &fname1;input;putlog _infile_;run;
+  data _null_;infile &fname2;input;putlog _infile_;run;
 %end;
 
 %if &SYS_PROCHTTP_STATUS_CODE=400 %then %do;
   /* fetch log from parent session */
   %let logloc=%substr(&logloc,1,%index(&logloc,%str(/jobs/))-1);
   %&dbg.put &sysmacroname: Now querying &base_uri&logloc/log/content;
-  proc http method='GET' out=&fname1 &oauth_bearer
+  proc http method='GET' out=&fname2 &oauth_bearer
     url="&base_uri&logloc/log/content?limit=10000";
     headers
     %if &grant_type=authorization_code %then %do;
@@ -260,47 +243,32 @@ run;
   run;
   %if &mdebug=1 %then %do;
     %put &sysmacroname: fetching log content from &base_uri&logloc/log/content;
-    data _null_;infile &fname1;input;putlog _infile_;run;
+    data _null_;infile &fname2;input;putlog _infile_;run;
   %end;
 %end;
 
 %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201
 %then %do;
   %if &mdebug ne 1 %then %do; /* have already output above */
-    data _null_;infile &fname1;input;putlog _infile_;run;
+    data _null_;infile &fname2;input;putlog _infile_;run;
   %end;
   %mp_abort(mac=&sysmacroname
     ,msg=%str(logfetch: &SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
   )
 %end;
-data _null_;
-  file "&fpath3..lua";
-  put '
-    infile = io.open (sas.symget("fpath1"), "r")
-    outfile = io.open (sas.symget("fpath2"), "w")
-    io.input(infile)
-    local resp=json.decode(io.read())
-    for i, v in pairs(resp["items"]) do
-      outfile:write(v.line,"\n")
-    end
-    io.close(infile)
-    io.close(outfile)
-  ';
-run;
-%inc "&fpath3..lua";
 
-/* write log out to the specified fileref */
+%let libref2=%mf_getuniquelibref();
+libname &libref2 JSON fileref=&fname2;
 data _null_;
-  infile &fname2 end=last;
   file &outref mod;
   if _n_=1 then do;
     put "/** SASJS Viya Job Log Extract start: &uri **/";
   end;
-  input;
-  put _infile_;
+  set &libref2..items end=last;
   %if &mdebug=1 %then %do;
-    putlog _infile_;
+    putlog line;
   %end;
+  put line;
   if last then do;
     put "/** SASJS Viya Job Log Extract end: &uri **/";
   end;
@@ -309,7 +277,8 @@ run;
 %if &mdebug=0 %then %do;
   filename &fname1 clear;
   filename &fname2 clear;
-  filename &fname3 clear;
+  libname &libref1 clear;
+  libname &libref2 clear;
 %end;
 %else %do;
   %put &sysmacroname exit vars:;

viya/mv_registerclient.sas

@@ -1,10 +1,10 @@
 /**
   @file mv_registerclient.sas
   @brief Register Client and Secret (admin task)
-  @details When building apps on SAS Viya, an client id and secret are sometimes
-  required.  In order to generate them, filesystem access to the Consul Token
-  is needed (it is not enough to be in the SASAdministrator group in SAS
-  Environment Manager).
+  @details When building apps on SAS Viya, a client id and secret are usually
+  required.  In order to generate them, the Consul Token is required.  To access
+  this token, you need to be a system administrator (it is not enough to be in
+  the SASAdministrator group in SAS Environment Manager).
 
   If you are registering a lot of clients / secrets, you may find it more
   convenient to use the [Viya Token Generator]
@@ -25,62 +25,69 @@
         "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
 
+      %* generate random client using consul token as input parameter;
+      %mv_registerclient(consul_token=12x34sa43v2345n234lasd)
+
+      %* generate random client details with all scopes;
+      %mv_registerclient(scopes=openid *)
+
       %* specific client with just openid scope;
       %mv_registerclient(client_id=YourClient
         ,client_secret=YourSecret
         ,scopes=openid
       )
 
-      %* generate random client details with all scopes;
-      %mv_registerclient(scopes=openid *)
-
       %* generate random client with 90/180 second access/refresh token expiry;
       %mv_registerclient(scopes=openid *
         ,access_token_validity=90
         ,refresh_token_validity=180
       )
 
-  @param client_id= The client name.  Auto generated if blank.
-  @param client_secret= Client secret.  Auto generated if client is blank.
-  @param scopes=(openid) List of space-seperated unquoted scopes
-  @param grant_type=(authorization_code|refresh_token) Valid values are
-    "password" or "authorization_code" (unquoted)
-  @param outds=(mv_registerclient) The dataset to contain the registered client
-    id and secret
-  @param access_token_validity=(DEFAULT) The duration of validity of the access
-    token in seconds.  A value of DEFAULT will omit the entry (and use system
-    default)
-  @param refresh_token_validity=(DEFAULT)  The duration of validity of the
+  @param [in,out] client_id= The client name.  Auto generated if blank.
+  @param [in,out] client_secret= Client secret.  Auto generated if client is
+    blank.
+  @param [in] consul_token= (0) Provide the actual consul token value here if
+    using Viya 4 or above.
+  @param [in] scopes= (openid) List of space-seperated unquoted scopes
+  @param [in] grant_type= (authorization_code|refresh_token) Valid values are
+    "password" or "authorization_code" (unquoted).  Pipe seperated.
+  @param [out] outds=(mv_registerclient) The dataset to contain the registered
+    client id and secret
+  @param [in] access_token_validity= (DEFAULT) The access token duration in
+    seconds.  A value of DEFAULT will omit the entry (and use system default)
+  @param [in] refresh_token_validity= (DEFAULT)  The duration of validity of the
     refresh token in seconds.  A value of DEFAULT will omit the entry (and use
     system default)
-  @param name= An optional, human readable name for the client
-  @param required_user_groups= A list of group names. If a user does not belong
-    to all the required groups, the user will not be authenticated and no tokens
-    are issued to this client for that user. If this field is not specified,
-    authentication and token issuance proceeds normally.
-  @param autoapprove= During the auth step the user can choose which scope to
-    apply.  Setting this to true will autoapprove all the client scopes.
-  @param use_session= If true, access tokens issued to this client will be
+  @param [in] client_name= (DEFAULT) An optional, human readable name for the
+    client.
+  @param [in] required_user_groups= A list of group names. If a user does not
+    belong to all the required groups, the user will not be authenticated and no
+    tokens are issued to this client for that user. If this field is not
+    specified, authentication and token issuance proceeds normally.
+  @param [in] autoapprove= During the auth step the user can choose which scope
+    to apply.  Setting this to true will autoapprove all the client scopes.
+  @param [in] use_session= If true, access tokens issued to this client will be
     associated with an HTTP session and revoked upon logout or time-out.
-  @param outjson= (_null_) A dataset containing the lines of JSON submitted.
-    Useful for debugging.
+  @param [out] outjson= (_null_) A dataset containing the lines of JSON
+    submitted. Useful for debugging.
 
   @version VIYA V.03.04
   @author Allan Bowe, source: https://github.com/sasjs/core
 
   <h4> SAS Macros </h4>
-  @li mp_abort.sas
   @li mf_getplatform.sas
   @li mf_getuniquefileref.sas
   @li mf_getuniquelibref.sas
   @li mf_loc.sas
   @li mf_getquotedstr.sas
   @li mf_getuser.sas
+  @li mp_abort.sas
 
 **/
 
 %macro mv_registerclient(client_id=
     ,client_secret=
+    ,consul_token=0
     ,client_name=DEFAULT
     ,scopes=openid
     ,grant_type=authorization_code|refresh_token
@@ -92,33 +99,41 @@
     ,refresh_token_validity=DEFAULT
     ,outjson=_null_
   );
-%local consul_token fname1 fname2 fname3 libref access_token url tokloc;
+%local fname1 fname2 fname3 libref access_token url tokloc msg;
 
 %if client_name=DEFAULT %then %let client_name=
-  Generated by %mf_getuser() on %sysfunc(datetime(),datetime19.) using SASjs;
+  Generated by %mf_getuser() (&sysuserid) on %sysfunc(datetime(),datetime19.
+  ) using SASjs;
 
 options noquotelenmax;
-/* first, get consul token needed to get client id / secret */
-%let tokloc=/etc/SASSecurityCertificateFramework/tokens/consul/default;
-%let tokloc=%mf_loc(VIYACONFIG)&tokloc/client.token;
 
+%if "&consul_token"="0" %then %do;
+  /* first, get consul token needed to get client id / secret */
+  %let tokloc=/etc/SASSecurityCertificateFramework/tokens/consul/default;
+  %let tokloc=%mf_loc(VIYACONFIG)&tokloc/client.token;
 
-%mp_abort(iftrue=(%sysfunc(fileexist(&tokloc))=0)
-  ,mac=&sysmacroname
-  ,msg=%str(Unable to access the consul token at &tokloc)
-)
+  %if %sysfunc(fileexist(&tokloc))=0 %then %do;
+    %let msg=Unable to access the consul token at &tokloc;
+    %put &sysmacroname: &msg;
+    %put Try passing the value in the consul= macro parameter;
+    %put See docs:  https://core.sasjs.io/mv__registerclient_8sas.html;
+    %mp_abort(mac=mv_registerclient,msg=%str(&msg))
+  %end;
 
-%let consul_token=0;
-data _null_;
-  infile "&tokloc";
-  input token:$64.;
-  call symputx('consul_token',token);
-run;
+  data _null_;
+    infile "&tokloc";
+    input token:$64.;
+    call symputx('consul_token',token);
+  run;
 
-%mp_abort(iftrue=("&consul_token"="0")
-  ,mac=&sysmacroname
-  ,msg=%str(Unable to source the consul token from &tokloc)
-)
+  %if "&consul_token"="0" %then %do;
+    %put &sysmacroname: Unable to source the consul token from &tokloc;
+    %put It seems your account (&sysuserid) does not have admin rights;
+    %put Please speak with your platform adminstrator;
+    %put Docs:  https://core.sasjs.io/mv__registerclient_8sas.html;
+    %abort;
+  %end;
+%end;
 
 %local base_uri; /* location of rest apis */
 %let base_uri=%mf_getplatform(VIYARESTAPI);
@@ -131,6 +146,9 @@ proc http method='POST' out=&fname1
   headers "X-Consul-Token"="&consul_token";
 run;
 
+%put &=SYS_PROCHTTP_STATUS_CODE;
+%put &=SYS_PROCHTTP_STATUS_PHRASE;
+
 %let libref=%mf_getuniquelibref();
 libname &libref JSON fileref=&fname1;