fix: refreshed the testing toolkit, added debug options, updated documentation, included one new test (mv_getjobcode.sas)

feat: new (generic) mp_assert macro, and new feature (type filter) fo…

.gitignore

@@ -1,2 +1,13 @@
 node_modules
 .DS_Store
+sasjsbuild/
+sasjsresults/
+
+# avoid filenames with spaces being committed to source control
+**\ **
+
+# ignore the mc_* files - containing macros for individual libraries
+mc_*
+
+# ignore .env files as they can contain sasjs access tokens
+*.env*

.gitpod.dockerfile

@@ -0,0 +1,6 @@
+FROM gitpod/workspace-full
+
+RUN sudo apt-get update \
+ && sudo apt-get install -y \
+    doxygen \
+ && sudo rm -rf /var/lib/apt/lists/*

.gitpod.yml

@@ -0,0 +1,8 @@
+tasks:
+  - init: npm i -g @sasjs/cli
+
+image:
+  file: .gitpod.dockerfile
+vscode:
+  extensions:
+    - sasjs.sasjs-for-vscode

.sasjslint

@@ -0,0 +1,13 @@
+{
+    "noTrailingSpaces": true,
+    "noEncodedPasswords": true,
+    "hasDoxygenHeader": true,
+    "hasMacroNameInMend": false,
+    "hasMacroParentheses": true,
+    "noNestedMacros": false,
+    "noSpacesInFileNames": true,
+    "maxLineLength": 135,
+    "lowerCaseFileNames": true,
+    "noTabIndentation": true,
+    "indentationMultiple": 2
+}

.vscode/.editorconfig

@@ -0,0 +1,9 @@
+{
+  "search.exclude": {
+    "**/sasjsbuild/**": true,
+    "**/dist/**":true
+  },
+  "editor.insertSpaces": true,
+  "editor.tabSize": 2,
+  "trim_trailing_whitespace": true
+}

.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+  "recommendations": [
+    "sasjs.sasjs-for-vscode"
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,10 @@
+{
+  "editor.tabSize": 2,
+  "editor.insertSpaces": true,
+  "editor.detectIndentation": true,
+  "editor.formatOnSave": true,
+  "editor.rulers": [
+    80
+  ],
+  "files.trimTrailingWhitespace": true
+}

CONTRIBUTING.md

@@ -1,83 +1,31 @@
 # Contributing
 
-Contributions to the macrocore library are warmly welcomed!  To avoid any
+Contributions are warmly welcomed!  To avoid any misunderstandings, do please first discuss the change you wish to make via issue, email, or any other method with the owners of this repository before submitting a PR.
-misunderstandings, do please first discuss the change you wish to make via issue,
-email, or any other method with the owners of this repository before submitting
-a PR.
 
-Please note we have a code of conduct, please follow it in all your interactions
+Please note we have a [code of conduct](https://www.contributor-covenant.org/version/2/0/code_of_conduct/), please follow it in all your interactions with the project.
-with the project.
+
+# Environment Setup
+
+This repository makes use of the [SASjs](https://sasjs.io) framework for code organisation, compilation, documentation, and deployment.  The following tools are highly recommended:
+
+* [NPM](https://sasjs.io/windows/#npm) - the runtime and dependency manager for [SASjs CLI](https://cli.sasjs.io) (batteries included)
+* [VSCode](https://sasjs.io/windows/#vscode) - feature packed IDE for code editing (warning - highly effective!)
+* [GIT](https://sasjs.io/windows/#git) - a safety net you cannot (and should not) do without.
+
+For generating the documentation (`sasjs doc`) it is also necessary to install [doxygen](https://www.doxygen.nl/manual/install.html).
 
 
-## Code of Conduct
+To get configured:
 
-### Our Pledge
+1.  Clone the repository
+2.  Install local dependencies (`npm install`)
+3.  Install the SASjs CLI globally (`npm install -g @sasjs/cli`)
+4.  Add a target, and authentication (`npm add`).  See [docs](https://cli.sasjs.io/add/).
 
-In the interest of fostering an open and welcoming environment, we as
+To contribute:
-contributors and maintainers pledge to making participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, gender identity and expression, level of experience,
-nationality, personal appearance, race, religion, or sexual identity and
-orientation.
 
-### Our Standards
+1.  Create your feature branch (`git checkout -b myfeature`)
+2.  Make your change
+3.  Update the `all.sas` file (`python3 build.py`)
+4.  Push and make a PR
 
-Examples of behavior that contributes to creating a positive environment
-include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-### Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-### Scope
-
-This Code of Conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
-address, posting via an official social media account, or acting as an appointed
-representative at an online or offline event. Representation of a project may be
-further defined and clarified by project maintainers.
-
-### Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at support@macropeople.com. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-### Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at [http://contributor-covenant.org/version/1/4][version]
-
-[homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/4/

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2018 (Allan Bowe)
+Copyright 2020 (Allan Bowe)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

README.md

@@ -1,6 +1,6 @@
 # Macro Core
 
-Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of Application Development on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
+Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
 
 You can download and compile them all in just two lines of SAS code:
 
@@ -9,7 +9,7 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 %inc mc;
 ```
 
-Documentation: https://sasjs.github.io/core.github.io/files.html
+Documentation: https://core.sasjs.io
 
 # Components
 
@@ -40,6 +40,27 @@ Documentation: https://sasjs.github.io/core.github.io/files.html
 - X command enabled
 - Prefixes: _mmw_,_mmu_,_mmx_
 
+**lua** library
+
+Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
+
+To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert your LUA into a data step with put statements, and create the macro wrapper with a `ml_` prefix.  You can then use your module in any program by running:
+
+```
+/* compile the lua module */
+%ml_yourmodule()
+
+/* Execute.  Do not use the restart keyword! */
+proc lua;
+submit;
+  print(yourStuff);
+endsubmit;
+run;
+```
+
+- X command enabled
+- Prefixes: _mmw_,_mmu_,_mmx_
+
 # Installation
 
 First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available,eg:
@@ -63,7 +84,7 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 ## File Properties
 
 - filenames much match macro names
-- filenames must be lowercase
+- filenames must be lowercase, without spaces
 - macro names must be lowercase
 - one macro per file
 - prefixes:
@@ -72,12 +93,13 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
   - _mm_ for metadata macros (interface with the metadata server).
   - _mmx_ for macros that use metadata and are XCMD enabled
   - _mx_ for macros that are XCMD enabled
+  - _ml_ for macros that are used to compile LUA modules
   - _mv_ for macros that will only work in Viya
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
 - UTF-8
-- no trailing white space
+
 
 ## Header Properties
 
@@ -91,17 +113,38 @@ The **Macro Core** documentation is created using [doxygen](http://www.doxygen.n
 - version. The EARLIEST SAS version in which this macro is known to work.
 - author. Author name, contact details optional
 
-All macros must be commented in the doxygen format, to enable the [online documentation](https://sasjs.github.io/core.github.io/).
+All macros must be commented in the doxygen format, to enable the [online documentation](https://core.sasjs.io).
+
+### Dependencies
+SAS code can contain one of two types of dependency - SAS Macros, and SAS Programs.  When compiling projects using the [SASjs CLI](https://cli.sasjs.io) the doxygen header is scanned for `  @li` items under the following headers:
+
+```sas
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mm_assignlib.sas
+
+  <h4> SAS Programs </h4>
+  @li somefile.ddl SOMEFREF
+  @li someprogram.sas FREFTWO
+```
+
+The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Programs) when creating SAS Jobs and Services.
+
+When contributing to this library, it is therefore important to ensure that all dependencies are listed in the header in this format.
+
 
 ## Coding Standards
 
 - Indentation = 2 spaces. No tabs!
+- no trailing white space
+- no invisible characters, other than spaces. If invisibles are needed, use hex literals.
 - Macro variables should not have the trailing dot (`&var` not `&var.`) unless necessary to prevent incorrect resolution
-- The closing `%mend;` should not contain the macro name.
+- The closing `%mend;` should **not** contain the macro name.
 - All macros should be defined with brackets, even if no variables are needed - ie `%macro x();` not `%macro x;`
 - Mandatory parameters should be positional, all optional parameters should be keyword (var=) style.
 - All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect.
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
+- The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
 
 # General Notes
 

base/mf_abort.sas

@@ -21,6 +21,7 @@
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
 %macro mf_abort(mac=mf_abort.sas, type=, msg=, iftrue=%str(1=1)
@@ -48,7 +49,10 @@
         input; putlog _infile_;
         i=1;
         retain logonce 0;
-        if (_infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR") and logonce=0 then do;
+        if (
+            _infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR"
+          ) and logonce=0
+        then do;
           call symputx('logline',_n_);
           logonce+1;
         end;
@@ -111,7 +115,8 @@
     %let syscc=0;
     %if %symexist(SYS_JES_JOB_URI) %then %do;
       /* refer web service output to file service in one hit */
-      filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json";
+      filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
+        name="_webout.json";
       %let rc=%sysfunc(fcopy(_web,_webout));
     %end;
     %else %do;
@@ -139,3 +144,5 @@
     %abort cancel;
   %end;
 %mend;
+
+/** @endcond */

base/mf_existfeature.sas

@@ -1,5 +1,5 @@
 /**
-  @file mf_existfeature.sas
+  @file
   @brief Checks whether a feature exists
   @details Check to see if a feature is supported in your environment.
     Run without arguments to see a list of detectable features.
@@ -13,13 +13,14 @@
 
   @return output returns 1 or 0 (or -1 if not found)
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_getplatform.sas
 
 
   @version 8
   @author Allan Bowe
 **/
+/** @cond */
 
 %macro mf_existfeature(feature
 )/*/STORE SOURCE*/;
@@ -31,12 +32,16 @@
     %put Supported features:  PROCLUA;
   %end;
   %else %if &feature=PROCLUA %then %do;
+    /* https://blogs.sas.com/content/sasdummy/2015/08/03/using-lua-within-your-sas-programs */
     %if &platform=SASVIYA %then 1;
-    %else %if "&sysver"="9.3" or "&sysver"="9.4" %then 1;
+    %else %if "&sysver"="9.2" or "&sysver"="9.3" %then 0;
-    %else 0;
+    %else %if "&SYSVLONG" < "9.04.01M3" %then 0;
+    %else 1;
   %end;
   %else %do;
     -1
     %put &sysmacroname: &feature not found;
   %end;
 %mend;
+
+/** @endcond */

base/mf_existfileref.sas

@@ -0,0 +1,27 @@
+/**
+  @file
+  @brief Checks whether a fileref exists
+  @details You can probably do without this macro as it is just a one liner.
+  Mainly it is here as a convenient way to remember the syntax!
+
+  @param fref the fileref to detect
+
+  @return output Returns 1 if found and 0 if not found.  Note - it is possible
+  that the fileref is found, but the file does not (yet) exist. If you need
+  to test for this, you may as well use the fileref function directly.
+
+  @version 8
+  @author [Allan Bowe](https://www.linkedin.com/in/allanbowe/)
+**/
+
+%macro mf_existfileref(fref
+)/*/STORE SOURCE*/;
+
+  %if %sysfunc(fileref(&fref))=0 %then %do;
+    1
+  %end;
+  %else %do;
+    0
+  %end;
+
+%mend;

base/mf_existvar.sas

@@ -12,6 +12,7 @@
   @version 9.2
   @author Allan Bowe
 **/
+/** @cond */
 
 %macro mf_existvar(libds /* 2 part dataset name */
       , var /* variable name */
@@ -30,3 +31,5 @@
   %end;
 
 %mend;
+
+/** @endcond */

base/mf_existvarlist.sas

@@ -4,9 +4,9 @@
   @details Returns 0 if ANY of the variables do not exist, or 1 if they ALL do.
   Usage:
 
-        %put %mf_existVarList(sashelp.class, age sex name dummyvar)
+      %put %mf_existVarList(sashelp.class, age sex name dummyvar);
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_abort.sas
 
   @param libds 2 part dataset or view reference
@@ -14,6 +14,7 @@
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
 %macro mf_existvarlist(libds, varlist
@@ -28,7 +29,7 @@
   %let dsid=%sysfunc(open(&libds,is));
 
   %if &dsid=0 %then %do;
-    %put WARNING:  unable to open &libds in mf_existvarlist (&dsid);
+    %put %str(WARN)ING:  unable to open &libds in mf_existvarlist (&dsid);
   %end;
 
   %if %sysfunc(attrn(&dsid,NVARS))=0 %then %do;
@@ -54,3 +55,5 @@
     %put Vars not found: &found;
   %end;
 %mend;
+
+/** @endcond */

base/mf_getattrc.sas

@@ -23,7 +23,7 @@
   %local dsid rc;
   %let dsid=%sysfunc(open(&libds,is));
   %if &dsid = 0 %then %do;
-    %put WARNING: Cannot open %trim(&libds), system message below;
+    %put %str(WARN)ING: Cannot open %trim(&libds), system message below;
     %put %sysfunc(sysmsg());
     -1
   %end;

base/mf_getattrn.sas

@@ -23,7 +23,7 @@
   %local dsid rc;
   %let dsid=%sysfunc(open(&libds,is));
   %if &dsid = 0 %then %do;
-    %put WARNING: Cannot open %trim(&libds), system message below;
+    %put %str(WARN)ING: Cannot open %trim(&libds), system message below;
     %put %sysfunc(sysmsg());
     -1
   %end;

base/mf_getengine.sas

@@ -3,7 +3,7 @@
   @brief Returns the engine type of a SAS library
   @details Usage:
 
-      %put %mf_getEngine(SASHELP);
+      %put %mf_getengine(SASHELP);
 
   returns:
   > V9
@@ -21,16 +21,23 @@
 
   @version 9.2
   @author Allan Bowe
-**/
 
-%macro mf_getEngine(libref
+  <h4> Related Macros </h4>
+  @li mf_getxengine.sas
+
+**/
+/** @cond */
+
+%macro mf_getengine(libref
 )/*/STORE SOURCE*/;
   %local dsid engnum rc engine;
 
   /* in case the parameter is a libref.tablename, pull off just the libref */
   %let libref = %upcase(%scan(&libref, 1, %str(.)));
 
-  %let dsid=%sysfunc(open(sashelp.vlibnam(where=(libname="%upcase(&libref)")),i));
+  %let dsid=%sysfunc(
+    open(sashelp.vlibnam(where=(libname="%upcase(&libref)")),i)
+  );
   %if (&dsid ^= 0) %then %do;
     %let engnum=%sysfunc(varnum(&dsid,ENGINE));
     %let rc=%sysfunc(fetch(&dsid));
@@ -42,3 +49,5 @@
   &engine
 
 %mend;
+
+/** @endcond */

base/mf_getkeyvalue.sas

@@ -16,7 +16,7 @@
 
 %macro mf_getkeyvalue(key,libds=work.mp_setkeyvalue
 )/*/STORE SOURCE*/;
- %local ds dsid key valc valn type rc;
+%local ds dsid key valc valn type rc;
 %let dsid=%sysfunc(open(&libds(where=(key="&key"))));
 %syscall set(dsid);
 %let rc = %sysfunc(fetch(&dsid));

base/mf_getplatform.sas

@@ -10,8 +10,9 @@
 
   @param switch the param for which to return a platform specific variable
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_mval.sas
+  @li mf_trimstr.sas
 
   @version 9.4 / 3.4
   @author Allan Bowe
@@ -26,21 +27,23 @@
     or "&sysprocessmode"= "SAS Compute Server" %then %do;
         SASVIYA
     %end;
-    %else %if "&sysprocessmode"="SAS Stored Process Server" %then %do;
+    %else %if "&sysprocessmode"="SAS Stored Process Server"
+      or "&sysprocessmode"="SAS Workspace Server"
+    %then %do;
       SASMETA
       %return;
     %end;
     %else %do;
-      SAS
+      BASESAS
       %return;
     %end;
   %end;
-  %else %if %symexist(_metaport) %then %do;
+  %else %if %symexist(_metaport) or %symexist(_metauser) %then %do;
     SASMETA
     %return;
   %end;
   %else %do;
-    SAS
+    BASESAS
     %return;
   %end;
 %end;
@@ -57,6 +60,6 @@
   %else 0;
 %end;
 %else %if &switch=VIYARESTAPI %then %do;
-  %sysfunc(getoption(servicesbaseurl))
+  %mf_trimstr(%sysfunc(getoption(servicesbaseurl)),/)
 %end;
 %mend;

base/mf_getquotedstr.sas

@@ -8,6 +8,13 @@
   which returns:
 > 'blah','blah','blah'
 
+  Alternatively:
+
+      %put %mf_getquotedstr(these words are double quoted,quote=D)
+
+  for:
+> "these","words","are","double","quoted"
+
   @param in_str the unquoted, spaced delimited string to transform
   @param dlm= the delimeter to be applied to the output (default comma)
   @param indlm= the delimeter used for the input (default is space)

base/mf_getschema.sas

@@ -17,6 +17,7 @@
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
 %macro mf_getschema(libref
@@ -38,3 +39,5 @@
   &schema
 
 %mend;
+
+/** @endcond */

base/mf_getuniquefileref.sas

@@ -1,7 +1,8 @@
 /**
   @file
   @brief Assigns and returns an unused fileref
-  @details Use as follows:
+  @details
+  Use as follows:
 
       %let fileref1=%mf_getuniquefileref();
       %let fileref2=%mf_getuniquefileref();
@@ -11,7 +12,7 @@
 
 > mcref0 mcref1
 
-  @prefix= first part of fileref. Remember that filerefs can only be 8
+  @param prefix= first part of fileref. Remember that filerefs can only be 8
     characters, so a 7 letter prefix would mean that `maxtries` should be 10.
   @param maxtries= the last part of the libref.  Provide an integer value.
 

base/mf_getuniquelibref.sas

@@ -14,7 +14,7 @@
 
 > mclib3
 
-  @prefix= first part of libref.  Remember that librefs can only be 8 characters,
+  @param prefix= first part of libref.  Remember that librefs can only be 8 characters,
     so a 7 letter prefix would mean that maxtries should be 10.
   @param maxtries= the last part of the libref.  Provide an integer value.
 

base/mf_getvalue.sas

@@ -7,7 +7,7 @@
       %put %mf_getvalue(sashelp.class,name,filter=%quote(age=15));
       %put %mf_getvalue(sashelp.class,name);
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_getattrn.sas
 
   @param libds dataset to query

base/mf_getvarformat.sas

@@ -23,9 +23,9 @@
       8.
       NOTE: Variable renegade does not exist in test
 
-  @param libds Two part dataset (or view) reference.
+  @param [in] libds Two part dataset (or view) reference.
-  @param var Variable name for which a format should be returned
+  @param [in] var Variable name for which a format should be returned
-  @param force Set to 1 to supply a default if the variable has no format
+  @param [in] force=(0) Set to 1 to supply a default if the variable has no format
   @returns outputs format
 
   @author Allan Bowe
@@ -60,7 +60,7 @@
     %let vlen = %sysfunc(varlen(&dsid, &vnum));
     %let vtype = %sysfunc(vartype(&dsid, &vnum.));
     %if &vtype=C %then %let vformat=$&vlen..;
-    %else %let vformat=8.;
+    %else %let vformat=best.;
   %end;
 
 
@@ -68,4 +68,4 @@
   %let rc = %sysfunc(close(&dsid));
   /* Return variable format */
   &vformat
-%mend;
+%mend mf_getVarFormat;

base/mf_getvarlist.sas

@@ -10,14 +10,21 @@
   returns:
   > List of Variables=Name Sex Age Height Weight
 
+  For a seperated list of column values:
+
         %put %mf_getvarlist(sashelp.class,dlm=%str(,),quote=double);
 
   returns:
   > "Name","Sex","Age","Height","Weight"
 
-  @param libds Two part dataset (or view) reference.
+  @param [in] libds Two part dataset (or view) reference.
-  @param dlm= provide a delimiter (eg comma or space) to separate the vars
+  @param [in] dlm= ( ) Provide a delimiter (eg comma or space) to separate the
-  @param quote= use either DOUBLE or SINGLE to quote the results
+    variables
+  @param [in] quote= (none) use either DOUBLE or SINGLE to quote the results
+  @param [in] typefilter= (A) Filter for certain types of column.  Valid values:
+    @li A Return All columns
+    @li C Return Character columns
+    @li N Return Numeric columns
 
   @version 9.2
   @author Allan Bowe
@@ -27,9 +34,10 @@
 %macro mf_getvarlist(libds
       ,dlm=%str( )
       ,quote=no
+      ,typefilter=A
 )/*/STORE SOURCE*/;
   /* declare local vars */
-  %local outvar dsid nvars x rc dlm q var;
+  %local outvar dsid nvars x rc dlm q var vtype;
 
   /* credit Rowland Hale  - byte34 is double quote, 39 is single quote */
   %if %upcase(&quote)=DOUBLE %then %let q=%qsysfunc(byte(34));
@@ -37,23 +45,24 @@
   /* open dataset in macro */
   %let dsid=%sysfunc(open(&libds));
 
-
   %if &dsid %then %do;
     %let nvars=%sysfunc(attrn(&dsid,NVARS));
     %if &nvars>0 %then %do;
-      /* add first dataset variable to global macro variable */
+      /* add variables with supplied delimeter */
-      %let outvar=&q.%sysfunc(varname(&dsid,1))&q.;
-      /* add remaining variables with supplied delimeter */
       %do x=1 %to &nvars;
+        /* get variable type */
+        %let vtype=%sysfunc(vartype(&dsid,&x));
+        %if &vtype=&typefilter or &typefilter=A %then %do;
           %let var=&q.%sysfunc(varname(&dsid,&x))&q.;
           %if &var=&q&q %then %do;
             %put &sysmacroname: Empty column found in &libds!;
             %let var=&q. &q.;
           %end;
-        %if &x=1 %then %let outvar=&var;
+          %if %quote(&outvar)=%quote() %then %let outvar=&var;
           %else %let outvar=&outvar.&dlm.&var.;
         %end;
       %end;
+    %end;
     %let rc=%sysfunc(close(&dsid));
   %end;
   %else %do;
@@ -61,4 +70,4 @@
     %let rc=%sysfunc(close(&dsid));
   %end;
   &outvar
-%mend;
+%mend mf_getvarlist;

base/mf_getvartype.sas

@@ -45,4 +45,4 @@ Usage:
   %let rc = %sysfunc(close(&dsid));
   /* Return variable type */
   &vtype
-%mend;
+%mend mf_getvartype;

base/mf_getxengine.sas

@@ -0,0 +1,43 @@
+/**
+  @file
+  @brief Returns the engine type of a SAS fileref
+  @details Queries sashelp.vextfl to get the xengine value.
+  Usage:
+
+      filename feng temp;
+      %put %mf_getxengine(feng);
+
+  returns:
+  > TEMP
+
+  @param fref The fileref to check
+
+  @returns The XENGINE value in sashelp.vextfl or 0 if not found.
+
+  @version 9.2
+  @author Allan Bowe
+
+  <h4> Related Macros </h4>
+  @li mf_getengine.sas
+
+**/
+
+%macro mf_getxengine(fref
+)/*/STORE SOURCE*/;
+  %local dsid engnum rc engine;
+
+  %let dsid=%sysfunc(
+    open(sashelp.vextfl(where=(fileref="%upcase(&fref)")),i)
+  );
+  %if (&dsid ^= 0) %then %do;
+    %let engnum=%sysfunc(varnum(&dsid,XENGINE));
+    %let rc=%sysfunc(fetch(&dsid));
+    %let engine=%sysfunc(getvarc(&dsid,&engnum));
+    %* put &fref. ENGINE is &engine.;
+    %let rc= %sysfunc(close(&dsid));
+  %end;
+  %else %let engine=0;
+
+  &engine
+
+%mend;

base/mf_isblank.sas

@@ -9,7 +9,8 @@
 
       %put mf_isblank(&var);
 
-  inspiration:  https://support.sas.com/resources/papers/proceedings09/022-2009.pdf
+  inspiration:
+  https://support.sas.com/resources/papers/proceedings09/022-2009.pdf
 
   @param param VALUE to be checked
 

base/mf_isdir.sas

@@ -0,0 +1,34 @@
+/**
+  @file
+  @brief Checks whether a path is a valid directory
+  @details
+  Usage:
+
+      %let isdir=%mf_isdir(/tmp);
+
+  With thanks and full credit to Andrea Defronzo -
+  https://www.linkedin.com/in/andrea-defronzo-b1a47460/
+
+  @param path full path of the file/directory to be checked
+
+  @return output returns 1 if path is a directory, 0 if it is not
+
+  @version 9.2
+**/
+
+%macro mf_isdir(path
+)/*/STORE SOURCE*/;
+  %local rc did is_directory fref_t;
+
+  %let is_directory = 0;
+  %let rc = %sysfunc(filename(fref_t, %superq(path)));
+  %let did = %sysfunc(dopen(&fref_t.));
+  %if &did. ^= 0 %then %do;
+    %let is_directory = 1;
+    %let rc = %sysfunc(dclose(&did.));
+  %end;
+  %let rc = %sysfunc(filename(fref_t));
+
+  &is_directory
+
+%mend;

base/mf_mval.sas

@@ -1,7 +1,8 @@
 /**
   @file mf_mval.sas
   @brief Returns a macro variable value if the variable exists
-  @details Use this macro to avoid repetitive use of `%if %symexist(MACVAR) %then`
+  @details
+  Use this macro to avoid repetitive use of `%if %symexist(MACVAR) %then`
   type logic.
   Usage:
 

base/mf_nobs.sas

@@ -6,7 +6,7 @@
 
         %put Number of observations=%mf_nobs(sashelp.class);
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_getattrn.sas
 
   @param libds library.dataset

base/mf_trimstr.sas

@@ -0,0 +1,50 @@
+/**
+  @file mf_trimstr.sas
+  @brief Removes character(s) from the end, if they exist
+  @details If the designated characters exist at the end of the string, they
+  are removed
+
+        %put %mf_trimstr(/blah/,/); * /blah;
+        %put %mf_trimstr(/blah/,h); * /blah/;
+        %put %mf_trimstr(/blah/,h/);* /bla;
+
+  <h4> SAS Macros </h4>
+
+
+  @param basestr The string to be modified
+  @param trimstr The string to be removed from the end of `basestr`, if it
+    exists
+
+  @return output returns result with the value of `trimstr` removed from the end
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_trimstr(basestr,trimstr);
+%local baselen trimlen trimval;
+
+/* return if basestr is shorter than trimstr (or 0) */
+%let baselen=%length(%superq(basestr));
+%let trimlen=%length(%superq(trimstr));
+%if &baselen < &trimlen or &baselen=0 %then %return;
+
+/* obtain the characters from the end of basestr */
+%let trimval=%qsubstr(%superq(basestr)
+  ,%length(%superq(basestr))-&trimlen+1
+  ,&trimlen);
+
+/* compare and if matching, chop it off! */
+%if %superq(basestr)=%superq(trimstr) %then %do;
+  %return;
+%end;
+%else %if %superq(trimval)=%superq(trimstr) %then %do;
+  %qsubstr(%superq(basestr),1,%length(%superq(basestr))-&trimlen)
+%end;
+%else %do;
+  &basestr
+%end;
+
+%mend;

base/mf_uid.sas

@@ -1,11 +1,11 @@
 /**
   @file
-  @brief Creates a Unique ID based on system time in a friendly format
+  @brief Creates a unique ID based on system time in friendly format
   @details format = YYYYMMDD_HHMMSSmmm_<sysjobid>_<3randomDigits>
 
         %put %mf_uid();
 
-  @version 9.2
+  @version 9.3
   @author Allan Bowe
 
 **/
@@ -14,8 +14,8 @@
 )/*/STORE SOURCE*/;
   %local today now;
   %let today=%sysfunc(today(),yymmddn8.);
-  %let now=%sysfunc(compress(%sysfunc(time(),time12.3),:.));
+  %let now=%sysfunc(compress(%sysfunc(time(),tod12.3),:.));
 
   &today._&now._&sysjobid._%sysevalf(%sysfunc(ranuni(0))*999,CEIL)
 
-%mend;
+%mend mf_uid;

base/mf_verifymacvars.sas

@@ -11,7 +11,7 @@
   Returns:
   > 1
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_abort.sas
 
   @param verifyvars space separated list of macro variable names

base/mf_wordsinstr1butnotstr2.sas

@@ -30,7 +30,7 @@
 
 %local count_base count_extr i i2 extr_word base_word match outvar;
 %if %length(&str1)=0 or %length(&str2)=0 %then %do;
-  %put WARNING: empty string provided!;
+  %put %str(WARN)ING: empty string provided!;
   %put base string (str1)= &str1;
   %put compare string (str2) = &str2;
   %return;

base/mp_abort.sas

@@ -6,12 +6,19 @@
     results back to the client in an STP Web App context, or completely stop
     in the case of a batch run.
 
+  Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored
+  Process environments.  This macro takes a unique approach - we set the SAS
+  syscc to 0, run `stpsrvset('program error', 0)` (if SAS 9) and then - we open
+  a macro but don't close it!  This provides a graceful abort for SAS web
+  services in all web enabled environments.
+
   @param mac= to contain the name of the calling macro
   @param msg= message to be returned
   @param iftrue= supply a condition under which the macro should be executed.
 
   @version 9.4M3
   @author Allan Bowe
+  @cond
 **/
 
 %macro mp_abort(mac=mp_abort.sas, type=, msg=, iftrue=%str(1=1)
@@ -41,7 +48,10 @@
         input; putlog _infile_;
         i=1;
         retain logonce 0;
-        if (_infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR") and logonce=0 then do;
+        if (
+            _infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR"
+          ) and logonce=0 then
+        do;
           call symputx('logline',_n_);
           logonce+1;
         end;
@@ -65,8 +75,15 @@
     %end;
 
     %if %symexist(SYS_JES_JOB_URI) %then %do;
-      /* refer web service output to file service in one hit */
+      /* setup webout */
-      filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json";
+      OPTIONS NOBOMFILE;
+      %if "X&SYS_JES_JOB_URI.X"="XX" %then %do;
+          filename _webout temp lrecl=999999 mod;
+      %end;
+      %else %do;
+        filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
+          name="_webout.json" lrecl=999999 mod;
+      %end;
     %end;
 
     /* send response in SASjs JSON format */
@@ -138,3 +155,4 @@
   %end;
 %mend;
 
+/** @endcond */

base/mp_assert.sas

@@ -0,0 +1,56 @@
+/**
+  @file
+  @brief Generic assertion
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      %mp_assert(iftrue=(1=1),
+        desc=Obviously true
+      )
+
+      %mp_assert(iftrue=(1=0),
+        desc=Will fail
+      )
+
+  @param [in] iftrue= (1=1) A condition where, if true, the test is a PASS.
+  Else, the test is a fail.
+
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Column &inds contained ALL columns|
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assert(iftrue=(1=1),
+  desc=0,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  data ;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_comments="&sysmacroname: Test result of "!!symget('iftrue');
+  %if %eval(%unquote(&iftrue)) %then %do;
+    test_result='PASS';
+  %end;
+  %else %do;
+    test_result='FAIL';
+  %end;
+  run;
+
+  %local ds ;
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+
+%mend mp_assert;

base/mp_assertcols.sas

@@ -0,0 +1,145 @@
+/**
+  @file
+  @brief Asserts the existence (or not) of columns
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      %mp_assertcols(sashelp.class,
+        cols=name age sex,
+        test=ALL,
+        desc=check all columns exist
+      )
+
+      %mp_assertcols(sashelp.class,
+        cols=a b c,
+        test=NONE
+      )
+
+      %mp_assertcols(sashelp.class,
+        cols=age depth,
+        test=ANY
+      )
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_existvarlist.sas
+  @li mf_getvarlist.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_abort.sas
+
+
+  @param [in] inds The input library.dataset to test for values
+  @param [in] cols= The list of columns to check for
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [in] test= (ALL) The test to apply.  Valid values are:
+    @li ALL - Test is a PASS if ALL columns exist in &inds
+    @li ANY - Test is a PASS if ANY of the columns exist in &inds
+    @li NONE - Test is a PASS if NONE of the columns exist in &inds
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Column &inds contained ALL columns|
+
+
+  <h4> Related Macros </h4>
+  @li mp_assertdsobs.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertcols(inds,
+  cols=0,
+  test=ALL,
+  desc=0,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  %mp_abort(iftrue= (&syscc ne 0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc - on macro entry)
+  )
+
+  %local lib ds ;
+  %let lib=%scan(&inds,1,%str(.));
+  %let ds=%scan(&inds,2,%str(.));
+  %let cols=%upcase(&cols);
+
+  %mp_abort(iftrue= (%mf_existds(&lib..&ds)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(&lib..&ds not found!)
+  )
+
+  %mp_abort(iftrue= (&cols=0)
+    ,mac=&sysmacroname
+    ,msg=%str(No cols provided)
+  )
+
+
+  %let test=%upcase(&test);
+
+  %if &test ne ANY and &test ne ALL and &test ne NONE %then %do;
+    %mp_abort(
+      mac=&sysmacroname,
+      msg=%str(Invalid test - &test)
+    )
+  %end;
+
+  /**
+    * now do the actual test!
+    */
+  %local result;
+  %if %mf_existVarList(&inds,&cols)=1 %then %let result=ALL;
+  %else %do;
+    %local targetcols compare;
+    %let targetcols=%upcase(%mf_getvarlist(&inds));
+    %let compare=%mf_wordsinstr1butnotstr2(
+        Str1=&cols,
+        Str2=&targetcols
+      );
+    %if %cmpres(&compare)=%cmpres(&cols) %then %let result=NONE;
+    %else %let result=SOME;
+  %end;
+
+  data;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    if test_description='0'
+    then test_description="Testing &inds for existence of &test of: &cols";
+
+    test_result='FAIL';
+    test_comments="&sysmacroname: &inds has &result columns ";
+  %if &test=ALL %then %do;
+    %if &result=ALL %then %do;
+      test_result='PASS';
+    %end;
+  %end;
+  %else %if &test=ANY %then %do;
+    %if &result=SOME %then %do;
+      test_result='PASS';
+    %end;
+  %end;
+  %else %if &test=NONE %then %do;
+    %if &result=NONE %then %do;
+      test_result='PASS';
+    %end;
+  %end;
+  %else %do;
+    test_comments="&sysmacroname: Unsatisfied test condition - &test";
+  %end;
+  run;
+
+  %local ds;
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+
+%mend;

base/mp_assertcolvals.sas

@@ -0,0 +1,147 @@
+/**
+  @file
+  @brief Asserts the values in a column
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      data work.checkds;
+        do checkval='Jane','James','Jill';
+          output;
+        end;
+      run;
+      %mp_assertcolvals(sashelp.class.name,
+        checkvals=work.checkds.checkval,
+        desc=At least one value has a match,
+        test=ANYVAL
+      )
+
+      data work.check;
+        do val='M','F';
+          output;
+        end;
+      run;
+      %mp_assertcolvals(sashelp.class.sex,
+        checkvals=work.check.val,
+        desc=All values have a match,
+        test=ALLVALS
+      )
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+
+  @param [in] indscol The input library.dataset.column to test for values
+  @param [in] checkvals= A library.dataset.column value containing a UNIQUE
+    list of values to be compared against the source (indscol).
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [in] test= (ALLVALS) The test to apply.  Valid values are:
+    @li ALLVALS - Test is a PASS if ALL values have a match in checkvals
+    @li ANYVAL - Test is a PASS if at least 1 value has a match in checkvals
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Column &indscol contained ALL target vals|
+
+
+  <h4> Related Macros </h4>
+  @li mp_assertdsobs.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertcolvals(indscol,
+  checkvals=0,
+  test=ALLVALS,
+  desc=mp_assertcolvals - no desc provided,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  %mp_abort(iftrue= (&syscc ne 0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc - on macro entry)
+  )
+
+  %local lib ds col clib cds ccol nobs;
+  %let lib=%scan(&indscol,1,%str(.));
+  %let ds=%scan(&indscol,2,%str(.));
+  %let col=%scan(&indscol,3,%str(.));
+  %mp_abort(iftrue= (%mf_existds(&lib..&ds)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(&lib..&ds not found!)
+  )
+
+  %mp_abort(iftrue= (&checkvals=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Set CHECKVALS to a library.dataset.column containing check vals)
+  )
+  %let clib=%scan(&checkvals,1,%str(.));
+  %let cds=%scan(&checkvals,2,%str(.));
+  %let ccol=%scan(&checkvals,3,%str(.));
+  %mp_abort(iftrue= (%mf_existds(&clib..&cds)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(&clib..&cds not found!)
+  )
+  %let nobs=%mf_nobs(&clib..&cds);
+  %mp_abort(iftrue= (&nobs=0)
+    ,mac=&sysmacroname
+    ,msg=%str(&clib..&cds is empty!)
+  )
+
+  %let test=%upcase(&test);
+
+  %if &test ne ALLVALS and &test ne ANYVAL %then %do;
+    %mp_abort(
+      mac=&sysmacroname,
+      msg=%str(Invalid test - &test)
+    )
+  %end;
+
+  %local result orig;
+  %let result=-1;
+  %let orig=-1;
+  proc sql noprint;
+  select count(*) into: result
+    from &lib..&ds
+    where &col not in (
+      select &ccol from &clib..&cds
+    );
+  select count(*) into: orig from &lib..&ds;
+  quit;
+
+  %mp_abort(iftrue= (&syscc ne 0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc after macro query)
+  )
+
+  data;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_result='FAIL';
+    test_comments="&sysmacroname: &lib..&ds..&col has &result values "
+      !!"not in &clib..&cds..&ccol ";
+  %if &test=ANYVAL %then %do;
+    if &result < &orig then test_result='PASS';
+  %end;
+  %else %if &test=ALLVALS %then %do;
+    if &result=0 then test_result='PASS';
+  %end;
+  %else %do;
+    test_comments="&sysmacroname: Unsatisfied test condition - &test";
+  %end;
+  run;
+
+  %local ds;
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+
+%mend;

base/mp_assertdsobs.sas

@@ -0,0 +1,89 @@
+/**
+  @file
+  @brief Asserts the number of observations in a dataset
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      %mp_assertdsobs(sashelp.class) %* tests if any observations are present;
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+
+  @param [in] inds input dataset to test for presence of observations
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [in] test= (HASOBS) The test to apply.  Valid values are:
+    @li HASOBS - Test is a PASS if the input dataset has any observations
+    @li EMPTY - Test is a PASS if input dataset is empty
+    @li EQUALS [integer] - Test passes if obs count matches the provided integer
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Dataset &inds has XX obs|
+
+  <h4> Related Macros </h4>
+  @li mp_assertcolvals.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertdsobs(inds,
+  test=HASOBS,
+  desc=Testing observations,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  %local nobs;
+  %let nobs=%mf_nobs(&inds);
+  %let test=%upcase(&test);
+
+  %if %substr(&test.xxxxx,1,6)=EQUALS %then %do;
+    %let val=%scan(&test,2,%str( ));
+    %mp_abort(iftrue= (%DATATYP(&val)=CHAR)
+      ,mac=&sysmacroname
+      ,msg=%str(Invalid test - &test, expected EQUALS [integer])
+    )
+    %let test=EQUALS;
+  %end;
+  %else %if &test ne HASOBS and &test ne EMPTY %then %do;
+    %mp_abort(
+      mac=&sysmacroname,
+      msg=%str(Invalid test - &test)
+    )
+  %end;
+
+  data;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_result='FAIL';
+    test_comments="&sysmacroname: Dataset &inds has &nobs observations";
+  %if &test=HASOBS %then %do;
+    if &nobs>0 then test_result='PASS';
+  %end;
+  %else %if &test=EMPTY %then %do;
+    if &nobs=0 then test_result='PASS';
+  %end;
+  %else %if &test=EQUALS %then %do;
+    if &nobs=&val then test_result='PASS';
+  %end;
+  %else %do;
+    test_comments="&sysmacroname: Unsatisfied test condition - &test";
+  %end;
+  run;
+
+  %local ds;
+  %let ds=&syslast;
+
+  proc append base=&outds data=&ds;
+  run;
+
+  proc sql;
+  drop table &ds;
+
+%mend;

base/mp_binarycopy.sas

@@ -4,7 +4,8 @@
   @details Reads in a file byte by byte and writes it back out.  Is an
   os-independent method to copy files.  In case of naming collision, the
   default filerefs can be modified.
-    Based on http://stackoverflow.com/questions/13046116/using-sas-to-copy-a-text-file
+  Based on:
+  https://stackoverflow.com/questions/13046116/using-sas-to-copy-a-text-file
 
         %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
 

base/mp_cleancsv.sas

@@ -8,6 +8,7 @@
   applying CRLF line endings and converting embedded cr and crlf to lf.
 
   usage:
+
       fileref mycsv "/path/your/csv";
       %mp_cleancsv(in=mycsv,out=/path/new.csv)
 
@@ -17,11 +18,12 @@
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
 %macro mp_cleancsv(in=NOTPROVIDED,out=NOTPROVIDED,qchar='22'x);
 %if "&in"="NOTPROVIDED" or "&out"="NOTPROVIDED" %then %do;
-  %put %str(ERR)OR: Please provide valid input (&in) and output (&out) locations;
+  %put %str(ERR)OR: Please provide valid input (&in) & output (&out) locations;
   %return;
 %end;
 
@@ -66,3 +68,4 @@
     end;
   run;
 %mend;
+/** @endcond */

base/mp_createconstraints.sas

@@ -22,7 +22,7 @@
   @param outds= a table containing the create statements (create_statement column)
   @param execute= `YES|NO` - default is NO. To actually create, use YES.
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
 
   @version 9.2
   @author Allan Bowe

base/mp_createwebservice.sas

@@ -26,7 +26,7 @@ Usage:
     ;;;;
     %mp_createwebservice(path=/Public/app/common,name=appInit,code=ft15f001,replace=YES)
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_getplatform.sas
   @li mm_createwebservice.sas
   @li mv_createwebservice.sas

base/mp_csv2ds.sas

@@ -0,0 +1,144 @@
+/**
+  @file mp_csv2ds.sas
+  @brief Efficient import of arbitrary CSV using a dataset as template
+  @details Used to import relevant columns from a large CSV using
+  a dataset to provide the types and lengths.  Assumes that a header
+  row is provided, and datarows start on line 2.  Extra columns in
+  both the CSV and base dataset are ignored.
+
+  Usage:
+
+      filename mycsv temp;
+      data _null_;
+        file mycsv;
+        put 'name,age,nickname';
+        put 'John,48,Jonny';
+        put 'Jennifer,23,Jen';
+      run;
+
+      %mp_csv2ds(inref=mycsv,outds=myds,baseds=sashelp.class)
+
+
+  @param inref= fileref to the CSV
+  @param outds= output ds (lib.ds format)
+  @param view= Set to YES or NO to determine whether the output should be
+    a view or not.  Default is NO (not a view).
+  @param baseds= Template dataset on which to create the input statement.
+    Is used to determine types, lengths, and any informats.
+
+  @version 9.2
+  @author Allan Bowe
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_existds.sas
+
+**/
+
+%macro mp_csv2ds(inref=0,outds=0,baseds=0,view=NO);
+
+%mp_abort(iftrue=( &inref=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the INREF variable must be provided)
+)
+%mp_abort(iftrue=( %superq(outds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the OUTDS variable must be provided)
+)
+%mp_abort(iftrue=( &baseds=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS variable must be provided)
+)
+%mp_abort(iftrue=( &baseds=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS variable must be provided)
+)
+%mp_abort(iftrue=( %mf_existds(&baseds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS dataset (&baseds) needs to be assigned, and to exist)
+)
+
+/* count rows */
+%local hasheader; %let hasheader=0;
+data _null_;
+  if _N_ > 1 then do;
+    call symputx('hasheader',1,'l');
+    stop;
+  end;
+  infile &inref;
+  input;
+run;
+%mp_abort(iftrue=( &hasheader=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(No header row in &inref)
+)
+
+/* get the variables in the CSV */
+data _data_;
+  infile &inref;
+  input;
+  length name $32;
+  do i=1 to countc(_infile_,',')+1;
+    name=upcase(scan(_infile_,i,','));
+    output;
+  end;
+  stop;
+run;
+%local csv_vars;%let csv_vars=&syslast;
+
+/* get the variables in the dataset */
+proc contents noprint data=&baseds
+  out=_data_ (keep=name type length format: informat);
+run;
+%local base_vars; %let base_vars=&syslast;
+
+proc sql undo_policy=none;
+create table &csv_vars as
+  select a.*
+    ,b.type
+    ,b.length
+    ,b.format
+    ,b.formatd
+    ,b.formatl
+    ,b.informat
+  from &csv_vars a
+  left join &base_vars b
+  on a.name=upcase(b.name)
+  order by i;
+
+/* prepare the input statement */
+%local instat dropvars;
+data _null_;
+  set &syslast end=last;
+  length in dropvars $32767;
+  retain in dropvars;
+  if missing(type) then do;
+    informat='$1.';
+    dropvars=catx(' ',dropvars,name);
+  end;
+  else if missing(informat) then do;
+    if type=1 then informat='best.';
+    else informat=cats('$',length,'.');
+  end;
+  else informat=cats(informat,'.');
+  in=catx(' ',in,name,':',informat);
+  if last then do;
+    call symputx('instat',in,'l');
+    call symputx('dropvars',dropvars,'l');
+  end;
+run;
+
+/* import the CSV */
+data &outds
+  %if %upcase(&view)=YES %then %do;
+    /view=&outds
+  %end;
+  ;
+  infile &inref dsd firstobs=2;
+  input &instat;
+  %if %length(&dropvars)>0 %then %do;
+    drop &dropvars;
+  %end;
+run;
+
+%mend;

base/mp_dirlist.sas

@@ -20,6 +20,8 @@
 
       %mp_dirlist(outds=cwdfileprops, getattrs=YES)
 
+      %mp_dirlist(fref=MYFREF)
+
   @warning In a Unix environment, the existence of a named pipe will cause this
   macro to hang.  Therefore this tool should be used with caution in a SAS 9 web
   application, as it can use up all available multibridge sessions if requests
@@ -29,12 +31,14 @@
 
 
   @param path= for which to return contents
+  @param fref= Provide a DISK engine fileref as an alternative to PATH
   @param outds= the output dataset to create
   @param getattrs= YES/NO (default=NO).  Uses doptname and foptname to return
   all attributes for each file / folder.
 
 
   @returns outds contains the following variables:
+    - directory (containing folder)
     - file_or_folder (file / folder)
     - filepath (path/to/file.name)
     - filename (just the file name)
@@ -47,18 +51,29 @@
 **/
 
 %macro mp_dirlist(path=%sysfunc(pathname(work))
+    , fref=0
     , outds=work.mp_dirlist
     , getattrs=NO
 )/*/STORE SOURCE*/;
 %let getattrs=%upcase(&getattrs)XX;
 
-data &outds (compress=no keep=file_or_folder filepath filename ext msg);
+data &outds(compress=no
-  length filepath $500 fref fref2 $8 file_or_folder $6 filename $80 ext $20 msg $200;
+    keep=file_or_folder filepath filename ext msg directory
+  );
+  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
+    ext $20 msg $200;
+  %if &fref=0 %then %do;
     rc = filename(fref, "&path");
+  %end;
+  %else %do;
+    fref="&fref";
+    rc=0;
+  %end;
   if rc = 0 then do;
     did = dopen(fref);
+    directory=dinfo(did,'Directory');
     if did=0 then do;
-        putlog "NOTE: This directory is empty - &path";
+      putlog "NOTE: This directory is empty - " directory;
       msg=sysmsg();
       put _all_;
       stop;
@@ -73,7 +88,8 @@ data &outds (compress=no keep=file_or_folder filepath filename ext msg);
   dnum = dnum(did);
   do i = 1 to dnum;
     filename = dread(did, i);
-    rc = filename(fref2, "&path/"!!filename);
+    filepath=cats(directory,'/',filename);
+    rc = filename(fref2,filepath);
     midd=dopen(fref2);
     dmsg=sysmsg();
     if did > 0 then file_or_folder='folder';
@@ -85,7 +101,7 @@ data &outds (compress=no keep=file_or_folder filepath filename ext msg);
 
     if index(fmsg,'File is in use') or index(dmsg,'is not a directory')
       then file_or_folder='file';
-    else if index(fmsg, 'Insufficient authorization') then file_or_folder='file';
+    else if index(fmsg,'Insufficient authorization') then file_or_folder='file';
     else if file_or_folder='' then file_or_folder='locked';
 
     if file_or_folder='file' then do;
@@ -96,7 +112,6 @@ data &outds (compress=no keep=file_or_folder filepath filename ext msg);
       ext='';
       file_or_folder='folder';
     end;
-    filepath="&path/"!!filename;
     output;
   end;
   rc = dclose(did);

base/mp_dropmembers.sas

@@ -9,7 +9,7 @@
       create view view2 as select * from sashelp.class;
       %mp_dropmembers(list=data1 view2)
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_isblank.sas
 
 

base/mp_ds2cards.sas

@@ -3,25 +3,29 @@
   @brief Create a CARDS file from a SAS dataset.
   @details Uses dataset attributes to convert all data into datalines.
     Running the generated file will rebuild the original dataset.
-  usage:
+  Usage:
 
       %mp_ds2cards(base_ds=sashelp.class
         , cards_file= "C:\temp\class.sas"
         , maxobs=5)
 
-    stuff to add
+  TODO:
     - labelling the dataset
     - explicity setting a unix LF
     - constraints / indexes etc
 
-  @param base_ds= Should be two level - eg work.blah.  This is the table that
+  @param [in] base_ds= Should be two level - eg work.blah.  This is the table
-                   is converted to a cards file.
+                  that is converted to a cards file.
-  @param tgt_ds= Table that the generated cards file would create. Optional -
+  @param [in] tgt_ds= Table that the generated cards file would create.
-                  if omitted, will be same as BASE_DS.
+    Optional - if omitted, will be same as BASE_DS.
-  @param cards_file= Location in which to write the (.sas) cards file
+  @param [out] cards_file= Location in which to write the (.sas) cards file
-  @param maxobs= to limit output to the first <code>maxobs</code> observations
+  @param [in] maxobs= to limit output to the first <code>maxobs</code>
-  @param showlog= whether to show generated cards file in the SAS log (YES/NO)
+    observations
-  @param outencoding= provide encoding value for file statement (eg utf-8)
+  @param [in] showlog= whether to show generated cards file in the SAS log
+    (YES/NO)
+  @param [in] outencoding= provide encoding value for file statement (eg utf-8)
+  @param [in] append= If NO then will rebuild the cards file if it already
+    exists, otherwise will append to it.  Used by the mp_lib2cards.sas macro.
 
 
   @version 9.2
@@ -34,11 +38,12 @@
     ,random_sample=NO
     ,showlog=YES
     ,outencoding=
+    ,append=NO
 )/*/STORE SOURCE*/;
 %local i setds nvars;
 
 %if not %sysfunc(exist(&base_ds)) %then %do;
-   %put WARNING:  &base_ds does not exist;
+  %put %str(WARN)ING:  &base_ds does not exist;
   %return;
 %end;
 
@@ -46,6 +51,8 @@
 %if (&tgt_ds = ) %then %let tgt_ds=&base_ds;
 %if %index(&tgt_ds,.)=0 %then %let tgt_ds=WORK.%scan(&base_ds,2,.);
 %if ("&outencoding" ne "") %then %let outencoding=encoding="&outencoding";
+%if ("&append" = "") %then %let append=;
+%else %let append=mod;
 
 /* get varcount */
 %let nvars=0;
@@ -54,14 +61,16 @@ select count(*) into: nvars from dictionary.columns
   where libname="%scan(%upcase(&base_ds),1)"
     and memname="%scan(%upcase(&base_ds),2)";
 %if &nvars=0 %then %do;
-  %put WARNING:  Dataset &base_ds has no variables!  It will not be converted.;
+  %put %str(WARN)ING: Dataset &base_ds has no variables, will not be converted.;
   %return;
 %end;
 
 /* get indexes */
-proc sort data=sashelp.vindex
+proc sort
-    (where=(upcase(libname)="%scan(%upcase(&base_ds),1)"
+  data=sashelp.vindex(
-       and upcase(memname)="%scan(%upcase(&base_ds),2)"))
+    where=(upcase(libname)="%scan(%upcase(&base_ds),1)"
+      and upcase(memname)="%scan(%upcase(&base_ds),2)")
+    )
   out=_data_;
   by indxname indxpos;
 run;
@@ -103,8 +112,7 @@ proc sql
   order by ranuni(42)
 %end;
   ;
-
+reset outobs=max;
-
 create table datalines1 as
   select name,type,length,varnum,format,label from dictionary.columns
   where libname="%upcase(%scan(&base_ds,1))"
@@ -173,7 +181,7 @@ data _null_;
 run;
 
 data _null_;
-  file &cards_file. &outencoding lrecl=32767 termstr=nl;
+  file &cards_file. &outencoding lrecl=32767 termstr=nl &append;
   length __attrib $32767;
   if _n_=1 then do;
     put '/*******************************************************************';

base/mp_ds2csv.sas

@@ -0,0 +1,58 @@
+/**
+  @file
+  @brief Export a dataset to a CSV file
+  @details Export to a file or a fileref
+  Usage:
+
+      %mp_ds2csv(sashelp.class,outref="%sysfunc(pathname(work))/file.csv")
+
+  @param ds The dataset to be exported
+  @param outfile= The output filename - should be quoted.
+  @param outref= The output fileref (takes precedence if provided)
+  @param outencoding= The output encoding to use (unquoted)
+
+  @version 9.2
+  @author Allan Bowe (credit mjsq)
+**/
+
+%macro mp_ds2csv(ds, outref=0, outfile=, outencoding=0
+)/*/STORE SOURCE*/;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WARN)ING:  &ds does not exist;
+  %return;
+%end;
+
+%if %index(&ds,.)=0 %then %let ds=WORK.&ds;
+
+%if &outencoding=0 %then %let outencoding=;
+%else %let outencoding=encoding="&outencoding";
+
+%local outloc;
+%if &outref=0 %then %let outloc=&outfile;
+%else %let outloc=&outref;
+
+/* credit to mjsq - https://stackoverflow.com/a/55642267 */
+
+/* first get headers */
+data _null_;
+  file &outloc dlm=',' dsd &outencoding lrecl=32767;
+  length header $ 2000;
+  dsid=open("&ds.","i");
+  num=attrn(dsid,"nvars");
+  do i=1 to num;
+    header = trim(left(coalescec(varlabel(dsid,i),varname(dsid,i))));
+    put header @;
+  end;
+  rc=close(dsid);
+run;
+
+/* next, export data */
+data _null_;
+  set &ds.;
+  file &outloc mod dlm=',' dsd &outencoding lrecl=32767;
+  put (_all_) (+0);
+run;
+
+
+%mend;

base/mp_filtercheck.sas

@@ -0,0 +1,188 @@
+/**
+  @file
+  @brief Checks an input filter table for validity
+  @details Performs checks on the input table to ensure it arrives in the
+  correct format.  This is necessary to prevent code injection.  Will update
+  SYSCC to 1008 if bad records are found, and call mp_abort.sas for a
+  graceful service exit (configurable).
+
+  Used for dynamic filtering in [Data Controller for SAS®](https://datacontroller.io).
+
+  Usage:
+
+      %mp_filtercheck(work.filter,targetds=sashelp.class,outds=work.badrecords)
+
+  The input table should have the following format:
+
+  |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$4000|
+  |---|---|---|---|---|---|
+  |AND|AND|1|AGE|=|12|
+  |AND|AND|1|SEX|<=|'M'|
+  |AND|OR|2|Name|NOT IN|('Jane','Alfred')|
+  |AND|OR|2|Weight|>=|7|
+
+  Rules applied:
+
+  @li GROUP_LOGIC - only AND/OR
+  @li SUBGROUP_LOGIC - only AND/OR
+  @li SUBGROUP_ID - only integers
+  @li VARIABLE_NM - must be in the target table
+  @li OPERATOR_NM - only =/>/</<=/>=/BETWEEN/IN/NOT IN/NE/CONTAINS
+  @li RAW_VALUE - no unquoted values except integers, commas and spaces.
+
+  @returns The &outds table containing any bad rows, plus a REASON_CD column.
+
+  @param [in] inds The table to be checked, with the format above
+  @param [in] targetds= The target dataset against which to verify VARIABLE_NM
+  @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
+  @param [out] outds= The output table, which is a copy of the &inds. table
+  plus a REASON_CD column, containing only bad records.  If bad records found,
+  the SYSCC value will be set to 1008 (general data problem).  Downstream
+  processes should check this table (and return code) before continuing.
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
+  @li mf_nobs.sas
+  @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+  @todo Support date / hex / name literals and exponents in RAW_VALUE field
+**/
+
+%macro mp_filtercheck(inds,targetds=,outds=work.badrecords,abort=YES);
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(syscc=&syscc - on macro entry)
+)
+
+/* Validate input column */
+%local vtype;
+%let vtype=%mf_getvartype(&inds,RAW_VALUE);
+%mp_abort(iftrue=(&abort=YES and &vtype ne C),
+  mac=&sysmacroname,
+  msg=%str(%str(ERR)OR: RAW_VALUE must be character)
+)
+%if &vtype ne C %then %do;
+  %put &sysmacroname: RAW_VALUE must be character;
+  %let syscc=42;
+  %return;
+%end;
+
+
+/**
+  * Sanitise the values based on valid value lists, then strip out
+  * quotes, commas, periods and spaces.
+  * Only numeric values should remain
+  */
+%local reason_cd;
+data &outds;
+  /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
+    OPERATOR_NM $10 RAW_VALUE $4000;*/
+  set &inds;
+  length reason_cd $32;
+
+  /* closed list checks */
+  if GROUP_LOGIC not in ('AND','OR') then do;
+    REASON_CD='GROUP_LOGIC should be either AND or OR';
+    putlog REASON_CD= GROUP_LOGIC=;
+    output;
+  end;
+  if SUBGROUP_LOGIC not in ('AND','OR') then do;
+    REASON_CD='SUBGROUP_LOGIC should be either AND or OR';
+    putlog REASON_CD= SUBGROUP_LOGIC=;
+    output;
+  end;
+  if mod(SUBGROUP_ID,1) ne 0 then do;
+    REASON_CD='SUBGROUP_ID should be integer';
+    putlog REASON_CD= SUBGROUP_ID=;
+    output;
+  end;
+  if upcase(VARIABLE_NM) not in
+  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
+  then do;
+    REASON_CD="VARIABLE_NM not in &targetds";
+    putlog REASON_CD= VARIABLE_NM=;
+    output;
+  end;
+  if OPERATOR_NM not in
+  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
+  then do;
+    REASON_CD='Invalid OPERATOR_NM';
+    putlog REASON_CD= OPERATOR_NM=;
+    output;
+  end;
+
+  /* special logic */
+  if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ','');
+  else if OPERATOR_NM in ('IN','NOT IN') then do;
+    if substr(raw_value,1,1) ne '('
+    or substr(cats(reverse(raw_value)),1,1) ne ')'
+    then do;
+      REASON_CD='Missing brackets in RAW_VALUE';
+      putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+      output;
+    end;
+    else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
+  end;
+  else raw_value1=raw_value;
+
+  /* remove nested literals eg '' */
+  raw_value1=tranwrd(raw_value1,"''",'');
+
+  /* now match string literals (always single quotes) */
+  raw_value2=raw_value1;
+  regex = prxparse("s/(\').*?(\')//");
+  call prxchange(regex,-1,raw_value2);
+
+  /* remove commas and periods*/
+  raw_value3=compress(raw_value2,',.');
+
+  /* output records that contain values other than digits and spaces */
+  if notdigit(compress(raw_value3,' '))>0 then do;
+    putlog raw_value3= $hex32.;
+    REASON_CD='Invalid RAW_VALUE';
+    putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
+    output;
+  end;
+
+run;
+
+data _null_;
+  set &outds;
+  call symputx('REASON_CD',reason_cd,'l');
+  stop;
+run;
+
+%mp_abort(iftrue=(&abort=YES and %mf_nobs(&outds)>0),
+  mac=&sysmacroname,
+  msg=%str(Filter issues in &inds, reason: &reason_cd, details in &outds)
+)
+
+%if %mf_nobs(&outds)>0 %then %do;
+  %let syscc=1008;
+  %return;
+%end;
+
+/**
+  * syntax checking passed but it does not mean the filter is valid
+  * for that we can run a proc sql validate query
+  */
+%local fref1;
+%let fref1=%mf_getuniquefileref();
+%mp_filtergenerate(&inds,outref=&fref1)
+
+/* this macro will also set syscc to 1008 if any issues found */
+%mp_filtervalidate(&fref1,&targetds,outds=&outds,abort=&abort)
+
+%mend mp_filtercheck;

base/mp_filtergenerate.sas

@@ -0,0 +1,102 @@
+/**
+  @file
+  @brief Generates a filter clause from an input table, to a fileref
+  @details Uses the input table to generate an output filter clause.
+  This feature is used to create dynamic dropdowns in [Data Controller for SAS&reg](
+  https://datacontroller.io). The input table should be in the format below:
+
+  |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$4000|
+  |---|---|---|---|---|---|
+  |AND|AND|1|AGE|=|12|
+  |AND|AND|1|SEX|<=|'M'|
+  |AND|OR|2|Name|NOT IN|('Jane','Alfred')|
+  |AND|OR|2|Weight|>=|7|
+
+  Note - if the above table is received from an external client, the values
+  should first be validated using the mp_filtercheck.sas macro to avoid risk
+  of SQL injection.
+
+  To generate the filter, run the following code:
+
+      data work.filtertable;
+        infile datalines4 dsd;
+        input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+          OPERATOR_NM:$10. RAW_VALUE:$4000.;
+      datalines4;
+      AND,AND,1,AGE,=,12
+      AND,AND,1,SEX,<=,"'M'"
+      AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
+      AND,OR,2,Weight,>=,7
+      ;;;;
+      run;
+
+      %mp_filtergenerate(work.filtertable,outref=myfilter)
+
+      data _null_;
+        infile myfilter;
+        input;
+        put _infile_;
+      run;
+
+  Will write the following query to the log:
+
+  > (
+  >     AGE = 12
+  >   AND
+  >     SEX <= 'M'
+  > ) AND (
+  >     Name NOT IN ('Jane','Alfred')
+  >   OR
+  >     Weight >= 7
+  > )
+
+  @param [in] inds The input table with query values
+  @param [out] outref= The output fileref to contain the filter clause.  Will
+    be created (or replaced).
+
+  <h4> Related Macros </h4>
+  @li mp_filtercheck.sas
+  @li mp_filtervalidate.sas
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_nobs.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mp_filtergenerate(inds,outref=filter);
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(syscc=&syscc - on macro entry)
+)
+
+filename &outref temp;
+
+%if %mf_nobs(&inds)=0 %then %do;
+  /* ensure we have a default filter */
+  data _null_;
+    file &outref;
+    put '1=1';
+  run;
+%end;
+%else %do;
+  data _null_;
+    file &outref lrecl=32800;
+    set &inds end=last;
+    by SUBGROUP_ID;
+    if _n_=1 then put '((';
+    else if first.SUBGROUP_ID then put +1 GROUP_LOGIC '(';
+    else put +2 SUBGROUP_LOGIC;
+
+    put +4 VARIABLE_NM OPERATOR_NM RAW_VALUE;
+
+    if last.SUBGROUP_ID then put ')'@;
+    if last then put ')';
+  run;
+%end;
+
+%mend;

base/mp_filtervalidate.sas

@@ -0,0 +1,104 @@
+/**
+  @file
+  @brief Checks a generated filter query for validity
+  @details Runs a generated filter in proc sql with the validate option.
+  Used in mp_filtercheck.sas in an fcmp container.
+
+  Built to support dynamic filtering in
+  [Data Controller for SAS®](https://datacontroller.io).
+
+  Usage:
+
+      data work.filtertable;
+        infile datalines4 dsd;
+        input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+          OPERATOR_NM:$10. RAW_VALUE:$4000.;
+      datalines4;
+      AND,AND,1,AGE,=,12
+      AND,AND,1,SEX,<=,"'M'"
+      AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
+      AND,OR,2,Weight,>=,7
+      ;;;;
+      run;
+
+      %mp_filtergenerate(work.filtertable,outref=myfilter)
+
+      %mp_filtervalidate(myfilter,sashelp.class)
+
+
+  @returns The SYSCC value will be 1008 if there are validation issues.
+
+  @param [in] inref The input fileref to validate (generated by
+    mp_filtergenerate.sas)
+  @param [in] targetds The target dataset against which to verify the query
+  @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
+  @param [out] outds= (work.mp_filtervalidate) Output dataset containing the
+  error / warning message, if one exists.  If this table contains any rows,
+  there are problems!
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filtercheck.sas
+  @li mp_filtergenerate.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mp_filtervalidate(inref,targetds,abort=YES,outds=work.mp_filtervalidate);
+
+%mp_abort(iftrue= (&syscc ne 0 or &syserr ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(syscc=&syscc / syserr=&syserr - on macro entry)
+)
+
+%local fref1;
+%let fref1=%mf_getuniquefileref();
+
+data _null_;
+  file &fref1;
+  infile &inref end=eof;
+  if _n_=1 then do;
+    put "proc sql;";
+    put "validate select * from &targetds";
+    put "where " ;
+  end;
+  input;
+  put _infile_;
+  putlog _infile_;
+  if eof then put ";quit;";
+run;
+
+%inc &fref1;
+
+data &outds;
+  if &sqlrc or &syscc or &syserr then do;
+    REASON_CD=coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
+    output;
+  end;
+  else stop;
+run;
+
+filename &fref1 clear;
+
+%if %mf_nobs(&outds)>0 %then %do;
+  %if &abort=YES %then %do;
+    data _null_;
+      set &outds;
+      call symputx('REASON_CD',reason_cd,'l');
+      stop;
+    run;
+    %mp_abort(
+      mac=&sysmacroname,
+      msg=%str(Filter issues in &inref: %quote(&reason_cd))
+    )
+  %end;
+  %let syscc=1008;
+%end;
+
+%mend;

base/mp_getconstraints.sas

@@ -21,7 +21,7 @@
   @param ds= The target dataset.  Leave blank (default) for all datasets.
   @param outds the output dataset
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
 
   @version 9.2
   @author Allan Bowe

base/mp_getdbml.sas

@@ -0,0 +1,335 @@
+/**
+  @file
+  @brief Extract DBML from SAS Libraries
+  @details DBML is an open source markup format to represent databases.
+  More details: https://www.dbml.org/home/
+
+  Usage:
+
+
+      %mp_getdbml(liblist=SASHELP WORK,outref=mydbml,showlog=YES)
+
+  Take the log output and paste it into the renderer at https://dbdiagram.io
+  to view your data model diagram.  The code takes a "best guess" at
+  the one to one and one to many relationships (based on constraints
+  and indexes, and assuming that the column names would match).
+
+  You may need to adjust the rendered DBML to suit your needs.
+
+  ![dbml for sas](https://i.imgur.com/8T1tIZp.gif)
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getquotedstr.sas
+  @li mp_getconstraints.sas
+
+  @param liblist= Space seperated list of librefs to take as
+    input (Default=SASHELP)
+  @param outref= Fileref to contain the DBML (Default=getdbml)
+  @param showlog= set to YES to show the DBML in the log (Default is NO)
+
+  @version 9.3
+  @author Allan Bowe
+**/
+
+%macro mp_getdbml(liblist=SASHELP,outref=getdbml,showlog=NO
+)/*/STORE SOURCE*/;
+
+/* check fileref is assigned */
+%if %sysfunc(fileref(&outref)) > 0 %then %do;
+  filename &outref temp;
+%end;
+
+%let liblist=%upcase(&liblist);
+
+proc sql noprint;
+create table _data_ as
+  select * from dictionary.tables
+  where upcase(libname) in (%mf_getquotedstr(&liblist))
+  order by libname,memname;
+%local tabinfo; %let tabinfo=&syslast;
+
+create table _data_ as
+  select * from dictionary.columns
+  where upcase(libname) in (%mf_getquotedstr(&liblist))
+  order by libname,memname,varnum;
+%local colinfo; %let colinfo=&syslast;
+
+%local dsnlist;
+  select distinct upcase(cats(libname,'.',memname)) into: dsnlist
+  separated by ' '
+  from &syslast
+;
+
+create table _data_ as
+  select * from dictionary.indexes
+  where upcase(libname) in (%mf_getquotedstr(&liblist))
+  order by idxusage, indxname, indxpos;
+%local idxinfo; %let idxinfo=&syslast;
+
+/* Extract all Primary Key and Unique data constraints */
+%mp_getconstraints(lib=%scan(&liblist,1),outds=_data_)
+%local colconst; %let colconst=&syslast;
+
+%do x=2 %to %sysfunc(countw(&liblist));
+  %mp_getconstraints(lib=%scan(&liblist,&x),outds=_data_)
+  proc append base=&colconst data=&syslast;
+  run;
+%end;
+
+
+
+
+/* header info */
+data _null_;
+  file &outref;
+  put "// DBML generated by &sysuserid on %sysfunc(datetime(),datetime19.) ";
+  put "Project sasdbml {";
+  put "  database_type: 'SAS'";
+  put "  Note: 'Generated by the mp_getdbml() macro'";
+  put "}";
+run;
+
+/* create table groups */
+data _null_;
+  file &outref mod;
+  set &tabinfo;
+  by libname;
+  if first.libname then put "TableGroup " libname "{";
+  ds=quote(cats(libname,'.',memname));
+  put '   ' ds;
+  if last.libname then put "}";
+run;
+
+/* table for pks */
+data _data_;
+  length curds const col $39;
+  call missing (of _all_);
+  stop;
+run;
+%let pkds=&syslast;
+
+%local x curds constraints_used constcheck;
+%do x=1 %to %sysfunc(countw(&dsnlist,%str( )));
+  %let curds=%scan(&dsnlist,&x,%str( ));
+  %let constraints_used=;
+  %let constcheck=0;
+  data _null_;
+    file &outref mod;
+    length lab $1024 typ $20;
+    set &colinfo (where=(
+        libname="%scan(&curds,1,.)" and upcase(memname)="%scan(&curds,2,.)"
+    )) end=last;
+
+    if _n_=1 then do;
+      table='Table "'!!"&curds"!!'"{';
+      put table;
+    end;
+    name=upcase(name);
+    lab=" note:"!!quote(trim(tranwrd(label,'"',"'")));
+    if upcase(format)=:'DATETIME' then typ='datetime';
+    else if type='char' then typ=cats('char(',length,')');
+    else typ='num';
+
+    if notnull='yes' then notnul=' not null';
+    if notnull='no' and missing(label) then put '  ' name typ;
+    else if notnull='yes' and missing(label) then do;
+      put '  ' name typ '[' notnul ']';
+    end;
+    else if notnull='no' then put '  ' name typ '[' lab ']';
+    else put '  ' name typ '[' notnul ',' lab ']';
+
+  run;
+
+  data _data_(keep=curds const col);
+    length ctype $11 cols constraints_used $5000;
+    set &colconst (where=(
+      upcase(libref)="%scan(&curds,1,.)"
+      and upcase(table_name)="%scan(&curds,2,.)"
+      and constraint_type in ('PRIMARY','UNIQUE')
+    )) end=last;
+    file &outref mod;
+    by constraint_type constraint_name;
+    retain cols;
+    column_name=upcase(column_name);
+
+    if _n_=1 then put / '  indexes {';
+
+    if upcase(strip(constraint_type)) = 'PRIMARY' then ctype='[pk]';
+    else ctype='[unique]';
+
+    if first.constraint_name then cols = cats('(',column_name);
+    else cols=cats(cols,',',column_name);
+
+    if last.constraint_name then do;
+      cols=cats(cols,')',ctype)!!' //'!!constraint_name;
+      put '    ' cols;
+      constraints_used=catx(' ',constraints_used, constraint_name);
+      call symputx('constcheck',1);
+    end;
+
+    if last then call symput('constraints_used',cats(upcase(constraints_used)));
+
+    length curds const col $39;
+    curds="&curds";
+    const=constraint_name;
+    col=column_name;
+  run;
+
+  proc append base=&pkds data=&syslast;run;
+
+  /* Create Unique Indexes, but only if they were not already defined within
+    the Constraints section. */
+  data _data_(keep=curds const col);
+    set &idxinfo (where=(
+      libname="%scan(&curds,1,.)"
+      and upcase(memname)="%scan(&curds,2,.)"
+      and unique='yes'
+      and upcase(indxname) not in (%mf_getquotedstr(&constraints_used))
+    ));
+    file &outref mod;
+    by idxusage indxname;
+    name=upcase(name);
+    if &constcheck=1 then stop; /* we only care about PKs so stop if we have */
+    if _n_=1 and &constcheck=0 then put / '  indexes {';
+
+    length cols $5000;
+    retain cols;
+    if first.indxname then cols = cats('(',name);
+    else cols=cats(cols,',',name);
+
+    if last.indxname then do;
+      cols=cats(cols,')[unique]')!!' //'!!indxname;
+      put '    ' cols;
+      call symputx('constcheck',1);
+    end;
+
+    length curds const col $39;
+    curds="&curds";
+    const=indxname;
+    col=name;
+  run;
+  proc append base=&pkds data=&syslast;run;
+
+  data _null_;
+    file &outref mod;
+    if &constcheck =1 then put '  }';
+    put '}';
+  run;
+
+%end;
+
+/**
+  * now we need to figure out the relationships
+  */
+
+/* sort alphabetically so we can have one set of unique cols per table */
+proc sort data=&pkds nodupkey;
+  by curds const col;
+run;
+
+data &pkds.1 (keep=curds col)
+    &pkds.2 (keep=curds cols);
+  set &pkds;
+  by curds const;
+  length retconst $39 cols $5000;
+  retain retconst cols;
+  if first.curds then do;
+    retconst=const;
+    cols=upcase(col);
+  end;
+  else cols=catx(' ',cols,upcase(col));
+  if retconst=const then do;
+    output &pkds.1;
+    if last.const then output &pkds.2;
+  end;
+run;
+
+%let curdslist="0";
+%do x=1 %to %sysfunc(countw(&dsnlist,%str( )));
+  %let curds=%scan(&dsnlist,&x,%str( ));
+
+  %let pkcols=0;
+  data _null_;
+    set &pkds.2(where=(curds="&curds"));
+    call symputx('pkcols',cols);
+  run;
+  %if &pkcols ne 0 %then %do;
+    %let curdslist=&curdslist,"&curds";
+
+    /* start with one2one */
+    data &pkds.4;
+      file &outref mod;
+      set &pkds.2(where=(cols="&pkcols" and curds not in (&curdslist)));
+      line='Ref: "'!!"&curds"
+        !!cats('".(',"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))",')')
+        !!' - '
+        !!cats(quote(trim(curds))
+            ,'.('
+            ,"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))"
+            ,')'
+          );
+      put line;
+    run;
+
+    /* now many2one */
+    /* get table with one row per col */
+    data &pkds.5;
+      set &pkds.1(where=(curds="&curds"));
+    run;
+    /* get tables which contain the PK columns */
+    proc sql;
+    create table &pkds.5a as
+      select upcase(cats(b.libname,'.',b.memname)) as curds
+        ,b.name
+      from &pkds.5 a
+      inner join &colinfo b
+      on a.col=upcase(b.name);
+    /* count to make sure those tables contain ALL the columns */
+    create table &pkds.5b as
+      select curds,count(*) as cnt
+      from &pkds.5a
+      where curds not in (
+          select curds from &pkds.2 where cols="&pkcols"
+        ) /* not a one to one match */
+        and curds ne "&curds" /* exclude self */
+      group by 1;
+    create table &pkds.6 as
+      select a.*
+        ,b.cols
+      from &pkds.5b a
+      left join &pkds.4 b
+      on a.curds=b.curds;
+
+    data _null_;
+      set &pkds.6;
+      file &outref mod;
+      colcnt=%sysfunc(countw(&pkcols));
+      if cnt=colcnt then do;
+        /* table contains all the PK cols, and was not a direct / 121 match */
+        line='Ref: "'!!"&curds"
+          !!'".('
+          !!"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))"
+          !!') > '
+          !!cats(quote(trim(curds))
+              ,'.('
+              ,"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))"
+              ,')'
+          );
+        put line;
+      end;
+    run;
+  %end;
+%end;
+
+
+%if %upcase(&showlog)=YES %then %do;
+  options ps=max;
+  data _null_;
+    infile &outref;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend;

base/mp_getddl.sas

@@ -15,7 +15,7 @@
       proc sql; describe table &syslast;
       %mp_getddl(work,test,flavour=tsql,showlog=YES)
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mp_getconstraints.sas
 
   @param lib libref of the library to create DDL for.  Should be assigned.
@@ -30,7 +30,6 @@
     datetime2 format or regular decimal type
   @version 9.3
   @author Allan Bowe
-  @source https://github.com/sasjs/core
 **/
 
 %macro mp_getddl(libref,ds,fref=getddl,flavour=SAS,showlog=NO,schema=
@@ -87,7 +86,9 @@ create table _data_ as
   %global constraints_used;
   data _null_;
     length ctype $11 constraint_name_orig $256 constraints_used $5000;
-    set &colconst (where=(table_name="&curds" and constraint_type in ('PRIMARY','UNIQUE'))) end=last;
+    set &colconst(
+        where=(table_name="&curds" and constraint_type in ('PRIMARY','UNIQUE'))
+      ) end=last;
     file &fref mod;
     by constraint_type constraint_name;
     retain constraints_used;
@@ -132,7 +133,7 @@ run;
     %let curds=%scan(&dsnlist,&x);
     data _null_;
       file &fref mod;
-      length nm lab $1024;
+      length nm lab $1024 typ $20;
       set &colinfo (where=(upcase(memname)="&curds")) end=last;
 
       if _n_=1 then do;
@@ -146,10 +147,12 @@ run;
       end;
       else put "   ,"@@;
       if length(format)>1 then fmt=" format="!!cats(format);
-      len=" length="!!cats(length);
+      if length(label)>1 then lab=" label="!!quote(trim(label));
-      lab=" label="!!quote(trim(label));
       if notnull='yes' then notnul=' not null';
-      put name type len fmt notnul lab;
+      if type='char' then typ=cats('char(',length,')');
+      else if length ne 8 then typ='num length='!!left(length);
+      else typ='num';
+      put name typ fmt notnul lab;
     run;
 
     /* Extra step for data constraints */
@@ -160,10 +163,19 @@ run;
       put ');';
     run;
 
-    /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
+    /* Create Unique Indexes, but only if they were not already defined within
+      the Constraints section. */
     data _null_;
       *length ds $128;
-      set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
+      set &idxinfo(
+        where=(
+          memname="&curds"
+          and unique='yes'
+          and indxname not in (
+              %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+              )
+          )
+        );
       file &fref mod;
       by idxusage indxname;
 /*       ds=cats(libname,'.',memname); */
@@ -227,10 +239,19 @@ run;
     /* Extra step for data constraints */
     %addConst()
 
-    /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
+    /* Create Unique Indexes, but only if they were not already defined within
+      the Constraints section. */
     data _null_;
       *length ds $128;
-      set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
+      set &idxinfo(
+        where=(
+          memname="&curds"
+          and unique='yes'
+          and indxname not in (
+            %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+          )
+        )
+      );
       file &fref mod;
       by idxusage indxname;
       *ds=cats(libname,'.',memname);
@@ -319,15 +340,24 @@ run;
       put ');';
     run;
 
-    /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
+    /* Create Unique Indexes, but only if they were not already defined within
+      the Constraints section. */
     data _null_;
       *length ds $128;
-      set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
+      set &idxinfo(
+        where=(
+          memname="&curds"
+          and unique='yes'
+          and indxname not in (
+            %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+          )
+        )
+      );
       file &fref mod;
       by idxusage indxname;
 /*       ds=cats(libname,'.',memname); */
       if first.indxname then do;
-          put 'CREATE UNIQUE INDEX "' indxname +(-1) '" ' "ON &schema..&curds (" ;
+          put 'CREATE UNIQUE INDEX "' indxname +(-1) '" ' "ON &schema..&curds(";
           put '  "' name +(-1) '"' ;
       end;
       else put '  ,"' name +(-1) '"';
@@ -339,7 +369,7 @@ run;
 
   %end;
 %end;
-%if &showlog=YES %then %do;
+%if %upcase(&showlog)=YES %then %do;
   options ps=max;
   data _null_;
     infile &fref;

base/mp_getmaxvarlengths.sas

@@ -8,10 +8,11 @@
   - NAME    Name of the base dataset column
   - MAXLEN Maximum length of the data contained therein.
 
-  Character fields may be allocated very large widths (eg 32000) of which the maximum
+  Character fields may be allocated very large widths (eg 32000) of which the
-    value is likely to be much narrower.  This macro was designed to enable a HTML
+  maximum  value is likely to be much narrower.  This macro was designed to
-    table to be appropriately sized however this could be used as part of a data
+  enable a HTML table to be appropriately sized however this could be used as
-    audit to ensure we aren't over-sizing our tables in relation to the data therein.
+  part of a data audit to ensure we aren't over-sizing our tables in relation to
+  the data therein.
 
   Numeric fields are converted using the relevant format to determine the width.
   Usage:
@@ -21,7 +22,7 @@
   @param libds Two part dataset (or view) reference.
   @param outds= The output dataset to create
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
   @li mf_getvarformat.sas

base/mp_guesspk.sas

@@ -22,7 +22,7 @@
   @param min_rows= The minimum number of rows a table should have in order to try
     and guess the PK.  Default=5.
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_getvarlist.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas

base/mp_hashdataset.sas

@@ -0,0 +1,75 @@
+/**
+  @file
+  @brief Returns a unique hash for a dataset
+  @details Ignores metadata attributes, used only to hash values. Compared
+  datasets must be in the same order.
+
+      %mp_hashdataset(sashelp.class,outds=myhash)
+
+      data _null_;
+        set work.myhash;
+        put hashkey=;
+      run;
+
+  ![sas md5 hash dataset log results](https://i.imgur.com/MqF98vk.png)
+
+  <h4> SAS Macros </h4>
+  @li mf_getattrn.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
+
+  @param [in] libds dataset to hash
+  @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
+  will contain one column (hashkey) with one observation (a hex32.
+  representation of the input hash)
+  |hashkey:$32.|
+  |---|
+  |28ABC74ABFC45F50794237BA5566E6CA|
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_hashdataset(
+  libds,
+  outds=
+)/*/STORE SOURCE*/;
+  %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
+    %put %str(WARN)ING: Dataset &libds is empty;, or is not a dataset;
+  %end;
+  %else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
+    %put %str(ERR)OR: Dataset &libds is not a dataset;
+  %end;
+  %else %do;
+    %local keyvar /* roll up the md5 */
+      prevkeyvar /* retain prev record md5 */
+      lastvar /* last var in input ds */
+      varlist var i;
+    /* avoid naming conflict for hash key vars */
+    %let keyvar=%mf_getuniquename();
+    %let prevkeyvar=%mf_getuniquename();
+    %let lastvar=%mf_getuniquename();
+    %let varlist=%mf_getvarlist(&libds);
+    data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
+      length &prevkeyvar &keyvar $32;
+      retain &prevkeyvar;
+      set &libds end=&lastvar;
+      /* hash should include previous row */
+      &keyvar=put(md5(&prevkeyvar
+      /* loop every column, hashing every individual value */
+    %do i=1 %to %sysfunc(countw(&varlist));
+      %let var=%scan(&varlist,&i,%str( ));
+      %if %mf_getvartype(&libds,&var)=C %then %do;
+          !!put(md5(trim(&var)),$hex32.)
+      %end;
+      %else %do;
+          !!put(md5(trim(put(&var*1,binary64.))),$hex32.)
+      %end;
+    %end;
+      ),$hex32.);
+      &prevkeyvar=&keyvar;
+      if &lastvar then output;
+    run;
+  %end;
+%mend;

base/mp_jsonout.sas

@@ -4,8 +4,11 @@
   @details PROC JSON is faster but will produce errs like the ones below if
   special chars are encountered.
 
-     >An object or array close is not valid at this point in the JSON text.
+  > ERROR: Some code points did not transcode.
-     >Date value out of range
+
+  > An object or array close is not valid at this point in the JSON text.
+
+  > Date value out of range
 
   If this happens, try running with ENGINE=DATASTEP.
 
@@ -14,7 +17,9 @@
         filename tmp temp;
         data class; set sashelp.class;run;
 
+        %mp_jsonout(OPEN,jref=tmp)
         %mp_jsonout(OBJ,class,jref=tmp)
+        %mp_jsonout(CLOSE,jref=tmp)
 
         data _null_;
         infile tmp;
@@ -27,18 +32,18 @@
   For more information see https://sasjs.io
 
   @param action Valid values:
-    * OPEN - opens the JSON
+    @li OPEN - opens the JSON
-    * OBJ - sends a table with each row as an object
+    @li OBJ - sends a table with each row as an object
-    * ARR - sends a table with each row in an array
+    @li ARR - sends a table with each row in an array
-    * CLOSE - closes the JSON
+    @li CLOSE - closes the JSON
 
   @param ds the dataset to send.  Must be a work table.
   @param jref= the fileref to which to send the JSON
   @param dslabel= the name to give the table in the exported JSON
   @param fmt= Whether to keep or strip formats from the table
-  @param engine= Which engine to use to send the JSON, options are:
+  @param engine= Which engine to use to send the JSON, valid options are:
-  * PROCJSON (default)
+    @li PROCJSON (default)
-  * DATASTEP 
+    @li DATASTEP (more reliable when data has non standard characters)
 
   @param dbg= DEPRECATED - was used to conditionally add PRETTY to
     proc json but this can cause line truncation in large files.
@@ -83,7 +88,8 @@
     %end;
     data _null_;file &jref mod ;
       put "["; call symputx('cols',0,'l');
-    proc sort data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
+    proc sort
+      data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
       out=_data_;
       by varnum;
 
@@ -122,7 +128,8 @@
       %end;
     %end;
     run;
-    /* write to temp loc to avoid _webout truncation - https://support.sas.com/kb/49/325.html */
+    /* write to temp loc to avoid _webout truncation
+      - https://support.sas.com/kb/49/325.html */
     filename _sjs temp lrecl=131068 encoding='utf-8';
     data _null_; file _sjs lrecl=131068 encoding='utf-8' mod;
       set &tempds;
@@ -158,7 +165,7 @@
 %end;
 
 %else %if &action=CLOSE %then %do;
-  data _null_;file &jref encoding='utf-8';
+  data _null_;file &jref encoding='utf-8' mod;
     put "}";
   run;
 %end;

base/mp_lib2cards.sas

@@ -1,21 +1,35 @@
 /**
   @file
   @brief Convert all library members to CARDS files
-  @details Gets list of members then calls the <code>%mp_ds2cards()</code>
+  @details Gets list of members then calls the <code>%mp_ds2cards()</code> macro.
-            macro
+  Usage:
-    usage:
 
       %mp_lib2cards(lib=sashelp
           , outloc= C:\temp )
 
-  <h4> Dependencies </h4>
+  The output will be one cards file in the `outloc` directory per dataset in the
+  input `lib` library.  If the `outloc` directory does not exist, it is created.
+
+  To create a single SAS file with the first 1000 records of each table in a
+  library you could use this syntax:
+
+      %mp_lib2cards(lib=sashelp
+          , outloc= /tmp
+          , outfile= myfile.sas
+          , maxobs= 1000
+      )
+
+  <h4> SAS Macros </h4>
   @li mf_mkdir.sas
+  @li mf_trimstr.sas
   @li mp_ds2cards.sas
 
-  @param lib= Library in which to convert all datasets
+  @param [in] lib= Library in which to convert all datasets
-  @param outloc= Location in which to store output.  Defaults to WORK library.
+  @param [out] outloc= Location in which to store output.  Defaults to WORK
-    Do not use a trailing slash (my/path not my/path/).  No quotes.
+    library. No quotes.
-  @param maxobs= limit output to the first <code>maxobs</code> observations
+  @param [out] outfile= Optional output file NAME - if provided, then will create
+  a single output file instead of one file per input table.
+  @param [in] maxobs= limit output to the first <code>maxobs</code> observations
 
   @version 9.2
   @author Allan Bowe
@@ -25,6 +39,7 @@
     ,outloc=%sysfunc(pathname(work)) /* without trailing slash */
     ,maxobs=max
     ,random_sample=NO
+    ,outfile=0
 )/*/STORE SOURCE*/;
 
 /* Find the tables */
@@ -36,6 +51,10 @@ select distinct lowcase(memname)
   from dictionary.tables
   where upcase(libname)="%upcase(&lib)";
 
+/* trim trailing slash, if provided */
+%let outloc=%mf_trimstr(&outloc,/);
+%let outloc=%mf_trimstr(&outloc,\);
+
 /* create the output directory */
 %mf_mkdir(&outloc)
 
@@ -43,9 +62,17 @@ select distinct lowcase(memname)
 %do x=1 %to %sysfunc(countw(&memlist));
   %let ds=%scan(&memlist,&x);
   %mp_ds2cards(base_ds=&lib..&ds
-      ,cards_file="&outloc/&ds..sas"
     ,maxobs=&maxobs
-      ,random_sample=&random_sample)
+    ,random_sample=&random_sample
+  %if "&outfile" ne "0" %then %do;
+    ,append=YES
+    ,cards_file="&outloc/&outfile"
+  %end;
+  %else %do;
+    ,append=NO
+    ,cards_file="&outloc/&ds..sas"
+  %end;
+  )
 %end;
 
 %mend;

base/mp_mdtablewrite.sas

@@ -0,0 +1,98 @@
+/**
+  @file
+  @brief Create a Markdown Table from a dataset
+  @details A markdown table is a simple table representation for use in
+  documents written in markdown format.
+
+  An online generator is available here:
+  https://www.tablesgenerator.com/markdown_tables
+
+  This structure is also used by the Macro Core library for documenting input/
+  output datasets, as well as the sasjs/cli tool for documenting inputs/outputs
+  for web services.
+
+  We take the standard definition one step further by embedding the informat
+  in the table header row, like so:
+
+      |var1:$|var2:best.|var3:date9.|
+      |---|---|---|
+      |some text|42|01JAN1960|
+      |blah|1|31DEC1999|
+
+  Which resolves to:
+
+  |var1:$|var2:best.|var3:date9.|
+  |---|---|---|
+  |some text|42|01JAN1960|
+  |blah|1|31DEC1999|
+
+
+  Usage:
+
+      %mp_mdtablewrite(libds=sashelp.class,showlog=YES)
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getvarlist.sas
+  @li mf_getvarformat.sas
+
+  @param [in] libds= the library / dataset to create or read from.
+  @param [out] fref= Fileref to contain the markdown. Default=mdtable.
+  @param [out] showlog= set to YES to show the markdown in the log. Default=NO.
+
+  @version 9.3
+  @author Allan Bowe
+**/
+
+%macro mp_mdtablewrite(
+  libds=,
+  fref=mdtable,
+  showlog=NO
+)/*/STORE SOURCE*/;
+
+/* check fileref is assigned */
+%if %sysfunc(fileref(&fref)) > 0 %then %do;
+  filename &fref temp;
+%end;
+
+%local vars;
+%let vars=%mf_getvarlist(&libds);
+
+/* create the header row */
+data _null_;
+  file &fref;
+  length line $32767;
+  put '|'
+%local i var fmt;
+%do i=1 %to %sysfunc(countw(&vars));
+  %let var=%scan(&vars,&i);
+  %let fmt=%mf_getvarformat(&libds,&var,force=1);
+  "&var:&fmt|"
+%end;
+  ;
+  put '|'
+%do i=1 %to %sysfunc(countw(&vars));
+  "---|"
+%end;
+  ;
+run;
+
+/* write out the data */
+data _null_;
+  file &fref mod dlm='|' lrecl=32767;
+  set &libds ;
+  length line $32767;
+  line=cats('|',%mf_getvarlist(&libds,dlm=%str(,'|',)),'|');
+  put line;
+run;
+
+%if %upcase(&showlog)=YES %then %do;
+  options ps=max;
+  data _null_;
+    infile &fref;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_mdtablewrite;

base/mp_prevobs.sas

@@ -0,0 +1,88 @@
+/**
+  @file
+  @brief Enables previous observations to be re-instated
+  @details Remembers the last X observations by storing them in a hash table.
+  Is a convenience over the use of lag() or retain, when an entire observation
+  needs to be restored.
+
+  This macro will also restore automatic variables (such as _n_ and _error_).
+
+  Example Usage:
+
+      data example;
+        set sashelp.class;
+        calc_var=_n_*3;
+        %* initialise hash and save from PDV ;
+        %mp_prevobs(INIT,history=2)
+        if _n_ =10 then do;
+          %* fetch previous but 1 record;
+          %mp_prevobs(FETCH,-2)
+          put _n_= name= age= calc_var=;
+          %* fetch previous record;
+          %mp_prevobs(FETCH,-1)
+          put _n_= name= age= calc_var=;
+          %* reinstate current record ;
+          %mp_prevobs(FETCH,0)
+          put _n_= name= age= calc_var=;
+        end;
+      run;
+
+  Result:
+
+  <img src="https://imgur.com/PSjHoET.png" alt="mp_prevobs sas" width="400"/>
+
+  Credit is made to `data _null_` for authoring this very helpful paper:
+  https://www.lexjansen.com/pharmasug/2008/cc/CC08.pdf
+
+  @param action Either FETCH a current or previous record, or INITialise.
+  @param record The relative (to current) position of the previous observation
+    to return.
+  @param history= The number of records to retain in the hash table. Default=5
+  @param prefix= the prefix to give to the variables used to store the hash name
+    and index. Default=mp_prevobs
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_prevobs(action,record,history=5,prefix=mp_prevobs
+)/*/STORE SOURCE*/;
+%let action=%upcase(&action);
+%let prefix=%upcase(&prefix);
+%let record=%eval((&record+0) * -1);
+
+%if &action=INIT %then %do;
+
+  if _n_ eq 1 then do;
+    attrib &prefix._VAR length=$64;
+    dcl hash &prefix._HASH(ordered:'Y');
+    &prefix._KEY=0;
+    &prefix._HASH.defineKey("&prefix._KEY");
+    do while(1);
+      call vnext(&prefix._VAR);
+      if &prefix._VAR='' then leave;
+      if &prefix._VAR eq "&prefix._VAR" then continue;
+      else if &prefix._VAR eq "&prefix._KEY" then continue;
+      &prefix._HASH.defineData(&prefix._VAR);
+    end;
+    &prefix._HASH.defineDone();
+  end;
+  /* this part has to happen before FETCHing */
+  &prefix._KEY+1;
+  &prefix._rc=&prefix._HASH.add();
+  if &prefix._rc then putlog 'adding' &prefix._rc=;
+  %if &history>0 %then %do;
+    if &prefix._key>&history+1 then
+      &prefix._HASH.remove(key: &prefix._KEY - &history - 1);
+    if &prefix._rc then putlog 'removing' &prefix._rc=;
+  %end;
+%end;
+%else %if &action=FETCH %then %do;
+  if &record>&prefix._key then putlog "Not enough records in &Prefix._hash yet";
+  else &prefix._rc=&prefix._HASH.find(key: &prefix._KEY - &record);
+  if &prefix._rc then putlog &prefix._rc= " when fetching " &prefix._KEY=
+    "with record &record and " _n_=;
+%end;
+
+%mend;

base/mp_resetoption.sas

@@ -1,7 +1,8 @@
 /**
   @file
   @brief Reset an option to original value
-  @details Inspired by the SAS Jedi - https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options/
+  @details Inspired by the SAS Jedi -
+https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options
     Called as follows:
 
     options obs=30;

base/mp_searchdata.sas

@@ -3,7 +3,7 @@
   @brief Searches all data in a library
   @details
   Scans an entire library and creates a copy of any table
-    containing a specific string or numeric value.  Only 
+    containing a specific string OR numeric value.  Only
     matching records are written out.
     If both a string and numval are provided, the string
     will take precedence.
@@ -20,12 +20,13 @@
   @param ds= the dataset to search (leave blank to search entire library)
   @param string= the string value to search
   @param numval= the numeric value to search (must be exact)
-  @param outloc= the directory in which to create the output datasets with matching
+  @param outloc= the directory in which to create the output datasets with
-    rows.  Will default to a subfolder in the WORK library.
+    matching rows.  Will default to a subfolder in the WORK library.
-  @param outobs= set to a positive integer to restrict the number of observations
+  @param outobs= set to a positive integer to restrict the number of
+    observations
   @param filter_text= add a (valid) filter clause to further filter the results
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
   @li mf_mkdir.sas
@@ -44,9 +45,14 @@
   ,filter_text=%str(1=1)
 )/*/STORE SOURCE*/;
 
-%local table_list table table_num table colnum col start_tm vars type coltype;
+%local table_list table table_num table colnum col start_tm check_tm vars type
+  coltype;
 %put process began at %sysfunc(datetime(),datetime19.);
 
+%if &syscc ge 4 %then %do;
+  %put %str(WAR)NING: SYSCC=&syscc on macro entry;
+  %return;
+%end;
 
 %if &string = %then %let type=N;
 %else %let type=C;
@@ -78,6 +84,7 @@ proc sql
     %put NO COLUMNS IN &lib..&table!  This will be skipped.;
   %end;
   %else %do;
+    %let check_tm=%sysfunc(datetime());
     /* build sql statement */
     create table mpsearch.&table as select * from &lib..&table
       where %unquote(&filter_text) and
@@ -88,14 +95,20 @@ proc sql
       %let coltype=%mf_getvartype(&lib..&table,&col);
       %if &type=C and &coltype=C %then %do;
         /* if a char column, see if it contains the string */
-        or (&col ? "&string")
+        or ("&col"n ? "&string")
       %end;
       %else %if &type=N and &coltype=N %then %do;
         /* if numeric match exactly */
-        or (&col = &numval)
+        or ("&col"n = &numval)
       %end;
     %end;
     );
+    %put Search query for &table took
+      %sysevalf(%sysfunc(datetime())-&check_tm) seconds;
+    %if &sqlrc ne 0 %then %do;
+      %put %str(WAR)NING: SQLRC=&sqlrc when processing &table;
+      %return;
+    %end;
     %if %mf_nobs(mpsearch.&table)=0 %then %do;
       drop table mpsearch.&table;
     %end;

base/mp_setkeyvalue.sas

@@ -6,7 +6,7 @@
       %mp_setkeyvalue(someindex,22,type=N)
       %mp_setkeyvalue(somenewindex,somevalue)
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_existds.sas
 
   @param key Provide a key on which to perform the lookup
@@ -26,7 +26,7 @@
 
   %if not (%mf_existds(&libds)) %then %do;
     data &libds (index=(key/unique));
-      length key $32 valc $256 valn 8 type $1;
+      length key $64 valc $2048 valn 8 type $1;
       call missing(of _all_);
       stop;
     run;

base/mp_stprequests.sas

@@ -1,7 +1,8 @@
 /**
   @file
   @brief Capture session start / finish times and request details
-  @details For details, see http://www.rawsas.com/2015/09/logging-of-stored-process-server.html.
+  @details For details, see
+  https://rawsas.com/event-logging-of-stored-process-server-sessions.
     Requires a base table in the following structure (name can be changed):
 
     proc sql;

base/mp_streamfile.sas

@@ -1,22 +1,24 @@
 /**
-  @file mp_streamfile.sas
+  @file
   @brief Streams a file to _webout according to content type
   @details Will set headers using appropriate functions (SAS 9 vs Viya) and send
   content as a binary stream.
 
   Usage:
 
-      filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      filename mc url
+        "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
 
       %mp_streamfile(contenttype=csv,inloc=/some/where.txt,outname=myfile.txt)
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_getplatform.sas
   @li mp_binarycopy.sas
 
   @param contenttype= Either TEXT, ZIP, CSV, EXCEL (default TEXT)
   @param inloc= /path/to/file.ext to be sent
+  @param inref= fileref of file to be sent (if provided, overrides `inloc`)
   @param outname= the name of the file, as downloaded by the browser
 
   @author Allan Bowe
@@ -27,14 +29,27 @@
 %macro mp_streamfile(
   contenttype=TEXT
   ,inloc=
+  ,inref=0
   ,outname=
 )/*/STORE SOURCE*/;
 
 %let contentype=%upcase(&contenttype);
 %local platform; %let platform=%mf_getplatform();
 
+
+/**
+  * check engine type to avoid the below err message:
+  * > Function is only valid for filerefs using the CACHE access method.
+  */
+%local streamweb;
+%let streamweb=0;
+data _null_;
+  set sashelp.vextfl(where=(upcase(fileref)="_WEBOUT"));
+  if xengine='STREAM' then call symputx('streamweb',1,'l');
+run;
+
 %if &contentype=ZIP %then %do;
-  %if &platform=SASMETA %then %do;
+  %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/zip');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -48,7 +63,7 @@
 %end;
 %else %if &contentype=EXCEL %then %do;
   /* suitable for XLS format */
-  %if &platform=SASMETA %then %do;
+  %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/vnd.ms-excel');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -61,20 +76,22 @@
   %end;
 %end;
 %else %if &contentype=XLSX %then %do;
-  %if &platform=SASMETA %then %do;
+  %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
+      rc=stpsrv_header('Content-type',
+        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
-      contenttype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' 
+      contenttype=
+        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
 %else %if &contentype=TEXT %then %do;
-  %if &platform=SASMETA %then %do;
+  %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/text');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -87,7 +104,7 @@
   %end;
 %end;
 %else %if &contentype=CSV %then %do;
-  %if &platform=SASMETA %then %do;
+  %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/csv');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -110,6 +127,11 @@
   %return;
 %end;
 
-%mp_binarycopy(inloc="&inloc",outref=_webout)
+%if &inref ne 0 %then %do;
+  %mp_binarycopy(inref=&inref,outref=_webout)
+%end;
+%else %do;
+  %mp_binarycopy(inloc="&inloc",outref=_webout)
+%end;
 
 %mend;

base/mp_testjob.sas

@@ -0,0 +1,92 @@
+/**
+  @file
+  @brief Runs arbitrary code for a specified amount of time
+  @details Executes a series of procs and data steps to enable performance
+  testing of arbitrary jobs.
+
+      %mp_testjob(
+        duration=60*5
+      )
+
+  @param [in] duration= the time in seconds which the job should run for. Actual
+  time may vary, as the check is done in between steps.  Default = 30 (seconds).
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquelibref.sas
+  @li mf_getuniquename.sas
+  @li mf_mkdir.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mp_testjob(duration=30
+)/*/STORE SOURCE*/;
+%local lib dir ds1 ds2 ds3 start_tm i;
+
+%let start_tm=%sysfunc(datetime());
+%let duration=%sysevalf(&duration);
+
+/* create a temporary library in WORK */
+%let lib=%mf_getuniquelibref();
+%let dir=%mf_getuniquename();
+%mf_mkdir(%sysfunc(pathname(work))/&dir)
+libname &lib "%sysfunc(pathname(work))/&dir";
+
+/* loop through until time expires */
+%let ds1=%mf_getuniquename();
+%let ds2=%mf_getuniquename();
+%let ds3=%mf_getuniquename();
+%do i=0 %to 1;
+
+  /* create big dataset */
+  data &lib..&ds1(compress=no );
+    do x=1 to 1000000;
+      randnum0=ranuni(0)*3;
+      randnum1=ranuni(0)*2;
+      bigchar=repeat('A',300);
+      output;
+    end;
+  run;
+  %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
+
+  proc summary ;
+    class randnum0 randnum1;
+    output out=&lib..&ds2;
+  run;quit;
+  %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
+
+  /* add more data */
+  proc sql;
+  create table &lib..&ds3 as
+    select *, ranuni(0)*10 as randnum2
+  from &lib..&ds1
+  order by randnum1;
+  quit;
+  %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
+
+  proc sort data=&lib..&ds3;
+    by descending x;
+  run;
+  %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
+
+  /* wait 5 seconds */
+  data _null_;
+    call sleep(5,1);
+  run;
+  %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
+
+  %let i=0;
+
+%end;
+
+%gate:
+%put time is up!;
+proc datasets lib=&lib kill;
+run;
+quit;
+libname &lib clear;
+
+
+%mend;

base/mp_testservice.sas

@@ -0,0 +1,259 @@
+/**
+  @file mp_testservice.sas
+  @brief Will execute a test against a SASjs web service on SAS 9 or Viya
+  @details Prepares the input files and retrieves the resulting datasets from
+  the response JSON.
+
+      %mp_testjob(
+        duration=60*5
+      )
+
+  Note - the _webout fileref should NOT be assigned prior to running this macro.
+
+  @param [in] program The _PROGRAM endpoint to test
+  @param [in] inputfiles=(0) A list of space seperated fileref:filename pairs as
+    follows:
+        inputfiles=inref:filename inref2:filename2
+  @param [in] inputparams=(0) A dataset containing name/value pairs in the
+    following format:
+    |name:$32|value:$1000|
+    |---|---|
+    |stpmacname|some value|
+    |mustbevalidname|can be anything, oops, %abort!!|
+
+  @param [in] debug= (log) Provide the _debug value
+  @param [in] viyaresult=(WEBOUT_JSON) The Viya result type to return.  For
+    more info, see mv_getjobresult.sas
+  @param [out] outlib= (0) Output libref to contain the final tables.  Set to
+    0 if the service output is not in JSON format.
+  @param [out] outref= (0) Output fileref to create, to contain the full _webout
+    response.
+
+  <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+  @li mv_getjobresult.sas
+  @li mv_jobflow.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mp_testservice(program,
+  inputfiles=0,
+  inputparams=0,
+  debug=log,
+  outlib=0,
+  outref=0,
+  viyaresult=WEBOUT_JSON
+)/*/STORE SOURCE*/;
+%local mdebug;
+%if &debug ne 0 %then %do;
+  %let mdebug=1;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let mdebug=0;
+
+/* sanitise inputparams */
+%local pcnt;
+%let pcnt=0;
+%if &inputparams ne 0 %then %do;
+  data _null_;
+    set &inputparams;
+    if not nvalid(name,'v7') then putlog (_all_)(=);
+    else if name in (
+      'program','inputfiles','inputparams','debug','outlib','outref'
+    ) then putlog (_all_)(=);
+    else do;
+      x+1;
+      call symputx(name,quote(cats(value)),'l');
+      call symputx('pval'!!left(x),name,'l');
+      call symputx('pcnt',x,'l');
+    end;
+  run;
+  %mp_abort(iftrue= (%mf_nobs(&inputparams) ne &pcnt)
+    ,mac=&sysmacroname
+    ,msg=%str(Invalid values in &inputparams)
+  )
+%end;
+
+
+%local fref1 webref;
+%let fref1=%mf_getuniquefileref();
+%let webref=%mf_getuniquefileref();
+
+%local platform;
+%let platform=%mf_getplatform();
+%if &platform=SASMETA %then %do;
+
+  /* parse the input files */
+  %local webcount i var;
+  %if %quote(&inputfiles) ne 0 %then %do;
+    %let webcount=%sysfunc(countw(&inputfiles));
+    %put &=webcount;
+    %do i=1 %to &webcount;
+      %let var=%scan(&inputfiles,&i,%str( ));
+      %local webfref&i webname&i;
+      %let webref&i=%scan(&var,1,%str(:));
+      %let webname&i=%scan(&var,2,%str(:));
+      %put webref&i=&&webref&i;
+      %put webname&i=&&webname&i;
+    %end;
+  %end;
+  %else %let webcount=0;
+
+  proc stp program="&program";
+    inputparam _program="&program"
+  %do i=1 %to &webcount;
+    %if &webcount=1 %then %do;
+      _webin_fileref="&&webref&i"
+      _webin_name="&&webname&i"
+    %end;
+    %else %do;
+      _webin_fileref&i="&&webref&i"
+      _webin_name&i="&&webname&i"
+    %end;
+  %end;
+    _webin_file_count="&webcount"
+    _debug="&debug"
+  %do i=1 %to &pcnt;
+    /* resolve name only, proc stp fetches value */
+    &&pval&i=&&&&&&pval&i
+  %end;
+    ;
+  %do i=1 %to &webcount;
+    inputfile &&webref&i;
+  %end;
+    outputfile _webout=&webref;
+  run;
+
+  data _null_;
+    infile &webref;
+    file &fref1;
+    input;
+    length line $10000;
+    if index(_infile_,'>>weboutBEGIN<<') then do;
+        line=tranwrd(_infile_,'>>weboutBEGIN<<','');
+        put line;
+    end;
+    else if index(_infile_,'>>weboutEND<<') then do;
+        line=tranwrd(_infile_,'>>weboutEND<<','');
+        put line;
+        stop;
+    end;
+    else put _infile_;
+  run;
+  data _null_;
+    infile &fref1;
+    input;
+    put _infile_;
+  run;
+  %if &outlib ne 0 %then %do;
+    libname &outlib json (&fref1);
+  %end;
+  %if &outref ne 0 %then %do;
+    filename &outref temp;
+    %mp_binarycopy(inref=&webref,outref=&outref)
+  %end;
+
+%end;
+%else %if &platform=SASVIYA %then %do;
+
+  /* prepare inputparams */
+  %local ds1;
+  %let ds1=%mf_getuniquename();
+  %if "&inputparams" ne "0" %then %do;
+    proc transpose data=&inputparams out=&ds1;
+      id name;
+      var value;
+    run;
+  %end;
+  %else %do;
+    data &ds1;run;
+  %end;
+
+  /* parse the input files - convert to sasjs params */
+  %local webcount i var sasjs_tables;
+  %if %quote(&inputfiles) ne 0 %then %do;
+    %let webcount=%sysfunc(countw(&inputfiles));
+    %put &=webcount;
+    %do i=1 %to &webcount;
+      %let var=%scan(&inputfiles,&i,%str( ));
+      %local webfref&i webname&i sasjs&i.data;
+      %let webref&i=%scan(&var,1,%str(:));
+      %let webname&i=%scan(&var,2,%str(:));
+      %put webref&i=&&webref&i;
+      %put webname&i=&&webname&i;
+
+      %let sasjs_tables=&sasjs_tables &&webname&i;
+      data _null_;
+        infile &&webref&i lrecl=32767;
+        input;
+        if _n_=1 then call symputx("sasjs&i.data",_infile_);
+        else call symputx(
+          "sasjs&i.data",cats(symget("sasjs&i.data"),'0D0A'x,_infile_)
+        );
+        putlog "&sysmacroname infile: " _infile_;
+      run;
+      data &ds1;
+        set &ds1;
+        length sasjs&i.data $32767 sasjs_tables $1000;
+        sasjs&i.data=symget("sasjs&i.data");
+        sasjs_tables=symget("sasjs_tables");
+      run;
+    %end;
+  %end;
+  %else %let webcount=0;
+
+  data &ds1;
+    retain _program "&program";
+    set &ds1;
+    putlog "&sysmacroname inputparams:";
+    putlog (_all_)(=);
+  run;
+
+  %mv_jobflow(inds=&ds1
+    ,maxconcurrency=1
+    ,outds=work.results
+    ,outref=&fref1
+    ,mdebug=&mdebug
+  )
+  /* show the log */
+  data _null_;
+    infile &fref1;
+    input;
+    putlog _infile_;
+  run;
+  /* get the uri to fetch results */
+  data _null_;
+    set work.results;
+    call symputx('uri',uri);
+    putlog "&sysmacroname: fetching results for " uri;
+  run;
+  /* fetch results from webout.json */
+  %mv_getjobresult(uri=&uri,
+    result=&viyaresult,
+    outref=&outref,
+    outlib=&outlib,
+    mdebug=&mdebug
+  )
+
+%end;
+%else %do;
+  %put %str(ERR)OR: Unrecognised platform:  &platform;
+%end;
+
+%if &mdebug=0 %then %do;
+  filename &webref clear;
+%end;
+%else %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+
+%mend mp_testservice;

base/mp_testwritespeedlibrary.sas

@@ -0,0 +1,59 @@
+/**
+  @file mp_testwritespeedlibrary.sas
+  @brief Tests the write speed of a new table in a SAS library
+  @details Will create a new table of a certain size in an
+  existing SAS library.  The table will have one column,
+  and will be subsequently deleted.
+
+      %mp_testwritespeedlibrary(
+        lib=work
+        ,size=0.5
+        ,outds=work.results
+      )
+
+  @param lib= (WORK) The library in which to create the table
+  @param size= (0.1) The size in GB of the table to create
+  @param outds= (WORK.RESULTS) The output dataset to be created.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_existds.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mp_testwritespeedlibrary(lib=WORK
+  ,outds=work.results
+  ,size=0.1
+)/*/STORE SOURCE*/;
+%local ds start;
+
+/* find an unused, unique name for the new table */
+%let ds=%mf_getuniquename();
+%do %until(%mf_existds(&lib..&ds)=0);
+  %let ds=%mf_getuniquename();
+%end;
+
+%let start=%sysfunc(datetime());
+
+data &lib..&ds(compress=no keep=x);
+  header=128*1024;
+  size=(1073741824/8 * &size) - header;
+  do x=1 to size;
+    output;
+  end;
+run;
+
+proc sql;
+drop table &lib..&ds;
+
+data &outds;
+  lib="&lib";
+  start_dttm=put(&start,datetime19.);
+  end_dttm=put(datetime(),datetime19.);
+  duration_seconds=end_dttm-start_dttm;
+run;
+
+%mend;

base/mp_tree.sas

@@ -0,0 +1,71 @@
+/**
+  @file
+  @brief Recursively scans a directory tree to get all subfolders and content
+  @details
+  Usage:
+
+      %mp_tree(dir=/tmp, outds=work.tree)
+
+  Credits:
+
+  Roger Deangelis:
+https://communities.sas.com/t5/SAS-Programming/listing-all-files-within-a-directory-and-subdirectories/m-p/332616/highlight/true#M74887
+
+  Tom:
+https://communities.sas.com/t5/SAS-Programming/listing-all-files-of-all-types-from-all-subdirectories/m-p/334113/highlight/true#M75419
+
+
+  @param dir= Directory to be scanned (default=/tmp)
+  @param outds= Dataset to create (default=work.mp_tree)
+
+  @returns outds contains the following variables:
+
+    - `dir`: a flag (1/0) to say whether it is a directory or not.  This is not
+      reliable - folders that you do not have permission to open will be flagged
+      as directories.
+    - `ext`: file extension
+    - `filename`: file name
+    - `dirname`: directory name
+    - `fullpath`: directory + file name
+
+  @version 9.2
+**/
+
+%macro mp_tree(dir=/tmp
+  ,outds=work.mp_tree
+)/*/STORE SOURCE*/;
+
+data &outds ;
+  length dir 8 ext filename dirname $256 fullpath $512 ;
+  call missing(of _all_);
+  fullpath = "&dir";
+run;
+
+%local sep;
+%if &sysscp=WIN or &SYSSCP eq DNTHOST %then %let sep=\;
+%else %let sep=/;
+
+data &outds ;
+  modify &outds ;
+  retain sep "&sep";
+  rc=filename('tmp',fullpath);
+  dir_id=dopen('tmp');
+  dir = (dir_id ne 0) ;
+  if dir then dirname=fullpath;
+  else do;
+    filename=scan(fullpath,-1,sep) ;
+    dirname =substrn(fullpath,1,length(fullpath)-length(filename));
+    if index(filename,'.')>1 then ext=scan(filename,-1,'.');
+  end;
+  replace;
+  if dir then do;
+    do i=1 to dnum(dir_id);
+      fullpath=cats(dirname,sep,dread(dir_id,i));
+      output;
+    end;
+    rc=dclose(dir_id);
+  end;
+  rc=filename('tmp');
+run;
+
+%mend;

base/mp_unzip.sas

@@ -12,7 +12,7 @@
 
       %mp_unzip(ziploc="/some/file.zip",outdir=/some/folder)
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_mkdir.sas
   @li mf_getuniquefileref.sas
 

base/mp_updatevarlength.sas

@@ -18,7 +18,7 @@
   @param var The variable to modify
   @param len The new length to apply
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_existds.sas
   @li mp_abort.sas
   @li mf_existvar.sas

base/mp_validatecol.sas

@@ -0,0 +1,66 @@
+/**
+  @file
+  @brief Used to validate variables in a dataset
+  @details Useful when sanitising inputs, to ensure that they arrive with a
+  certain pattern.
+  Usage:
+
+      data test;
+        infile datalines4 dsd;
+        input;
+        libds=_infile_;
+        %mp_validatecol(libds,LIBDS,is_libds)
+      datalines4;
+      some.libname
+      !lib.blah
+      %abort
+      definite.ok
+      not.ok!
+      nineletrs._
+      ;;;;
+      run;
+
+  @param [in] incol The column to be validated
+  @param [in] rule The rule to apply.  Current rules:
+    @li ISNUM - checks if the variable is numeric
+    @li LIBDS - matches LIBREF.DATASET format
+  @param [out] outcol The variable to create, with the results of the match
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+
+  @version 9.3
+**/
+
+%macro mp_validatecol(incol,rule,outcol);
+
+/* tempcol is given a unique name with every invocation */
+%local tempcol;
+%let tempcol=%mf_getuniquename();
+
+%if &rule=ISNUM %then %do;
+  /*
+    credit SØREN LASSEN
+    https://sasmacro.blogspot.com/2009/06/welcome-isnum-macro.html
+  */
+  &tempcol=input(&incol,?? best32.);
+  if missing(&tempcol) then &outcol=0;
+  else &outcol=1;
+  drop &tempcol;
+%end;
+%else %if &rule=LIBDS %then %do;
+  /* match libref.dataset */
+  if _n_=1 then do;
+    retain &tempcol;
+    &tempcol=prxparse('/^[_a-z]\w{0,7}\.[_a-z]\w{0,31}$/i');
+    if missing(&tempcol) then do;
+      putlog "%str(ERR)OR: Invalid expression for LIBDS";
+      stop;
+    end;
+    drop &tempcol;
+  end;
+  if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
+  else &outcol=0;
+%end;
+
+%mend mp_validatecol;

base/mp_zip.sas

@@ -12,7 +12,7 @@
   be sure that _debug is not set (else the SPWA will send non zipped content
   as well).
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mp_dirlist.sas
 
   @param in= unquoted filepath, dataset of files or directory to zip

build.py

@@ -9,19 +9,22 @@ for file in files:
     ml = open('lua/' + name + '.sas', "w")
     ml.write("/**\n")
     ml.write("  @file " + name + '.sas\n')
-    ml.write("  @brief Creates the " + basename + " file\n")
+    ml.write("  @brief Compiles the " + basename + " lua file\n")
     ml.write("  @details Writes " + basename + " to the work directory\n")
+    ml.write("  and then includes it.\n")
     ml.write("  Usage:\n\n")
     ml.write("      %" + name + "()\n\n")
     ml.write("**/\n\n")
     ml.write("%macro " + name + "();\n")
     ml.write("data _null_;\n")
-    ml.write("  file \"%sysfunc(pathname(work))/" + basename + "\";\n")
+    ml.write("  file \"%sysfunc(pathname(work))/" + name + ".lua\";\n")
     with open(file) as infile:
         for line in infile:
           ml.write("  put '" + line.rstrip().replace("'","''") + " ';\n")
-    ml.write("run;\n")
+    ml.write("run;\n\n")
+    ml.write("%inc \"%sysfunc(pathname(work))/" + name + ".lua\";\n\n")
     ml.write("%mend;\n")
+
 ml.close()
 
 # prepare web files

doxy/batch.bat

@@ -1,6 +0,0 @@
-ECHO on
-
-rmdir /s /q C:\DEVELOPMENT\output
-mkdir C:\DEVELOPMENT\output
-
-doxygen.exe Doxyfile

doxy/build.sh

@@ -1,40 +0,0 @@
-#!/bin/bash
-####################################################################
-# PROJECT: Macro Core Docs Build                                   #
-####################################################################
-
-BUILD_FOLDER="/tmp/macrocore_docs"
-
-# move to project root
-cd ..
-
-# create build directory
-rm -rf $BUILD_FOLDER
-mkdir $BUILD_FOLDER
-
-# copy relevant files
-cp -r base $BUILD_FOLDER
-cp -r meta $BUILD_FOLDER
-cp -r metax $BUILD_FOLDER
-cp -r viya $BUILD_FOLDER
-cp -r doxy $BUILD_FOLDER
-cp main.dox $BUILD_FOLDER
-cp doxy/Doxyfile $BUILD_FOLDER
-
-# update Doxyfile and generate
-cd $BUILD_FOLDER
-echo "OUTPUT_DIRECTORY=$BUILD_FOLDER/out" >> $BUILD_FOLDER/Doxyfile
-echo "INPUT+=main.dox" >> $BUILD_FOLDER/Doxyfile
-doxygen Doxyfile
-
-# refresh github pages site
-git clone git@github.com:sasjs/core.github.io.git
-cd core.github.io
-rm -r *
-mv $BUILD_FOLDER/out/doxy/* .
-echo 'core.sasjs.io' > CNAME
-git add *
-git commit -m "build.sh build on $(date +%F:%H:%M:%S)"
-git push
-
-echo "check it out:  https://sasjs.github.io/core.github.io/files.html"

doxy/new_footer.html

@@ -1,22 +0,0 @@
-<!--BEGIN GENERATE_TREEVIEW-->
-    <li class="footer"><b>$generatedby</b>
-    <a href="http://www.doxygen.org/index.html">
-    <img class="footer" src="doxygen.png" alt="doxygen"/></a>
-   <i> For more information visit the </i> <a href="https://github.com/sasjs/core">Macro Core library</a>.</li>
-
-   </ul>
- </div>
-<!--END GENERATE_TREEVIEW-->
-<!--BEGIN !GENERATE_TREEVIEW-->
-<hr class="footer"/>
-<table width="100%"><tbody><tr><td>
-For more information visit the <a href="https://github.com/sasjs/core">Macro Core library</a>.
-</td><td><address class="footer"><small>
-&copy;$year<br/>
-$generatedby <a href="http://www.doxygen.org/index.html">
-<!--<img class="footer" src="$relpath$doxygen.png" alt="doxygen"/>-->doxygen
-</a> $doxygenversion
-</small></address></tr></tbody></table>
-<!--END !GENERATE_TREEVIEW-->
-</body>
-</html>

doxy/new_header.html

@@ -1,72 +0,0 @@
-<!-- HTML header for doxygen 1.8.14-->
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml">
-<head>
-<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
-<meta http-equiv="X-UA-Compatible" content="IE=9"/>
-<meta name="generator" content="Doxygen $doxygenversion"/>
-<meta name="viewport" content="width=device-width, initial-scale=1"/>
-<!--BEGIN PROJECT_NAME--><title>$projectname: $title</title><!--END PROJECT_NAME-->
-<!--BEGIN !PROJECT_NAME--><title>$title</title><!--END !PROJECT_NAME-->
-<link href="$relpath^tabs.css" rel="stylesheet" type="text/css"/>
-<script type="text/javascript" src="$relpath^jquery.js"></script>
-<script type="text/javascript" src="$relpath^dynsections.js"></script>
-$treeview
-$search
-$mathjax
-<link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
-<link REL="icon" HREF="https://sasjs.io/img/runningman.jpg">
-
-$extrastylesheet
-</head>
-<body>
-<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
-
-<!--BEGIN TITLEAREA-->
-<div id="titlearea" style='background-color:white' >
-<table cellspacing="0" cellpadding="0">
- <tbody>
- <tr style="height: 26px;">
-  <!--BEGIN PROJECT_LOGO-->
-  <td id="projectlogo"></td>
-
-  <!--END PROJECT_LOGO-->
-<!--BEGIN PROJECT_NAME-->
-  <td>
-  <div id="projectname">
-    <!--a href=".">
-      <img alt="Macro Core" src="https://macropeople.com/wp-content/uploads/2018/05/macropeople2014retina_V2.png" height=60/>
-     </a-->
-     <a href=".">
-        <img alt="Macro Core" src="./Macro_core_website_1.png" height=60/>
-       </a>
-   </div>
-  </td>
-  <td>
-    <table style="padding-left: 2em;" cellspacing="0" cellpadding="0">
-      <tr><td> Production Ready Macros for SAS Application Developers</td></tr>
-      <tr><td><a href="https://github.com/sasjs/core">
-          https://github.com/sasjs/core
-      </a></td></tr>
-    </table>
-  </td>
-  <!--END PROJECT_NAME-->
-  <!--BEGIN !PROJECT_NAME-->
-   <!--BEGIN PROJECT_BRIEF-->
-   <!--END PROJECT_BRIEF-->
-  <!--END !PROJECT_NAME-->
-  <div class="header">
-  <div class="headertitle">
-
-  <!--BEGIN DISABLE_INDEX-->
-   <!--BEGIN SEARCHENGINE-->
-
-   <td>$searchbox</td>
-   <!--END SEARCHENGINE-->
-  <!--END DISABLE_INDEX-->
- </tr>
- </tbody>
-</table>
-</div>
-<!--END TITLEAREA-->
-<!-- end header part -->

lua/json.lua

@@ -1,5 +1,5 @@
 --
--- json2sas.lua  (modified from json.lua)
+-- json.lua
 --
 -- Copyright (c) 2019 rxi
 --
@@ -22,7 +22,7 @@
 -- SOFTWARE.
 --
 
-local json2sas = { _version = "0.1.2" }
+json = { _version = "0.1.2" }
 
 -------------------------------------------------------------------------------
 -- Encode
@@ -122,7 +122,7 @@ encode = function(val, stack)
   error("unexpected type '" .. t .. "'")
 end
 
-function json2sas.encode(val)
+function json.encode(val)
   return ( encode(val) )
 end
 
@@ -356,7 +356,7 @@ parse = function(str, idx)
   decode_error(str, idx, "unexpected character '" .. chr .. "'")
 end
 
-function json2sas.decode(str)
+function json.decode(str)
   if type(str) ~= "string" then
     error("expected argument of type string, got " .. type(str))
   end
@@ -368,88 +368,4 @@ function json2sas.decode(str)
   return res
 end
 
--- convert macro variable array into one variable and decode
+return json
-function json2sas.go(macvar)
-  local x=1
-  local cnt=0
-  local mac=sas.symget(macvar..'0')
-  local newstr=''
-  if mac and mac ~= '' then
-    cnt=mac
-    for x=1,cnt,1 do
-      mac=sas.symget(macvar..x)
-      if mac and mac ~= '' then
-        newstr=newstr..mac
-      else
-        return print(macvar..x..' NOT FOUND!!')
-      end
-    end
-  else
-    return print(macvar..'0 NOT FOUND!!')
-  end
-  -- print('mac:'..mac..'cnt:'..cnt..'newstr'..newstr)
-  local oneVar=json2sas.decode(newstr)
-  local jsdata=oneVar["data"]
-  local meta={}
-  local attrs={}
-  for tablename, data in pairs(jsdata) do -- each table
-    print("Processing table: "..tablename)
-    attrs[tablename]={}
-    for k, v in ipairs(data) do -- each row
-      if(k==1) then  -- column names
-        for a, b in pairs(v) do
-          attrs[tablename][a]={}
-          attrs[tablename][a]["name"]=b
-        end
-      elseif(k==2) then  -- get types
-        for a, b in pairs(v) do
-  	      if type(b)=='number' then
-  	        attrs[tablename][a]["type"]="N"
-  	        attrs[tablename][a]["length"]=8
-  	      else
-            attrs[tablename][a]["type"]="C"
-            attrs[tablename][a]["length"]=string.len(b)
-  	      end
-        end
-      else  --update lengths
-        for a, b in pairs(v) do
-          if (type(b)=='string' and string.len(b)>attrs[tablename][a]["length"])
-          then
-            attrs[tablename][a]["length"]=string.len(b)
-          end
-        end
-  	  end
-  	end
-    print(json2sas.encode(attrs[tablename])) -- show results
-
-    -- Now create the SAS table
-    sas.new_table("work."..tablename,attrs[tablename])
-    local dsid=sas.open("work."..tablename, "u")
-    for k, v in ipairs(data) do
-      if k>1 then
-        sas.append(dsid)
-        for a, b in pairs(v) do
-          sas.put_value(dsid, attrs[tablename][a]["name"], b)
-        end
-        sas.update(dsid)
-      end
-    end
-    sas.close(dsid)
-  end
-  return json2sas.decode(newstr)
-end
-
-
-function quote(str)
-  return sas.quote(str)
-end
-function sasvar(str)
-  print("processing: "..str)
-  print(sas.symexist(str))
-  if sas.symexist(str)==1 then
-    return quote(str)..':'..quote(sas.symget(str))..','
-  end
-  return ''
-end
-
-return json2sas

lua/ml_json.sas

@@ -1,18 +1,19 @@
 /**
-  @file ml_json2sas.sas
+  @file ml_json.sas
-  @brief Creates the json2sas.lua file
+  @brief Compiles the json.lua lua file
-  @details Writes json2sas.lua to the work directory
+  @details Writes json.lua to the work directory
+  and then includes it.
   Usage:
 
-      %ml_json2sas()
+      %ml_json()
 
 **/
 
-%macro ml_json2sas();
+%macro ml_json();
 data _null_;
-  file "%sysfunc(pathname(work))/json2sas.lua";
+  file "%sysfunc(pathname(work))/ml_json.lua";
   put '-- ';
-  put '-- json2sas.lua  (modified from json.lua) ';
+  put '-- json.lua ';
   put '-- ';
   put '-- Copyright (c) 2019 rxi ';
   put '-- ';
@@ -35,7 +36,7 @@ data _null_;
   put '-- SOFTWARE. ';
   put '-- ';
   put ' ';
-  put 'local json2sas = { _version = "0.1.2" } ';
+  put 'json = { _version = "0.1.2" } ';
   put ' ';
   put '------------------------------------------------------------------------------- ';
   put '-- Encode ';
@@ -135,7 +136,7 @@ data _null_;
   put '  error("unexpected type ''" .. t .. "''") ';
   put 'end ';
   put ' ';
-  put 'function json2sas.encode(val) ';
+  put 'function json.encode(val) ';
   put '  return ( encode(val) ) ';
   put 'end ';
   put ' ';
@@ -369,7 +370,7 @@ data _null_;
   put '  decode_error(str, idx, "unexpected character ''" .. chr .. "''") ';
   put 'end ';
   put ' ';
-  put 'function json2sas.decode(str) ';
+  put 'function json.decode(str) ';
   put '  if type(str) ~= "string" then ';
   put '    error("expected argument of type string, got " .. type(str)) ';
   put '  end ';
@@ -381,90 +382,9 @@ data _null_;
   put '  return res ';
   put 'end ';
   put ' ';
-  put '-- convert macro variable array into one variable and decode ';
+  put 'return json ';
-  put 'function json2sas.go(macvar) ';
-  put '  local x=1 ';
-  put '  local cnt=0 ';
-  put '  local mac=sas.symget(macvar..''0'') ';
-  put '  local newstr='''' ';
-  put '  if mac and mac ~= '''' then ';
-  put '    cnt=mac ';
-  put '    for x=1,cnt,1 do ';
-  put '      mac=sas.symget(macvar..x) ';
-  put '      if mac and mac ~= '''' then ';
-  put '        newstr=newstr..mac ';
-  put '      else ';
-  put '        return print(macvar..x..'' NOT FOUND!!'') ';
-  put '      end ';
-  put '    end ';
-  put '  else ';
-  put '    return print(macvar..''0 NOT FOUND!!'') ';
-  put '  end ';
-  put '  -- print(''mac:''..mac..''cnt:''..cnt..''newstr''..newstr) ';
-  put '  local oneVar=json2sas.decode(newstr) ';
-  put '  local jsdata=oneVar["data"] ';
-  put '  local meta={} ';
-  put '  local attrs={} ';
-  put '  for tablename, data in pairs(jsdata) do -- each table ';
-  put '    print("Processing table: "..tablename) ';
-  put '    attrs[tablename]={} ';
-  put '    for k, v in ipairs(data) do -- each row ';
-  put '      if(k==1) then  -- column names ';
-  put '        for a, b in pairs(v) do ';
-  put '          attrs[tablename][a]={} ';
-  put '          attrs[tablename][a]["name"]=b ';
-  put '        end ';
-  put '      elseif(k==2) then  -- get types ';
-  put '        for a, b in pairs(v) do ';
-  put '  	      if type(b)==''number'' then ';
-  put '  	        attrs[tablename][a]["type"]="N" ';
-  put '  	        attrs[tablename][a]["length"]=8 ';
-  put '  	      else ';
-  put '            attrs[tablename][a]["type"]="C" ';
-  put '            attrs[tablename][a]["length"]=string.len(b) ';
-  put '  	      end ';
-  put '        end ';
-  put '      else  --update lengths ';
-  put '        for a, b in pairs(v) do ';
-  put '          if (type(b)==''string'' and string.len(b)>attrs[tablename][a]["length"]) ';
-  put '          then ';
-  put '            attrs[tablename][a]["length"]=string.len(b) ';
-  put '          end ';
-  put '        end ';
-  put '  	  end ';
-  put '  	end ';
-  put '    print(json2sas.encode(attrs[tablename])) -- show results ';
-  put ' ';
-  put '    -- Now create the SAS table ';
-  put '    sas.new_table("work."..tablename,attrs[tablename]) ';
-  put '    local dsid=sas.open("work."..tablename, "u") ';
-  put '    for k, v in ipairs(data) do ';
-  put '      if k>1 then ';
-  put '        sas.append(dsid) ';
-  put '        for a, b in pairs(v) do ';
-  put '          sas.put_value(dsid, attrs[tablename][a]["name"], b) ';
-  put '        end ';
-  put '        sas.update(dsid) ';
-  put '      end ';
-  put '    end ';
-  put '    sas.close(dsid) ';
-  put '  end ';
-  put '  return json2sas.decode(newstr) ';
-  put 'end ';
-  put ' ';
-  put ' ';
-  put 'function quote(str) ';
-  put '  return sas.quote(str) ';
-  put 'end ';
-  put 'function sasvar(str) ';
-  put '  print("processing: "..str) ';
-  put '  print(sas.symexist(str)) ';
-  put '  if sas.symexist(str)==1 then ';
-  put '    return quote(str)..'':''..quote(sas.symget(str))..'','' ';
-  put '  end ';
-  put '  return '''' ';
-  put 'end ';
-  put ' ';
-  put 'return json2sas ';
 run;
+
+%inc "%sysfunc(pathname(work))/ml_json.lua";
+
 %mend;

main.dox

@@ -13,6 +13,11 @@
         * Not metadata aware
         * No X command
         * Prefixes:  _mf_, _mp_
+
+    Macros starting `mf_` are macro _functions_ and can be used in assignment
+    statements.  Those starting `mp_` are macro _procedures_, which generate
+    SAS statements, and must therefore be applied accordingly.
+
  */
 
 /*! \dir meta
@@ -46,3 +51,14 @@
         * Prefixes: _mv_
 
  */
+
+ /*! \dir lua
+ *  \brief Lua macros
+ *  \details These macros have the following attributes:
+
+        * OS independent
+        * Work as LUA functions (they are immediately executed/compiled)
+        * Auto-generated from the plain source `.lua` files in the same directory
+        * Prefixes: _ml_
+
+ */

meta/mm_adduser2group.sas

@@ -70,7 +70,7 @@ run;
 %end;
 
 %if &syscc ge 4 %then %do;
-  %put WARNING:  SYSCC=&syscc, exiting &sysmacroname;
+  %put %str(WARN)ING:  SYSCC=&syscc, exiting &sysmacroname;
   %return;
 %end;
 

meta/mm_assigndirectlib.sas

@@ -14,7 +14,7 @@
       disconnect from MyAlias;
       quit;
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_getengine.sas
   @li mp_abort.sas
 
@@ -117,7 +117,8 @@ run;
 %end;
 %else %if &engine=REMOTE %then %do;
   data x;
-    length rcCon rcProp rc k 3 uriCon uriProp PropertyValue PropertyName Delimiter $256 properties $2048;
+    length rcCon rcProp rc k 3 uriCon uriProp PropertyValue PropertyName
+      Delimiter $256 properties $2048;
     retain properties;
     rcCon = metadata_getnasn("&liburi", "LibraryConnection", 1, uriCon);
 
@@ -129,7 +130,8 @@ run;
       rc = metadata_getattr(uriProp , "DefaultValue",PropertyValue);
       rc = metadata_getattr(uriProp , "PropertyName",PropertyName);
       rc = metadata_getattr(uriProp , "Delimiter",Delimiter);
-      properties = trim(properties) !! " " !! trim(PropertyName) !! trim(Delimiter) !! trim(PropertyValue);
+      properties = trim(properties) !! " " !! trim(PropertyName)
+        !! trim(Delimiter) !! trim(PropertyValue);
       output;
       k+1;
       rcProp = metadata_getnasn(uriCon, "Properties", k, uriProp);
@@ -170,7 +172,8 @@ run;
       else if value='Connection.OLE.Property.PROVIDER.Name.xmlKey.txt' then do;
         rc4=metadata_getattr(conprop_uri,'DefaultValue',provider);
       end;
-      else if value='Connection.OLE.Property.PROPERTIES.Name.xmlKey.txt' then do;
+      else if value='Connection.OLE.Property.PROPERTIES.Name.xmlKey.txt' then
+      do;
         rc5=metadata_getattr(conprop_uri,'DefaultValue',properties);
       end;
     end;
@@ -329,7 +332,7 @@ run;
   run;
 
   %if %length(&open_passthrough)>0 %then %do;
-    %put WARNING:  Passthrough option for postgres not yet supported;
+    %put %str(WARN)ING: Passthrough option for postgres not yet supported;
     %return;
   %end;
   %else %do;
@@ -357,7 +360,8 @@ run;
     call symputx('authdomain',authdomain,'l');
 
     /* path */
-    rc=metadata_getprop(assocuri1,'Connection.Oracle.Property.PATH.Name.xmlKey.txt',path);
+    rc=metadata_getprop(assocuri1,
+      'Connection.Oracle.Property.PATH.Name.xmlKey.txt',path);
     call symputx('path',path,'l');
 
     /* schema */
@@ -366,14 +370,16 @@ run;
     call symputx('schema',schema,'l');
   run;
   %put NOTE: Executing the following:/; %put NOTE-;
-  %put NOTE- libname &libref ORACLE path=&path schema=&schema authdomain=&authdomain;
+  %put NOTE- libname &libref ORACLE path=&path schema=&schema;
+  %put NOTE-     authdomain=&authdomain;
   %put NOTE-;
   libname &libref ORACLE path=&path schema=&schema authdomain=&authdomain;
 %end;
 %else %if &engine=SQLSVR %then %do;
   %put NOTE: Obtaining &engine library details;
   data _null;
-    length assocuri1 assocuri2 assocuri3 authdomain path schema userid passwd $256;
+    length assocuri1 assocuri2 assocuri3 authdomain path schema userid
+      passwd $256;
     call missing (of _all_);
 
     rc=metadata_getnasn("&liburi",'DefaultLogin',1,assocuri1);
@@ -384,7 +390,8 @@ run;
 
     /* path */
     rc=metadata_getnasn("&liburi",'LibraryConnection',1,assocuri2);
-    rc=metadata_getprop(assocuri2,'Connection.SQL.Property.Datasrc.Name.xmlKey.txt',path);
+    rc=metadata_getprop(assocuri2,
+      'Connection.SQL.Property.Datasrc.Name.xmlKey.txt',path);
     call symputx('path',path,'l');
 
     /* schema */
@@ -394,15 +401,17 @@ run;
   run;
 
   %put NOTE: Executing the following:/; %put NOTE-;
-  %put NOTE- libname &libref SQLSVR datasrc=&path schema=&schema user="&user" pass="XXX";
+  %put NOTE- libname &libref SQLSVR datasrc=&path schema=&schema ;
+  %put NOTE-    user="&user" pass="XXX";
   %put NOTE-;
 
-  libname &libref SQLSVR datasrc=&path schema=&schema user="&user" pass="&pass" ;
+  libname &libref SQLSVR datasrc=&path schema=&schema user="&user" pass="&pass";
 %end;
 %else %if &engine=TERADATA %then %do;
   %put NOTE: Obtaining &engine library details;
   data _null;
-    length assocuri1 assocuri2 assocuri3 authdomain path schema userid passwd $256;
+    length assocuri1 assocuri2 assocuri3 authdomain path schema userid
+      passwd $256;
     call missing (of _all_);
 
         /* get auth domain */
@@ -421,7 +430,8 @@ run;
 
     /* path */
     rc=metadata_getnasn("&liburi",'LibraryConnection',1,assocuri2);
-    rc=metadata_getprop(assocuri2,'Connection.Teradata.Property.SERVER.Name.xmlKey.txt',path);
+    rc=metadata_getprop(assocuri2,
+      'Connection.Teradata.Property.SERVER.Name.xmlKey.txt',path);
     call symputx('path',path,'l');
 
     /* schema */
@@ -431,7 +441,8 @@ run;
   run;
 
   %put NOTE: Executing the following:/; %put NOTE-;
-  %put NOTE- libname &libref TERADATA server=&path schema=&schema authdomain=&authdomain;
+  %put NOTE- libname &libref TERADATA server=&path schema=&schema ;
+  %put NOTe-   authdomain=&authdomain;
   %put NOTE-;
 
   libname &libref TERADATA server=&path schema=&schema authdomain=&authdomain;
@@ -444,8 +455,8 @@ run;
   %return;
 %end;
 %else %do;
-  %put WARNING: Engine &engine is currently unsupported;
+  %put %str(WARN)ING: Engine &engine is currently unsupported;
-  %put WARNING- Please contact your support team.;
+  %put %str(WARN)ING- Please contact your support team.;
   %return;
 %end;
 

meta/mm_assignlib.sas

@@ -10,11 +10,12 @@
 
       %mm_assignlib(SOMEREF)
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mp_abort.sas
 
   @param libref the libref (not name) of the metadata library
-  @param mAbort= If not assigned, HARD will call %mp_abort(), SOFT will silently return
+  @param mAbort= If not assigned, HARD will call %mp_abort(), SOFT will
+    silently return
 
   @returns libname statement
 
@@ -38,6 +39,7 @@
       rc=metadata_getattr(liburi,"Name",LibName);
       /* now try and assign it */
       if libname("&libref",,'meta',cats('liburi="',liburi,'";')) ne 0 then do;
+        putlog "&libref could not be assigned";
         call symputx('msg',sysmsg(),'l');
         if "&mabort"='HARD' then call symputx('mp_abort',1,'l');
       end;

meta/mm_createapplication.sas

@@ -13,9 +13,11 @@
         ,params= name1=value1
name2=value2
emptyvalue=
       )
 
-  @warning application components do not get deleted when removing the container folder!  be sure you have the administrative priviliges to remove this kind of metadata from the SMC plugin (or be ready to do to so programmatically).
+  @warning application components do not get deleted when removing the container
+  folder!  be sure you have the administrative priviliges to remove this kind of
+  metadata from the SMC plugin (or be ready to do to so programmatically).
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mp_abort.sas
   @li mf_verifymacvars.sas
 

meta/mm_createdataset.sas

@@ -18,7 +18,7 @@
 
     %mm_createdataset(tableuri=G5X8AFW1.BE00015Y)
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mm_getlibs.sas
   @li mm_gettables.sas
   @li mm_getcols.sas

meta/mm_createdocument.sas

@@ -12,7 +12,7 @@
       %mm_createdocument(tree=/User Folders/sasdemo
         ,name=MyNote)
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mp_abort.sas
   @li mf_verifymacvars.sas
 

meta/mm_createfolder.sas

@@ -55,7 +55,7 @@ data _null_;
 
   * must have a starting slash ;
   if ( substr(folderPath,1,1) ne '/' ) then do;
-    put "%str(ERR)OR: &sysmacroname PATH parameter value must have starting slash";
+    put "%str(ERR)OR: &sysmacroname PATH param value must have starting slash";
     stop;
   end;
 
@@ -123,9 +123,10 @@ run;
   %local inmeta;
   %put creating: &path;
   %let inmeta=<AddMetadata><Reposid>$METAREPOSITORY</Reposid><Metadata>
-    <Tree Name='&child' PublicType='Folder' TreeType='BIP Folder' UsageVersion='1000000'>
+    <Tree Name='&child' PublicType='Folder' TreeType='BIP Folder'
-    <ParentTree><Tree ObjRef='&parentFolderObjId'/></ParentTree></Tree></Metadata>
+    UsageVersion='1000000'><ParentTree><Tree ObjRef='&parentFolderObjId'/>
-    <NS>SAS</NS><Flags>268435456</Flags></AddMetadata>;
+    </ParentTree></Tree></Metadata><NS>SAS</NS><Flags>268435456</Flags>
+    </AddMetadata>;
 
   proc metadata in="&inmeta" out=__newdir verbose;
   run ;

meta/mm_createlibrary.sas

@@ -19,7 +19,7 @@
         ,directory=/tmp/tests
         ,mDebug=1)
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_verifymacvars.sas
   @li mm_createfolder.sas
 
@@ -88,7 +88,7 @@ data _null_;
   putlog (_all_)(=);
 run;
 %if &checktype = SASLibrary %then %do;
-  %put WARNING: Library (&liburi) already exists with libname (&libname)  ;
+  %put %str(WARN)ING: Library (&liburi) already exists with libname (&libname);
   %return;
 %end;
 
@@ -103,7 +103,7 @@ data _null_;
   putlog (_all_)(=);
 run;
 %if &checktype = SASLibrary %then %do;
-  %put WARNING: Library (&liburi) already exists with libref (&libref)  ;
+  %put %str(WARN)ING: Library (&liburi) already exists with libref (&libref)  ;
   %return;
 %end;
 
@@ -123,7 +123,7 @@ data _null_;
   call symputx('treeuri',uri,'l');
 run;
 %if &foldertype ne Tree %then %do;
-  %put WARNING: Tree &tree does not exist!;
+  %put %str(WARN)ING: Tree &tree does not exist!;
   %return;
 %end;
 
@@ -166,7 +166,7 @@ filename &frefout temp;
     putlog (_all_)(=);
   run;
   %if &checktype ne Prototype %then %do;
-    %put %str(ERR)OR: Prototype (Library.SAS.Prototype.Name.xmlKey.txt) not found!;
+    %put %str(ERR)OR: Prototype Library.SAS.Prototype.Name.xmlKey.txt not found;
     %return;
   %end;
 
@@ -231,7 +231,7 @@ filename &frefout temp;
     *  check SAS version
     */
   %if %sysevalf(&sysver lt 9.3) %then %do;
-    %put WARNING: Version 9.3 or later required;
+    %put %str(WARN)ING: Version 9.3 or later required;
     %return;
   %end;
 

meta/mm_createstp.sas

@@ -39,7 +39,7 @@
         ,Server=SASApp
         ,stptype=2)
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mf_nobs.sas
   @li mf_verifymacvars.sas
   @li mm_getdirectories.sas
@@ -118,7 +118,7 @@ data _null_;
   call symputx('treeuri',uri,'l');
 run;
 %if &foldertype ne Tree %then %do;
-  %put WARNING: Tree &tree does not exist!;
+  %put %str(WARN)ING: Tree &tree does not exist!;
   %return;
 %end;
 
@@ -133,7 +133,7 @@ data _null_;
   call symputx('stpuri',uri,'l');
 run;
 %if &cmtype = ClassifierMap %then %do;
-  %put WARNING: Stored Process &stpname already exists in &tree!;
+  %put %str(WARN)ING: Stored Process &stpname already exists in &tree!;
   %return;
 %end;
 
@@ -141,14 +141,14 @@ run;
   * Check that the physical file exists
   */
 %if %sysfunc(fileexist(&directory/&filename)) ne 1 %then %do;
-  %put WARNING: FILE *&directory/&filename* NOT FOUND!;
+  %put %str(WARN)ING: FILE *&directory/&filename* NOT FOUND!;
   %return;
 %end;
 
 %if &stptype=1 %then %do;
   /* type 1 STP - where code is stored on filesystem */
   %if %sysevalf(&sysver lt 9.2) %then %do;
-    %put WARNING: Version 9.2 or later required;
+    %put %str(WARN)ING: Version 9.2 or later required;
     %return;
   %end;
 
@@ -162,7 +162,7 @@ run;
   %if &checkdirtype ne Directory %then %do;
     %mm_getdirectories(path=&directory,outds=&outds ,mDebug=&mDebug)
     %if %mf_nobs(&outds)=0 or %sysfunc(exist(&outds))=0 %then %do;
-      %put WARNING: The directory object does not exist for &directory;
+      %put %str(WARN)ING: The directory object does not exist for &directory;
       %return;
     %end;
   %end;
@@ -180,12 +180,12 @@ run;
     length id $20 type $256;
     __rc=metadata_resolve("&treeuri",type,id);
     if type ne 'Tree' then do;
-      putlog "WARNING:  Invalid tree URI: &treeuri";
+      putlog "%str(WARN)ING:  Invalid tree URI: &treeuri";
       stopme=1;
     end;
     __rc=metadata_resolve(directoryuri,type,id);
     if type ne 'Directory' then do;
-      putlog 'WARNING:  Invalid directory URI: ' directoryuri;
+      putlog "%str(WARN)ING:  Invalid directory URI: " directoryuri;
       stopme=1;
     end;
 
@@ -194,7 +194,7 @@ run;
     if type ne 'LogicalServer' then do;
       __rc=metadata_getnobj("omsobj:LogicalServer?@Name='&server'",1,serveruri);
       if serveruri='' then do;
-        putlog "WARNING:  Invalid server: &server";
+        putlog "%str(WARN)ING:  Invalid server: &server";
         stopme=1;
       end;
     end;
@@ -210,13 +210,14 @@ run;
     rc3=METADATA_SETATTR(prompturi, 'GroupType','2');
     rc4=METADATA_SETATTR(prompturi, 'Name','Parameters');
     rc5=METADATA_SETATTR(prompturi, 'PublicType','Embedded:PromptGroup');
-    GroupInfo="<PromptGroup promptId='PromptGroup_%sysfunc(datetime())_&sysprocessid'"
+    GroupInfo=
+      "<PromptGroup promptId='PromptGroup_%sysfunc(datetime())_&sysprocessid'"
       !!" version='1.0'><Label><Text xml:lang='en-GB'>Parameters</Text>"
       !!"</Label></PromptGroup>";
     rc6 = METADATA_SETATTR(prompturi, 'GroupInfo',groupinfo);
 
     if sum(of rc1-rc6) ne 0 then do;
-      putlog 'WARNING: Issue creating prompt.';
+      putlog "%str(WARN)ING: Issue creating prompt.";
       if prompturi ne . then do;
         putlog '  Removing orphan: ' prompturi;
         rc = METADATA_DELOBJ(prompturi);
@@ -231,7 +232,7 @@ run;
     rc9=METADATA_SETATTR(fileuri, 'IsARelativeName','1');
     rc10=METADATA_SETASSN(fileuri, 'Directories','MODIFY',directoryuri);
     if sum(of rc7-rc10) ne 0 then do;
-      putlog 'WARNING: Issue creating file.';
+      putlog "%str(WARN)ING: Issue creating file.";
       if fileuri ne . then do;
         putlog '  Removing orphans:' prompturi fileuri;
         rc = METADATA_DELOBJ(prompturi);
@@ -250,7 +251,7 @@ run;
       !!"<OutputParameters/></StoredProcess>";
     rc14= METADATA_SETATTR(texturi, 'StoredText',storedtext);
     if sum(of rc11-rc14) ne 0 then do;
-      putlog 'WARNING: Issue creating TextStore.';
+      putlog "%str(WARN)ING: Issue creating TextStore.";
       if texturi ne . then do;
         putlog '  Removing orphans: ' prompturi fileuri texturi;
         rc = METADATA_DELOBJ(prompturi);
@@ -298,7 +299,7 @@ run;
 %else %if &stptype=2 %then %do;
   /* type 2 stp - code is stored in metadata */
   %if %sysevalf(&sysver lt 9.3) %then %do;
-    %put WARNING: SAS version 9.3 or later required to create type2 STPs;
+    %put %str(WARN)ING: SAS version 9.3 or later required to create type2 STPs;
     %return;
   %end;
   /* check we have the correct ServerContext */
@@ -310,7 +311,7 @@ run;
     call symputx('serveruri',serveruri);
   run;
   %if &serveruri=NOTFOUND %then %do;
-    %put WARNING: ServerContext *&server* not found!;
+    %put %str(WARN)ING: ServerContext *&server* not found!;
     %return;
   %end;
 
@@ -375,15 +376,13 @@ run;
     */
   %mm_updatestpsourcecode(stp=&tree/&stpname
     ,stpcode="&directory/&filename"
-    ,frefin=&frefin.
-    ,frefout=&frefout.
     ,mdebug=&mdebug
     ,minify=&minify)
 
 
 %end;
 %else %do;
-  %put WARNING:  STPTYPE=*&stptype* not recognised!;
+  %put %str(WARN)ING:  STPTYPE=*&stptype* not recognised!;
 %end;
 
 %mend;

meta/mm_createwebservice.sas

@@ -22,9 +22,9 @@ Usage:
         %webout(OBJ,example2) * Object format, easier to work with ;
         %webout(CLOSE)
     ;;;;
-    %mm_createwebservice(path=/Public/app/common,name=appInit,code=ft15f001,replace=YES)
+    %mm_createwebservice(path=/Public/app/common,name=appInit)
 
-  <h4> Dependencies </h4>
+  <h4> SAS Macros </h4>
   @li mm_createstp.sas
   @li mf_getuser.sas
   @li mm_createfolder.sas
@@ -119,7 +119,8 @@ data _null_;
   put '    %end; ';
   put '    data _null_;file &jref mod ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
-  put '    proc sort data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
+  put '    proc sort ';
+  put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
   put '      out=_data_; ';
   put '      by varnum; ';
   put ' ';
@@ -158,7 +159,8 @@ data _null_;
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
-  put '    /* write to temp loc to avoid _webout truncation - https://support.sas.com/kb/49/325.html */ ';
+  put '    /* write to temp loc to avoid _webout truncation ';
+  put '      - https://support.sas.com/kb/49/325.html */ ';
   put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
   put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod; ';
   put '      set &tempds; ';
@@ -194,7 +196,7 @@ data _null_;
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8''; ';
+  put '  data _null_;file &jref encoding=''utf-8'' mod; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
@@ -235,8 +237,16 @@ data _null_;
   put '%else %if &action=OPEN %then %do; ';
   put '  /* fix encoding */ ';
   put '  OPTIONS NOBOMFILE; ';
+  put ' ';
+  put '  /** ';
+  put '    * check engine type to avoid the below err message: ';
+  put '    * > Function is only valid for filerefs using the CACHE access method. ';
+  put '    */ ';
   put '  data _null_; ';
-  put '    rc = stpsrv_header(''Content-type'',"text/html; encoding=utf-8"); ';
+  put '    set sashelp.vextfl(where=(fileref="_WEBOUT")); ';
+  put '    if xengine=''STREAM'' then do; ';
+  put '      rc=stpsrv_header(''Content-type'',"text/html; encoding=utf-8"); ';
+  put '    end; ';
   put '  run; ';
   put ' ';
   put '  /* setup json */ ';
@@ -250,16 +260,9 @@ data _null_;
   put '%end; ';
   put ' ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
-  put '  %if &sysver=9.4 %then %do; ';
-  put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
-  put '      ,engine=PROCJSON,dbg=%str(&_debug) ';
-  put '    ) ';
-  put '  %end; ';
-  put '  %else %do; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
   put '    ,engine=DATASTEP,dbg=%str(&_debug) ';
   put '  ) ';
-  put '  %end; ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
   put '  %if %str(&_debug) ge 131 %then %do; ';
@@ -275,14 +278,14 @@ data _null_;
   put '      i+1; ';
   put '      call symputx(''wt''!!left(i),name,''l''); ';
   put '      call symputx(''wtcnt'',i,''l''); ';
-  put '    data _null_; file &fref encoding=''utf-8''; ';
+  put '    data _null_; file &fref mod encoding=''utf-8''; ';
   put '      put ",""WORK"":{"; ';
   put '    %do i=1 %to &wtcnt; ';
   put '      %let wt=&&wt&i; ';
   put '      proc contents noprint data=&wt ';
   put '        out=_data_ (keep=name type length format:); ';
   put '      run;%let tempds=%scan(&syslast,2,.); ';
-  put '      data _null_; file &fref encoding=''utf-8''; ';
+  put '      data _null_; file &fref mod encoding=''utf-8''; ';
   put '        dsid=open("WORK.&wt",''is''); ';
   put '        nlobs=attrn(dsid,''NLOBS''); ';
   put '        nvars=attrn(dsid,''NVARS''); ';
@@ -293,10 +296,10 @@ data _null_;
   put '        put '',"nvars":'' nvars; ';
   put '      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP) ';
   put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP) ';
-  put '      data _null_; file &fref encoding=''utf-8''; ';
+  put '      data _null_; file &fref mod encoding=''utf-8''; ';
   put '        put "}"; ';
   put '    %end; ';
-  put '    data _null_; file &fref encoding=''utf-8''; ';
+  put '    data _null_; file &fref mod encoding=''utf-8''; ';
   put '      put "}"; ';
   put '    run; ';
   put '  %end; ';