feat: two new macros for checking a filter query, and then generating a filter query. One test to cover the generation part. One more macro to provide assertions on the number of rows in a table, compatible with the upcoming 'sasjs test' feature.

fix: switching to data step for JSON generation in mp_jsonout and the…

.gitignore

@@ -1,2 +1,12 @@
 node_modules
 .DS_Store
+sasjsbuild/
+
+# avoid filenames with spaces being committed to source control
+**\ **
+
+# ignore the mc_* files - containing macros for individual libraries
+mc_*
+
+# ignore .env files as they can contain sasjs access tokens
+*.env*

.gitpod.dockerfile

@@ -0,0 +1,6 @@
+FROM gitpod/workspace-full
+
+RUN sudo apt-get update \
+ && sudo apt-get install -y \
+    doxygen \
+ && sudo rm -rf /var/lib/apt/lists/*

.gitpod.yml

@@ -0,0 +1,8 @@
+tasks:
+  - init: npm i -g @sasjs/cli
+
+image:
+  file: .gitpod.dockerfile
+vscode:
+  extensions:
+    - sasjs.sasjs-for-vscode

.sasjslint

@@ -0,0 +1,13 @@
+{
+    "noTrailingSpaces": true,
+    "noEncodedPasswords": true,
+    "hasDoxygenHeader": true,
+    "hasMacroNameInMend": false,
+    "hasMacroParentheses": true,
+    "noNestedMacros": false,
+    "noSpacesInFileNames": true,
+    "maxLineLength": 135,
+    "lowerCaseFileNames": true,
+    "noTabIndentation": true,
+    "indentationMultiple": 2
+}

.vscode/.editorconfig

@@ -0,0 +1,9 @@
+{
+  "search.exclude": {
+    "**/sasjsbuild/**": true,
+    "**/dist/**":true
+  },
+  "editor.insertSpaces": true,
+  "editor.tabSize": 2,
+  "trim_trailing_whitespace": true
+}

.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+  "recommendations": [
+    "sasjs.sasjs-for-vscode"
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,10 @@
+{
+  "editor.tabSize": 2,
+  "editor.insertSpaces": true,
+  "editor.detectIndentation": true,
+  "editor.formatOnSave": true,
+  "editor.rulers": [
+    80
+  ],
+  "files.trimTrailingWhitespace": true
+}

CONTRIBUTING.md

@@ -1,83 +1,31 @@
 # Contributing
 
-Contributions to the macrocore library are warmly welcomed!  To avoid any
-misunderstandings, do please first discuss the change you wish to make via issue,
-email, or any other method with the owners of this repository before submitting
-a PR.
+Contributions are warmly welcomed!  To avoid any misunderstandings, do please first discuss the change you wish to make via issue, email, or any other method with the owners of this repository before submitting a PR.
 
-Please note we have a code of conduct, please follow it in all your interactions
-with the project.
+Please note we have a [code of conduct](https://www.contributor-covenant.org/version/2/0/code_of_conduct/), please follow it in all your interactions with the project.
+
+# Environment Setup
+
+This repository makes use of the [SASjs](https://sasjs.io) framework for code organisation, compilation, documentation, and deployment.  The following tools are highly recommended:
+
+* [NPM](https://sasjs.io/windows/#npm) - the runtime and dependency manager for [SASjs CLI](https://cli.sasjs.io) (batteries included)
+* [VSCode](https://sasjs.io/windows/#vscode) - feature packed IDE for code editing (warning - highly effective!)
+* [GIT](https://sasjs.io/windows/#git) - a safety net you cannot (and should not) do without.
+
+For generating the documentation (`sasjs doc`) it is also necessary to install [doxygen](https://www.doxygen.nl/manual/install.html).
 
 
-## Code of Conduct
+To get configured:
 
-### Our Pledge
+1.  Clone the repository
+2.  Install local dependencies (`npm install`)
+3.  Install the SASjs CLI globally (`npm install -g @sasjs/cli`)
+4.  Add a target, and authentication (`npm add`).  See [docs](https://cli.sasjs.io/add/).
 
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to making participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, gender identity and expression, level of experience,
-nationality, personal appearance, race, religion, or sexual identity and
-orientation.
+To contribute:
 
-### Our Standards
+1.  Create your feature branch (`git checkout -b myfeature`)
+2.  Make your change
+3.  Update the `all.sas` file (`python3 build.py`)
+4.  Push and make a PR
 
-Examples of behavior that contributes to creating a positive environment
-include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-### Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-### Scope
-
-This Code of Conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
-address, posting via an official social media account, or acting as an appointed
-representative at an online or offline event. Representation of a project may be
-further defined and clarified by project maintainers.
-
-### Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at support@macropeople.com. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-### Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at [http://contributor-covenant.org/version/1/4][version]
-
-[homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/4/

README.md

@@ -1,6 +1,6 @@
 # Macro Core
 
-Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of Application Development on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
+Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
 
 You can download and compile them all in just two lines of SAS code:
 
@@ -9,7 +9,7 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 %inc mc;
 ```
 
-Documentation: https://sasjs.github.io/core.github.io/files.html
+Documentation: https://core.sasjs.io
 
 # Components
 
@@ -40,6 +40,27 @@ Documentation: https://sasjs.github.io/core.github.io/files.html
 - X command enabled
 - Prefixes: _mmw_,_mmu_,_mmx_
 
+**lua** library
+
+Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
+
+To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert your LUA into a data step with put statements, and create the macro wrapper with a `ml_` prefix.  You can then use your module in any program by running:
+
+```
+/* compile the lua module */
+%ml_yourmodule()
+
+/* Execute.  Do not use the restart keyword! */
+proc lua;
+submit;
+  print(yourStuff);
+endsubmit;
+run;
+```
+
+- X command enabled
+- Prefixes: _mmw_,_mmu_,_mmx_
+
 # Installation
 
 First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available,eg:
@@ -63,7 +84,7 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 ## File Properties
 
 - filenames much match macro names
-- filenames must be lowercase
+- filenames must be lowercase, without spaces
 - macro names must be lowercase
 - one macro per file
 - prefixes:
@@ -72,12 +93,13 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
   - _mm_ for metadata macros (interface with the metadata server).
   - _mmx_ for macros that use metadata and are XCMD enabled
   - _mx_ for macros that are XCMD enabled
+  - _ml_ for macros that are used to compile LUA modules
   - _mv_ for macros that will only work in Viya
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
 - UTF-8
-- no trailing white space
+
 
 ## Header Properties
 
@@ -96,7 +118,7 @@ All macros must be commented in the doxygen format, to enable the [online docume
 ### Dependencies
 SAS code can contain one of two types of dependency - SAS Macros, and SAS Programs.  When compiling projects using the [SASjs CLI](https://cli.sasjs.io) the doxygen header is scanned for `  @li` items under the following headers:
 
-```
+```sas
   <h4> SAS Macros </h4>
   @li mf_nobs.sas
   @li mm_assignlib.sas
@@ -114,13 +136,15 @@ When contributing to this library, it is therefore important to ensure that all
 ## Coding Standards
 
 - Indentation = 2 spaces. No tabs!
+- no trailing white space
+- no invisible characters, other than spaces. If invisibles are needed, use hex literals.
 - Macro variables should not have the trailing dot (`&var` not `&var.`) unless necessary to prevent incorrect resolution
-- The closing `%mend;` should not contain the macro name.
+- The closing `%mend;` should **not** contain the macro name.
 - All macros should be defined with brackets, even if no variables are needed - ie `%macro x();` not `%macro x;`
 - Mandatory parameters should be positional, all optional parameters should be keyword (var=) style.
 - All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect.
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
-- If you have a long-running SQL query, the use of a `quit;` statement is recommended in order to benefit from the timing statistics.
+- The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
 
 # General Notes
 

base/mf_abort.sas

@@ -49,7 +49,10 @@
         input; putlog _infile_;
         i=1;
         retain logonce 0;
-        if (_infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR") and logonce=0 then do;
+        if (
+            _infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR"
+          ) and logonce=0
+        then do;
           call symputx('logline',_n_);
           logonce+1;
         end;
@@ -112,7 +115,8 @@
     %let syscc=0;
     %if %symexist(SYS_JES_JOB_URI) %then %do;
       /* refer web service output to file service in one hit */
-      filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json";
+      filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
+        name="_webout.json";
       %let rc=%sysfunc(fcopy(_web,_webout));
     %end;
     %else %do;

base/mf_existfeature.sas

@@ -32,9 +32,11 @@
     %put Supported features:  PROCLUA;
   %end;
   %else %if &feature=PROCLUA %then %do;
+    /* https://blogs.sas.com/content/sasdummy/2015/08/03/using-lua-within-your-sas-programs */
     %if &platform=SASVIYA %then 1;
-    %else %if "&sysver"="9.3" or "&sysver"="9.4" %then 1;
-    %else 0;
+    %else %if "&sysver"="9.2" or "&sysver"="9.3" %then 0;
+    %else %if "&SYSVLONG" < "9.04.01M3" %then 0;
+    %else 1;
   %end;
   %else %do;
     -1

base/mf_existfileref.sas

@@ -0,0 +1,27 @@
+/**
+  @file
+  @brief Checks whether a fileref exists
+  @details You can probably do without this macro as it is just a one liner.
+  Mainly it is here as a convenient way to remember the syntax!
+
+  @param fref the fileref to detect
+
+  @return output Returns 1 if found and 0 if not found.  Note - it is possible
+  that the fileref is found, but the file does not (yet) exist. If you need
+  to test for this, you may as well use the fileref function directly.
+
+  @version 8
+  @author [Allan Bowe](https://www.linkedin.com/in/allanbowe/)
+**/
+
+%macro mf_existfileref(fref
+)/*/STORE SOURCE*/;
+
+  %if %sysfunc(fileref(&fref))=0 %then %do;
+    1
+  %end;
+  %else %do;
+    0
+  %end;
+
+%mend;

base/mf_getattrn.sas

@@ -23,7 +23,7 @@
   %local dsid rc;
   %let dsid=%sysfunc(open(&libds,is));
   %if &dsid = 0 %then %do;
-    %put WARNING: Cannot open %trim(&libds), system message below;
+    %put %str(WARN)ING: Cannot open %trim(&libds), system message below;
     %put %sysfunc(sysmsg());
     -1
   %end;

base/mf_getengine.sas

@@ -22,6 +22,9 @@
   @version 9.2
   @author Allan Bowe
 
+  <h4> Related Macros </h4>
+  @li mf_getxengine.sas
+
 **/
 /** @cond */
 
@@ -32,7 +35,9 @@
   /* in case the parameter is a libref.tablename, pull off just the libref */
   %let libref = %upcase(%scan(&libref, 1, %str(.)));
 
-  %let dsid=%sysfunc(open(sashelp.vlibnam(where=(libname="%upcase(&libref)")),i));
+  %let dsid=%sysfunc(
+    open(sashelp.vlibnam(where=(libname="%upcase(&libref)")),i)
+  );
   %if (&dsid ^= 0) %then %do;
     %let engnum=%sysfunc(varnum(&dsid,ENGINE));
     %let rc=%sysfunc(fetch(&dsid));

base/mf_getplatform.sas

@@ -27,21 +27,23 @@
     or "&sysprocessmode"= "SAS Compute Server" %then %do;
         SASVIYA
     %end;
-    %else %if "&sysprocessmode"="SAS Stored Process Server" %then %do;
+    %else %if "&sysprocessmode"="SAS Stored Process Server"
+      or "&sysprocessmode"="SAS Workspace Server"
+    %then %do;
       SASMETA
       %return;
     %end;
     %else %do;
-      SAS
+      BASESAS
       %return;
     %end;
   %end;
-  %else %if %symexist(_metaport) %then %do;
+  %else %if %symexist(_metaport) or %symexist(_metauser) %then %do;
     SASMETA
     %return;
   %end;
   %else %do;
-    SAS
+    BASESAS
     %return;
   %end;
 %end;

base/mf_getquotedstr.sas

@@ -8,6 +8,13 @@
   which returns:
 > 'blah','blah','blah'
 
+  Alternatively:
+
+      %put %mf_getquotedstr(these words are double quoted,quote=D)
+
+  for:
+> "these","words","are","double","quoted"
+
   @param in_str the unquoted, spaced delimited string to transform
   @param dlm= the delimeter to be applied to the output (default comma)
   @param indlm= the delimeter used for the input (default is space)

base/mf_getvarlist.sas

@@ -10,14 +10,17 @@
   returns:
   > List of Variables=Name Sex Age Height Weight
 
+  For a seperated list of column values:
+
         %put %mf_getvarlist(sashelp.class,dlm=%str(,),quote=double);
 
   returns:
   > "Name","Sex","Age","Height","Weight"
 
-  @param libds Two part dataset (or view) reference.
-  @param dlm= provide a delimiter (eg comma or space) to separate the vars
-  @param quote= use either DOUBLE or SINGLE to quote the results
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] dlm= ( ) Provide a delimiter (eg comma or space) to separate the
+    variables
+  @param [in] quote= (none) use either DOUBLE or SINGLE to quote the results
 
   @version 9.2
   @author Allan Bowe

base/mf_getxengine.sas

@@ -0,0 +1,43 @@
+/**
+  @file
+  @brief Returns the engine type of a SAS fileref
+  @details Queries sashelp.vextfl to get the xengine value.
+  Usage:
+
+      filename feng temp;
+      %put %mf_getxengine(feng);
+
+  returns:
+  > TEMP
+
+  @param fref The fileref to check
+
+  @returns The XENGINE value in sashelp.vextfl or 0 if not found.
+
+  @version 9.2
+  @author Allan Bowe
+
+  <h4> Related Macros </h4>
+  @li mf_getengine.sas
+
+**/
+
+%macro mf_getxengine(fref
+)/*/STORE SOURCE*/;
+  %local dsid engnum rc engine;
+
+  %let dsid=%sysfunc(
+    open(sashelp.vextfl(where=(fileref="%upcase(&fref)")),i)
+  );
+  %if (&dsid ^= 0) %then %do;
+    %let engnum=%sysfunc(varnum(&dsid,XENGINE));
+    %let rc=%sysfunc(fetch(&dsid));
+    %let engine=%sysfunc(getvarc(&dsid,&engnum));
+    %* put &fref. ENGINE is &engine.;
+    %let rc= %sysfunc(close(&dsid));
+  %end;
+  %else %let engine=0;
+
+  &engine
+
+%mend;

base/mf_isblank.sas

@@ -9,7 +9,8 @@
 
       %put mf_isblank(&var);
 
-  inspiration:  https://support.sas.com/resources/papers/proceedings09/022-2009.pdf
+  inspiration:
+  https://support.sas.com/resources/papers/proceedings09/022-2009.pdf
 
   @param param VALUE to be checked
 

base/mf_isdir.sas

@@ -6,7 +6,8 @@
 
       %let isdir=%mf_isdir(/tmp);
 
-  With thanks and full credit to Andrea Defronzo - https://www.linkedin.com/in/andrea-defronzo-b1a47460/
+  With thanks and full credit to Andrea Defronzo -
+  https://www.linkedin.com/in/andrea-defronzo-b1a47460/
 
   @param path full path of the file/directory to be checked
 

base/mf_mval.sas

@@ -1,7 +1,8 @@
 /**
   @file mf_mval.sas
   @brief Returns a macro variable value if the variable exists
-  @details Use this macro to avoid repetitive use of `%if %symexist(MACVAR) %then`
+  @details
+  Use this macro to avoid repetitive use of `%if %symexist(MACVAR) %then`
   type logic.
   Usage:
 

base/mf_trimstr.sas

@@ -12,7 +12,8 @@
 
 
   @param basestr The string to be modified
-  @param trimstr The string to be removed from the end of `basestr`, if it exists
+  @param trimstr The string to be removed from the end of `basestr`, if it
+    exists
 
   @return output returns result with the value of `trimstr` removed from the end
 

base/mf_uid.sas

@@ -1,11 +1,11 @@
 /**
   @file
-  @brief Creates a Unique ID based on system time in a friendly format
+  @brief Creates a unique ID based on system time in friendly format
   @details format = YYYYMMDD_HHMMSSmmm_<sysjobid>_<3randomDigits>
 
         %put %mf_uid();
 
-  @version 9.2
+  @version 9.3
   @author Allan Bowe
 
 **/
@@ -14,7 +14,7 @@
 )/*/STORE SOURCE*/;
   %local today now;
   %let today=%sysfunc(today(),yymmddn8.);
-  %let now=%sysfunc(compress(%sysfunc(time(),time12.3),:.));
+  %let now=%sysfunc(compress(%sysfunc(time(),tod12.3),:.));
 
   &today._&now._&sysjobid._%sysevalf(%sysfunc(ranuni(0))*999,CEIL)
 

base/mp_abort.sas

@@ -6,11 +6,11 @@
     results back to the client in an STP Web App context, or completely stop
     in the case of a batch run.
 
-  Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored Process
-  environments.  This macro takes a unique approach - we set the SAS syscc to 0,
-  run `stpsrvset('program error', 0)` (if SAS 9) and then - we open a macro
-  but don't close it!  This provides a graceful abort for SAS web services in all
-  web enabled environments.
+  Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored
+  Process environments.  This macro takes a unique approach - we set the SAS
+  syscc to 0, run `stpsrvset('program error', 0)` (if SAS 9) and then - we open
+  a macro but don't close it!  This provides a graceful abort for SAS web
+  services in all web enabled environments.
 
   @param mac= to contain the name of the calling macro
   @param msg= message to be returned
@@ -48,7 +48,10 @@
         input; putlog _infile_;
         i=1;
         retain logonce 0;
-        if (_infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR") and logonce=0 then do;
+        if (
+            _infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR"
+          ) and logonce=0 then
+        do;
           call symputx('logline',_n_);
           logonce+1;
         end;

base/mp_assertdsobs.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Asserts the number of observations in a dataset
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      %mp_assertdsobs(sashelp.class) %* tests if any observations are present;
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+
+
+  @param [in] inds input dataset to test for presence of observations
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [in] test= (HASOBS) The test to apply.  Valid values are:
+    @li HASOBS Test is a PASS if the input dataset has any observations
+    @li EMPTY Test is a PASS if input dataset is empty
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Dataset &inds has XX obs|
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertdsobs(inds,
+  test=HASOBS,
+  desc=Testing observations,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  %local nobs;
+  %let nobs=%mf_nobs(&inds);
+  %let test=%upcase(&test);
+
+  data;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_result='FAIL';
+    test_comments="&sysmacroname: Dataset &inds has &nobs observations";
+  %if &test=HASOBS %then %do;
+    if &nobs>0 then test_result='PASS';
+  %end;
+  %else %if &test=EMPTY %then %do;
+    if &nobs=0 then test_result='PASS';
+  %end;
+  %else %do;
+    test_comments="&sysmacroname: Unsatisfied test condition - &test";
+  %end;
+  run;
+
+  %local ds;
+  %let ds=&syslast;
+
+  proc append base=&outds data=&ds;
+  run;
+
+  proc sql;
+  drop table &ds;
+
+%mend;

base/mp_binarycopy.sas

@@ -4,7 +4,8 @@
   @details Reads in a file byte by byte and writes it back out.  Is an
   os-independent method to copy files.  In case of naming collision, the
   default filerefs can be modified.
-    Based on http://stackoverflow.com/questions/13046116/using-sas-to-copy-a-text-file
+  Based on:
+  https://stackoverflow.com/questions/13046116/using-sas-to-copy-a-text-file
 
         %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
 

base/mp_cleancsv.sas

@@ -23,7 +23,7 @@
 
 %macro mp_cleancsv(in=NOTPROVIDED,out=NOTPROVIDED,qchar='22'x);
 %if "&in"="NOTPROVIDED" or "&out"="NOTPROVIDED" %then %do;
-  %put %str(ERR)OR: Please provide valid input (&in) and output (&out) locations;
+  %put %str(ERR)OR: Please provide valid input (&in) & output (&out) locations;
   %return;
 %end;
 

base/mp_dirlist.sas

@@ -57,8 +57,11 @@
 )/*/STORE SOURCE*/;
 %let getattrs=%upcase(&getattrs)XX;
 
-data &outds (compress=no keep=file_or_folder filepath filename ext msg directory);
-  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80 ext $20 msg $200;
+data &outds(compress=no
+    keep=file_or_folder filepath filename ext msg directory
+  );
+  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
+    ext $20 msg $200;
   %if &fref=0 %then %do;
     rc = filename(fref, "&path");
   %end;

base/mp_ds2cards.sas

@@ -3,25 +3,29 @@
   @brief Create a CARDS file from a SAS dataset.
   @details Uses dataset attributes to convert all data into datalines.
     Running the generated file will rebuild the original dataset.
-  usage:
+  Usage:
 
       %mp_ds2cards(base_ds=sashelp.class
         , cards_file= "C:\temp\class.sas"
         , maxobs=5)
 
-    stuff to add
+  TODO:
     - labelling the dataset
     - explicity setting a unix LF
     - constraints / indexes etc
 
-  @param base_ds= Should be two level - eg work.blah.  This is the table that
-                   is converted to a cards file.
-  @param tgt_ds= Table that the generated cards file would create. Optional -
-                  if omitted, will be same as BASE_DS.
-  @param cards_file= Location in which to write the (.sas) cards file
-  @param maxobs= to limit output to the first <code>maxobs</code> observations
-  @param showlog= whether to show generated cards file in the SAS log (YES/NO)
-  @param outencoding= provide encoding value for file statement (eg utf-8)
+  @param [in] base_ds= Should be two level - eg work.blah.  This is the table
+                  that is converted to a cards file.
+  @param [in] tgt_ds= Table that the generated cards file would create.
+    Optional - if omitted, will be same as BASE_DS.
+  @param [out] cards_file= Location in which to write the (.sas) cards file
+  @param [in] maxobs= to limit output to the first <code>maxobs</code>
+    observations
+  @param [in] showlog= whether to show generated cards file in the SAS log
+    (YES/NO)
+  @param [in] outencoding= provide encoding value for file statement (eg utf-8)
+  @param [in] append= If NO then will rebuild the cards file if it already
+    exists, otherwise will append to it.  Used by the mp_lib2cards.sas macro.
 
 
   @version 9.2
@@ -34,6 +38,7 @@
     ,random_sample=NO
     ,showlog=YES
     ,outencoding=
+    ,append=NO
 )/*/STORE SOURCE*/;
 %local i setds nvars;
 
@@ -46,6 +51,8 @@
 %if (&tgt_ds = ) %then %let tgt_ds=&base_ds;
 %if %index(&tgt_ds,.)=0 %then %let tgt_ds=WORK.%scan(&base_ds,2,.);
 %if ("&outencoding" ne "") %then %let outencoding=encoding="&outencoding";
+%if ("&append" = "") %then %let append=;
+%else %let append=mod;
 
 /* get varcount */
 %let nvars=0;
@@ -59,9 +66,11 @@ select count(*) into: nvars from dictionary.columns
 %end;
 
 /* get indexes */
-proc sort data=sashelp.vindex
-    (where=(upcase(libname)="%scan(%upcase(&base_ds),1)"
-       and upcase(memname)="%scan(%upcase(&base_ds),2)"))
+proc sort
+  data=sashelp.vindex(
+    where=(upcase(libname)="%scan(%upcase(&base_ds),1)"
+      and upcase(memname)="%scan(%upcase(&base_ds),2)")
+    )
   out=_data_;
   by indxname indxpos;
 run;
@@ -172,7 +181,7 @@ data _null_;
 run;
 
 data _null_;
-  file &cards_file. &outencoding lrecl=32767 termstr=nl;
+  file &cards_file. &outencoding lrecl=32767 termstr=nl &append;
   length __attrib $32767;
   if _n_=1 then do;
     put '/*******************************************************************';

base/mp_filtercheck.sas

@@ -0,0 +1,143 @@
+/**
+  @file
+  @brief Checks an input filter table for validity
+  @details Performs checks on the input table to ensure it arrives in the
+  correct format.  This is necessary to prevent code injection.  Will update
+  SYSCC to 1008 if bad records are found.
+
+  Used for dynamic filtering in [Data Controller for SAS®](https://datacontroller.io).
+
+  Usage:
+
+      %mp_filtercheck(work.filter,targetds=sashelp.class,outds=work.badrecords)
+
+  The input table should have the following format:
+
+  |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
+  |---|---|---|---|---|---|
+  |AND|AND|1|AGE|=|12|
+  |AND|AND|1|SEX|<=|'M'|
+  |AND|OR|2|Name|NOT IN|('Jane','Alfred')|
+  |AND|OR|2|Weight|>=|7|
+
+  Rules applied:
+
+  @li GROUP_LOGIC - only AND/OR
+  @li SUBGROUP_LOGIC - only AND/OR
+  @li SUBGROUP_ID - only integers
+  @li VARIABLE_NM - must be in the target table
+  @li OPERATOR_NM - only =/>/</<=/>=/BETWEEN/IN/NOT IN/NOT EQUAL/CONTAINS
+  @li RAW_VALUE - no unquoted values except integers, commas and spaces.
+
+  @returns The &outds table containing any bad rows, plus a REASON_CD column.
+
+  @param [in] inds The table to be checked, with the format above
+  @param [in] targetds= The target dataset against which to verify VARIABLE_NM
+  @param [out] outds= The output table, which is a copy of the &inds. table
+  plus a REASON_CD column, containing only bad records.  If bad records found,
+  the SYSCC value will be set to 1008 (general data problem).  Downstream
+  processes should check this table (and return code) before continuing.
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_getvarlist.sas
+  @li mf_nobs.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filtergenerate.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+  @todo Support date / hex / name literals and exponents in RAW_VALUE field
+**/
+
+%macro mp_filtercheck(inds,targetds=,outds=work.badrecords);
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(syscc=&syscc - on macro entry)
+)
+
+/**
+  * Sanitise the values based on valid value lists, then strip out
+  * quotes, commas, periods and spaces.
+  * Only numeric values should remain
+  */
+
+data &outds;
+  set &inds;
+  length reason_cd $32;
+
+  /* closed list checks */
+  if GROUP_LOGIC not in ('AND','OR') then do;
+    REASON_CD='GROUP_LOGIC should be either AND or OR';
+    putlog REASON_CD= GROUP_LOGIC=;
+    output;
+  end;
+  if SUBGROUP_LOGIC not in ('AND','OR') then do;
+    REASON_CD='SUBGROUP_LOGIC should be either AND or OR';
+    putlog REASON_CD= SUBGROUP_LOGIC=;
+    output;
+  end;
+  if mod(SUBGROUP_ID,1) ne 0 then do;
+    REASON_CD='SUBGROUP_ID should be integer';
+    putlog REASON_CD= SUBGROUP_ID=;
+    output;
+  end;
+  if upcase(VARIABLE_NM) not in
+  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
+  then do;
+    REASON_CD="VARIABLE_NM not in &targetds";
+    putlog REASON_CD= VARIABLE_NM=;
+    output;
+  end;
+  if OPERATOR_NM not in
+  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NOT EQUAL','CONTAINS')
+  then do;
+    REASON_CD='Invalid OPERATOR_NM';
+    putlog REASON_CD= OPERATOR_NM=;
+    output;
+  end;
+
+  /* special logic */
+  if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ','');
+  else if OPERATOR_NM in ('IN','NOT IN') then do;
+    if substr(raw_value,1,1) ne '('
+    or substr(cats(reverse(raw_value)),1,1) ne ')'
+    then do;
+      REASON_CD='Missing brackets in RAW_VALUE';
+      putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+      output;
+    end;
+    else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
+  end;
+  else raw_value1=raw_value;
+
+  /* remove nested literals eg '' */
+  raw_value1=tranwrd(raw_value1,"''",'');
+
+  /* now match string literals (always single quotes) */
+  raw_value2=raw_value1;
+  regex = prxparse("s/(\').*?(\')//");
+  call prxchange(regex,-1,raw_value2);
+
+  /* remove commas */
+  raw_value3=compress(raw_value2,',');
+
+
+
+
+  /* output records that contain values other than digits and spaces */
+  if notdigit(compress(raw_value3,' '))>0 then do;
+    putlog raw_value3= $hex32.;
+    REASON_CD='Invalid RAW_VALUE';
+    putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
+    output;
+  end;
+
+run;
+
+%if %mf_nobs(&outds)>0 %then %let syscc=1008;
+
+%mend;

base/mp_filtergenerate.sas

@@ -0,0 +1,90 @@
+/**
+  @file
+  @brief Generates a filter clause from an input table, to a fileref
+  @details Uses the input table to generate an output filter clause.
+  This feature is used to create dynamic dropdowns in [Data Controller for SAS&reg](
+  https://datacontroller.io). The input table should be in the format below:
+
+  |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
+  |---|---|---|---|---|---|
+  |AND|AND|1|AGE|=|12|
+  |AND|AND|1|SEX|<=|'M'|
+  |AND|OR|2|Name|NOT IN|('Jane','Alfred')|
+  |AND|OR|2|Weight|>=|7|
+
+  Note - if the above table is received from an external client, the values
+  should first be validated using the mp_filtercheck.sas macro to avoid risk
+  of SQL injection.
+
+  To generate the filter, run the following code:
+
+      data work.filtertable;
+        infile datalines4 dsd;
+        input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+          OPERATOR_NM:$10. RAW_VALUE:$32767.;
+      datalines4;
+      AND,AND,1,AGE,=,12
+      AND,AND,1,SEX,<=,"'M'"
+      AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
+      AND,OR,2,Weight,>=,7
+      ;;;;
+      run;
+
+      %mp_filtergenerate(work.filtertable,outref=myfilter)
+
+      data _null_;
+        infile myfilter;
+        input;
+        put _infile_;
+      run;
+
+  Will write the following query to the log:
+
+  > (
+  >     AGE = 12
+  >   AND
+  >     SEX <= 'M'
+  > ) AND (
+  >     Name NOT IN ('Jane','Alfred')
+  >   OR
+  >     Weight >= 7
+  > )
+
+  @param [in] inds The input table with query values
+  @param [out] outref= The output fileref to contain the filter clause.  Will
+    be created (or replaced).
+
+  <h4> Related Macros </h4>
+  @li mp_filtercheck.sas
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mp_filtergenerate(inds,outref=filter);
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(syscc=&syscc - on macro entry)
+)
+
+filename &outref temp;
+
+data _null_;
+  file &outref lrecl=32800;
+  set &inds end=last;
+  by SUBGROUP_ID;
+  if _n_=1 then put '(';
+  else if first.SUBGROUP_ID then put +1 GROUP_LOGIC '(';
+  else put +2 SUBGROUP_LOGIC;
+
+  put +4 VARIABLE_NM OPERATOR_NM RAW_VALUE;
+
+  if last.SUBGROUP_ID then put ')'@;
+run;
+
+%mend;

base/mp_getdbml.sas

@@ -133,7 +133,9 @@ run;
 
     if notnull='yes' then notnul=' not null';
     if notnull='no' and missing(label) then put '  ' name typ;
-    else if notnull='yes' and missing(label) then put '  ' name typ '[' notnul ']';
+    else if notnull='yes' and missing(label) then do;
+      put '  ' name typ '[' notnul ']';
+    end;
     else if notnull='no' then put '  ' name typ '[' lab ']';
     else put '  ' name typ '[' notnul ',' lab ']';
 
@@ -166,7 +168,7 @@ run;
       call symputx('constcheck',1);
     end;
 
-    if last then call symputx('constraints_used',cats(upcase(constraints_used)));
+    if last then call symput('constraints_used',cats(upcase(constraints_used)));
 
     length curds const col $39;
     curds="&curds";
@@ -176,7 +178,8 @@ run;
 
   proc append base=&pkds data=&syslast;run;
 
-  /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
+  /* Create Unique Indexes, but only if they were not already defined within
+    the Constraints section. */
   data _data_(keep=curds const col);
     set &idxinfo (where=(
       libname="%scan(&curds,1,.)"
@@ -187,7 +190,7 @@ run;
     file &outref mod;
     by idxusage indxname;
     name=upcase(name);
-    if &constcheck=1 then stop; /* in fact we only care about PKs so stop if we have */
+    if &constcheck=1 then stop; /* we only care about PKs so stop if we have */
     if _n_=1 and &constcheck=0 then put / '  indexes {';
 
     length cols $5000;
@@ -261,7 +264,11 @@ run;
       line='Ref: "'!!"&curds"
         !!cats('".(',"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))",')')
         !!' - '
-        !!cats(quote(trim(curds)),'.(',"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))",')');
+        !!cats(quote(trim(curds))
+            ,'.('
+            ,"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))"
+            ,')'
+          );
       put line;
     run;
 
@@ -282,7 +289,9 @@ run;
     create table &pkds.5b as
       select curds,count(*) as cnt
       from &pkds.5a
-      where curds not in (select curds from &pkds.2 where cols="&pkcols") /* not a one to one match */
+      where curds not in (
+          select curds from &pkds.2 where cols="&pkcols"
+        ) /* not a one to one match */
         and curds ne "&curds" /* exclude self */
       group by 1;
     create table &pkds.6 as

base/mp_getddl.sas

@@ -86,7 +86,9 @@ create table _data_ as
   %global constraints_used;
   data _null_;
     length ctype $11 constraint_name_orig $256 constraints_used $5000;
-    set &colconst (where=(table_name="&curds" and constraint_type in ('PRIMARY','UNIQUE'))) end=last;
+    set &colconst(
+        where=(table_name="&curds" and constraint_type in ('PRIMARY','UNIQUE'))
+      ) end=last;
     file &fref mod;
     by constraint_type constraint_name;
     retain constraints_used;
@@ -161,10 +163,19 @@ run;
       put ');';
     run;
 
-    /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
+    /* Create Unique Indexes, but only if they were not already defined within
+      the Constraints section. */
     data _null_;
       *length ds $128;
-      set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
+      set &idxinfo(
+        where=(
+          memname="&curds"
+          and unique='yes'
+          and indxname not in (
+              %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+              )
+          )
+        );
       file &fref mod;
       by idxusage indxname;
 /*       ds=cats(libname,'.',memname); */
@@ -228,10 +239,19 @@ run;
     /* Extra step for data constraints */
     %addConst()
 
-    /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
+    /* Create Unique Indexes, but only if they were not already defined within
+      the Constraints section. */
     data _null_;
       *length ds $128;
-      set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
+      set &idxinfo(
+        where=(
+          memname="&curds"
+          and unique='yes'
+          and indxname not in (
+            %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+          )
+        )
+      );
       file &fref mod;
       by idxusage indxname;
       *ds=cats(libname,'.',memname);
@@ -320,10 +340,19 @@ run;
       put ');';
     run;
 
-    /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
+    /* Create Unique Indexes, but only if they were not already defined within
+      the Constraints section. */
     data _null_;
       *length ds $128;
-      set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
+      set &idxinfo(
+        where=(
+          memname="&curds"
+          and unique='yes'
+          and indxname not in (
+            %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+          )
+        )
+      );
       file &fref mod;
       by idxusage indxname;
 /*       ds=cats(libname,'.',memname); */

base/mp_getmaxvarlengths.sas

@@ -8,10 +8,11 @@
   - NAME    Name of the base dataset column
   - MAXLEN Maximum length of the data contained therein.
 
-  Character fields may be allocated very large widths (eg 32000) of which the maximum
-    value is likely to be much narrower.  This macro was designed to enable a HTML
-    table to be appropriately sized however this could be used as part of a data
-    audit to ensure we aren't over-sizing our tables in relation to the data therein.
+  Character fields may be allocated very large widths (eg 32000) of which the
+  maximum  value is likely to be much narrower.  This macro was designed to
+  enable a HTML table to be appropriately sized however this could be used as
+  part of a data audit to ensure we aren't over-sizing our tables in relation to
+  the data therein.
 
   Numeric fields are converted using the relevant format to determine the width.
   Usage:

base/mp_hashdataset.sas

@@ -0,0 +1,75 @@
+/**
+  @file
+  @brief Returns a unique hash for a dataset
+  @details Ignores metadata attributes, used only to hash values. Compared
+  datasets must be in the same order.
+
+      %mp_hashdataset(sashelp.class,outds=myhash)
+
+      data _null_;
+        set work.myhash;
+        put hashkey=;
+      run;
+
+  ![sas md5 hash dataset log results](https://i.imgur.com/MqF98vk.png)
+
+  <h4> SAS Macros </h4>
+  @li mf_getattrn.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
+
+  @param [in] libds dataset to hash
+  @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
+  will contain one column (hashkey) with one observation (a hex32.
+  representation of the input hash)
+  |hashkey:$32.|
+  |---|
+  |28ABC74ABFC45F50794237BA5566E6CA|
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_hashdataset(
+  libds,
+  outds=
+)/*/STORE SOURCE*/;
+  %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
+    %put %str(WARN)ING: Dataset &libds is empty;, or is not a dataset;
+  %end;
+  %else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
+    %put %str(ERR)OR: Dataset &libds is not a dataset;
+  %end;
+  %else %do;
+    %local keyvar /* roll up the md5 */
+      prevkeyvar /* retain prev record md5 */
+      lastvar /* last var in input ds */
+      varlist var i;
+    /* avoid naming conflict for hash key vars */
+    %let keyvar=%mf_getuniquename();
+    %let prevkeyvar=%mf_getuniquename();
+    %let lastvar=%mf_getuniquename();
+    %let varlist=%mf_getvarlist(&libds);
+    data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
+      length &prevkeyvar &keyvar $32;
+      retain &prevkeyvar;
+      set &libds end=&lastvar;
+      /* hash should include previous row */
+      if _n_>1 then &keyvar=put(md5(&prevkeyvar
+      /* loop every column, hashing every individual value */
+    %do i=1 %to %sysfunc(countw(&varlist));
+      %let var=%scan(&varlist,&i,%str( ));
+      %if %mf_getvartype(&libds,&var)=C %then %do;
+          !!put(md5(trim(&var)),$hex32.)
+      %end;
+      %else %do;
+          !!put(md5(trim(put(&var*1,binary64.))),$hex32.)
+      %end;
+    %end;
+      ),$hex32.);
+      &prevkeyvar=&keyvar;
+      if &lastvar then output;
+    run;
+  %end;
+%mend;

base/mp_jsonout.sas

@@ -4,7 +4,10 @@
   @details PROC JSON is faster but will produce errs like the ones below if
   special chars are encountered.
 
+  > ERROR: Some code points did not transcode.
+
   > An object or array close is not valid at this point in the JSON text.
+
   > Date value out of range
 
   If this happens, try running with ENGINE=DATASTEP.
@@ -14,7 +17,9 @@
         filename tmp temp;
         data class; set sashelp.class;run;
 
+        %mp_jsonout(OPEN,jref=tmp)
         %mp_jsonout(OBJ,class,jref=tmp)
+        %mp_jsonout(CLOSE,jref=tmp)
 
         data _null_;
         infile tmp;
@@ -27,18 +32,18 @@
   For more information see https://sasjs.io
 
   @param action Valid values:
-    * OPEN - opens the JSON
-    * OBJ - sends a table with each row as an object
-    * ARR - sends a table with each row in an array
-    * CLOSE - closes the JSON
+    @li OPEN - opens the JSON
+    @li OBJ - sends a table with each row as an object
+    @li ARR - sends a table with each row in an array
+    @li CLOSE - closes the JSON
 
   @param ds the dataset to send.  Must be a work table.
   @param jref= the fileref to which to send the JSON
   @param dslabel= the name to give the table in the exported JSON
   @param fmt= Whether to keep or strip formats from the table
-  @param engine= Which engine to use to send the JSON, options are:
-  * PROCJSON (default)
-  * DATASTEP 
+  @param engine= Which engine to use to send the JSON, valid options are:
+    @li PROCJSON (default)
+    @li DATASTEP (more reliable when data has non standard characters)
 
   @param dbg= DEPRECATED - was used to conditionally add PRETTY to
     proc json but this can cause line truncation in large files.
@@ -83,7 +88,8 @@
     %end;
     data _null_;file &jref mod ;
       put "["; call symputx('cols',0,'l');
-    proc sort data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
+    proc sort
+      data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
       out=_data_;
       by varnum;
 
@@ -122,7 +128,8 @@
       %end;
     %end;
     run;
-    /* write to temp loc to avoid _webout truncation - https://support.sas.com/kb/49/325.html */
+    /* write to temp loc to avoid _webout truncation
+      - https://support.sas.com/kb/49/325.html */
     filename _sjs temp lrecl=131068 encoding='utf-8';
     data _null_; file _sjs lrecl=131068 encoding='utf-8' mod;
       set &tempds;
@@ -158,7 +165,7 @@
 %end;
 
 %else %if &action=CLOSE %then %do;
-  data _null_;file &jref encoding='utf-8';
+  data _null_;file &jref encoding='utf-8' mod;
     put "}";
   run;
 %end;

base/mp_lib2cards.sas

@@ -1,21 +1,35 @@
 /**
   @file
   @brief Convert all library members to CARDS files
-  @details Gets list of members then calls the <code>%mp_ds2cards()</code>
-            macro
-    usage:
+  @details Gets list of members then calls the <code>%mp_ds2cards()</code> macro.
+  Usage:
 
       %mp_lib2cards(lib=sashelp
           , outloc= C:\temp )
 
+  The output will be one cards file in the `outloc` directory per dataset in the
+  input `lib` library.  If the `outloc` directory does not exist, it is created.
+
+  To create a single SAS file with the first 1000 records of each table in a
+  library you could use this syntax:
+
+      %mp_lib2cards(lib=sashelp
+          , outloc= /tmp
+          , outfile= myfile.sas
+          , maxobs= 1000
+      )
+
   <h4> SAS Macros </h4>
   @li mf_mkdir.sas
+  @li mf_trimstr.sas
   @li mp_ds2cards.sas
 
-  @param lib= Library in which to convert all datasets
-  @param outloc= Location in which to store output.  Defaults to WORK library.
-    Do not use a trailing slash (my/path not my/path/).  No quotes.
-  @param maxobs= limit output to the first <code>maxobs</code> observations
+  @param [in] lib= Library in which to convert all datasets
+  @param [out] outloc= Location in which to store output.  Defaults to WORK
+    library. No quotes.
+  @param [out] outfile= Optional output file NAME - if provided, then will create
+  a single output file instead of one file per input table.
+  @param [in] maxobs= limit output to the first <code>maxobs</code> observations
 
   @version 9.2
   @author Allan Bowe
@@ -25,6 +39,7 @@
     ,outloc=%sysfunc(pathname(work)) /* without trailing slash */
     ,maxobs=max
     ,random_sample=NO
+    ,outfile=0
 )/*/STORE SOURCE*/;
 
 /* Find the tables */
@@ -36,6 +51,10 @@ select distinct lowcase(memname)
   from dictionary.tables
   where upcase(libname)="%upcase(&lib)";
 
+/* trim trailing slash, if provided */
+%let outloc=%mf_trimstr(&outloc,/);
+%let outloc=%mf_trimstr(&outloc,\);
+
 /* create the output directory */
 %mf_mkdir(&outloc)
 
@@ -43,9 +62,17 @@ select distinct lowcase(memname)
 %do x=1 %to %sysfunc(countw(&memlist));
   %let ds=%scan(&memlist,&x);
   %mp_ds2cards(base_ds=&lib..&ds
-      ,cards_file="&outloc/&ds..sas"
     ,maxobs=&maxobs
-      ,random_sample=&random_sample)
+    ,random_sample=&random_sample
+  %if "&outfile" ne "0" %then %do;
+    ,append=YES
+    ,cards_file="&outloc/&outfile"
+  %end;
+  %else %do;
+    ,append=NO
+    ,cards_file="&outloc/&ds..sas"
+  %end;
+  )
 %end;
 
 %mend;

base/mp_mdtablewrite.sas

@@ -0,0 +1,98 @@
+/**
+  @file
+  @brief Create a Markdown Table from a dataset
+  @details A markdown table is a simple table representation for use in
+  documents written in markdown format.
+
+  An online generator is available here:
+  https://www.tablesgenerator.com/markdown_tables
+
+  This structure is also used by the Macro Core library for documenting input/
+  output datasets, as well as the sasjs/cli tool for documenting inputs/outputs
+  for web services.
+
+  We take the standard definition one step further by embedding the informat
+  in the table header row, like so:
+
+      |var1:$|var2:best.|var3:date9.|
+      |---|---|---|
+      |some text|42|01JAN1960|
+      |blah|1|31DEC1999|
+
+  Which resolves to:
+
+  |var1:$|var2:best.|var3:date9.|
+  |---|---|---|
+  |some text|42|01JAN1960|
+  |blah|1|31DEC1999|
+
+
+  Usage:
+
+      %mp_mdtablewrite(libds=sashelp.class,showlog=YES)
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getvarlist.sas
+  @li mf_getvarformat.sas
+
+  @param [in] libds= the library / dataset to create or read from.
+  @param [out] fref= Fileref to contain the markdown. Default=mdtable.
+  @param [out] showlog= set to YES to show the markdown in the log. Default=NO.
+
+  @version 9.3
+  @author Allan Bowe
+**/
+
+%macro mp_mdtablewrite(
+  libds=,
+  fref=mdtable,
+  showlog=NO
+)/*/STORE SOURCE*/;
+
+/* check fileref is assigned */
+%if %sysfunc(fileref(&fref)) > 0 %then %do;
+  filename &fref temp;
+%end;
+
+%local vars;
+%let vars=%mf_getvarlist(&libds);
+
+/* create the header row */
+data _null_;
+  file &fref;
+  length line $32767;
+  put '|'
+%local i var fmt;
+%do i=1 %to %sysfunc(countw(&vars));
+  %let var=%scan(&vars,&i);
+  %let fmt=%mf_getvarformat(&libds,&var,force=1);
+  "&var:&fmt|"
+%end;
+  ;
+  put '|'
+%do i=1 %to %sysfunc(countw(&vars));
+  "---|"
+%end;
+  ;
+run;
+
+/* write out the data */
+data _null_;
+  file &fref mod dlm='|' lrecl=32767;
+  set &libds ;
+  length line $32767;
+  line=cats('|',%mf_getvarlist(&libds,dlm=%str(,'|',)),'|');
+  put line;
+run;
+
+%if %upcase(&showlog)=YES %then %do;
+  options ps=max;
+  data _null_;
+    infile &fref;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_mdtablewrite;

base/mp_resetoption.sas

@@ -1,7 +1,8 @@
 /**
   @file
   @brief Reset an option to original value
-  @details Inspired by the SAS Jedi - https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options/
+  @details Inspired by the SAS Jedi -
+https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options
     Called as follows:
 
     options obs=30;

base/mp_searchdata.sas

@@ -20,9 +20,10 @@
   @param ds= the dataset to search (leave blank to search entire library)
   @param string= the string value to search
   @param numval= the numeric value to search (must be exact)
-  @param outloc= the directory in which to create the output datasets with matching
-    rows.  Will default to a subfolder in the WORK library.
-  @param outobs= set to a positive integer to restrict the number of observations
+  @param outloc= the directory in which to create the output datasets with
+    matching rows.  Will default to a subfolder in the WORK library.
+  @param outobs= set to a positive integer to restrict the number of
+    observations
   @param filter_text= add a (valid) filter clause to further filter the results
 
   <h4> SAS Macros </h4>
@@ -44,7 +45,8 @@
   ,filter_text=%str(1=1)
 )/*/STORE SOURCE*/;
 
-%local table_list table table_num table colnum col start_tm check_tm vars type coltype;
+%local table_list table table_num table colnum col start_tm check_tm vars type
+  coltype;
 %put process began at %sysfunc(datetime(),datetime19.);
 
 %if &syscc ge 4 %then %do;
@@ -101,7 +103,8 @@ proc sql
       %end;
     %end;
     );
-    %put Search query for &table took %sysevalf(%sysfunc(datetime())-&check_tm) seconds;
+    %put Search query for &table took
+      %sysevalf(%sysfunc(datetime())-&check_tm) seconds;
     %if &sqlrc ne 0 %then %do;
       %put %str(WAR)NING: SQLRC=&sqlrc when processing &table;
       %return;

base/mp_setkeyvalue.sas

@@ -26,7 +26,7 @@
 
   %if not (%mf_existds(&libds)) %then %do;
     data &libds (index=(key/unique));
-      length key $32 valc $256 valn 8 type $1;
+      length key $64 valc $2048 valn 8 type $1;
       call missing(of _all_);
       stop;
     run;

base/mp_stprequests.sas

@@ -1,7 +1,8 @@
 /**
   @file
   @brief Capture session start / finish times and request details
-  @details For details, see http://www.rawsas.com/2015/09/logging-of-stored-process-server.html.
+  @details For details, see
+  https://rawsas.com/event-logging-of-stored-process-server-sessions.
     Requires a base table in the following structure (name can be changed):
 
     proc sql;

base/mp_streamfile.sas

@@ -6,7 +6,8 @@
 
   Usage:
 
-      filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      filename mc url
+        "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
 
       %mp_streamfile(contenttype=csv,inloc=/some/where.txt,outname=myfile.txt)
@@ -65,13 +66,15 @@
 %else %if &contentype=XLSX %then %do;
   %if &platform=SASMETA %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
+      rc=stpsrv_header('Content-type',
+        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
-      contenttype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+      contenttype=
+        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;

base/mp_testjob.sas

@@ -0,0 +1,92 @@
+/**
+  @file
+  @brief Runs arbitrary code for a specified amount of time
+  @details Executes a series of procs and data steps to enable performance
+  testing of arbitrary jobs.
+
+      %mp_testjob(
+        duration=60*5
+      )
+
+  @param [in] duration= the time in seconds which the job should run for. Actual
+  time may vary, as the check is done in between steps.  Default = 30 (seconds).
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquelibref.sas
+  @li mf_getuniquename.sas
+  @li mf_mkdir.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mp_testjob(duration=30
+)/*/STORE SOURCE*/;
+%local lib dir ds1 ds2 ds3 start_tm i;
+
+%let start_tm=%sysfunc(datetime());
+%let duration=%sysevalf(&duration);
+
+/* create a temporary library in WORK */
+%let lib=%mf_getuniquelibref();
+%let dir=%mf_getuniquename();
+%mf_mkdir(%sysfunc(pathname(work))/&dir)
+libname &lib "%sysfunc(pathname(work))/&dir";
+
+/* loop through until time expires */
+%let ds1=%mf_getuniquename();
+%let ds2=%mf_getuniquename();
+%let ds3=%mf_getuniquename();
+%do i=0 %to 1;
+
+  /* create big dataset */
+  data &lib..&ds1(compress=no );
+    do x=1 to 1000000;
+      randnum0=ranuni(0)*3;
+      randnum1=ranuni(0)*2;
+      bigchar=repeat('A',300);
+      output;
+    end;
+  run;
+  %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
+
+  proc summary ;
+    class randnum0 randnum1;
+    output out=&lib..&ds2;
+  run;quit;
+  %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
+
+  /* add more data */
+  proc sql;
+  create table &lib..&ds3 as
+    select *, ranuni(0)*10 as randnum2
+  from &lib..&ds1
+  order by randnum1;
+  quit;
+  %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
+
+  proc sort data=&lib..&ds3;
+    by descending x;
+  run;
+  %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
+
+  /* wait 5 seconds */
+  data _null_;
+    call sleep(5,1);
+  run;
+  %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
+
+  %let i=0;
+
+%end;
+
+%gate:
+%put time is up!;
+proc datasets lib=&lib kill;
+run;
+quit;
+libname &lib clear;
+
+
+%mend;

base/mp_tree.sas

@@ -8,8 +8,11 @@
 
   Credits:
 
-  * Roger Deangelis, https://communities.sas.com/t5/SAS-Programming/listing-all-files-within-a-directory-and-subdirectories/m-p/332616/highlight/true#M74887
-  * Tom, https://communities.sas.com/t5/SAS-Programming/listing-all-files-of-all-types-from-all-subdirectories/m-p/334113/highlight/true#M75419
+  Roger Deangelis:
+https://communities.sas.com/t5/SAS-Programming/listing-all-files-within-a-directory-and-subdirectories/m-p/332616/highlight/true#M74887
+
+  Tom:
+https://communities.sas.com/t5/SAS-Programming/listing-all-files-of-all-types-from-all-subdirectories/m-p/334113/highlight/true#M75419
 
 
   @param dir= Directory to be scanned (default=/tmp)

build.py

@@ -9,19 +9,22 @@ for file in files:
     ml = open('lua/' + name + '.sas', "w")
     ml.write("/**\n")
     ml.write("  @file " + name + '.sas\n')
-    ml.write("  @brief Creates the " + basename + " file\n")
+    ml.write("  @brief Compiles the " + basename + " lua file\n")
     ml.write("  @details Writes " + basename + " to the work directory\n")
+    ml.write("  and then includes it.\n")
     ml.write("  Usage:\n\n")
     ml.write("      %" + name + "()\n\n")
     ml.write("**/\n\n")
     ml.write("%macro " + name + "();\n")
     ml.write("data _null_;\n")
-    ml.write("  file \"%sysfunc(pathname(work))/" + basename + "\";\n")
+    ml.write("  file \"%sysfunc(pathname(work))/" + name + ".lua\";\n")
     with open(file) as infile:
         for line in infile:
           ml.write("  put '" + line.rstrip().replace("'","''") + " ';\n")
-    ml.write("run;\n")
+    ml.write("run;\n\n")
+    ml.write("%inc \"%sysfunc(pathname(work))/" + name + ".lua\";\n\n")
     ml.write("%mend;\n")
+
 ml.close()
 
 # prepare web files

doxy/batch.bat

@@ -1,6 +0,0 @@
-ECHO on
-
-rmdir /s /q C:\DEVELOPMENT\output
-mkdir C:\DEVELOPMENT\output
-
-doxygen.exe Doxyfile

doxy/build.sh

@@ -1,44 +0,0 @@
-#!/bin/bash
-####################################################################
-# PROJECT: Macro Core Docs Build                                   #
-####################################################################
-
-BUILD_FOLDER="/tmp/macrocore_docs"
-
-# move to project root
-cd ..
-
-# create build directory
-rm -rf $BUILD_FOLDER
-mkdir $BUILD_FOLDER
-
-# copy relevant files
-cp -r base $BUILD_FOLDER
-cp -r meta $BUILD_FOLDER
-cp -r metax $BUILD_FOLDER
-cp -r viya $BUILD_FOLDER
-cp -r doxy $BUILD_FOLDER
-cp main.dox $BUILD_FOLDER
-cp doxy/Doxyfile $BUILD_FOLDER
-
-# update Doxyfile and generate
-cd $BUILD_FOLDER
-echo "OUTPUT_DIRECTORY=$BUILD_FOLDER/out" >> $BUILD_FOLDER/Doxyfile
-echo "INPUT+=main.dox" >> $BUILD_FOLDER/Doxyfile
-doxygen Doxyfile
-
-# refresh github pages site
-git clone git@github.com:sasjs/core.github.io.git
-cd core.github.io
-rm -r *
-mv $BUILD_FOLDER/out/doxy/* .
-echo 'core.sasjs.io' > CNAME
-git add .
-git commit -m "build.sh build on $(date +%F:%H:%M:%S)"
-git push
-npx sitemap-generator-cli https://core.sasjs.io
-git add .
-git commit -m "adding sitemap"
-git push
-
-echo "check it out:  https://sasjs.github.io/core.github.io/files.html"

doxy/new_footer.html

@@ -1,23 +0,0 @@
-<!-- HTML footer for doxygen 1.8.17-->
-<!-- start footer part -->
-<!--BEGIN GENERATE_TREEVIEW-->
-<div id="nav-path" class="navpath"><!-- id is needed for treeview function! -->
-  <ul>
-    $navpath
-    <li class="footer">$generatedby
-    <a href="https://www.doxygen.org/index.html">
-      <img class="footer" src="$relpath^doxygen.png" alt="doxygen"/></a> $doxygenversion </li>
-   <i> For more information visit the </i> <a href="https://github.com/sasjs/core">Macro Core library</a>.</li>
-
-   </ul>
- </div>
-<!--END GENERATE_TREEVIEW-->
-<!--BEGIN !GENERATE_TREEVIEW-->
-<hr class="footer"/><address class="footer"><small>
-  $generatedby &#160;<a href="http://www.doxygen.org/index.html">
-  <img class="footer" src="$relpath^doxygen.png" alt="doxygen"/>
-  </a> $doxygenversion
-  </small></address>
-<!--END !GENERATE_TREEVIEW-->
-</body>
-</html>

doxy/new_header.html

@@ -1,72 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<!-- HTML header for doxygen 1.8.17-->
-<html xmlns="https://www.w3.org/1999/xhtml" lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
-<meta http-equiv="X-UA-Compatible" content="IE=9"/>
-<meta name="generator" content="Doxygen $doxygenversion"/>
-<meta name="viewport" content="width=device-width, initial-scale=1"/>
-<!--BEGIN PROJECT_NAME--><title>$projectname: $title</title><!--END PROJECT_NAME-->
-<!--BEGIN !PROJECT_NAME--><title>$title</title><!--END !PROJECT_NAME-->
-<link href="$relpath^tabs.css" rel="stylesheet" type="text/css"/>
-<script type="text/javascript" src="$relpath^jquery.js"></script>
-<script type="text/javascript" src="$relpath^dynsections.js"></script>
-$treeview
-$search
-$mathjax
-<link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
-<link REL="icon" HREF="https://sasjs.io/img/runningman.jpg">
-$extrastylesheet
-</head>
-<body>
-<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
-
-<!--BEGIN TITLEAREA-->
-<div id="titlearea">
-<table cellspacing="0" cellpadding="0">
- <tbody>
-  <tr style="height: 56px;">
-  <!--BEGIN PROJECT_LOGO-->
-  <td id="projectlogo">
-    <img alt="Logo" src="$relpath^$projectlogo"/></td>
-  <!--END PROJECT_LOGO-->
-<!--BEGIN PROJECT_NAME-->
-<td id="projectalign" style="padding-left: 0.5em;">
-  <div id="projectname">
-   <!--BEGIN PROJECT_NUMBER-->&#160;<span id="projectnumber">$projectnumber</span><!--END PROJECT_NUMBER-->
-   </div>
-   <!--BEGIN PROJECT_BRIEF--><div id="projectbrief">
-    Production Ready Macros for SAS Application Developers</br>
-    <a href="https://github.com/sasjs/core">
-        https://github.com/sasjs/core
-    </a>
-
-   </div>
-   <meta name="Description" content="$projectbrief">
-   <!--END PROJECT_BRIEF-->
-  </td>
-  <!--END PROJECT_NAME-->
-  <!--BEGIN !PROJECT_NAME-->
-   <!--BEGIN PROJECT_BRIEF-->
-   <td style="padding-left: 0.5em;">
-    <div id="projectbrief">$projectbrief</div>
-    <table style="padding-left: 2em;" cellspacing="0" cellpadding="0">
-      <tr><td> Production Ready Macros for SAS Application Developers</td></tr>
-      <tr><td><a href="https://github.com/sasjs/core">
-          https://github.com/sasjs/core
-      </a></td></tr>
-    </table>
-    </td>
-   <!--END PROJECT_BRIEF-->
-  <!--END !PROJECT_NAME-->
-  <!--BEGIN DISABLE_INDEX-->
-   <!--BEGIN SEARCHENGINE-->
-   <td>$searchbox</td>
-   <!--END SEARCHENGINE-->
-  <!--END DISABLE_INDEX-->
- </tr>
- </tbody>
-</table>
-</div>
-<!--END TITLEAREA-->
-<!-- end header part -->

doxy/new_stylesheet.css

@@ -1,5 +0,0 @@
-#projectlogo img
-{
-	border: 0px none;
-	max-height:70px
-}

lua/json.lua

@@ -1,5 +1,5 @@
 --
--- json2sas.lua  (modified from json.lua)
+-- json.lua
 --
 -- Copyright (c) 2019 rxi
 --
@@ -22,7 +22,7 @@
 -- SOFTWARE.
 --
 
-local json2sas = { _version = "0.1.2" }
+json = { _version = "0.1.2" }
 
 -------------------------------------------------------------------------------
 -- Encode
@@ -122,7 +122,7 @@ encode = function(val, stack)
   error("unexpected type '" .. t .. "'")
 end
 
-function json2sas.encode(val)
+function json.encode(val)
   return ( encode(val) )
 end
 
@@ -356,7 +356,7 @@ parse = function(str, idx)
   decode_error(str, idx, "unexpected character '" .. chr .. "'")
 end
 
-function json2sas.decode(str)
+function json.decode(str)
   if type(str) ~= "string" then
     error("expected argument of type string, got " .. type(str))
   end
@@ -368,88 +368,4 @@ function json2sas.decode(str)
   return res
 end
 
--- convert macro variable array into one variable and decode
-function json2sas.go(macvar)
-  local x=1
-  local cnt=0
-  local mac=sas.symget(macvar..'0')
-  local newstr=''
-  if mac and mac ~= '' then
-    cnt=mac
-    for x=1,cnt,1 do
-      mac=sas.symget(macvar..x)
-      if mac and mac ~= '' then
-        newstr=newstr..mac
-      else
-        return print(macvar..x..' NOT FOUND!!')
-      end
-    end
-  else
-    return print(macvar..'0 NOT FOUND!!')
-  end
-  -- print('mac:'..mac..'cnt:'..cnt..'newstr'..newstr)
-  local oneVar=json2sas.decode(newstr)
-  local jsdata=oneVar["data"]
-  local meta={}
-  local attrs={}
-  for tablename, data in pairs(jsdata) do -- each table
-    print("Processing table: "..tablename)
-    attrs[tablename]={}
-    for k, v in ipairs(data) do -- each row
-      if(k==1) then  -- column names
-        for a, b in pairs(v) do
-          attrs[tablename][a]={}
-          attrs[tablename][a]["name"]=b
-        end
-      elseif(k==2) then  -- get types
-        for a, b in pairs(v) do
-  	      if type(b)=='number' then
-  	        attrs[tablename][a]["type"]="N"
-  	        attrs[tablename][a]["length"]=8
-  	      else
-            attrs[tablename][a]["type"]="C"
-            attrs[tablename][a]["length"]=string.len(b)
-  	      end
-        end
-      else  --update lengths
-        for a, b in pairs(v) do
-          if (type(b)=='string' and string.len(b)>attrs[tablename][a]["length"])
-          then
-            attrs[tablename][a]["length"]=string.len(b)
-          end
-        end
-  	  end
-  	end
-    print(json2sas.encode(attrs[tablename])) -- show results
-
-    -- Now create the SAS table
-    sas.new_table("work."..tablename,attrs[tablename])
-    local dsid=sas.open("work."..tablename, "u")
-    for k, v in ipairs(data) do
-      if k>1 then
-        sas.append(dsid)
-        for a, b in pairs(v) do
-          sas.put_value(dsid, attrs[tablename][a]["name"], b)
-        end
-        sas.update(dsid)
-      end
-    end
-    sas.close(dsid)
-  end
-  return json2sas.decode(newstr)
-end
-
-
-function quote(str)
-  return sas.quote(str)
-end
-function sasvar(str)
-  print("processing: "..str)
-  print(sas.symexist(str))
-  if sas.symexist(str)==1 then
-    return quote(str)..':'..quote(sas.symget(str))..','
-  end
-  return ''
-end
-
-return json2sas
+return json

lua/ml_json.sas

@@ -1,18 +1,19 @@
 /**
-  @file ml_json2sas.sas
-  @brief Creates the json2sas.lua file
-  @details Writes json2sas.lua to the work directory
+  @file ml_json.sas
+  @brief Compiles the json.lua lua file
+  @details Writes json.lua to the work directory
+  and then includes it.
   Usage:
 
-      %ml_json2sas()
+      %ml_json()
 
 **/
 
-%macro ml_json2sas();
+%macro ml_json();
 data _null_;
-  file "%sysfunc(pathname(work))/json2sas.lua";
+  file "%sysfunc(pathname(work))/ml_json.lua";
   put '-- ';
-  put '-- json2sas.lua  (modified from json.lua) ';
+  put '-- json.lua ';
   put '-- ';
   put '-- Copyright (c) 2019 rxi ';
   put '-- ';
@@ -35,7 +36,7 @@ data _null_;
   put '-- SOFTWARE. ';
   put '-- ';
   put ' ';
-  put 'local json2sas = { _version = "0.1.2" } ';
+  put 'json = { _version = "0.1.2" } ';
   put ' ';
   put '------------------------------------------------------------------------------- ';
   put '-- Encode ';
@@ -135,7 +136,7 @@ data _null_;
   put '  error("unexpected type ''" .. t .. "''") ';
   put 'end ';
   put ' ';
-  put 'function json2sas.encode(val) ';
+  put 'function json.encode(val) ';
   put '  return ( encode(val) ) ';
   put 'end ';
   put ' ';
@@ -369,7 +370,7 @@ data _null_;
   put '  decode_error(str, idx, "unexpected character ''" .. chr .. "''") ';
   put 'end ';
   put ' ';
-  put 'function json2sas.decode(str) ';
+  put 'function json.decode(str) ';
   put '  if type(str) ~= "string" then ';
   put '    error("expected argument of type string, got " .. type(str)) ';
   put '  end ';
@@ -381,90 +382,9 @@ data _null_;
   put '  return res ';
   put 'end ';
   put ' ';
-  put '-- convert macro variable array into one variable and decode ';
-  put 'function json2sas.go(macvar) ';
-  put '  local x=1 ';
-  put '  local cnt=0 ';
-  put '  local mac=sas.symget(macvar..''0'') ';
-  put '  local newstr='''' ';
-  put '  if mac and mac ~= '''' then ';
-  put '    cnt=mac ';
-  put '    for x=1,cnt,1 do ';
-  put '      mac=sas.symget(macvar..x) ';
-  put '      if mac and mac ~= '''' then ';
-  put '        newstr=newstr..mac ';
-  put '      else ';
-  put '        return print(macvar..x..'' NOT FOUND!!'') ';
-  put '      end ';
-  put '    end ';
-  put '  else ';
-  put '    return print(macvar..''0 NOT FOUND!!'') ';
-  put '  end ';
-  put '  -- print(''mac:''..mac..''cnt:''..cnt..''newstr''..newstr) ';
-  put '  local oneVar=json2sas.decode(newstr) ';
-  put '  local jsdata=oneVar["data"] ';
-  put '  local meta={} ';
-  put '  local attrs={} ';
-  put '  for tablename, data in pairs(jsdata) do -- each table ';
-  put '    print("Processing table: "..tablename) ';
-  put '    attrs[tablename]={} ';
-  put '    for k, v in ipairs(data) do -- each row ';
-  put '      if(k==1) then  -- column names ';
-  put '        for a, b in pairs(v) do ';
-  put '          attrs[tablename][a]={} ';
-  put '          attrs[tablename][a]["name"]=b ';
-  put '        end ';
-  put '      elseif(k==2) then  -- get types ';
-  put '        for a, b in pairs(v) do ';
-  put '  	      if type(b)==''number'' then ';
-  put '  	        attrs[tablename][a]["type"]="N" ';
-  put '  	        attrs[tablename][a]["length"]=8 ';
-  put '  	      else ';
-  put '            attrs[tablename][a]["type"]="C" ';
-  put '            attrs[tablename][a]["length"]=string.len(b) ';
-  put '  	      end ';
-  put '        end ';
-  put '      else  --update lengths ';
-  put '        for a, b in pairs(v) do ';
-  put '          if (type(b)==''string'' and string.len(b)>attrs[tablename][a]["length"]) ';
-  put '          then ';
-  put '            attrs[tablename][a]["length"]=string.len(b) ';
-  put '          end ';
-  put '        end ';
-  put '  	  end ';
-  put '  	end ';
-  put '    print(json2sas.encode(attrs[tablename])) -- show results ';
-  put ' ';
-  put '    -- Now create the SAS table ';
-  put '    sas.new_table("work."..tablename,attrs[tablename]) ';
-  put '    local dsid=sas.open("work."..tablename, "u") ';
-  put '    for k, v in ipairs(data) do ';
-  put '      if k>1 then ';
-  put '        sas.append(dsid) ';
-  put '        for a, b in pairs(v) do ';
-  put '          sas.put_value(dsid, attrs[tablename][a]["name"], b) ';
-  put '        end ';
-  put '        sas.update(dsid) ';
-  put '      end ';
-  put '    end ';
-  put '    sas.close(dsid) ';
-  put '  end ';
-  put '  return json2sas.decode(newstr) ';
-  put 'end ';
-  put ' ';
-  put ' ';
-  put 'function quote(str) ';
-  put '  return sas.quote(str) ';
-  put 'end ';
-  put 'function sasvar(str) ';
-  put '  print("processing: "..str) ';
-  put '  print(sas.symexist(str)) ';
-  put '  if sas.symexist(str)==1 then ';
-  put '    return quote(str)..'':''..quote(sas.symget(str))..'','' ';
-  put '  end ';
-  put '  return '''' ';
-  put 'end ';
-  put ' ';
-  put 'return json2sas ';
+  put 'return json ';
 run;
+
+%inc "%sysfunc(pathname(work))/ml_json.lua";
+
 %mend;

main.dox

@@ -51,3 +51,14 @@
         * Prefixes: _mv_
 
  */
+
+ /*! \dir lua
+ *  \brief Lua macros
+ *  \details These macros have the following attributes:
+
+        * OS independent
+        * Work as LUA functions (they are immediately executed/compiled)
+        * Auto-generated from the plain source `.lua` files in the same directory
+        * Prefixes: _ml_
+
+ */

meta/mm_assigndirectlib.sas

@@ -117,7 +117,8 @@ run;
 %end;
 %else %if &engine=REMOTE %then %do;
   data x;
-    length rcCon rcProp rc k 3 uriCon uriProp PropertyValue PropertyName Delimiter $256 properties $2048;
+    length rcCon rcProp rc k 3 uriCon uriProp PropertyValue PropertyName
+      Delimiter $256 properties $2048;
     retain properties;
     rcCon = metadata_getnasn("&liburi", "LibraryConnection", 1, uriCon);
 
@@ -129,7 +130,8 @@ run;
       rc = metadata_getattr(uriProp , "DefaultValue",PropertyValue);
       rc = metadata_getattr(uriProp , "PropertyName",PropertyName);
       rc = metadata_getattr(uriProp , "Delimiter",Delimiter);
-      properties = trim(properties) !! " " !! trim(PropertyName) !! trim(Delimiter) !! trim(PropertyValue);
+      properties = trim(properties) !! " " !! trim(PropertyName)
+        !! trim(Delimiter) !! trim(PropertyValue);
       output;
       k+1;
       rcProp = metadata_getnasn(uriCon, "Properties", k, uriProp);
@@ -170,7 +172,8 @@ run;
       else if value='Connection.OLE.Property.PROVIDER.Name.xmlKey.txt' then do;
         rc4=metadata_getattr(conprop_uri,'DefaultValue',provider);
       end;
-      else if value='Connection.OLE.Property.PROPERTIES.Name.xmlKey.txt' then do;
+      else if value='Connection.OLE.Property.PROPERTIES.Name.xmlKey.txt' then
+      do;
         rc5=metadata_getattr(conprop_uri,'DefaultValue',properties);
       end;
     end;
@@ -357,7 +360,8 @@ run;
     call symputx('authdomain',authdomain,'l');
 
     /* path */
-    rc=metadata_getprop(assocuri1,'Connection.Oracle.Property.PATH.Name.xmlKey.txt',path);
+    rc=metadata_getprop(assocuri1,
+      'Connection.Oracle.Property.PATH.Name.xmlKey.txt',path);
     call symputx('path',path,'l');
 
     /* schema */
@@ -366,14 +370,16 @@ run;
     call symputx('schema',schema,'l');
   run;
   %put NOTE: Executing the following:/; %put NOTE-;
-  %put NOTE- libname &libref ORACLE path=&path schema=&schema authdomain=&authdomain;
+  %put NOTE- libname &libref ORACLE path=&path schema=&schema;
+  %put NOTE-     authdomain=&authdomain;
   %put NOTE-;
   libname &libref ORACLE path=&path schema=&schema authdomain=&authdomain;
 %end;
 %else %if &engine=SQLSVR %then %do;
   %put NOTE: Obtaining &engine library details;
   data _null;
-    length assocuri1 assocuri2 assocuri3 authdomain path schema userid passwd $256;
+    length assocuri1 assocuri2 assocuri3 authdomain path schema userid
+      passwd $256;
     call missing (of _all_);
 
     rc=metadata_getnasn("&liburi",'DefaultLogin',1,assocuri1);
@@ -384,7 +390,8 @@ run;
 
     /* path */
     rc=metadata_getnasn("&liburi",'LibraryConnection',1,assocuri2);
-    rc=metadata_getprop(assocuri2,'Connection.SQL.Property.Datasrc.Name.xmlKey.txt',path);
+    rc=metadata_getprop(assocuri2,
+      'Connection.SQL.Property.Datasrc.Name.xmlKey.txt',path);
     call symputx('path',path,'l');
 
     /* schema */
@@ -394,7 +401,8 @@ run;
   run;
 
   %put NOTE: Executing the following:/; %put NOTE-;
-  %put NOTE- libname &libref SQLSVR datasrc=&path schema=&schema user="&user" pass="XXX";
+  %put NOTE- libname &libref SQLSVR datasrc=&path schema=&schema ;
+  %put NOTE-    user="&user" pass="XXX";
   %put NOTE-;
 
   libname &libref SQLSVR datasrc=&path schema=&schema user="&user" pass="&pass";
@@ -402,7 +410,8 @@ run;
 %else %if &engine=TERADATA %then %do;
   %put NOTE: Obtaining &engine library details;
   data _null;
-    length assocuri1 assocuri2 assocuri3 authdomain path schema userid passwd $256;
+    length assocuri1 assocuri2 assocuri3 authdomain path schema userid
+      passwd $256;
     call missing (of _all_);
 
         /* get auth domain */
@@ -421,7 +430,8 @@ run;
 
     /* path */
     rc=metadata_getnasn("&liburi",'LibraryConnection',1,assocuri2);
-    rc=metadata_getprop(assocuri2,'Connection.Teradata.Property.SERVER.Name.xmlKey.txt',path);
+    rc=metadata_getprop(assocuri2,
+      'Connection.Teradata.Property.SERVER.Name.xmlKey.txt',path);
     call symputx('path',path,'l');
 
     /* schema */
@@ -431,7 +441,8 @@ run;
   run;
 
   %put NOTE: Executing the following:/; %put NOTE-;
-  %put NOTE- libname &libref TERADATA server=&path schema=&schema authdomain=&authdomain;
+  %put NOTE- libname &libref TERADATA server=&path schema=&schema ;
+  %put NOTe-   authdomain=&authdomain;
   %put NOTE-;
 
   libname &libref TERADATA server=&path schema=&schema authdomain=&authdomain;

meta/mm_assignlib.sas

@@ -14,7 +14,8 @@
   @li mp_abort.sas
 
   @param libref the libref (not name) of the metadata library
-  @param mAbort= If not assigned, HARD will call %mp_abort(), SOFT will silently return
+  @param mAbort= If not assigned, HARD will call %mp_abort(), SOFT will
+    silently return
 
   @returns libname statement
 

meta/mm_createapplication.sas

@@ -13,7 +13,9 @@
         ,params= name1=value1
name2=value2
emptyvalue=
       )
 
-  @warning application components do not get deleted when removing the container folder!  be sure you have the administrative priviliges to remove this kind of metadata from the SMC plugin (or be ready to do to so programmatically).
+  @warning application components do not get deleted when removing the container
+  folder!  be sure you have the administrative priviliges to remove this kind of
+  metadata from the SMC plugin (or be ready to do to so programmatically).
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas

meta/mm_createfolder.sas

@@ -55,7 +55,7 @@ data _null_;
 
   * must have a starting slash ;
   if ( substr(folderPath,1,1) ne '/' ) then do;
-    put "%str(ERR)OR: &sysmacroname PATH parameter value must have starting slash";
+    put "%str(ERR)OR: &sysmacroname PATH param value must have starting slash";
     stop;
   end;
 
@@ -123,9 +123,10 @@ run;
   %local inmeta;
   %put creating: &path;
   %let inmeta=<AddMetadata><Reposid>$METAREPOSITORY</Reposid><Metadata>
-    <Tree Name='&child' PublicType='Folder' TreeType='BIP Folder' UsageVersion='1000000'>
-    <ParentTree><Tree ObjRef='&parentFolderObjId'/></ParentTree></Tree></Metadata>
-    <NS>SAS</NS><Flags>268435456</Flags></AddMetadata>;
+    <Tree Name='&child' PublicType='Folder' TreeType='BIP Folder'
+    UsageVersion='1000000'><ParentTree><Tree ObjRef='&parentFolderObjId'/>
+    </ParentTree></Tree></Metadata><NS>SAS</NS><Flags>268435456</Flags>
+    </AddMetadata>;
 
   proc metadata in="&inmeta" out=__newdir verbose;
   run ;

meta/mm_createlibrary.sas

@@ -166,7 +166,7 @@ filename &frefout temp;
     putlog (_all_)(=);
   run;
   %if &checktype ne Prototype %then %do;
-    %put %str(ERR)OR: Prototype (Library.SAS.Prototype.Name.xmlKey.txt) not found!;
+    %put %str(ERR)OR: Prototype Library.SAS.Prototype.Name.xmlKey.txt not found;
     %return;
   %end;
 

meta/mm_createstp.sas

@@ -210,7 +210,8 @@ run;
     rc3=METADATA_SETATTR(prompturi, 'GroupType','2');
     rc4=METADATA_SETATTR(prompturi, 'Name','Parameters');
     rc5=METADATA_SETATTR(prompturi, 'PublicType','Embedded:PromptGroup');
-    GroupInfo="<PromptGroup promptId='PromptGroup_%sysfunc(datetime())_&sysprocessid'"
+    GroupInfo=
+      "<PromptGroup promptId='PromptGroup_%sysfunc(datetime())_&sysprocessid'"
       !!" version='1.0'><Label><Text xml:lang='en-GB'>Parameters</Text>"
       !!"</Label></PromptGroup>";
     rc6 = METADATA_SETATTR(prompturi, 'GroupInfo',groupinfo);
@@ -375,8 +376,6 @@ run;
     */
   %mm_updatestpsourcecode(stp=&tree/&stpname
     ,stpcode="&directory/&filename"
-    ,frefin=&frefin.
-    ,frefout=&frefout.
     ,mdebug=&mdebug
     ,minify=&minify)
 

meta/mm_createwebservice.sas

@@ -22,7 +22,7 @@ Usage:
         %webout(OBJ,example2) * Object format, easier to work with ;
         %webout(CLOSE)
     ;;;;
-    %mm_createwebservice(path=/Public/app/common,name=appInit,code=ft15f001,replace=YES)
+    %mm_createwebservice(path=/Public/app/common,name=appInit)
 
   <h4> SAS Macros </h4>
   @li mm_createstp.sas
@@ -119,7 +119,8 @@ data _null_;
   put '    %end; ';
   put '    data _null_;file &jref mod ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
-  put '    proc sort data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
+  put '    proc sort ';
+  put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
   put '      out=_data_; ';
   put '      by varnum; ';
   put ' ';
@@ -158,7 +159,8 @@ data _null_;
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
-  put '    /* write to temp loc to avoid _webout truncation - https://support.sas.com/kb/49/325.html */ ';
+  put '    /* write to temp loc to avoid _webout truncation ';
+  put '      - https://support.sas.com/kb/49/325.html */ ';
   put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
   put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod; ';
   put '      set &tempds; ';
@@ -194,7 +196,7 @@ data _null_;
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8''; ';
+  put '  data _null_;file &jref encoding=''utf-8'' mod; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
@@ -235,8 +237,16 @@ data _null_;
   put '%else %if &action=OPEN %then %do; ';
   put '  /* fix encoding */ ';
   put '  OPTIONS NOBOMFILE; ';
+  put ' ';
+  put '  /** ';
+  put '    * check engine type to avoid the below err message: ';
+  put '    * > Function is only valid for filerefs using the CACHE access method. ';
+  put '    */ ';
   put '  data _null_; ';
+  put '    set sashelp.vextfl(where=(fileref="_WEBOUT")); ';
+  put '    if xengine=''STREAM'' then do; ';
   put '      rc=stpsrv_header(''Content-type'',"text/html; encoding=utf-8"); ';
+  put '    end; ';
   put '  run; ';
   put ' ';
   put '  /* setup json */ ';
@@ -250,17 +260,10 @@ data _null_;
   put '%end; ';
   put ' ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
-  put '  %if &sysver=9.4 %then %do; ';
-  put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
-  put '      ,engine=PROCJSON,dbg=%str(&_debug) ';
-  put '    ) ';
-  put '  %end; ';
-  put '  %else %do; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
   put '    ,engine=DATASTEP,dbg=%str(&_debug) ';
   put '  ) ';
   put '%end; ';
-  put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
   put '  %if %str(&_debug) ge 131 %then %do; ';
   put '    /* if debug mode, send back first 10 records of each work table also */ ';
@@ -275,14 +278,14 @@ data _null_;
   put '      i+1; ';
   put '      call symputx(''wt''!!left(i),name,''l''); ';
   put '      call symputx(''wtcnt'',i,''l''); ';
-  put '    data _null_; file &fref encoding=''utf-8''; ';
+  put '    data _null_; file &fref mod encoding=''utf-8''; ';
   put '      put ",""WORK"":{"; ';
   put '    %do i=1 %to &wtcnt; ';
   put '      %let wt=&&wt&i; ';
   put '      proc contents noprint data=&wt ';
   put '        out=_data_ (keep=name type length format:); ';
   put '      run;%let tempds=%scan(&syslast,2,.); ';
-  put '      data _null_; file &fref encoding=''utf-8''; ';
+  put '      data _null_; file &fref mod encoding=''utf-8''; ';
   put '        dsid=open("WORK.&wt",''is''); ';
   put '        nlobs=attrn(dsid,''NLOBS''); ';
   put '        nvars=attrn(dsid,''NVARS''); ';
@@ -293,10 +296,10 @@ data _null_;
   put '        put '',"nvars":'' nvars; ';
   put '      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP) ';
   put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP) ';
-  put '      data _null_; file &fref encoding=''utf-8''; ';
+  put '      data _null_; file &fref mod encoding=''utf-8''; ';
   put '        put "}"; ';
   put '    %end; ';
-  put '    data _null_; file &fref encoding=''utf-8''; ';
+  put '    data _null_; file &fref mod encoding=''utf-8''; ';
   put '      put "}"; ';
   put '    run; ';
   put '  %end; ';

meta/mm_deletelibrary.sas

@@ -0,0 +1,92 @@
+/**
+  @file
+  @brief Deletes a library by Name
+
+  @details  Used to delete a library.
+  Usage:
+
+      %* create a library in the home directory ;
+      %mm_createlibrary(
+        libname=My Temp Library,
+        libref=XXTEMPXX,
+        tree=/User Folders/&sysuserid,
+        directory=%sysfunc(pathname(work))
+      )
+
+      %* delete the library ;
+      %mm_deletelibrary(name=My Temp Library)
+
+  After running the above, the following will be shown in the log:
+
+  ![](https://i.imgur.com/Y4Tog24.png)
+
+  @param [in] name= the name (not libref) of the library to be deleted
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_abort.sas
+
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mm_deletelibrary(
+      name=
+)/*/STORE SOURCE*/;
+
+
+/**
+  * Check if library exists and get uri
+  */
+data _null_;
+  length type uri $256;
+  rc=metadata_resolve("omsobj:SASLibrary?@Name='&name'",type,uri);
+  call symputx('checktype',type,'l');
+  call symputx('liburi',uri,'l');
+  putlog (_all_)(=);
+run;
+%if &checktype ne SASLibrary %then %do;
+  %put &sysmacroname: Library (&name) was not found, and so will not be deleted;
+  %return;
+%end;
+
+%local fname1 fname2;
+%let fname1=%mf_getuniquefileref();
+%let fname2=%mf_getuniquefileref();
+
+filename &fname1 temp lrecl=10000;
+filename &fname2 temp lrecl=10000;
+data _null_ ;
+  file &fname1 ;
+  put "<DeleteMetadata><Metadata><SASLibrary Id='&liburi'/>";
+  put "</Metadata><NS>SAS</NS><Flags>268436480</Flags><Options/>";
+  put "</DeleteMetadata>";
+run ;
+proc metadata in=&fname1 out=&fname2 verbose;run;
+
+/* list the result */
+data _null_;infile &fname2; input; list; run;
+
+filename &fname1 clear;
+filename &fname2 clear;
+
+/**
+  * Check deletion
+  */
+%local isgone;
+data _null_;
+  length type uri $256;
+  rc=metadata_resolve("omsobj:SASLibrary?@Id='&liburi'",type,uri);
+  call symputx('isgone',type,'l');
+run;
+
+%mp_abort(iftrue=(&isgone = SASLibrary)
+  ,mac=&sysmacroname
+  ,msg=%str(Library (&name) NOT deleted)
+)
+
+%put &sysmacroname: Library &name (&liburi) was successfully deleted;
+
+%mend;

meta/mm_getauthinfo.sas

@@ -21,8 +21,8 @@
 )/*/STORE SOURCE*/;
 
 %if %length(&outds)>30 %then %do;
-  %put %str(ERR)OR: Temp tables are created with the &outds prefix, which therefore
-  needs to be 30 characters or less;
+  %put %str(ERR)OR: Temp tables are created with the &outds prefix, which
+    therefore needs to be 30 characters or less;
   %return;
 %end;
 %if %index(&outds,'.')>0 %then %do;

meta/mm_getdirectories.sas

@@ -39,8 +39,10 @@ data &outds (keep=directoryuri name directoryname directorydesc );
   do while
   (metadata_getnobj("omsobj:Directory?@Id contains '.'",__i,directoryuri)>0);
 %end; %else %do;
-  do while
-  (metadata_getnobj("omsobj:Directory?@DirectoryName='&path'",__i,directoryuri)>0);
+  do while(
+    metadata_getnobj("omsobj:Directory?@DirectoryName='&path'",__i,directoryuri)
+    >0
+  );
 %end;
     __rc1=metadata_getattr(directoryuri, "Name", name);
     __rc2=metadata_getattr(directoryuri, "DirectoryName", directoryname);

meta/mm_getfoldermembers.sas

@@ -0,0 +1,96 @@
+/**
+  @file
+  @brief Returns all direct child members of a particular folder
+  @details Displays the children for a particular folder, in a similar fashion
+  to the viya counterpart (mv_getfoldermembers.sas)
+
+  Usage:
+
+      %mm_getfoldermembers(root=/, outds=rootfolders)
+
+      %mm_getfoldermembers(root=/User Folders/&sysuserid, outds=usercontent)
+
+  @param [in] root= the parent folder under which to return all contents
+  @param [out] outds= the dataset to create that contains the list of
+    directories
+  @param [in] mDebug= set to 1 to show debug messages in the log
+
+  <h4> Data Outputs </h4>
+
+  Example for `root=/`:
+
+  |metauri $17|metaname $256|metatype $32|
+  |---|---|---|
+  |A5XLSNXI.AA000001|Products  |Folder|
+  |A5XLSNXI.AA000002|Shared Data  |Folder|
+  |A5XLSNXI.AA000003|User Folders  |Folder|
+  |A5XLSNXI.AA000004|System  |Folder|
+  |A5XLSNXI.AA00003K|30.SASApps  |Folder|
+  |A5XLSNXI.AA00006A|Public|Folder|
+
+  <h4> SAS Macros </h4>
+  @li mm_getfoldertree.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+%macro mm_getfoldermembers(
+    root=
+    ,outds=work.mm_getfoldertree
+)/*/STORE SOURCE*/;
+
+%if "&root" = "/" %then %do;
+  %local fname1 fname2 fname3;
+  %let fname1=%mf_getuniquefileref();
+  %let fname2=%mf_getuniquefileref();
+  %let fname3=%mf_getuniquefileref();
+  data _null_ ;
+    file &fname1 ;
+    put '<GetMetadataObjects>' ;
+    put '<Reposid>$METAREPOSITORY</Reposid>' ;
+    put '<Type>Tree</Type>' ;
+    put '<NS>SAS</NS>' ;
+    put '<Flags>388</Flags>' ;
+    put '<Options>' ;
+    put '<XMLSelect search="Tree[SoftwareComponents/SoftwareComponent'@;
+    put '[@Name=''BIP Service'']]"/>';
+    put '</Options>' ;
+    put '</GetMetadataObjects>' ;
+  run ;
+  proc metadata in=&fname1 out=&fname2 verbose;run;
+
+  /* create an XML map to read the response */
+  data _null_;
+    file &fname3;
+    put '<SXLEMAP version="1.2" name="SASFolders">';
+    put '<TABLE name="SASFolders">';
+    put '<TABLE-PATH syntax="XPath">//Objects/Tree</TABLE-PATH>';
+    put '<COLUMN name="metauri">><LENGTH>17</LENGTH>';
+    put '<PATH syntax="XPath">//Objects/Tree/@Id</PATH></COLUMN>';
+    put '<COLUMN name="metaname"><LENGTH>256</LENGTH>>';
+    put '<PATH syntax="XPath">//Objects/Tree/@Name</PATH></COLUMN>';
+    put '</TABLE></SXLEMAP>';
+  run;
+  %local libref1;
+  %let libref1=%mf_getuniquelibref();
+  libname &libref1 xml xmlfileref=&fname2 xmlmap=&fname3;
+
+  data &outds;
+    length metatype $32;
+    retain metatype 'Folder';
+    set &libref1..sasfolders;
+  run;
+
+%end;
+%else %do;
+  %mm_getfoldertree(root=&root, outds=&outds,depth=1)
+  data &outds;
+    set &outds(rename=(name=metaname publictype=metatype));
+    keep metaname metauri metatype;
+  run;
+%end;
+
+%mend;

meta/mm_getfoldertree.sas

@@ -1,18 +1,21 @@
 /**
-  @file mm_getfoldertree.sas
+  @file
   @brief Returns all folders / subfolder content for a particular root
   @details Shows all members and SubTrees recursively for a particular root.
   Note - for big sites, this returns a lot of data!  So you may wish to reduce
-  the logging to speed up the process (see example below)
+  the logging to speed up the process (see example below), OR - use mm_tree.sas
+  which uses proc metadata and is far more efficient.
+
   Usage:
 
     options ps=max nonotes nosource;
     %mm_getfoldertree(root=/My/Meta/Path, outds=iwantthisdataset)
     options notes source;
 
-  @param root= the parent folder under which to return all contents
-  @param outds= the dataset to create that contains the list of directories
-  @param mDebug= set to 1 to show debug messages in the log
+  @param [in] root= the parent folder under which to return all contents
+  @param [out] outds= the dataset to create that contains the list of
+    directories
+  @param [in] mDebug= set to 1 to show debug messages in the log
 
   <h4> SAS Macros </h4>
 

meta/mm_getgroups.sas

@@ -5,17 +5,16 @@
     blank to return all groups.
   Usage:
 
-    - all groups
-    %mm_getGroups()
+    - all groups: `%mm_getGroups()`
 
-    - all groups for a particular user
-    %mm_getgroups(user=&sysuserid)
+    - all groups for a particular user: `%mm_getgroups(user=&sysuserid)`
 
-  @param user= the metadata user to return groups for.  Leave blank for all
+  @param [in] user= the metadata user to return groups for.  Leave blank for all
     groups.
-  @param outds= the dataset to create that contains the list of groups
-  @param repo= the metadata repository that contains the user/group information
-  @param mDebug= set to 1 to show debug messages in the log
+  @param [in] repo= the metadata repository that contains the user/group
+    information
+  @param [in] mDebug= set to 1 to show debug messages in the log
+  @param [out] outds= the dataset to create that contains the list of groups
 
   @returns outds  dataset containing all groups in a column named "metagroup"
     - groupuri
@@ -41,7 +40,8 @@
 %&mD.put Executing mm_getGroups.sas;
 %&mD.put _local_;
 
-/* on some sites, user / group info is in a different metadata repo to the default */
+/* on some sites, user / group info is in a different metadata repo to the
+    default */
 %if &oldrepo ne &repo %then %do;
   options metarepository=&repo;
 %end;

meta/mm_getlibmetadiffs.sas

@@ -0,0 +1,129 @@
+/**
+  @file
+  @brief Compares the metadata of a library with the physical tables
+  @details Creates a series of output tables that show the differences between
+  metadata and physical tables.
+  Each output can be created with an optional prefix.
+
+  Credit - Paul Homes
+  https://platformadmin.com/blogs/paul/2012/11/sas-proc-metalib-ods-output
+
+  Usage:
+
+      %* create (and assign) a library for testing purposes ;
+      %mm_createlibrary(
+        libname=My Temp Library,
+        libref=XXTEMPXX,
+        tree=/User Folders/&sysuserid,
+        directory=%sysfunc(pathname(work))
+      )
+
+      %* create some tables;
+      data work.table1 table2 table3;
+        a=1;b='two';c=3;
+      run;
+
+      %* register the tables;
+      proc metalib;
+        omr=(library="My Temp Library");
+        report(type=detail);
+        update_rule (delete);
+      run;
+
+      %* modify the tables;
+      proc sql;
+      drop table table3;
+      alter table table2 drop c;
+      alter table table2 add d num;
+
+      %* run the macro;
+      %mm_getlibmetadiffs(libname=My Temp Library)
+
+      %* delete the library ;
+      %mm_deletelibrary(name=My Temp Library)
+
+  The program will create four output tables, with the following structure (and
+  example data):
+
+  #### &prefix.added
+  |name:$32.|metaID:$17.|SAStabName:$32.|
+  |---|---|---|
+  |||DATA1|
+
+  #### &prefix.deleted
+  |name:$32.|metaID:$17.|SAStabName:$32.|
+  |---|---|---|
+  |TABLE3|A5XLSNXI.BK0001HO|TABLE3|
+
+  #### &prefix.updated
+  |tabName:$32.|tabMetaID:$17.|SAStabName:$32.|metaName:$32.|metaID:$17.|sasname:$32.|metaType:$16.|change:$64.|
+  |---|---|---|---|---|---|---|---|
+  |TABLE2|A5XLSNXI.BK0001HN|TABLE2|c|A5XLSNXI.BM000MA9|c|Column|Deleted|
+  ||||d||d|Column|Added|
+
+  #### &prefix.meta
+  |Label1:$28.|cValue1:$1.|nValue1:D12.3|
+  |---|---|---|
+  |Total tables analyzed|4|4|
+  |Tables to be Updated|1|1|
+  |Tables to be Deleted|1|1|
+  |Tables to be Added|1|1|
+  |Tables matching data source|1|1|
+  |Tables not processed|0|0|
+
+  If you are interested in more functionality like this (checking the health of
+  SAS metadata and your SAS 9 environment) then do contact [Allan Bowe](
+  https://www.linkedin.com/in/allanbowe) for details of our SAS 9 Health Check
+  service.
+
+  Our system scan will perform hundreds of checks to identify common issues,
+  such as dangling metadata, embedded passwords, security issues and more.
+
+  @param [in] libname= the metadata name of the library to be compared
+  @param [out] outlib= The output library in which to store the output tables.
+  Default=WORK.
+  @param [out] prefix The prefix for the four tables created. Default=metadiff.
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mm_getlibmetadiffs(
+  libname= ,
+  prefix=metadiff,
+  outlib=work
+)/*/STORE SOURCE*/;
+
+  /* create tempds */
+  data;run;
+  %local tempds;
+  %let tempds=&syslast;
+
+  /* save options */
+  proc optsave out=&tempds;
+  run;
+
+  options VALIDVARNAME=ANY VALIDMEMNAME=EXTEND;
+
+  ods output
+    factoid1=&outlib..&prefix.meta
+    updtab=&outlib..&prefix.updated
+    addtab=&outlib..&prefix.added
+    deltab=&outlib..&prefix.deleted
+  ;
+
+  proc metalib;
+    omr=(library="&libname");
+    noexec;
+    report(type=detail);
+    update_rule (delete);
+  run;
+
+  ods output close;
+
+  /* restore options */
+  proc optload data=&tempds;
+  run;
+
+%mend mm_getlibmetadiffs;

meta/mm_getobjects.sas

@@ -46,7 +46,8 @@ filename sxlemap temp;
 data _null_;
   file sxlemap;
   put '<SXLEMAP version="1.2" name="SASObjects"><TABLE name="SASObjects">';
-  put "<TABLE-PATH syntax='XPath'>/GetMetadataObjects/Objects/&type</TABLE-PATH>";
+  put "<TABLE-PATH syntax='XPath'>/GetMetadataObjects/Objects/&type";
+  put "</TABLE-PATH>";
   put '<COLUMN name="id">';
   put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/&type/@Id</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>200</LENGTH>";

meta/mm_getpublictypes.sas

@@ -1,7 +1,8 @@
 /**
   @file mm_getpublictypes.sas
   @brief Creates a dataset with all deployable public types
-  @details More info: https://support.sas.com/documentation/cdl/en/bisag/65422/HTML/default/viewer.htm#n1nkrdzsq5iunln18bk2236istkb.htm
+  @details More info:
+  https://support.sas.com/documentation/cdl/en/bisag/65422/HTML/default/viewer.htm#n1nkrdzsq5iunln18bk2236istkb.htm
 
   Usage:
 

meta/mm_getrepos.sas

@@ -44,61 +44,76 @@ filename sxlemap temp;
 data _null_;
   file sxlemap;
   put '<SXLEMAP version="1.2" name="SASRepos"><TABLE name="SASRepos">';
-  put "<TABLE-PATH syntax='XPath'>/GetRepositories/Repositories/Repository</TABLE-PATH>";
+  put "<TABLE-PATH syntax='XPath'>/GetRepositories/Repositories/Repository";
+  put "</TABLE-PATH>";
   put '<COLUMN name="id">';
-  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@Id</PATH>";
+  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@Id";
+  put "</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>200</LENGTH>";
   put '</COLUMN>';
   put '<COLUMN name="name">';
-  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@Name</PATH>";
+  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@Name";
+  put "</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>200</LENGTH>";
   put '</COLUMN>';
   put '<COLUMN name="desc">';
-  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@Desc</PATH>";
+  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@Desc";
+  put "</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>200</LENGTH>";
   put '</COLUMN>';
   put '<COLUMN name="DefaultNS">';
-  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@DefaultNS</PATH>";
+  put "<PATH syntax='XPath'>";
+  put "/GetRepositories/Repositories/Repository/@DefaultNS</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>200</LENGTH>";
   put '</COLUMN>';
   put '<COLUMN name="RepositoryType">';
-  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@RepositoryType</PATH>";
+  put "<PATH syntax='XPath'>";
+  put "/GetRepositories/Repositories/Repository/@RepositoryType</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>20</LENGTH>";
   put '</COLUMN>';
   put '<COLUMN name="RepositoryFormat">';
-  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@RepositoryFormat</PATH>";
+  put "<PATH syntax='XPath'>";
+  put "/GetRepositories/Repositories/Repository/@RepositoryFormat</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>10</LENGTH>";
   put '</COLUMN>';
   put '<COLUMN name="Access">';
-  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@Access</PATH>";
+  put "<PATH syntax='XPath'>";
+  put "/GetRepositories/Repositories/Repository/@Access</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>16</LENGTH>";
   put '</COLUMN>';
   put '<COLUMN name="CurrentAccess">';
-  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@CurrentAccess</PATH>";
+  put "<PATH syntax='XPath'>";
+  put "/GetRepositories/Repositories/Repository/@CurrentAccess</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>16</LENGTH>";
   put '</COLUMN>';
   put '<COLUMN name="PauseState">';
-  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@PauseState</PATH>";
+  put "<PATH syntax='XPath'>";
+  put "/GetRepositories/Repositories/Repository/@PauseState</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>16</LENGTH>";
   put '</COLUMN>';
   put '<COLUMN name="Path">';
-  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@Path</PATH>";
+  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@Path";
+  put "</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>256</LENGTH>";
   put '</COLUMN>';
   put '<COLUMN name="Engine">';
-  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@Engine</PATH>";
+  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@Engine";
+  put "</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>8</LENGTH>";
   put '</COLUMN>';
   put '<COLUMN name="Options">';
-  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@Options</PATH>";
+  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@Options";
+  put "</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>32</LENGTH>";
   put '</COLUMN>';
   put '<COLUMN name="MetadataCreated">';
-  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@MetadataCreated</PATH>";
+  put "<PATH syntax='XPath'>";
+  put "/GetRepositories/Repositories/Repository/@MetadataCreated</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>24</LENGTH>";
   put '</COLUMN>';
   put '<COLUMN name="MetadataUpdated">';
-  put "<PATH syntax='XPath'>/GetRepositories/Repositories/Repository/@MetadataUpdated</PATH>";
+  put "<PATH syntax='XPath'>";
+  put "/GetRepositories/Repositories/Repository/@MetadataUpdated</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>24</LENGTH>";
   put '</COLUMN>';
   put '</TABLE></SXLEMAP>';

meta/mm_getroles.sas

@@ -7,7 +7,7 @@
 
       %mm_getroles()
 
-  @param outds the dataset to create that contains the list of roles
+  @param [out] outds the dataset to create that contains the list of roles
 
   @returns outds  dataset containing all roles, with the following columns:
     - uri
@@ -44,15 +44,19 @@ filename sxlemap temp;
 data _null_;
   file sxlemap;
   put '<SXLEMAP version="1.2" name="roles"><TABLE name="roles">';
-  put "<TABLE-PATH syntax='XPath'>/GetMetadataObjects/Objects/IdentityGroup</TABLE-PATH>";
+  put "<TABLE-PATH syntax='XPath'>/GetMetadataObjects/Objects/IdentityGroup";
+  put "</TABLE-PATH>";
   put '<COLUMN name="roleuri">';
-  put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/IdentityGroup/@Id</PATH>";
+  put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/IdentityGroup/@Id";
+  put "</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>32</LENGTH>";
   put '</COLUMN><COLUMN name="rolename">';
-  put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/IdentityGroup/@Name</PATH>";
+  put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/IdentityGroup/@Name";
+  put "</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>256</LENGTH>";
   put '</COLUMN><COLUMN name="roledesc">';
-  put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/IdentityGroup/@Desc</PATH>";
+  put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/IdentityGroup/@Desc";
+  put "</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>500</LENGTH>";
   put '</COLUMN></TABLE></SXLEMAP>';
 run;

meta/mm_getstps.sas

@@ -23,9 +23,10 @@
     combine with the <code>tree=</code> parameter.
   @param outds= the dataset to create that contains the list of stps.
   @param mDebug= set to 1 to show debug messages in the log
-  @param showDesc= provide a non blank value to return stored process descriptions
-  @param showUsageVersion= provide a non blank value to return the UsageVersion.  This
-    is either 1000000 (type 1, 9.2) or 2000000 (type2, 9.3 onwards).
+  @param showDesc= provide a non blank value to return stored process
+    descriptions
+  @param showUsageVersion= provide a non blank value to return the UsageVersion.
+    This is either 1000000 (type 1, 9.2) or 2000000 (type2, 9.3 onwards).
 
   @returns outds  dataset containing the following columns
     - stpuri

meta/mm_gettableid.sas

@@ -1,7 +1,7 @@
 /**
   @file mm_gettableid.sas
   @brief Get the metadata id for a particular table
-  @details Provide a libref and table name to return the corresponding metadata id
+  @details Provide a libref and table name to return the corresponding metadata
   in an output datataset.
 
   Usage:

meta/mm_getusers.sas

@@ -48,7 +48,8 @@ filename sxlemap temp;
 data _null_;
   file sxlemap;
   put '<SXLEMAP version="1.2" name="SASObjects"><TABLE name="SASObjects">';
-  put "<TABLE-PATH syntax='XPath'>/GetMetadataObjects/Objects/Person</TABLE-PATH>";
+  put "<TABLE-PATH syntax='XPath'>/GetMetadataObjects/Objects/Person";
+  put "</TABLE-PATH>";
   put '<COLUMN name="uri">';
   put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/Person/@Id</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>32</LENGTH>";

meta/mm_spkexport.sas

@@ -12,7 +12,8 @@
   Usage:
 
       %* import the macros (or make them available some other way);
-      filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      filename mc url
+        "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
 
       %* create sample text file as input to the macro;
@@ -92,7 +93,8 @@
 %let port=%sysfunc(getoption(metaport));
 %let platform_object_path=%mf_loc(POF);
 
-%let connx_string=%str(-host &host -port &port -user &mmxuser -password &mmxpass);
+%let connx_string=%str(-host &host -port &port -user &mmxuser %trim(
+  )-password &mmxpass);
 
 %mm_tree(root=%str(&metaloc) ,types=EXPORTABLE ,outds=exportable)
 

meta/mm_tree.sas

@@ -15,7 +15,8 @@
   Usage:
 
       %* load macros;
-      filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      filename mc url
+        "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
 
       %* export everything;
@@ -104,7 +105,8 @@ filename sxlemap temp;
 data _null_;
   file sxlemap;
   put '<SXLEMAP version="1.2" name="SASObjects"><TABLE name="SASObjects">';
-  put "<TABLE-PATH syntax='XPath'>/GetMetadataObjects/Objects/Tree</TABLE-PATH>";
+  put "<TABLE-PATH syntax='XPath'>/GetMetadataObjects/Objects/Tree";
+  put "</TABLE-PATH>";
   put '<COLUMN name="pathuri">';
   put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/Tree/@Id</PATH>";
   put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>64</LENGTH>";

meta/mm_updatestpservertype.sas

@@ -1,5 +1,5 @@
 /**
-  @file mm_updatestpservertype.sas
+  @file
   @brief Updates a type 2 stored process to run on STP or WKS context
   @details Only works on Type 2 (9.3 compatible) STPs
 
@@ -8,11 +8,10 @@
       %mm_updatestpservertype(target=/some/meta/path/myStoredProcess
         ,type=WKS)
 
-  <h4> SAS Macros </h4>
 
   @param target= full path to the STP being deleted
-  @param type= Either WKS or STP depending on whether Workspace or Stored Process
-        type required
+  @param type= Either WKS or STP depending on whether Workspace or
+    Stored Process type required
 
   @version 9.4
   @author Allan Bowe
@@ -52,7 +51,8 @@ data _null_;
     n+1;
     rc=metadata_getattr(uri,"Name",name);
     if name='Stored Process' then do;
-      rc = METADATA_SETATTR(uri,'StoredText','<?xml version="1.0" encoding="UTF-8"?>'
+      rc = METADATA_SETATTR(uri,'StoredText'
+        ,'<?xml version="1.0" encoding="UTF-8"?>'
         !!'<StoredProcess><ServerContext LogicalServerType="'!!"&newtype"
         !!'" OtherAllowed="false"/><ResultCapabilities Package="false" '
         !!' Streaming="true"/><OutputParameters/></StoredProcess>');

meta/mm_updatestpsourcecode.sas

@@ -9,28 +9,37 @@
       %mm_updatestpsourcecode(stp=/my/metadata/path/mystpname
         ,stpcode="/file/system/source.sas")
 
-
-  @param stp= the BIP Tree folder path plus Stored Process Name
-  @param stpcode= the source file (or fileref) containing the SAS code to load
+  @param [in] stp= the BIP Tree folder path plus Stored Process Name
+  @param [in] stpcode= the source file (or fileref) containing the SAS code to load
     into the stp.  For multiple files, they should simply be concatenated first.
-  @param minify= set to YES in order to strip comments, blank lines, and CRLFs.
+  @param [in] minify= set to YES in order to strip comments, blank lines, and CRLFs.
 
-  @param frefin= change default inref if it clashes with an existing one
-  @param frefout= change default outref if it clashes with an existing one
+  @param frefin= deprecated - a unique fileref is now always used
+  @param frefout= deprecated - a unique fileref is now always used
   @param mDebug= set to 1 to show debug messages in the log
 
   @version 9.3
   @author Allan Bowe
 
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+
 **/
 
 %macro mm_updatestpsourcecode(stp=
   ,stpcode=
   ,minify=NO
+  ,mdebug=0
+  /* deprecated */
   ,frefin=inmeta
   ,frefout=outmeta
-  ,mdebug=0
 );
+
+%if &frefin ne inmeta or &frefout ne outmeta %then %do;
+  %put %str(WARN)ING: the frefin and frefout parameters will be deprecated in
+    an upcoming release.;
+%end;
+
 /* first, check if STP exists */
 %local tsuri;
 %let tsuri=stopifempty ;
@@ -68,7 +77,9 @@ run;
   %return;
 %end;
 
-filename &frefin temp lrecl=32767;
+%local frefin frefout;
+%let frefin=%mf_getuniquefileref();
+%let frefout=%mf_getuniquefileref();
 
 /* write header XML */
 data _null_;
@@ -81,7 +92,7 @@ run;
 /* write contents */
 %if %length(&stpcode)>2 %then %do;
   data _null_;
-    file &frefin mod;
+    file &frefin lrecl=32767 mod;
     infile &stpcode lrecl=32767;
     length outstr $32767;
     input outstr ;
@@ -110,9 +121,6 @@ data _null_;
     </UpdateMetadata>";
 run;
 
-
-filename &frefout temp;
-
 proc metadata in= &frefin out=&frefout;
 run;
 
@@ -124,5 +132,9 @@ run;
     put _infile_;
   run;
 %end;
+%else %do;
+  filename &frefin clear;
+  filename &frefout clear;
+%end;
 
 %mend;

meta/mm_webout.sas

@@ -68,8 +68,16 @@
 %else %if &action=OPEN %then %do;
   /* fix encoding */
   OPTIONS NOBOMFILE;
+
+  /**
+    * check engine type to avoid the below err message:
+    * > Function is only valid for filerefs using the CACHE access method.
+    */
   data _null_;
+    set sashelp.vextfl(where=(fileref="_WEBOUT"));
+    if xengine='STREAM' then do;
       rc=stpsrv_header('Content-type',"text/html; encoding=utf-8");
+    end;
   run;
 
   /* setup json */
@@ -83,17 +91,10 @@
 %end;
 
 %else %if &action=ARR or &action=OBJ %then %do;
-  %if &sysver=9.4 %then %do;
-    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt
-      ,engine=PROCJSON,dbg=%str(&_debug)
-    )
-  %end;
-  %else %do;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt
     ,engine=DATASTEP,dbg=%str(&_debug)
   )
 %end;
-%end;
 %else %if &action=CLOSE %then %do;
   %if %str(&_debug) ge 131 %then %do;
     /* if debug mode, send back first 10 records of each work table also */
@@ -108,14 +109,14 @@
       i+1;
       call symputx('wt'!!left(i),name,'l');
       call symputx('wtcnt',i,'l');
-    data _null_; file &fref encoding='utf-8'; 
+    data _null_; file &fref mod encoding='utf-8';
       put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
       proc contents noprint data=&wt
         out=_data_ (keep=name type length format:);
       run;%let tempds=%scan(&syslast,2,.);
-      data _null_; file &fref encoding='utf-8';
+      data _null_; file &fref mod encoding='utf-8';
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
         nvars=attrn(dsid,'NVARS');
@@ -126,10 +127,10 @@
         put ',"nvars":' nvars;
       %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
       %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
-      data _null_; file &fref encoding='utf-8';
+      data _null_; file &fref mod encoding='utf-8';
         put "}";
     %end;
-    data _null_; file &fref encoding='utf-8';
+    data _null_; file &fref mod encoding='utf-8';
       put "}";
     run;
   %end;

metax/mmx_spkexport.sas

@@ -20,7 +20,8 @@ Usage:
     run;
 
     filename outref "%sysfunc(pathname(work))";
-    %mmx_spkexport(metaloc=%str(/30.Projects/3001.Internal/300115.DataController/dc1)
+    %mmx_spkexport(
+        metaloc=%str(/30.Projects/3001.Internal/300115.DataController/dc1)
         ,secureref=tmp
         ,outspkpath=%str(/tmp)
     )
@@ -56,7 +57,8 @@ Usage:
 /* get creds */
 %inc &secureref/nosource;
 
-%let connx_string=%str(-host &host -port &port -user '&mmxuser' -password '&mmxpass');
+%let connx_string=
+  %str(-host &host -port &port -user '&mmxuser' -password '&mmxpass');
 
 %mm_tree(root=%str(&metaloc) ,types=EXPORTABLE ,outds=exportable)
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sasjs/core",
-  "description": "SAS Macro Library for Application Development",
+  "description": "Production Ready Macros for SAS Application Developers",
   "license": "MIT",
   "keywords": [
     "SAS",
@@ -10,13 +10,33 @@
   "author": "Allan Bowe <support@macropeople.com>",
   "repository": {
     "type": "git",
-    "url": "https://github.com/sasjs/core"
+    "url": "git+https://github.com/sasjs/core.git"
   },
   "release": {
-    "branches": ["main"]
+    "branches": [
+      "main"
+    ]
   },
   "publishConfig": {
     "access": "public"
   },
-  "devDependencies": {}
+  "bugs": {
+    "url": "https://github.com/sasjs/core/issues"
+  },
+  "homepage": "https://github.com/sasjs/core#readme",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "docs": "sasjs doc && ./sasjs/utils/build.sh"
+  },
+  "devDependencies": {
+    "@sasjs/cli": "^2.14.2",
+    "ghooks": "^2.0.4"
+  },
+  "config": {
+    "ghooks": {
+      "pre-commit": "sasjs lint"
+    }
+  }
 }

sasjs/doxy/Doxyfile

@@ -1,5 +1,5 @@
 ALPHABETICAL_INDEX     = NO
-DISABLE_INDEX          = YES
+DISABLE_INDEX          = NO
 ENABLE_PREPROCESSING   = NO
 EXTENSION_MAPPING      = sas=Java ddl=Java
 EXTRACT_LOCAL_CLASSES  = NO
@@ -13,18 +13,21 @@ HIDE_IN_BODY_DOCS      = YES
 HIDE_SCOPE_NAMES       = YES
 HIDE_UNDOC_CLASSES     = YES
 HIDE_UNDOC_MEMBERS     = YES
-HTML_OUTPUT            = doxy
-HTML_HEADER            = ./doxy/new_header.html
-HTML_FOOTER            = ./doxy/new_footer.html
-HTML_EXTRA_STYLESHEET  = ./doxy/new_stylesheet.css
+HTML_OUTPUT            = $(DOXY_HTML_OUTPUT)
+HTML_HEADER            = $(HTML_HEADER)
+HTML_EXTRA_FILES       = $(HTML_EXTRA_FILES)
+HTML_FOOTER            = $(HTML_FOOTER)
+HTML_EXTRA_STYLESHEET  = $(HTML_EXTRA_STYLESHEET)
 INHERIT_DOCS           = NO
 INLINE_INFO            = NO
-INPUT                  = base meta metax viya
-LAYOUT_FILE            = ./doxy/DoxygenLayout.xml
+INPUT                  = $(DOXY_INPUT)
+INPUT                 += main.dox
+LAYOUT_FILE            = $(LAYOUT_FILE)
+USE_MDFILE_AS_MAINPAGE = README.md
 MAX_INITIALIZER_LINES  = 0
-PROJECT_NAME           = Macro Core
-PROJECT_LOGO           = doxy/Macro_core_website_1.png
-PROJECT_BRIEF          = "Production Ready Macros for SAS Application Developers"
+PROJECT_NAME           = $(PROJECT_NAME)
+PROJECT_LOGO           = $(PROJECT_LOGO)
+PROJECT_BRIEF          = $(PROJECT_BRIEF)
 RECURSIVE              = YES
 REPEAT_BRIEF           = NO
 SHOW_NAMESPACES        = NO

sasjs/doxy/DoxygenLayout.xml

@@ -2,7 +2,7 @@
   <!-- Generated by doxygen 1.8.14 -->
   <!-- Navigation index tabs for HTML output -->
   <navindex>
-    <tab type="mainpage" visible="no" title=""/>
+    <tab type="mainpage" visible="yes" title="Home"/>
     <tab type="pages" visible="no" title="" intro=""/>
     <tab type="modules" visible="no" title="" intro=""/>
     <tab type="namespaces" visible="no" title="">

sasjs/doxy/new_footer.html

@@ -0,0 +1,33 @@
+<!-- HTML footer for doxygen 1.8.17-->
+<!-- start footer part -->
+<!--BEGIN GENERATE_TREEVIEW-->
+<div id="nav-path" class="navpath">
+  <!-- id is needed for treeview function! -->
+  <ul>
+    $navpath
+    <li class="footer">
+      $generatedby
+      <a href="https://www.doxygen.org/index.html">
+        <img class="footer" src="$relpath^doxygen.svg" alt="doxygen"
+      /></a>
+      $doxygenversion
+    </li>
+    <li>
+      <i> For more information visit the </i>
+      <a href="https://github.com/sasjs/core">Macro Core library</a>.
+    </li>
+  </ul>
+</div>
+<!--END GENERATE_TREEVIEW-->
+<!--BEGIN !GENERATE_TREEVIEW-->
+<hr class="footer" />
+<address class="footer">
+  <small>
+    $generatedby &#160;<a href="http://www.doxygen.org/index.html">
+      <img class="footer" src="$relpath^doxygen.svg" alt="doxygen" />
+    </a>
+    $doxygenversion
+  </small>
+</address>
+
+<!--END !GENERATE_TREEVIEW-->

sasjs/doxy/new_header.html

@@ -0,0 +1,67 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!-- HTML header for doxygen 1.8.17-->
+<html xmlns="https://www.w3.org/1999/xhtml" lang="en">
+  <head>
+    <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=9" />
+    <meta property="og:type" content="website">
+    <meta name="author" content="Allan Bowe">
+    <meta name="generator" content="Doxygen $doxygenversion" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <!--BEGIN PROJECT_NAME-->
+    <meta name="description" content="$projectbrief" />
+    <!--END PROJECT_NAME-->
+    <!--BEGIN !PROJECT_NAME-->
+    <title>$title</title>
+    <!--END !PROJECT_NAME-->
+    <link href="$relpath^tabs.css" rel="stylesheet" type="text/css" />
+    <script type="text/javascript" src="$relpath^jquery.js"></script>
+    <script type="text/javascript" src="$relpath^dynsections.js"></script>
+    $treeview $search $mathjax
+    <link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
+    <link rel="shortcut icon" href="$relpath^favicon.ico" type="image/x-icon" />
+    $extrastylesheet
+  </head>
+  <body>
+    <div id="top">
+      <!-- do not remove this div, it is closed by doxygen! -->
+
+      <!--BEGIN TITLEAREA-->
+      <div id="titlearea">
+        <table cellspacing="0" cellpadding="0">
+          <tbody>
+            <tr style="height: 56px">
+              <!--BEGIN PROJECT_LOGO-->
+              <td id="projectlogo">
+                <a href="$relpath^"
+                  ><img alt="Logo" src="$relpath^$projectlogo"
+                /></a>
+              </td>
+              <!--END PROJECT_LOGO-->
+              <td id="projectalign" style="padding-left: 0.5em">
+                <div id="projectbrief">
+                  Production Ready Macros for SAS Application Developers<br />
+                  <a href="https://github.com/sasjs/core">
+                    https://github.com/sasjs/core
+                  </a>
+                </div>
+              </td>
+                  </tr>
+                </table>
+              </td>
+
+
+              <!--BEGIN DISABLE_INDEX-->
+              <!--BEGIN SEARCHENGINE-->
+              <td>$searchbox</td>
+              <!--END SEARCHENGINE-->
+              <!--END DISABLE_INDEX-->
+            </tr>
+          </tbody>
+        </table>
+      </div>
+      <!--END TITLEAREA-->
+      <!-- end header part -->
+    </div>
+  </body>
+</html>

sasjs/doxy/new_stylesheet.css

@@ -0,0 +1,5 @@
+#projectlogo img
+{
+  border: 0px none;
+  max-height:70px
+}

sasjs/sasjsconfig.json

@@ -0,0 +1,44 @@
+{
+  "$schema": "https://cli.sasjs.io/sasjsconfig-schema.json",
+  "macroFolders": [
+    "base",
+    "meta",
+    "metax",
+    "viya",
+    "lua"
+  ],
+  "docConfig": {
+    "displayMacroCore": false,
+    "enableLineage": false,
+    "doxyContent": {
+      "favIcon": "runningman.jpg",
+      "logo": "Macro_core_website_1.png",
+      "readMe": "../../README.md"
+    }
+  },
+  "serviceConfig": {
+    "initProgram": "tests/testinit.sas"
+  },
+  "defaultTarget": "viya",
+  "targets": [
+    {
+      "name": "viya",
+      "serverUrl": "https://sas.analytium.co.uk",
+      "serverType": "SASVIYA",
+      "appLoc": "/Public/temp/macrocore",
+      "serviceConfig": {
+        "serviceFolders": [
+          "tests/base",
+          "tests/viya"
+        ],
+        "macroVars": {
+          "mcTestAppLoc": "/Public/temp/macrocore"
+        }
+      },
+      "deployConfig": {
+        "deployServicePack": true
+      },
+      "contextName": "SAS Job Execution compute context"
+    }
+  ]
+}

sasjs/utils/build.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+####################################################################
+# PROJECT: Macro Core Docs Build                                   #
+# To execute, use the npm command (npm run docs)                   #
+####################################################################
+
+# refresh github pages site
+rm -rf sasjsbuild/docsite
+git clone git@github.com:sasjs/core.github.io.git sasjsbuild/docsite
+rm -rf sasjsbuild/docsite/*
+mv sasjsbuild/docs/* sasjsbuild/docsite/
+cd sasjsbuild/docsite/
+echo 'core.sasjs.io' > CNAME
+git add .
+git commit -m "build.sh build on $(date +%F:%H:%M:%S)"
+git push
+npx sitemap-generator-cli https://core.sasjs.io
+git add .
+git commit -m "adding sitemap"
+git push
+
+echo "check it out:  https://sasjs.github.io/core.github.io/files.html"

tests/base/mp_filtercheck.test.sas

@@ -0,0 +1,128 @@
+/**
+  @file
+  @brief Testing mp_filtercheck macro
+
+  <h4> SAS Macros </h4>
+  @li mp_filtercheck.sas
+  @li mp_assertdsobs.sas
+
+**/
+
+
+/* valid filter */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+datalines4;
+AND,AND,1,AGE,=,12
+AND,AND,1,SEX,<=,"'M'"
+AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
+AND,OR,2,Weight,>=,7
+;;;;
+run;
+
+%mp_filtercheck(work.inds,
+  targetds=sashelp.class,
+  outds=work.badrecords
+)
+%let syscc=0;
+%mp_assertdsobs(work.badrecords,
+  desc=Valid filter query,
+  test=EMPTY,
+  outds=work.test_results
+)
+
+/* invalid column */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+datalines4;
+AND,AND,1,invalid,=,12
+AND,AND,1,SEX,<=,"'M'"
+AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
+AND,OR,2,Weight,>=,7
+;;;;
+run;
+
+%mp_filtercheck(work.inds,
+  targetds=sashelp.class,
+  outds=work.badrecords
+)
+%let syscc=0;
+%mp_assertdsobs(work.badrecords,
+  desc=Invalid column name,
+  test=HASOBS,
+  outds=work.test_results
+)
+
+/* invalid raw value */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+datalines4;
+AND,OR,2,Name,NOT IN,"(''''Jane','Alfred')"
+;;;;
+run;
+
+%mp_filtercheck(work.inds,
+  targetds=sashelp.class,
+  outds=work.badrecords
+)
+%let syscc=0;
+%mp_assertdsobs(work.badrecords,
+  desc=Invalid raw value,
+  test=HASOBS,
+  outds=work.test_results
+)
+
+/* Code injection - column name */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+datalines4;
+AND,AND,1,%abort,=,12
+AND,OR,2,Weight,>=,7
+;;;;
+run;
+
+%mp_filtercheck(work.inds,
+  targetds=sashelp.class,
+  outds=work.badrecords
+)
+%let syscc=0;
+%mp_assertdsobs(work.badrecords,
+  desc=Code injection - column name,
+  test=HASOBS,
+  outds=work.test_results
+)
+
+/* Code injection - raw values*/
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+datalines4;
+AND,AND,1,age,=,;;%abort
+;;;;
+run;
+
+%mp_filtercheck(work.inds,
+  targetds=sashelp.class,
+  outds=work.badrecords
+)
+%let syscc=0;
+%mp_assertdsobs(work.badrecords,
+  desc=Code injection - raw value abort,
+  test=HASOBS,
+  outds=work.test_results
+)
+
+
+
+%webout(OPEN)
+%webout(OBJ, TEST_RESULTS)
+%webout(CLOSE)

tests/testinit.sas

@@ -0,0 +1,8 @@
+/**
+  @file
+  @brief init file for tests
+
+**/
+
+/* location in metadata or SAS Drive for temporary files */
+%let mcTestAppLoc=/Public/temp/test;

tests/viya/mv_createwebservice.test.sas

@@ -0,0 +1,24 @@
+/**
+  @file
+  @brief Testing mv_createwebservice macro
+
+  <h4> SAS Macros </h4>
+  @li mv_createwebservice.sas
+
+**/
+
+/**
+  * Test Case 1
+  * Send special char in a service
+  */
+
+filename testref temp;
+data _null_;
+  file testref;
+  put '01'x;
+run;
+%mv_createwebservice(
+  path=&mcTestAppLoc/tests/macros,
+  code=testref,
+  name=mv_createwebservice
+)