feat: two new macros, mp_mdtablewrite.sas (will create a markdown table from a sas dataset) and mm_getlibmetadiffs.sas (will create a set of datasets to describe the differences between a library in metadata and the physical library)

> @sasjs/core@1.0.0 postinstall
> node-git-hooks

Installing Git hooks...

up to date, audited 2 packages in 1s

found 0 vulnerabilities.

.githooks/pre-commit

@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# A hook script to verify that no filenames with capital letters are committed.
+# Called by "git commit" with no arguments.  The hook should
+# exit with non-zero status after issuing an appropriate message if
+# it wants to stop the commit.
+#
+# Go through all the changed files (except for deleted and unmerged)
+
+# Save exit code of last executed action
+exit_code=0
+
+# Check if file is one of SAS|DDL|CSV|SH and check for uppercase letters
+mime_pattern="\.(sas|ddl|csv|sh)"
+# Check for capital letters only in file names
+extra_pattern="(^|/)[^/]*([A-Z]+)[^/]*\.[A-Za-z]{3}$"
+# Grep git diff of files to commit
+files=$(	git diff --cached --find-copies --find-renames --name-only --diff-filter=ACMRTXBU |
+			grep -Ei "$mime_pattern" |
+			grep -E  "$extra_pattern" )
+echo "$files"
+if [[ -n "$files"  ]];
+then
+	echo
+	echo "Found files that contain capital letters."
+	echo "Please rename the following files in lowercase, and commit again:"
+
+	for file in $files; do
+		echo -e '- \E[0;32m'"$file"'\033[0m'
+	done
+	# Abort commit
+	exit_code=1
+fi
+
+if [ "$exit_code" == "0" ]; then
+	echo
+	echo -e '\033[1m'"Pre-commit validation Passed"'\033[0m'
+	echo
+else
+	echo
+	echo -e '\033[1m'"Commit Aborted!"'\033[0m'
+	echo
+fi
+exit $exit_code

.gitignore

@@ -1,3 +1,9 @@
 node_modules
 .DS_Store
 sasjsbuild/
+
+# avoid filenames with spaces being committed to source control
+**\ **
+
+# ignore the mc_* files - containing macros for individual libraries
+mc_*

.gitpod.dockerfile

@@ -0,0 +1,10 @@
+
+FROM gitpod/workspace-full
+
+RUN sudo apt-get update \
+ && sudo apt-get install -y \
+    doxygen \
+ && npm i -g npm@latest \
+ && npm i -g @sasjs/cli \
+ && npm i \
+ && sudo rm -rf /var/lib/apt/lists/*

.gitpod.yml

@@ -0,0 +1,8 @@
+tasks:
+  - init: npm i && clear
+
+image:
+  file: .gitpod.dockerfile
+vscode:
+  extensions:
+    - sasjs.sasjs-for-vscode@1.6.0:V4hJpMtbpekMcPRNhh4SXQ==

.vscode/.editorconfig

@@ -0,0 +1,9 @@
+{
+  "search.exclude": {
+    "**/sasjsbuild/**": true,
+    "**/dist/**":true
+  },
+  "editor.insertSpaces": true,
+  "editor.tabSize": 2,
+  "trim_trailing_whitespace": true
+}

.vscode/settings.json

@@ -0,0 +1,10 @@
+{
+  "editor.tabSize": 2,
+  "editor.insertSpaces": true,
+  "editor.detectIndentation": true,
+  "editor.formatOnSave": true,
+  "editor.rulers": [
+    80
+  ],
+  "files.trimTrailingWhitespace": true
+}

CONTRIBUTING.md

@@ -27,6 +27,5 @@ To contribute:
 1.  Create your feature branch (`git checkout -b myfeature`)
 2.  Make your change
 3.  Update the `all.sas` file (`python3 build.py`)
-4.  Commit the change, using the [conventional commit](https://www.conventionalcommits.org/en/v1.0.0) standard
-5.  Push and make a PR
+4.  Push and make a PR
 

README.md

@@ -1,6 +1,6 @@
 # Macro Core
 
-Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of Application Development on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
+Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
 
 You can download and compile them all in just two lines of SAS code:
 
@@ -9,7 +9,7 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 %inc mc;
 ```
 
-Documentation: https://sasjs.github.io/core.github.io/files.html
+Documentation: https://core.sasjs.io
 
 # Components
 
@@ -84,7 +84,7 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 ## File Properties
 
 - filenames much match macro names
-- filenames must be lowercase
+- filenames must be lowercase, without spaces
 - macro names must be lowercase
 - one macro per file
 - prefixes:
@@ -99,7 +99,7 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
 - UTF-8
-- no trailing white space
+
 
 ## Header Properties
 
@@ -136,13 +136,15 @@ When contributing to this library, it is therefore important to ensure that all
 ## Coding Standards
 
 - Indentation = 2 spaces. No tabs!
+- no trailing white space
+- no invisible characters, other than spaces. If invisibles are needed, use hex literals.
 - Macro variables should not have the trailing dot (`&var` not `&var.`) unless necessary to prevent incorrect resolution
-- The closing `%mend;` should not contain the macro name.
+- The closing `%mend;` should **not** contain the macro name.
 - All macros should be defined with brackets, even if no variables are needed - ie `%macro x();` not `%macro x;`
 - Mandatory parameters should be positional, all optional parameters should be keyword (var=) style.
 - All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect.
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
-- If you have a long-running SQL query, the use of a `quit;` statement is recommended in order to benefit from the timing statistics.
+- The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
 
 # General Notes
 

all.sas

@@ -1426,12 +1426,12 @@ Usage:
 
 %mend;/**
   @file
-  @brief Creates a Unique ID based on system time in a friendly format
+  @brief Creates a unique ID based on system time in friendly format
   @details format = YYYYMMDD_HHMMSSmmm_<sysjobid>_<3randomDigits>
 
         %put %mf_uid();
 
-  @version 9.2
+  @version 9.3
   @author Allan Bowe
 
 **/
@@ -1440,7 +1440,7 @@ Usage:
 )/*/STORE SOURCE*/;
   %local today now;
   %let today=%sysfunc(today(),yymmddn8.);
-  %let now=%sysfunc(compress(%sysfunc(time(),time12.3),:.));
+  %let now=%sysfunc(compress(%sysfunc(time(),tod12.3),:.));
 
   &today._&now._&sysjobid._%sysevalf(%sysfunc(ranuni(0))*999,CEIL)
 
@@ -4093,6 +4093,103 @@ select distinct lowcase(memname)
 %end;
 
 %mend;/**
+  @file
+  @brief Create a Markdown Table from a dataset
+  @details A markdown table is a simple table representation for use in
+  documents written in markdown format.
+
+  An online generator is available here:
+  https://www.tablesgenerator.com/markdown_tables
+
+  This structure is also used by the Macro Core library for documenting input/
+  output datasets, as well as the sasjs/cli tool for documenting inputs/outputs
+  for web services.
+
+  We take the standard definition one step further by embedding the informat
+  in the table header row, like so:
+
+      |var1:$|var2:best.|var3:date9.|
+      |---|---|---|
+      |some text|42|01JAN1960|
+      |blah|1|31DEC1999|
+
+  Which resolves to:
+
+  |var1:$|var2:best.|var3:date9.|
+  |---|---|---|
+  |some text|42|01JAN1960|
+  |blah|1|31DEC1999|
+
+
+  Usage:
+
+      %mp_mdtablewrite(libds=sashelp.class,showlog=YES)
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getvarlist.sas
+  @li mf_getvarformat.sas
+
+  @param [in] libds= the library / dataset to create or read from.
+  @param [out] fref= Fileref to contain the markdown. Default=mdtable.
+  @param [out] showlog= set to YES to show the markdown in the log. Default=NO.
+
+  @version 9.3
+  @author Allan Bowe
+**/
+
+%macro mp_mdtablewrite(
+  libds=,
+  fref=mdtable,
+  showlog=NO
+)/*/STORE SOURCE*/;
+
+/* check fileref is assigned */
+%if %sysfunc(fileref(&fref)) > 0 %then %do;
+  filename &fref temp;
+%end;
+
+%local vars;
+%let vars=%mf_getvarlist(&libds);
+
+/* create the header row */
+data _null_;
+  file &fref;
+  length line $32767;
+  put '|'
+%local i var fmt;
+%do i=1 %to %sysfunc(countw(&vars));
+  %let var=%scan(&vars,&i);
+  %let fmt=%mf_getvarformat(&libds,&var,force=1);
+  "&var:&fmt|"
+%end;
+  ;
+  put '|'
+%do i=1 %to %sysfunc(countw(&vars));
+  "---|"
+%end;
+  ;
+run;
+
+/* write out the data */
+data _null_;
+  file &fref mod dlm='|' lrecl=32767;
+  set &libds ;
+  length line $32767;
+  line=cats('|',%mf_getvarlist(&libds,dlm=%str(,'|',)),'|');
+  put line;
+run;
+
+%if %upcase(&showlog)=YES %then %do;
+  options ps=max;
+  data _null_;
+    infile &fref;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_mdtablewrite;/**
   @file
   @brief Logs the time the macro was executed in a control dataset.
   @details If the dataset does not exist, it is created.  Usage:
@@ -7601,6 +7698,98 @@ run;
   %return;
 %end;
 
+%mend;
+/**
+  @file
+  @brief Deletes a library by Name
+
+  @details  Used to delete a library.
+  Usage:
+
+      %* create a library in the home directory ;
+      %mm_createlibrary(
+        libname=My Temp Library,
+        libref=XXTEMPXX,
+        tree=/User Folders/&sysuserid,
+        directory=%sysfunc(pathname(work))
+      )
+
+      %* delete the library ;
+      %mm_deletelibrary(name=My Temp Library)
+
+  After running the above, the following will be shown in the log:
+
+  ![](https://i.imgur.com/Y4Tog24.png)
+
+  @param [in] name= the name (not libref) of the library to be deleted
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_abort.sas
+
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mm_deletelibrary(
+      name=
+)/*/STORE SOURCE*/;
+
+
+/**
+  * Check if library exists and get uri
+  */
+data _null_;
+  length type uri $256;
+  rc=metadata_resolve("omsobj:SASLibrary?@Name='&name'",type,uri);
+  call symputx('checktype',type,'l');
+  call symputx('liburi',uri,'l');
+  putlog (_all_)(=);
+run;
+%if &checktype ne SASLibrary %then %do;
+  %put &sysmacroname: Library (&name) was not found, and so will not be deleted;
+  %return;
+%end;
+
+%local fname1 fname2;
+%let fname1=%mf_getuniquefileref();
+%let fname2=%mf_getuniquefileref();
+
+filename &fname1 temp lrecl=10000;
+filename &fname2 temp lrecl=10000;
+data _null_ ;
+  file &fname1 ;
+  put "<DeleteMetadata><Metadata><SASLibrary Id='&liburi'/>";
+  put "</Metadata><NS>SAS</NS><Flags>268436480</Flags><Options/>";
+  put "</DeleteMetadata>";
+run ;
+proc metadata in=&fname1 out=&fname2 verbose;run;
+
+/* list the result */
+data _null_;infile &fname2; input; list; run;
+
+filename &fname1 clear;
+filename &fname2 clear;
+
+/**
+  * Check deletion
+  */
+%local isgone;
+data _null_;
+  length type uri $256;
+  rc=metadata_resolve("omsobj:SASLibrary?@Id='&liburi'",type,uri);
+  call symputx('isgone',type,'l');
+run;
+
+%mp_abort(iftrue=(&isgone = SASLibrary)
+  ,mac=&sysmacroname
+  ,msg=%str(Library (&name) NOT deleted)
+)
+
+%put &sysmacroname: Library &name (&liburi) was successfully deleted;
+
 %mend;
 /**
   @file mm_deletestp.sas
@@ -8105,20 +8294,117 @@ filename __outdoc clear;
 
 %mend;
 /**
-  @file mm_getfoldertree.sas
+  @file
+  @brief Returns all direct child members of a particular folder
+  @details Displays the children for a particular folder, in a similar fashion
+  to the viya counterpart (mv_getfoldermembers.sas)
+
+  Usage:
+
+      %mm_getfoldermembers(root=/, outds=rootfolders)
+
+      %mm_getfoldermembers(root=/User Folders/&sysuserid, outds=usercontent)
+
+  @param [in] root= the parent folder under which to return all contents
+  @param [out] outds= the dataset to create that contains the list of directories
+  @param [in] mDebug= set to 1 to show debug messages in the log
+
+  <h4> Data Outputs </h4>
+
+  Example for `root=/`:
+
+  |metauri $17|metaname $256|metatype $32|
+  |---|---|---|
+  |A5XLSNXI.AA000001|Products	|Folder|
+  |A5XLSNXI.AA000002|Shared Data	|Folder|
+  |A5XLSNXI.AA000003|User Folders	|Folder|
+  |A5XLSNXI.AA000004|System	|Folder|
+  |A5XLSNXI.AA00003K|30.SASApps	|Folder|
+  |A5XLSNXI.AA00006A|Public|Folder|
+
+  <h4> SAS Macros </h4>
+  @li mm_getfoldertree.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+%macro mm_getfoldermembers(
+     root=
+    ,outds=work.mm_getfoldertree
+)/*/STORE SOURCE*/;
+
+%if "&root" = "/" %then %do;
+  %local fname1 fname2 fname3;
+  %let fname1=%mf_getuniquefileref();
+  %let fname2=%mf_getuniquefileref();
+  %let fname3=%mf_getuniquefileref();
+  data _null_ ;
+    file &fname1 ;
+    put '<GetMetadataObjects>' ;
+    put '<Reposid>$METAREPOSITORY</Reposid>' ;
+    put '<Type>Tree</Type>' ;
+    put '<NS>SAS</NS>' ;
+    put '<Flags>388</Flags>' ;
+    put '<Options>' ;
+    put '<XMLSelect search="Tree[SoftwareComponents/SoftwareComponent'@;
+    put '[@Name=''BIP Service'']]"/>';
+    put '</Options>' ;
+    put '</GetMetadataObjects>' ;
+  run ;
+  proc metadata in=&fname1 out=&fname2 verbose;run;
+
+  /* create an XML map to read the response */
+  data _null_;
+    file &fname3;
+    put '<SXLEMAP version="1.2" name="SASFolders">';
+    put '<TABLE name="SASFolders">';
+    put '<TABLE-PATH syntax="XPath">//Objects/Tree</TABLE-PATH>';
+    put '<COLUMN name="metauri">><LENGTH>17</LENGTH>';
+    put '<PATH syntax="XPath">//Objects/Tree/@Id</PATH></COLUMN>';
+    put '<COLUMN name="metaname"><LENGTH>256</LENGTH>>';
+    put '<PATH syntax="XPath">//Objects/Tree/@Name</PATH></COLUMN>';
+    put '</TABLE></SXLEMAP>';
+  run;
+  %local libref1;
+  %let libref1=%mf_getuniquelibref();
+  libname &libref1 xml xmlfileref=&fname2 xmlmap=&fname3;
+
+  data &outds;
+    length metatype $32;
+    retain metatype 'Folder';
+    set &libref1..sasfolders;
+  run;
+
+%end;
+%else %do;
+  %mm_getfoldertree(root=&root, outds=&outds,depth=1)
+  data &outds;
+    set &outds(rename=(name=metaname publictype=metatype));
+    keep metaname metauri metatype;
+  run;
+%end;
+
+%mend;
+/**
+  @file
   @brief Returns all folders / subfolder content for a particular root
   @details Shows all members and SubTrees recursively for a particular root.
   Note - for big sites, this returns a lot of data!  So you may wish to reduce
-  the logging to speed up the process (see example below)
+  the logging to speed up the process (see example below), OR - use mm_tree.sas
+  which uses proc metadata and is far more efficient.
+
   Usage:
 
     options ps=max nonotes nosource;
     %mm_getfoldertree(root=/My/Meta/Path, outds=iwantthisdataset)
     options notes source;
-    
-  @param root= the parent folder under which to return all contents
-  @param outds= the dataset to create that contains the list of directories
-  @param mDebug= set to 1 to show debug messages in the log
+
+  @param [in] root= the parent folder under which to return all contents
+  @param [out] outds= the dataset to create that contains the list of directories
+  @param [in] mDebug= set to 1 to show debug messages in the log
 
   <h4> SAS Macros </h4>
 
@@ -8166,7 +8452,7 @@ data &outds.TMP/view=&outds.TMP;
     __n1+1;
     /* Walk through all possible associations of this object. */
     __n2=1;
-    if assoctype in ('Members','SubTrees') then 
+    if assoctype in ('Members','SubTrees') then
     do while(metadata_getnasn(pathuri,assoctype,__n2,metauri)>0);
       __n2+1;
       call missing(name,publictype,MetadataUpdated,MetadataCreated);
@@ -8357,6 +8643,127 @@ run;
 %end;
 
 %mend;/**
+  @file
+  @brief Compares the metadata of a library with the physical tables
+  @details Creates a series of output tables that show the differences between
+  metadata and physical tables.
+  Each output can be created with an optional prefix.
+
+  Credit - Paul Homes
+  https://platformadmin.com/blogs/paul/2012/11/sas-proc-metalib-ods-output
+
+  Usage:
+
+      %* create (and assign) a library for testing purposes ;
+      %mm_createlibrary(
+        libname=My Temp Library,
+        libref=XXTEMPXX,
+        tree=/User Folders/&sysuserid,
+        directory=%sysfunc(pathname(work))
+      )
+
+      %* create some tables;
+      data work.table1 table2 table3;
+        a=1;b='two';c=3;
+      run;
+
+      %* register the tables;
+      proc metalib;
+        omr=(library="My Temp Library");
+        report(type=detail);
+        update_rule (delete);
+      run;
+
+      %* modify the tables;
+      proc sql;
+      drop table table3;
+      alter table table2 drop c;
+      alter table table2 add d num;
+
+      %* run the macro;
+      %mm_getlibmetadiffs(libname=My Temp Library)
+
+      %* delete the library ;
+      %mm_deletelibrary(name=My Temp Library)
+
+  The program will create four output tables, with the following structure (and
+  example data):
+
+  #### &prefix.added
+  |name:$32.|metaID:$17.|SAStabName:$32.|
+  |---|---|---|
+  |||DATA1|
+
+  #### &prefix.deleted
+  |name:$32.|metaID:$17.|SAStabName:$32.|
+  |---|---|---|
+  |TABLE3|A5XLSNXI.BK0001HO|TABLE3|
+
+  #### &prefix.updated
+  |tabName:$32.|tabMetaID:$17.|SAStabName:$32.|metaName:$32.|metaID:$17.|sasname:$32.|metaType:$16.|change:$64.|
+  |---|---|---|---|---|---|---|---|
+  |TABLE2|A5XLSNXI.BK0001HN|TABLE2|c|A5XLSNXI.BM000MA9|c|Column|Deleted|
+  ||||d||d|Column|Added|
+
+  #### &prefix.meta
+  |Label1:$28.|cValue1:$1.|nValue1:D12.3|
+  |---|---|---|
+  |Total tables analyzed|4|4|
+  |Tables to be Updated|1|1|
+  |Tables to be Deleted|1|1|
+  |Tables to be Added|1|1|
+  |Tables matching data source|1|1|
+  |Tables not processed|0|0|
+
+  @param [in] libname= the metadata name of the library to be compared
+  @param [out] outlib= The output library in which to store the output tables.
+  Default=WORK.
+  @param [out] prefix The prefix for the four tables created. Default=metadiff.
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mm_getlibmetadiffs(
+  libname= ,
+  prefix=metadiff,
+  outlib=work
+)/*/STORE SOURCE*/;
+
+  /* create tempds */
+  data;run;
+  %local tempds;
+  %let tempds=&syslast;
+
+  /* save options */
+  proc optsave out=&tempds;
+  run;
+
+  options VALIDVARNAME=ANY VALIDMEMNAME=EXTEND;
+
+  ods output
+    factoid1=&outlib..&prefix.meta
+    updtab=&outlib..&prefix.updated
+    addtab=&outlib..&prefix.added
+    deltab=&outlib..&prefix.deleted
+  ;
+
+  proc metalib;
+    omr=(library="&libname");
+    noexec;
+    report(type=detail);
+    update_rule (delete);
+  run;
+
+  ods output close;
+
+  /* restore options */
+  proc optload data=&tempds;
+  run;
+
+%mend mm_getlibmetadiffs;
+/**
   @file
   @brief Creates a dataset with all metadata libraries
   @details Will only show the libraries to which a user has the requisite
@@ -10204,16 +10611,15 @@ run;
 %end;
 
 %mend;/**
-  @file mm_updatestpservertype.sas
+  @file
   @brief Updates a type 2 stored process to run on STP or WKS context
   @details Only works on Type 2 (9.3 compatible) STPs
 
   Usage:
 
-    %mm_updatestpservertype(target=/some/meta/path/myStoredProcess
-      ,type=WKS)
+      %mm_updatestpservertype(target=/some/meta/path/myStoredProcess
+        ,type=WKS)
 
-  <h4> SAS Macros </h4>
 
   @param target= full path to the STP being deleted
   @param type= Either WKS or STP depending on whether Workspace or Stored Process
@@ -10278,31 +10684,40 @@ run;
 
   Usage:
 
-    %mm_updatestpsourcecode(stp=/my/metadata/path/mystpname
-      ,stpcode="/file/system/source.sas")
+      %mm_updatestpsourcecode(stp=/my/metadata/path/mystpname
+        ,stpcode="/file/system/source.sas")
 
-
-  @param stp= the BIP Tree folder path plus Stored Process Name
-  @param stpcode= the source file (or fileref) containing the SAS code to load
+  @param [in] stp= the BIP Tree folder path plus Stored Process Name
+  @param [in] stpcode= the source file (or fileref) containing the SAS code to load
     into the stp.  For multiple files, they should simply be concatenated first.
-  @param minify= set to YES in order to strip comments, blank lines, and CRLFs.
+  @param [in] minify= set to YES in order to strip comments, blank lines, and CRLFs.
 
-  @param frefin= change default inref if it clashes with an existing one
-  @param frefout= change default outref if it clashes with an existing one
+  @param frefin= deprecated - a unique fileref is now always used
+  @param frefout= deprecated - a unique fileref is now always used
   @param mDebug= set to 1 to show debug messages in the log
 
   @version 9.3
   @author Allan Bowe
 
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+
 **/
 
 %macro mm_updatestpsourcecode(stp=
   ,stpcode=
   ,minify=NO
+  ,mdebug=0
+  /* deprecated */
   ,frefin=inmeta
   ,frefout=outmeta
-  ,mdebug=0
 );
+
+%if &frefin ne inmeta or &frefout ne outmeta %then %do;
+  %put %str(WARN)ING: the frefin and frefout parameters will be deprecated in
+    an upcoming release.;
+%end;
+
 /* first, check if STP exists */
 %local tsuri;
 %let tsuri=stopifempty ;
@@ -10340,7 +10755,9 @@ run;
   %return;
 %end;
 
-filename &frefin temp lrecl=32767;
+%local frefin frefout;
+%let frefin=%mf_getuniquefileref();
+%let frefout=%mf_getuniquefileref();
 
 /* write header XML */
 data _null_;
@@ -10353,7 +10770,7 @@ run;
 /* write contents */
 %if %length(&stpcode)>2 %then %do;
   data _null_;
-    file &frefin mod;
+    file &frefin lrecl=32767 mod;
     infile &stpcode lrecl=32767;
     length outstr $32767;
     input outstr ;
@@ -10382,9 +10799,6 @@ data _null_;
     </UpdateMetadata>";
 run;
 
-
-filename &frefout temp;
-
 proc metadata in= &frefin out=&frefout;
 run;
 
@@ -10396,6 +10810,10 @@ run;
     put _infile_;
   run;
 %end;
+%else %do;
+  filename &frefin clear;
+  filename &frefout clear;
+%end;
 
 %mend;/**
   @file mm_webout.sas
@@ -11056,23 +11474,23 @@ run;
     rec = "20"x;
     do while(fread(filein)=0);
       rc = fget(filein,rec,1);
-      if rec='"' then do;
+      if rec='"' then do;  /* DOUBLE QUOTE */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'"');rc =fwrite(fileid);
       end;
-      else if rec='0A'x then do;
-        rc =fput(fileid,'\');rc =fwrite(fileid);
-        rc =fput(fileid,'r');rc =fwrite(fileid);
-      end;
-      else if rec='0D'x then do;
+      else if rec='0A'x then do; /* LF */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'n');rc =fwrite(fileid);
       end;
-      else if rec='09'x then do;
+      else if rec='0D'x then do; /* CR */
+        rc =fput(fileid,'\');rc =fwrite(fileid);
+        rc =fput(fileid,'r');rc =fwrite(fileid);
+      end;
+      else if rec='09'x then do; /* TAB */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'t');rc =fwrite(fileid);
       end;
-      else if rec='5C'x then do;
+      else if rec='5C'x then do; /* BACKSLASH */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'\');rc =fwrite(fileid);
       end;
@@ -11712,23 +12130,23 @@ run;
     rec = "20"x;
     do while(fread(filein)=0);
       rc = fget(filein,rec,1);
-      if rec='"' then do;
+      if rec='"' then do;  /* DOUBLE QUOTE */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'"');rc =fwrite(fileid);
       end;
-      else if rec='0A'x then do;
-        rc =fput(fileid,'\');rc =fwrite(fileid);
-        rc =fput(fileid,'r');rc =fwrite(fileid);
-      end;
-      else if rec='0D'x then do;
+      else if rec='0A'x then do; /* LF */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'n');rc =fwrite(fileid);
       end;
-      else if rec='09'x then do;
+      else if rec='0D'x then do; /* CR */
+        rc =fput(fileid,'\');rc =fwrite(fileid);
+        rc =fput(fileid,'r');rc =fwrite(fileid);
+      end;
+      else if rec='09'x then do; /* TAB */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'t');rc =fwrite(fileid);
       end;
-      else if rec='5C'x then do;
+      else if rec='5C'x then do; /* BACKSLASH */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'\');rc =fwrite(fileid);
       end;
@@ -13802,6 +14220,8 @@ libname &libref;
       @li sas_services - will use oauth_bearer=sas_services
   @param [in] inds= The input dataset containing a list of jobs and parameters
   @param [in] maxconcurrency= The max number of parallel jobs to run.  Default=8.
+  @param [in] raise_err=0 Set to 1 to raise SYSCC when a job does not complete
+            succcessfully
   @param [in] mdebug= set to 1 to enable DEBUG messages
   @param [out] outds= The output dataset containing the results
   @param [out] outref= The output fileref to which to append the log file(s).
@@ -13825,6 +14245,7 @@ libname &libref;
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
     ,outref=0
+    ,raise_err=0
     ,mdebug=0
   );
 %local oauth_bearer;
@@ -14003,7 +14424,8 @@ data;run;%let jdswaitfor=&syslast;
     %end;
     %if &jid=&jcnt %then %do;
       /* we are at the end of the loop - time to see which jobs have finished */
-      %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref)
+      %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref
+                    ,raise_err=&raise_err)
       %local done;
       %let done=%mf_nobs(&jdswaitfor);
       %if &done>0 %then %do;
@@ -14110,6 +14532,8 @@ data;run;%let jdswaitfor=&syslast;
     following format:  `/jobExecution/jobs/&JOBID./state` and the corresponding
     job name.  The uri should be in a `uri` variable, and the job path/name
     should be in a `_program` variable.
+  @param [in] raise_err=0 Set to 1 to raise SYSCC when a job does not complete
+              succcessfully
   @param [out] outds= The output dataset containing the list of states by job
     (default=work.mv_jobexecute)
   @param [out] outref= A fileref to which the spawned job logs should be appended.
@@ -14121,6 +14545,7 @@ data;run;%let jdswaitfor=&syslast;
   @li mp_abort.sas
   @li mf_getplatform.sas
   @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
   @li mf_existvar.sas
   @li mf_nobs.sas
   @li mv_getjoblog.sas
@@ -14133,6 +14558,7 @@ data;run;%let jdswaitfor=&syslast;
     ,inds=0
     ,outds=work.mv_jobwaitfor
     ,outref=0
+    ,raise_err=0
   );
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
@@ -14176,7 +14602,7 @@ options noquotelenmax;
 data _null_;
   length jobparams $32767;
   set &inds end=last;
-  call symputx(cats('joburi',_n_),substr(uri,1,55)!!'/state','l');
+  call symputx(cats('joburi',_n_),substr(uri,1,55),'l');
   call symputx(cats('jobname',_n_),_program,'l');
   call symputx(cats('jobparams',_n_),jobparams,'l');
   if last then call symputx('uricnt',_n_,'l');
@@ -14191,7 +14617,7 @@ run;
 %let fname0=%mf_getuniquefileref();
 
 data &outds;
-  format _program uri $128. state $32. timestamp datetime19. jobparams $32767.;
+  format _program uri $128. state $32. stateDetails $32. timestamp datetime19. jobparams $32767.;
   stop;
 run;
 
@@ -14199,7 +14625,7 @@ run;
 %do i=1 %to &uricnt;
   %if "&&joburi&i" ne "0" %then %do;
     proc http method='GET' out=&fname0 &oauth_bearer url="&base_uri/&&joburi&i";
-      headers "Accept"="text/plain"
+      headers "Accept"="application/json"
       %if &grant_type=authorization_code %then %do;
               "Authorization"="Bearer &&&access_token_var"
       %end;  ;
@@ -14213,12 +14639,20 @@ run;
     %end;
 
     %let status=notset;
+
+    %local libref1;
+    %let libref1=%mf_getuniquelibref();
+    libname &libref1 json fileref=&fname0;
+
     data _null_;
-      infile &fname0;
-      input;
-      call symputx('status',_infile_,'l');
+      length state stateDetails $32;
+      set &libref1..root;
+      call symputx('status',state,'l');
+      call symputx('stateDetails',stateDetails,'l');
     run;
 
+    libname &libref1 clear;
+
     %if &status=completed or &status=failed or &status=canceled %then %do;
       %local plainuri;
       %let plainuri=%substr(&&joburi&i,1,55);
@@ -14227,6 +14661,7 @@ run;
         _program="&&jobname&i",
         uri="&plainuri",
         state="&status",
+        stateDetails=symget("stateDetails"),
         timestamp=datetime(),
         jobparams=symget("jobparams&i");
       %let joburi&i=0; /* do not re-check */
@@ -14245,6 +14680,16 @@ run;
         ,msg=%str(status &status not expected!!)
       )
     %end;
+
+    %if (&raise_err) %then %do;
+      %if (&status = canceled or &status = failed or %length(&stateDetails)>0) %then %do;
+        %if ("&stateDetails" = "%str(war)ning") %then %let SYSCC=4;
+        %else %let SYSCC=5;
+        %put %str(ERR)OR: Job &&jobname&i. did not complete successfully. &stateDetails;
+        %return;
+      %end; 
+    %end;
+
   %end;
   %if &i=&uricnt %then %do;
     %local goback;

base/mf_uid.sas

@@ -1,11 +1,11 @@
 /**
   @file
-  @brief Creates a Unique ID based on system time in a friendly format
+  @brief Creates a unique ID based on system time in friendly format
   @details format = YYYYMMDD_HHMMSSmmm_<sysjobid>_<3randomDigits>
 
         %put %mf_uid();
 
-  @version 9.2
+  @version 9.3
   @author Allan Bowe
 
 **/
@@ -14,7 +14,7 @@
 )/*/STORE SOURCE*/;
   %local today now;
   %let today=%sysfunc(today(),yymmddn8.);
-  %let now=%sysfunc(compress(%sysfunc(time(),time12.3),:.));
+  %let now=%sysfunc(compress(%sysfunc(time(),tod12.3),:.));
 
   &today._&now._&sysjobid._%sysevalf(%sysfunc(ranuni(0))*999,CEIL)
 

base/mp_mdtablewrite.sas

@@ -0,0 +1,98 @@
+/**
+  @file
+  @brief Create a Markdown Table from a dataset
+  @details A markdown table is a simple table representation for use in
+  documents written in markdown format.
+
+  An online generator is available here:
+  https://www.tablesgenerator.com/markdown_tables
+
+  This structure is also used by the Macro Core library for documenting input/
+  output datasets, as well as the sasjs/cli tool for documenting inputs/outputs
+  for web services.
+
+  We take the standard definition one step further by embedding the informat
+  in the table header row, like so:
+
+      |var1:$|var2:best.|var3:date9.|
+      |---|---|---|
+      |some text|42|01JAN1960|
+      |blah|1|31DEC1999|
+
+  Which resolves to:
+
+  |var1:$|var2:best.|var3:date9.|
+  |---|---|---|
+  |some text|42|01JAN1960|
+  |blah|1|31DEC1999|
+
+
+  Usage:
+
+      %mp_mdtablewrite(libds=sashelp.class,showlog=YES)
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getvarlist.sas
+  @li mf_getvarformat.sas
+
+  @param [in] libds= the library / dataset to create or read from.
+  @param [out] fref= Fileref to contain the markdown. Default=mdtable.
+  @param [out] showlog= set to YES to show the markdown in the log. Default=NO.
+
+  @version 9.3
+  @author Allan Bowe
+**/
+
+%macro mp_mdtablewrite(
+  libds=,
+  fref=mdtable,
+  showlog=NO
+)/*/STORE SOURCE*/;
+
+/* check fileref is assigned */
+%if %sysfunc(fileref(&fref)) > 0 %then %do;
+  filename &fref temp;
+%end;
+
+%local vars;
+%let vars=%mf_getvarlist(&libds);
+
+/* create the header row */
+data _null_;
+  file &fref;
+  length line $32767;
+  put '|'
+%local i var fmt;
+%do i=1 %to %sysfunc(countw(&vars));
+  %let var=%scan(&vars,&i);
+  %let fmt=%mf_getvarformat(&libds,&var,force=1);
+  "&var:&fmt|"
+%end;
+  ;
+  put '|'
+%do i=1 %to %sysfunc(countw(&vars));
+  "---|"
+%end;
+  ;
+run;
+
+/* write out the data */
+data _null_;
+  file &fref mod dlm='|' lrecl=32767;
+  set &libds ;
+  length line $32767;
+  line=cats('|',%mf_getvarlist(&libds,dlm=%str(,'|',)),'|');
+  put line;
+run;
+
+%if %upcase(&showlog)=YES %then %do;
+  options ps=max;
+  data _null_;
+    infile &fref;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_mdtablewrite;

meta/mm_deletelibrary.sas

@@ -0,0 +1,92 @@
+/**
+  @file
+  @brief Deletes a library by Name
+
+  @details  Used to delete a library.
+  Usage:
+
+      %* create a library in the home directory ;
+      %mm_createlibrary(
+        libname=My Temp Library,
+        libref=XXTEMPXX,
+        tree=/User Folders/&sysuserid,
+        directory=%sysfunc(pathname(work))
+      )
+
+      %* delete the library ;
+      %mm_deletelibrary(name=My Temp Library)
+
+  After running the above, the following will be shown in the log:
+
+  ![](https://i.imgur.com/Y4Tog24.png)
+
+  @param [in] name= the name (not libref) of the library to be deleted
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_abort.sas
+
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mm_deletelibrary(
+      name=
+)/*/STORE SOURCE*/;
+
+
+/**
+  * Check if library exists and get uri
+  */
+data _null_;
+  length type uri $256;
+  rc=metadata_resolve("omsobj:SASLibrary?@Name='&name'",type,uri);
+  call symputx('checktype',type,'l');
+  call symputx('liburi',uri,'l');
+  putlog (_all_)(=);
+run;
+%if &checktype ne SASLibrary %then %do;
+  %put &sysmacroname: Library (&name) was not found, and so will not be deleted;
+  %return;
+%end;
+
+%local fname1 fname2;
+%let fname1=%mf_getuniquefileref();
+%let fname2=%mf_getuniquefileref();
+
+filename &fname1 temp lrecl=10000;
+filename &fname2 temp lrecl=10000;
+data _null_ ;
+  file &fname1 ;
+  put "<DeleteMetadata><Metadata><SASLibrary Id='&liburi'/>";
+  put "</Metadata><NS>SAS</NS><Flags>268436480</Flags><Options/>";
+  put "</DeleteMetadata>";
+run ;
+proc metadata in=&fname1 out=&fname2 verbose;run;
+
+/* list the result */
+data _null_;infile &fname2; input; list; run;
+
+filename &fname1 clear;
+filename &fname2 clear;
+
+/**
+  * Check deletion
+  */
+%local isgone;
+data _null_;
+  length type uri $256;
+  rc=metadata_resolve("omsobj:SASLibrary?@Id='&liburi'",type,uri);
+  call symputx('isgone',type,'l');
+run;
+
+%mp_abort(iftrue=(&isgone = SASLibrary)
+  ,mac=&sysmacroname
+  ,msg=%str(Library (&name) NOT deleted)
+)
+
+%put &sysmacroname: Library &name (&liburi) was successfully deleted;
+
+%mend;

meta/mm_getfoldermembers.sas

@@ -0,0 +1,95 @@
+/**
+  @file
+  @brief Returns all direct child members of a particular folder
+  @details Displays the children for a particular folder, in a similar fashion
+  to the viya counterpart (mv_getfoldermembers.sas)
+
+  Usage:
+
+      %mm_getfoldermembers(root=/, outds=rootfolders)
+
+      %mm_getfoldermembers(root=/User Folders/&sysuserid, outds=usercontent)
+
+  @param [in] root= the parent folder under which to return all contents
+  @param [out] outds= the dataset to create that contains the list of directories
+  @param [in] mDebug= set to 1 to show debug messages in the log
+
+  <h4> Data Outputs </h4>
+
+  Example for `root=/`:
+
+  |metauri $17|metaname $256|metatype $32|
+  |---|---|---|
+  |A5XLSNXI.AA000001|Products	|Folder|
+  |A5XLSNXI.AA000002|Shared Data	|Folder|
+  |A5XLSNXI.AA000003|User Folders	|Folder|
+  |A5XLSNXI.AA000004|System	|Folder|
+  |A5XLSNXI.AA00003K|30.SASApps	|Folder|
+  |A5XLSNXI.AA00006A|Public|Folder|
+
+  <h4> SAS Macros </h4>
+  @li mm_getfoldertree.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+%macro mm_getfoldermembers(
+     root=
+    ,outds=work.mm_getfoldertree
+)/*/STORE SOURCE*/;
+
+%if "&root" = "/" %then %do;
+  %local fname1 fname2 fname3;
+  %let fname1=%mf_getuniquefileref();
+  %let fname2=%mf_getuniquefileref();
+  %let fname3=%mf_getuniquefileref();
+  data _null_ ;
+    file &fname1 ;
+    put '<GetMetadataObjects>' ;
+    put '<Reposid>$METAREPOSITORY</Reposid>' ;
+    put '<Type>Tree</Type>' ;
+    put '<NS>SAS</NS>' ;
+    put '<Flags>388</Flags>' ;
+    put '<Options>' ;
+    put '<XMLSelect search="Tree[SoftwareComponents/SoftwareComponent'@;
+    put '[@Name=''BIP Service'']]"/>';
+    put '</Options>' ;
+    put '</GetMetadataObjects>' ;
+  run ;
+  proc metadata in=&fname1 out=&fname2 verbose;run;
+
+  /* create an XML map to read the response */
+  data _null_;
+    file &fname3;
+    put '<SXLEMAP version="1.2" name="SASFolders">';
+    put '<TABLE name="SASFolders">';
+    put '<TABLE-PATH syntax="XPath">//Objects/Tree</TABLE-PATH>';
+    put '<COLUMN name="metauri">><LENGTH>17</LENGTH>';
+    put '<PATH syntax="XPath">//Objects/Tree/@Id</PATH></COLUMN>';
+    put '<COLUMN name="metaname"><LENGTH>256</LENGTH>>';
+    put '<PATH syntax="XPath">//Objects/Tree/@Name</PATH></COLUMN>';
+    put '</TABLE></SXLEMAP>';
+  run;
+  %local libref1;
+  %let libref1=%mf_getuniquelibref();
+  libname &libref1 xml xmlfileref=&fname2 xmlmap=&fname3;
+
+  data &outds;
+    length metatype $32;
+    retain metatype 'Folder';
+    set &libref1..sasfolders;
+  run;
+
+%end;
+%else %do;
+  %mm_getfoldertree(root=&root, outds=&outds,depth=1)
+  data &outds;
+    set &outds(rename=(name=metaname publictype=metatype));
+    keep metaname metauri metatype;
+  run;
+%end;
+
+%mend;

meta/mm_getfoldertree.sas

@@ -1,18 +1,20 @@
 /**
-  @file mm_getfoldertree.sas
+  @file
   @brief Returns all folders / subfolder content for a particular root
   @details Shows all members and SubTrees recursively for a particular root.
   Note - for big sites, this returns a lot of data!  So you may wish to reduce
-  the logging to speed up the process (see example below)
+  the logging to speed up the process (see example below), OR - use mm_tree.sas
+  which uses proc metadata and is far more efficient.
+
   Usage:
 
     options ps=max nonotes nosource;
     %mm_getfoldertree(root=/My/Meta/Path, outds=iwantthisdataset)
     options notes source;
-    
-  @param root= the parent folder under which to return all contents
-  @param outds= the dataset to create that contains the list of directories
-  @param mDebug= set to 1 to show debug messages in the log
+
+  @param [in] root= the parent folder under which to return all contents
+  @param [out] outds= the dataset to create that contains the list of directories
+  @param [in] mDebug= set to 1 to show debug messages in the log
 
   <h4> SAS Macros </h4>
 
@@ -60,7 +62,7 @@ data &outds.TMP/view=&outds.TMP;
     __n1+1;
     /* Walk through all possible associations of this object. */
     __n2=1;
-    if assoctype in ('Members','SubTrees') then 
+    if assoctype in ('Members','SubTrees') then
     do while(metadata_getnasn(pathuri,assoctype,__n2,metauri)>0);
       __n2+1;
       call missing(name,publictype,MetadataUpdated,MetadataCreated);

meta/mm_getlibmetadiffs.sas

@@ -0,0 +1,129 @@
+/**
+  @file
+  @brief Compares the metadata of a library with the physical tables
+  @details Creates a series of output tables that show the differences between
+  metadata and physical tables.
+  Each output can be created with an optional prefix.
+
+  Credit - Paul Homes
+  https://platformadmin.com/blogs/paul/2012/11/sas-proc-metalib-ods-output
+
+  Usage:
+
+      %* create (and assign) a library for testing purposes ;
+      %mm_createlibrary(
+        libname=My Temp Library,
+        libref=XXTEMPXX,
+        tree=/User Folders/&sysuserid,
+        directory=%sysfunc(pathname(work))
+      )
+
+      %* create some tables;
+      data work.table1 table2 table3;
+        a=1;b='two';c=3;
+      run;
+
+      %* register the tables;
+      proc metalib;
+        omr=(library="My Temp Library");
+        report(type=detail);
+        update_rule (delete);
+      run;
+
+      %* modify the tables;
+      proc sql;
+      drop table table3;
+      alter table table2 drop c;
+      alter table table2 add d num;
+
+      %* run the macro;
+      %mm_getlibmetadiffs(libname=My Temp Library)
+
+      %* delete the library ;
+      %mm_deletelibrary(name=My Temp Library)
+
+  The program will create four output tables, with the following structure (and
+  example data):
+
+  #### &prefix.added
+  |name:$32.|metaID:$17.|SAStabName:$32.|
+  |---|---|---|
+  |||DATA1|
+
+  #### &prefix.deleted
+  |name:$32.|metaID:$17.|SAStabName:$32.|
+  |---|---|---|
+  |TABLE3|A5XLSNXI.BK0001HO|TABLE3|
+
+  #### &prefix.updated
+  |tabName:$32.|tabMetaID:$17.|SAStabName:$32.|metaName:$32.|metaID:$17.|sasname:$32.|metaType:$16.|change:$64.|
+  |---|---|---|---|---|---|---|---|
+  |TABLE2|A5XLSNXI.BK0001HN|TABLE2|c|A5XLSNXI.BM000MA9|c|Column|Deleted|
+  ||||d||d|Column|Added|
+
+  #### &prefix.meta
+  |Label1:$28.|cValue1:$1.|nValue1:D12.3|
+  |---|---|---|
+  |Total tables analyzed|4|4|
+  |Tables to be Updated|1|1|
+  |Tables to be Deleted|1|1|
+  |Tables to be Added|1|1|
+  |Tables matching data source|1|1|
+  |Tables not processed|0|0|
+
+  If you are interested in more functionality like this (checking the health of
+  SAS metadata and your SAS 9 environment) then do contact [Allan Bowe](
+  https://www.linkedin.com/in/allanbowe) for details of our SAS 9 Health Check
+  service.
+
+  Our system scan will perform hundreds of checks to identify common issues,
+  such as dangling metadata, embedded passwords, security issues and more.
+
+  @param [in] libname= the metadata name of the library to be compared
+  @param [out] outlib= The output library in which to store the output tables.
+  Default=WORK.
+  @param [out] prefix The prefix for the four tables created. Default=metadiff.
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mm_getlibmetadiffs(
+  libname= ,
+  prefix=metadiff,
+  outlib=work
+)/*/STORE SOURCE*/;
+
+  /* create tempds */
+  data;run;
+  %local tempds;
+  %let tempds=&syslast;
+
+  /* save options */
+  proc optsave out=&tempds;
+  run;
+
+  options VALIDVARNAME=ANY VALIDMEMNAME=EXTEND;
+
+  ods output
+    factoid1=&outlib..&prefix.meta
+    updtab=&outlib..&prefix.updated
+    addtab=&outlib..&prefix.added
+    deltab=&outlib..&prefix.deleted
+  ;
+
+  proc metalib;
+    omr=(library="&libname");
+    noexec;
+    report(type=detail);
+    update_rule (delete);
+  run;
+
+  ods output close;
+
+  /* restore options */
+  proc optload data=&tempds;
+  run;
+
+%mend mm_getlibmetadiffs;

meta/mm_updatestpservertype.sas

@@ -1,14 +1,13 @@
 /**
-  @file mm_updatestpservertype.sas
+  @file
   @brief Updates a type 2 stored process to run on STP or WKS context
   @details Only works on Type 2 (9.3 compatible) STPs
 
   Usage:
 
-    %mm_updatestpservertype(target=/some/meta/path/myStoredProcess
-      ,type=WKS)
+      %mm_updatestpservertype(target=/some/meta/path/myStoredProcess
+        ,type=WKS)
 
-  <h4> SAS Macros </h4>
 
   @param target= full path to the STP being deleted
   @param type= Either WKS or STP depending on whether Workspace or Stored Process

meta/mm_updatestpsourcecode.sas

@@ -6,31 +6,40 @@
 
   Usage:
 
-    %mm_updatestpsourcecode(stp=/my/metadata/path/mystpname
-      ,stpcode="/file/system/source.sas")
+      %mm_updatestpsourcecode(stp=/my/metadata/path/mystpname
+        ,stpcode="/file/system/source.sas")
 
-
-  @param stp= the BIP Tree folder path plus Stored Process Name
-  @param stpcode= the source file (or fileref) containing the SAS code to load
+  @param [in] stp= the BIP Tree folder path plus Stored Process Name
+  @param [in] stpcode= the source file (or fileref) containing the SAS code to load
     into the stp.  For multiple files, they should simply be concatenated first.
-  @param minify= set to YES in order to strip comments, blank lines, and CRLFs.
+  @param [in] minify= set to YES in order to strip comments, blank lines, and CRLFs.
 
-  @param frefin= change default inref if it clashes with an existing one
-  @param frefout= change default outref if it clashes with an existing one
+  @param frefin= deprecated - a unique fileref is now always used
+  @param frefout= deprecated - a unique fileref is now always used
   @param mDebug= set to 1 to show debug messages in the log
 
   @version 9.3
   @author Allan Bowe
 
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+
 **/
 
 %macro mm_updatestpsourcecode(stp=
   ,stpcode=
   ,minify=NO
+  ,mdebug=0
+  /* deprecated */
   ,frefin=inmeta
   ,frefout=outmeta
-  ,mdebug=0
 );
+
+%if &frefin ne inmeta or &frefout ne outmeta %then %do;
+  %put %str(WARN)ING: the frefin and frefout parameters will be deprecated in
+    an upcoming release.;
+%end;
+
 /* first, check if STP exists */
 %local tsuri;
 %let tsuri=stopifempty ;
@@ -68,7 +77,9 @@ run;
   %return;
 %end;
 
-filename &frefin temp lrecl=32767;
+%local frefin frefout;
+%let frefin=%mf_getuniquefileref();
+%let frefout=%mf_getuniquefileref();
 
 /* write header XML */
 data _null_;
@@ -81,7 +92,7 @@ run;
 /* write contents */
 %if %length(&stpcode)>2 %then %do;
   data _null_;
-    file &frefin mod;
+    file &frefin lrecl=32767 mod;
     infile &stpcode lrecl=32767;
     length outstr $32767;
     input outstr ;
@@ -110,9 +121,6 @@ data _null_;
     </UpdateMetadata>";
 run;
 
-
-filename &frefout temp;
-
 proc metadata in= &frefin out=&frefout;
 run;
 
@@ -124,5 +132,9 @@ run;
     put _infile_;
   run;
 %end;
+%else %do;
+  filename &frefin clear;
+  filename &frefout clear;
+%end;
 
 %mend;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sasjs/core",
-  "description": "SAS Macro Library for Application Development",
+  "description": "Production Ready Macros for SAS Application Developers",
   "license": "MIT",
   "keywords": [
     "SAS",
@@ -28,9 +28,10 @@
   "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-    "docs": "./sasjs/utils/build.sh"
+    "docs": "sasjs doc && ./sasjs/utils/build.sh",
+    "postinstall": "node-git-hooks"
   },
-  "devDependencies": {
-    "@sasjs/cli": "^2.4.0"
+  "dependencies": {
+    "node-git-hooks": "^1.0.5"
   }
 }

sasjs/doxy/Doxyfile

@@ -14,20 +14,20 @@ HIDE_SCOPE_NAMES       = YES
 HIDE_UNDOC_CLASSES     = YES
 HIDE_UNDOC_MEMBERS     = YES
 HTML_OUTPUT            = $(DOXY_HTML_OUTPUT)
-HTML_HEADER            = $(DOXY_CONTENT)new_header.html
-HTML_FOOTER            = $(DOXY_CONTENT)new_footer.html
-HTML_EXTRA_STYLESHEET  = $(DOXY_CONTENT)new_stylesheet.css
+HTML_HEADER            = $(HTML_HEADER)
+HTML_EXTRA_FILES       = $(HTML_EXTRA_FILES)
+HTML_FOOTER            = $(HTML_FOOTER)
+HTML_EXTRA_STYLESHEET  = $(HTML_EXTRA_STYLESHEET)
 INHERIT_DOCS           = NO
 INLINE_INFO            = NO
-INPUT                  = $(DOXY_CONTENT)../../README.md \
-                       = $(DOXY_CONTENT)../../main.dox \
-                         $(DOXY_INPUT)
+INPUT                  = $(DOXY_INPUT)
+INPUT                 += main.dox
+LAYOUT_FILE            = $(LAYOUT_FILE)
 USE_MDFILE_AS_MAINPAGE = README.md
-LAYOUT_FILE            = $(DOXY_CONTENT)DoxygenLayout.xml
 MAX_INITIALIZER_LINES  = 0
-PROJECT_NAME           = Macro Core
-PROJECT_LOGO           = $(DOXY_CONTENT)Macro_core_website_1.png
-PROJECT_BRIEF          = "Production Ready Macros for SAS Application Developers"
+PROJECT_NAME           = $(PROJECT_NAME)
+PROJECT_LOGO           = $(PROJECT_LOGO)
+PROJECT_BRIEF          = $(PROJECT_BRIEF)
 RECURSIVE              = YES
 REPEAT_BRIEF           = NO
 SHOW_NAMESPACES        = NO
@@ -38,4 +38,4 @@ STRICT_PROTO_MATCHING  = YES
 STRIP_CODE_COMMENTS    = NO
 SUBGROUPING            = NO
 TAB_SIZE               = 2
-VERBATIM_HEADERS       = NO
+VERBATIM_HEADERS       = NO

sasjs/doxy/batch.bat

@@ -1,6 +0,0 @@
-ECHO on
-
-rmdir /s /q C:\DEVELOPMENT\output
-mkdir C:\DEVELOPMENT\output
-
-doxygen.exe Doxyfile

sasjs/doxy/new_footer.html

@@ -8,7 +8,7 @@
     <li class="footer">
       $generatedby
       <a href="https://www.doxygen.org/index.html">
-        <img class="footer" src="$relpath^doxygen.png" alt="doxygen"
+        <img class="footer" src="$relpath^doxygen.svg" alt="doxygen"
       /></a>
       $doxygenversion
     </li>
@@ -24,9 +24,10 @@
 <address class="footer">
   <small>
     $generatedby &#160;<a href="http://www.doxygen.org/index.html">
-      <img class="footer" src="$relpath^doxygen.png" alt="doxygen" />
+      <img class="footer" src="$relpath^doxygen.svg" alt="doxygen" />
     </a>
     $doxygenversion
   </small>
 </address>
+
 <!--END !GENERATE_TREEVIEW-->

sasjs/doxy/new_header.html

@@ -4,10 +4,12 @@
   <head>
     <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8" />
     <meta http-equiv="X-UA-Compatible" content="IE=9" />
+    <meta property="og:type" content="website">
+    <meta name="author" content="Allan Bowe">
     <meta name="generator" content="Doxygen $doxygenversion" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <!--BEGIN PROJECT_NAME-->
-    <title>$projectname: $title</title>
+    <meta name="description" content="$projectbrief" />
     <!--END PROJECT_NAME-->
     <!--BEGIN !PROJECT_NAME-->
     <title>$title</title>
@@ -17,7 +19,7 @@
     <script type="text/javascript" src="$relpath^dynsections.js"></script>
     $treeview $search $mathjax
     <link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
-    <link rel="icon" href="https://sasjs.io/img/runningman.jpg" />
+    <link rel="shortcut icon" href="$relpath^favicon.ico" type="image/x-icon" />
     $extrastylesheet
   </head>
   <body>
@@ -31,52 +33,24 @@
             <tr style="height: 56px">
               <!--BEGIN PROJECT_LOGO-->
               <td id="projectlogo">
-                <img alt="Logo" src="$relpath^$projectlogo" />
+                <a href="$relpath^"
+                  ><img alt="Logo" src="$relpath^$projectlogo"
+                /></a>
               </td>
               <!--END PROJECT_LOGO-->
-              <!--BEGIN PROJECT_NAME-->
               <td id="projectalign" style="padding-left: 0.5em">
-                <div id="projectname">
-                  <!--BEGIN PROJECT_NUMBER-->&#160;<span id="projectnumber"
-                    >$projectnumber</span
-                  ><!--END PROJECT_NUMBER-->
-                </div>
-                <!--BEGIN PROJECT_BRIEF-->
                 <div id="projectbrief">
                   Production Ready Macros for SAS Application Developers<br />
                   <a href="https://github.com/sasjs/core">
                     https://github.com/sasjs/core
                   </a>
                 </div>
-                <meta name="Description" content="$projectbrief" />
-                <!--END PROJECT_BRIEF-->
               </td>
-              <!--END PROJECT_NAME-->
-              <!--BEGIN !PROJECT_NAME-->
-              <!--BEGIN PROJECT_BRIEF-->
-              <td style="padding-left: 0.5em">
-                <div id="projectbrief">$projectbrief</div>
-                <table
-                  style="padding-left: 2em"
-                  cellspacing="0"
-                  cellpadding="0"
-                >
-                  <tr>
-                    <td>
-                      Production Ready Macros for SAS Application Developers
-                    </td>
-                  </tr>
-                  <tr>
-                    <td>
-                      <a href="https://github.com/sasjs/core">
-                        https://github.com/sasjs/core
-                      </a>
-                    </td>
                   </tr>
                 </table>
               </td>
-              <!--END PROJECT_BRIEF-->
-              <!--END !PROJECT_NAME-->
+
+
               <!--BEGIN DISABLE_INDEX-->
               <!--BEGIN SEARCHENGINE-->
               <td>$searchbox</td>

sasjs/sasjsconfig.json

@@ -2,6 +2,12 @@
   "$schema": "https://cli.sasjs.io/sasjsconfig-schema.json",
   "macroFolders": ["base", "meta", "metax", "viya", "lua"],
   "docConfig":{
-    "displayMacroCore": false
+    "displayMacroCore": false,
+    "enableLineage": false,
+    "doxyContent": {
+      "favIcon": "runningman.jpg",
+      "logo": "Macro_core_website_1.png",
+      "readMe": "../../README.md"
+    }
   }
 }

sasjs/utils/build.sh

@@ -1,12 +1,9 @@
 #!/bin/bash
 ####################################################################
 # PROJECT: Macro Core Docs Build                                   #
+# To execute, use the npm command (npm run docs)                   #
 ####################################################################
 
-cd ../..
-
-sasjs doc
-
 # refresh github pages site
 rm -rf sasjsbuild/docsite
 git clone git@github.com:sasjs/core.github.io.git sasjsbuild/docsite

viya/mv_createjob.sas

@@ -217,23 +217,23 @@ run;
     rec = "20"x;
     do while(fread(filein)=0);
       rc = fget(filein,rec,1);
-      if rec='"' then do;
+      if rec='"' then do;  /* DOUBLE QUOTE */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'"');rc =fwrite(fileid);
       end;
-      else if rec='0A'x then do;
-        rc =fput(fileid,'\');rc =fwrite(fileid);
-        rc =fput(fileid,'r');rc =fwrite(fileid);
-      end;
-      else if rec='0D'x then do;
+      else if rec='0A'x then do; /* LF */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'n');rc =fwrite(fileid);
       end;
-      else if rec='09'x then do;
+      else if rec='0D'x then do; /* CR */
+        rc =fput(fileid,'\');rc =fwrite(fileid);
+        rc =fput(fileid,'r');rc =fwrite(fileid);
+      end;
+      else if rec='09'x then do; /* TAB */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'t');rc =fwrite(fileid);
       end;
-      else if rec='5C'x then do;
+      else if rec='5C'x then do; /* BACKSLASH */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'\');rc =fwrite(fileid);
       end;

viya/mv_createwebservice.sas

@@ -566,23 +566,23 @@ run;
     rec = "20"x;
     do while(fread(filein)=0);
       rc = fget(filein,rec,1);
-      if rec='"' then do;
+      if rec='"' then do;  /* DOUBLE QUOTE */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'"');rc =fwrite(fileid);
       end;
-      else if rec='0A'x then do;
-        rc =fput(fileid,'\');rc =fwrite(fileid);
-        rc =fput(fileid,'r');rc =fwrite(fileid);
-      end;
-      else if rec='0D'x then do;
+      else if rec='0A'x then do; /* LF */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'n');rc =fwrite(fileid);
       end;
-      else if rec='09'x then do;
+      else if rec='0D'x then do; /* CR */
+        rc =fput(fileid,'\');rc =fwrite(fileid);
+        rc =fput(fileid,'r');rc =fwrite(fileid);
+      end;
+      else if rec='09'x then do; /* TAB */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'t');rc =fwrite(fileid);
       end;
-      else if rec='5C'x then do;
+      else if rec='5C'x then do; /* BACKSLASH */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'\');rc =fwrite(fileid);
       end;

viya/mv_jobflow.sas

@@ -106,6 +106,8 @@
       @li sas_services - will use oauth_bearer=sas_services
   @param [in] inds= The input dataset containing a list of jobs and parameters
   @param [in] maxconcurrency= The max number of parallel jobs to run.  Default=8.
+  @param [in] raise_err=0 Set to 1 to raise SYSCC when a job does not complete
+            succcessfully
   @param [in] mdebug= set to 1 to enable DEBUG messages
   @param [out] outds= The output dataset containing the results
   @param [out] outref= The output fileref to which to append the log file(s).
@@ -129,6 +131,7 @@
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
     ,outref=0
+    ,raise_err=0
     ,mdebug=0
   );
 %local oauth_bearer;
@@ -307,7 +310,8 @@ data;run;%let jdswaitfor=&syslast;
     %end;
     %if &jid=&jcnt %then %do;
       /* we are at the end of the loop - time to see which jobs have finished */
-      %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref)
+      %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref
+                    ,raise_err=&raise_err)
       %local done;
       %let done=%mf_nobs(&jdswaitfor);
       %if &done>0 %then %do;

viya/mv_jobwaitfor.sas

@@ -70,6 +70,8 @@
     following format:  `/jobExecution/jobs/&JOBID./state` and the corresponding
     job name.  The uri should be in a `uri` variable, and the job path/name
     should be in a `_program` variable.
+  @param [in] raise_err=0 Set to 1 to raise SYSCC when a job does not complete
+              succcessfully
   @param [out] outds= The output dataset containing the list of states by job
     (default=work.mv_jobexecute)
   @param [out] outref= A fileref to which the spawned job logs should be appended.
@@ -81,6 +83,7 @@
   @li mp_abort.sas
   @li mf_getplatform.sas
   @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
   @li mf_existvar.sas
   @li mf_nobs.sas
   @li mv_getjoblog.sas
@@ -93,6 +96,7 @@
     ,inds=0
     ,outds=work.mv_jobwaitfor
     ,outref=0
+    ,raise_err=0
   );
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
@@ -136,7 +140,7 @@ options noquotelenmax;
 data _null_;
   length jobparams $32767;
   set &inds end=last;
-  call symputx(cats('joburi',_n_),substr(uri,1,55)!!'/state','l');
+  call symputx(cats('joburi',_n_),substr(uri,1,55),'l');
   call symputx(cats('jobname',_n_),_program,'l');
   call symputx(cats('jobparams',_n_),jobparams,'l');
   if last then call symputx('uricnt',_n_,'l');
@@ -151,7 +155,7 @@ run;
 %let fname0=%mf_getuniquefileref();
 
 data &outds;
-  format _program uri $128. state $32. timestamp datetime19. jobparams $32767.;
+  format _program uri $128. state $32. stateDetails $32. timestamp datetime19. jobparams $32767.;
   stop;
 run;
 
@@ -159,7 +163,7 @@ run;
 %do i=1 %to &uricnt;
   %if "&&joburi&i" ne "0" %then %do;
     proc http method='GET' out=&fname0 &oauth_bearer url="&base_uri/&&joburi&i";
-      headers "Accept"="text/plain"
+      headers "Accept"="application/json"
       %if &grant_type=authorization_code %then %do;
               "Authorization"="Bearer &&&access_token_var"
       %end;  ;
@@ -173,12 +177,20 @@ run;
     %end;
 
     %let status=notset;
+
+    %local libref1;
+    %let libref1=%mf_getuniquelibref();
+    libname &libref1 json fileref=&fname0;
+
     data _null_;
-      infile &fname0;
-      input;
-      call symputx('status',_infile_,'l');
+      length state stateDetails $32;
+      set &libref1..root;
+      call symputx('status',state,'l');
+      call symputx('stateDetails',stateDetails,'l');
     run;
 
+    libname &libref1 clear;
+
     %if &status=completed or &status=failed or &status=canceled %then %do;
       %local plainuri;
       %let plainuri=%substr(&&joburi&i,1,55);
@@ -187,6 +199,7 @@ run;
         _program="&&jobname&i",
         uri="&plainuri",
         state="&status",
+        stateDetails=symget("stateDetails"),
         timestamp=datetime(),
         jobparams=symget("jobparams&i");
       %let joburi&i=0; /* do not re-check */
@@ -205,6 +218,16 @@ run;
         ,msg=%str(status &status not expected!!)
       )
     %end;
+
+    %if (&raise_err) %then %do;
+      %if (&status = canceled or &status = failed or %length(&stateDetails)>0) %then %do;
+        %if ("&stateDetails" = "%str(war)ning") %then %let SYSCC=4;
+        %else %let SYSCC=5;
+        %put %str(ERR)OR: Job &&jobname&i. did not complete successfully. &stateDetails;
+        %return;
+      %end; 
+    %end;
+
   %end;
   %if &i=&uricnt %then %do;
     %local goback;