feat: adding mm_deletelibrary.sas, a macro to easily delete a SAS Library from metadata. Documentation: https://core.sasjs.io/mm__deletelibrary_8sas.html

> @sasjs/core@1.0.0 postinstall
> node-git-hooks

Installing Git hooks...

up to date, audited 2 packages in 1s

found 0 vulnerabilities.

.githooks/pre-commit

@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# A hook script to verify that no filenames with capital letters are committed.
+# Called by "git commit" with no arguments.  The hook should
+# exit with non-zero status after issuing an appropriate message if
+# it wants to stop the commit.
+#
+# Go through all the changed files (except for deleted and unmerged)
+
+# Save exit code of last executed action
+exit_code=0
+
+# Check if file is one of SAS|DDL|CSV|SH and check for uppercase letters
+mime_pattern="\.(sas|ddl|csv|sh)"
+# Check for capital letters only in file names
+extra_pattern="(^|/)[^/]*([A-Z]+)[^/]*\.[A-Za-z]{3}$"
+# Grep git diff of files to commit
+files=$(	git diff --cached --find-copies --find-renames --name-only --diff-filter=ACMRTXBU |
+			grep -Ei "$mime_pattern" |
+			grep -E  "$extra_pattern" )
+echo "$files"
+if [[ -n "$files"  ]];
+then
+	echo
+	echo "Found files that contain capital letters."
+	echo "Please rename the following files in lowercase, and commit again:"
+
+	for file in $files; do
+		echo -e '- \E[0;32m'"$file"'\033[0m'
+	done
+	# Abort commit
+	exit_code=1
+fi
+
+if [ "$exit_code" == "0" ]; then
+	echo
+	echo -e '\033[1m'"Pre-commit validation Passed"'\033[0m'
+	echo
+else
+	echo
+	echo -e '\033[1m'"Commit Aborted!"'\033[0m'
+	echo
+fi
+exit $exit_code

.gitignore

@@ -1,2 +1,9 @@
 node_modules
-.DS_Store
+.DS_Store
+sasjsbuild/
+
+# avoid filenames with spaces being committed to source control
+**\ **
+
+# ignore the mc_* files - containing macros for individual libraries
+mc_*

.gitpod.dockerfile

@@ -0,0 +1,10 @@
+
+FROM gitpod/workspace-full
+
+RUN sudo apt-get update \
+ && sudo apt-get install -y \
+    doxygen \
+ && npm i -g npm@latest \
+ && npm i -g @sasjs/cli \
+ && npm i \
+ && sudo rm -rf /var/lib/apt/lists/*

.gitpod.yml

@@ -0,0 +1,8 @@
+tasks:
+  - init: npm i && clear
+
+image:
+  file: .gitpod.dockerfile
+vscode:
+  extensions:
+    - sasjs.sasjs-for-vscode@1.6.0:V4hJpMtbpekMcPRNhh4SXQ==

.vscode/.editorconfig

@@ -0,0 +1,9 @@
+{
+  "search.exclude": {
+    "**/sasjsbuild/**": true,
+    "**/dist/**":true
+  },
+  "editor.insertSpaces": true,
+  "editor.tabSize": 2,
+  "trim_trailing_whitespace": true
+}

.vscode/settings.json

@@ -0,0 +1,10 @@
+{
+  "editor.tabSize": 2,
+  "editor.insertSpaces": true,
+  "editor.detectIndentation": true,
+  "editor.formatOnSave": true,
+  "editor.rulers": [
+    80
+  ],
+  "files.trimTrailingWhitespace": true
+}

CONTRIBUTING.md

@@ -1,83 +1,31 @@
 # Contributing
 
-Contributions to the macrocore library are warmly welcomed!  To avoid any
+Contributions are warmly welcomed!  To avoid any misunderstandings, do please first discuss the change you wish to make via issue, email, or any other method with the owners of this repository before submitting a PR.
-misunderstandings, do please first discuss the change you wish to make via issue,
-email, or any other method with the owners of this repository before submitting
-a PR.
 
-Please note we have a code of conduct, please follow it in all your interactions
+Please note we have a [code of conduct](https://www.contributor-covenant.org/version/2/0/code_of_conduct/), please follow it in all your interactions with the project.
-with the project.
+
+# Environment Setup
+
+This repository makes use of the [SASjs](https://sasjs.io) framework for code organisation, compilation, documentation, and deployment.  The following tools are highly recommended:
+
+* [NPM](https://sasjs.io/windows/#npm) - the runtime and dependency manager for [SASjs CLI](https://cli.sasjs.io) (batteries included)
+* [VSCode](https://sasjs.io/windows/#vscode) - feature packed IDE for code editing (warning - highly effective!)
+* [GIT](https://sasjs.io/windows/#git) - a safety net you cannot (and should not) do without.
+
+For generating the documentation (`sasjs doc`) it is also necessary to install [doxygen](https://www.doxygen.nl/manual/install.html).
 
 
-## Code of Conduct
+To get configured:
 
-### Our Pledge
+1.  Clone the repository
+2.  Install local dependencies (`npm install`)
+3.  Install the SASjs CLI globally (`npm install -g @sasjs/cli`)
+4.  Add a target, and authentication (`npm add`).  See [docs](https://cli.sasjs.io/add/).
 
-In the interest of fostering an open and welcoming environment, we as
+To contribute:
-contributors and maintainers pledge to making participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, gender identity and expression, level of experience,
-nationality, personal appearance, race, religion, or sexual identity and
-orientation.
 
-### Our Standards
+1.  Create your feature branch (`git checkout -b myfeature`)
+2.  Make your change
+3.  Update the `all.sas` file (`python3 build.py`)
+4.  Push and make a PR
 
-Examples of behavior that contributes to creating a positive environment
-include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-### Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-### Scope
-
-This Code of Conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
-address, posting via an official social media account, or acting as an appointed
-representative at an online or offline event. Representation of a project may be
-further defined and clarified by project maintainers.
-
-### Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at support@macropeople.com. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-### Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at [http://contributor-covenant.org/version/1/4][version]
-
-[homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/4/

README.md

@@ -1,6 +1,6 @@
 # Macro Core
 
-Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of Application Development on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
+Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
 
 You can download and compile them all in just two lines of SAS code:
 
@@ -9,7 +9,7 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 %inc mc;
 ```
 
-Documentation: https://sasjs.github.io/core.github.io/files.html
+Documentation: https://core.sasjs.io
 
 # Components
 
@@ -84,7 +84,7 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 ## File Properties
 
 - filenames much match macro names
-- filenames must be lowercase
+- filenames must be lowercase, without spaces
 - macro names must be lowercase
 - one macro per file
 - prefixes:
@@ -99,7 +99,7 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
 - UTF-8
-- no trailing white space
+
 
 ## Header Properties
 
@@ -136,13 +136,15 @@ When contributing to this library, it is therefore important to ensure that all
 ## Coding Standards
 
 - Indentation = 2 spaces. No tabs!
+- no trailing white space
+- no invisible characters, other than spaces. If invisibles are needed, use hex literals.
 - Macro variables should not have the trailing dot (`&var` not `&var.`) unless necessary to prevent incorrect resolution
-- The closing `%mend;` should not contain the macro name.
+- The closing `%mend;` should **not** contain the macro name.
 - All macros should be defined with brackets, even if no variables are needed - ie `%macro x();` not `%macro x;`
 - Mandatory parameters should be positional, all optional parameters should be keyword (var=) style.
 - All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect.
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
-- If you have a long-running SQL query, the use of a `quit;` statement is recommended in order to benefit from the timing statistics.
+- The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
 
 # General Notes
 

base/mf_existfeature.sas

@@ -32,9 +32,11 @@
     %put Supported features:  PROCLUA;
   %end;
   %else %if &feature=PROCLUA %then %do;
+    /* https://blogs.sas.com/content/sasdummy/2015/08/03/using-lua-within-your-sas-programs */
     %if &platform=SASVIYA %then 1;
-    %else %if "&sysver"="9.3" or "&sysver"="9.4" %then 1;
+    %else %if "&sysver"="9.2" or "&sysver"="9.3" %then 0;
-    %else 0;
+    %else %if "&SYSVLONG" < "9.04.01M3" %then 0;
+    %else 1;
   %end;
   %else %do;
     -1

base/mf_existfileref.sas

@@ -0,0 +1,27 @@
+/**
+  @file
+  @brief Checks whether a fileref exists
+  @details You can probably do without this macro as it is just a one liner.
+  Mainly it is here as a convenient way to remember the syntax!
+
+  @param fref the fileref to detect
+
+  @return output Returns 1 if found and 0 if not found.  Note - it is possible
+  that the fileref is found, but the file does not (yet) exist. If you need
+  to test for this, you may as well use the fileref function directly.
+
+  @version 8
+  @author [Allan Bowe](https://www.linkedin.com/in/allanbowe/)
+**/
+
+%macro mf_existfileref(fref
+)/*/STORE SOURCE*/;
+
+  %if %sysfunc(fileref(&fref))=0 %then %do;
+    1
+  %end;
+  %else %do;
+    0
+  %end;
+
+%mend;

base/mf_getattrn.sas

@@ -23,7 +23,7 @@
   %local dsid rc;
   %let dsid=%sysfunc(open(&libds,is));
   %if &dsid = 0 %then %do;
-    %put WARNING: Cannot open %trim(&libds), system message below;
+    %put %str(WARN)ING: Cannot open %trim(&libds), system message below;
     %put %sysfunc(sysmsg());
     -1
   %end;

base/mf_getkeyvalue.sas

@@ -3,8 +3,8 @@
   @brief retrieves a key value pair from a control dataset
   @details By default, control dataset is work.mp_setkeyvalue.  Usage:
 
-    %mp_setkeyvalue(someindex,22,type=N)
+      %mp_setkeyvalue(someindex,22,type=N)
-    %put %mf_getkeyvalue(someindex)
+      %put %mf_getkeyvalue(someindex)
 
 
   @param key Provide a key on which to perform the lookup

base/mf_getquotedstr.sas

@@ -3,11 +3,18 @@
   @brief Adds custom quotes / delimiters to a  delimited string
   @details Can be used in open code, eg as follows:
 
-    %put %mf_getquotedstr(blah   blah  blah);
+      %put %mf_getquotedstr(blah   blah  blah);
 
   which returns:
 > 'blah','blah','blah'
 
+  Alternatively:
+
+      %put %mf_getquotedstr(these words are double quoted,quote=D)
+
+  for:
+> "these","words","are","double","quoted"
+
   @param in_str the unquoted, spaced delimited string to transform
   @param dlm= the delimeter to be applied to the output (default comma)
   @param indlm= the delimeter used for the input (default is space)

base/mf_uid.sas

@@ -1,11 +1,11 @@
 /**
   @file
-  @brief Creates a Unique ID based on system time in a friendly format
+  @brief Creates a unique ID based on system time in friendly format
   @details format = YYYYMMDD_HHMMSSmmm_<sysjobid>_<3randomDigits>
 
         %put %mf_uid();
 
-  @version 9.2
+  @version 9.3
   @author Allan Bowe
 
 **/
@@ -14,7 +14,7 @@
 )/*/STORE SOURCE*/;
   %local today now;
   %let today=%sysfunc(today(),yymmddn8.);
-  %let now=%sysfunc(compress(%sysfunc(time(),time12.3),:.));
+  %let now=%sysfunc(compress(%sysfunc(time(),tod12.3),:.));
 
   &today._&now._&sysjobid._%sysevalf(%sysfunc(ranuni(0))*999,CEIL)
 

base/mp_ds2cards.sas

@@ -3,25 +3,27 @@
   @brief Create a CARDS file from a SAS dataset.
   @details Uses dataset attributes to convert all data into datalines.
     Running the generated file will rebuild the original dataset.
-  usage:
+  Usage:
 
       %mp_ds2cards(base_ds=sashelp.class
         , cards_file= "C:\temp\class.sas"
         , maxobs=5)
 
-    stuff to add
+  TODO:
      - labelling the dataset
      - explicity setting a unix LF
      - constraints / indexes etc
 
-  @param base_ds= Should be two level - eg work.blah.  This is the table that
+  @param [in] base_ds= Should be two level - eg work.blah.  This is the table that
                    is converted to a cards file.
-  @param tgt_ds= Table that the generated cards file would create. Optional -
+  @param [in] tgt_ds= Table that the generated cards file would create. Optional -
                   if omitted, will be same as BASE_DS.
-  @param cards_file= Location in which to write the (.sas) cards file
+  @param [out] cards_file= Location in which to write the (.sas) cards file
-  @param maxobs= to limit output to the first <code>maxobs</code> observations
+  @param [in] maxobs= to limit output to the first <code>maxobs</code> observations
-  @param showlog= whether to show generated cards file in the SAS log (YES/NO)
+  @param [in] showlog= whether to show generated cards file in the SAS log (YES/NO)
-  @param outencoding= provide encoding value for file statement (eg utf-8)
+  @param [in] outencoding= provide encoding value for file statement (eg utf-8)
+  @param [in] append= If NO then will rebuild the cards file if it already exists,
+  otherwise will append to it.  Used by the mp_lib2cards.sas macro.
 
 
   @version 9.2
@@ -34,6 +36,7 @@
     ,random_sample=NO
     ,showlog=YES
     ,outencoding=
+    ,append=NO
 )/*/STORE SOURCE*/;
 %local i setds nvars;
 
@@ -46,6 +49,8 @@
 %if (&tgt_ds = ) %then %let tgt_ds=&base_ds;
 %if %index(&tgt_ds,.)=0 %then %let tgt_ds=WORK.%scan(&base_ds,2,.);
 %if ("&outencoding" ne "") %then %let outencoding=encoding="&outencoding";
+%if ("&append" = "") %then %let append=;
+%else %let append=mod;
 
 /* get varcount */
 %let nvars=0;
@@ -172,7 +177,7 @@ data _null_;
 run;
 
 data _null_;
-  file &cards_file. &outencoding lrecl=32767 termstr=nl;
+  file &cards_file. &outencoding lrecl=32767 termstr=nl &append;
   length __attrib $32767;
   if _n_=1 then do;
     put '/*******************************************************************';

base/mp_lib2cards.sas

@@ -1,21 +1,35 @@
 /**
   @file
   @brief Convert all library members to CARDS files
-  @details Gets list of members then calls the <code>%mp_ds2cards()</code>
+  @details Gets list of members then calls the <code>%mp_ds2cards()</code> macro.
-            macro
+  Usage:
-    usage:
 
-    %mp_lib2cards(lib=sashelp
+      %mp_lib2cards(lib=sashelp
-        , outloc= C:\temp )
+          , outloc= C:\temp )
+
+  The output will be one cards file in the `outloc` directory per dataset in the
+  input `lib` library.  If the `outloc` directory does not exist, it is created.
+
+  To create a single SAS file with the first 1000 records of each table in a
+  library you could use this syntax:
+
+      %mp_lib2cards(lib=sashelp
+          , outloc= /tmp
+          , outfile= myfile.sas
+          , maxobs= 1000
+      )
 
   <h4> SAS Macros </h4>
   @li mf_mkdir.sas
+  @li mf_trimstr.sas
   @li mp_ds2cards.sas
 
-  @param lib= Library in which to convert all datasets
+  @param [in] lib= Library in which to convert all datasets
-  @param outloc= Location in which to store output.  Defaults to WORK library.
+  @param [out] outloc= Location in which to store output.  Defaults to WORK
-    Do not use a trailing slash (my/path not my/path/).  No quotes.
+    library. No quotes.
-  @param maxobs= limit output to the first <code>maxobs</code> observations
+  @param [out] outfile= Optional output file NAME - if provided, then will create
+  a single output file instead of one file per input table.
+  @param [in] maxobs= limit output to the first <code>maxobs</code> observations
 
   @version 9.2
   @author Allan Bowe
@@ -25,6 +39,7 @@
     ,outloc=%sysfunc(pathname(work)) /* without trailing slash */
     ,maxobs=max
     ,random_sample=NO
+    ,outfile=0
 )/*/STORE SOURCE*/;
 
 /* Find the tables */
@@ -36,16 +51,28 @@ select distinct lowcase(memname)
   from dictionary.tables
   where upcase(libname)="%upcase(&lib)";
 
+/* trim trailing slash, if provided */
+%let outloc=%mf_trimstr(&outloc,/);
+%let outloc=%mf_trimstr(&outloc,\);
+
 /* create the output directory */
 %mf_mkdir(&outloc)
 
 /* create the cards files */
 %do x=1 %to %sysfunc(countw(&memlist));
-   %let ds=%scan(&memlist,&x);
+  %let ds=%scan(&memlist,&x);
-   %mp_ds2cards(base_ds=&lib..&ds
+  %mp_ds2cards(base_ds=&lib..&ds
-      ,cards_file="&outloc/&ds..sas"
+    ,maxobs=&maxobs
-      ,maxobs=&maxobs
+    ,random_sample=&random_sample
-      ,random_sample=&random_sample)
+  %if "&outfile" ne "0" %then %do;
+    ,append=YES
+    ,cards_file="&outloc/&outfile"
+  %end;
+  %else %do;
+    ,append=NO
+    ,cards_file="&outloc/&ds..sas"
+  %end;
+  )
 %end;
 
 %mend;

base/mp_setkeyvalue.sas

@@ -3,8 +3,8 @@
   @brief Logs a key value pair a control dataset
   @details If the dataset does not exist, it is created.  Usage:
 
-    %mp_setkeyvalue(someindex,22,type=N)
+      %mp_setkeyvalue(someindex,22,type=N)
-    %mp_setkeyvalue(somenewindex,somevalue)
+      %mp_setkeyvalue(somenewindex,somevalue)
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
@@ -26,7 +26,7 @@
 
   %if not (%mf_existds(&libds)) %then %do;
     data &libds (index=(key/unique));
-      length key $32 valc $256 valn 8 type $1;
+      length key $64 valc $2048 valn 8 type $1;
       call missing(of _all_);
       stop;
     run;

base/mp_testjob.sas

@@ -0,0 +1,92 @@
+/**
+  @file
+  @brief Runs arbitrary code for a specified amount of time
+  @details Executes a series of procs and data steps to enable performance
+  testing of arbitrary jobs.
+
+      %mp_testjob(
+         duration=60*5
+      )
+
+  @param [in] duration= the time in seconds which the job should run for. Actual
+  time may vary, as the check is done in between steps.  Default = 30 (seconds).
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquelibref.sas
+  @li mf_getuniquename.sas
+  @li mf_mkdir.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mp_testjob(duration=30
+)/*/STORE SOURCE*/;
+%local lib dir ds1 ds2 ds3 start_tm i;
+
+%let start_tm=%sysfunc(datetime());
+%let duration=%sysevalf(&duration);
+
+/* create a temporary library in WORK */
+%let lib=%mf_getuniquelibref();
+%let dir=%mf_getuniquename();
+%mf_mkdir(%sysfunc(pathname(work))/&dir)
+libname &lib "%sysfunc(pathname(work))/&dir";
+
+/* loop through until time expires */
+%let ds1=%mf_getuniquename();
+%let ds2=%mf_getuniquename();
+%let ds3=%mf_getuniquename();
+%do i=0 %to 1;
+
+  /* create big dataset */
+  data &lib..&ds1(compress=no );
+    do x=1 to 1000000;
+      randnum0=ranuni(0)*3;
+      randnum1=ranuni(0)*2;
+      bigchar=repeat('A',300);
+      output;
+    end;
+  run;
+  %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
+
+  proc summary ;
+    class randnum0 randnum1;
+    output out=&lib..&ds2;
+  run;quit;
+  %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
+
+  /* add more data */
+  proc sql;
+  create table &lib..&ds3 as
+    select *, ranuni(0)*10 as randnum2
+  from &lib..&ds1
+  order by randnum1;
+  quit;
+  %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
+
+  proc sort data=&lib..&ds3;
+    by descending x;
+  run;
+  %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
+
+  /* wait 5 seconds */
+  data _null_;
+    call sleep(5,1);
+  run;
+  %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
+
+  %let i=0;
+
+%end;
+
+%gate:
+%put time is up!;
+proc datasets lib=&lib kill;
+run;
+quit;
+libname &lib clear;
+
+
+%mend;

doxy/batch.bat

@@ -1,6 +0,0 @@
-ECHO on
-
-rmdir /s /q C:\DEVELOPMENT\output
-mkdir C:\DEVELOPMENT\output
-
-doxygen.exe Doxyfile

doxy/build.sh

@@ -1,45 +0,0 @@
-#!/bin/bash
-####################################################################
-# PROJECT: Macro Core Docs Build                                   #
-####################################################################
-
-BUILD_FOLDER="/tmp/macrocore_docs"
-
-# move to project root
-cd ..
-
-# create build directory
-rm -rf $BUILD_FOLDER
-mkdir $BUILD_FOLDER
-
-# copy relevant files
-cp -r base $BUILD_FOLDER
-cp -r meta $BUILD_FOLDER
-cp -r metax $BUILD_FOLDER
-cp -r lua $BUILD_FOLDER
-cp -r viya $BUILD_FOLDER
-cp -r doxy $BUILD_FOLDER
-cp main.dox $BUILD_FOLDER
-cp doxy/Doxyfile $BUILD_FOLDER
-
-# update Doxyfile and generate
-cd $BUILD_FOLDER
-echo "OUTPUT_DIRECTORY=$BUILD_FOLDER/out" >> $BUILD_FOLDER/Doxyfile
-echo "INPUT+=main.dox" >> $BUILD_FOLDER/Doxyfile
-doxygen Doxyfile
-
-# refresh github pages site
-git clone git@github.com:sasjs/core.github.io.git
-cd core.github.io
-rm -r *
-mv $BUILD_FOLDER/out/doxy/* .
-echo 'core.sasjs.io' > CNAME
-git add .
-git commit -m "build.sh build on $(date +%F:%H:%M:%S)"
-git push
-npx sitemap-generator-cli https://core.sasjs.io
-git add .
-git commit -m "adding sitemap"
-git push
-
-echo "check it out:  https://sasjs.github.io/core.github.io/files.html"

doxy/new_footer.html

@@ -1,23 +0,0 @@
-<!-- HTML footer for doxygen 1.8.17-->
-<!-- start footer part -->
-<!--BEGIN GENERATE_TREEVIEW-->
-<div id="nav-path" class="navpath"><!-- id is needed for treeview function! -->
-  <ul>
-    $navpath
-    <li class="footer">$generatedby
-    <a href="https://www.doxygen.org/index.html">
-      <img class="footer" src="$relpath^doxygen.png" alt="doxygen"/></a> $doxygenversion </li>
-   <i> For more information visit the </i> <a href="https://github.com/sasjs/core">Macro Core library</a>.</li>
-
-   </ul>
- </div>
-<!--END GENERATE_TREEVIEW-->
-<!--BEGIN !GENERATE_TREEVIEW-->
-<hr class="footer"/><address class="footer"><small>
-  $generatedby &#160;<a href="http://www.doxygen.org/index.html">
-  <img class="footer" src="$relpath^doxygen.png" alt="doxygen"/>
-  </a> $doxygenversion
-  </small></address>
-<!--END !GENERATE_TREEVIEW-->
-</body>
-</html>

doxy/new_header.html

@@ -1,72 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<!-- HTML header for doxygen 1.8.17-->
-<html xmlns="https://www.w3.org/1999/xhtml" lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
-<meta http-equiv="X-UA-Compatible" content="IE=9"/>
-<meta name="generator" content="Doxygen $doxygenversion"/>
-<meta name="viewport" content="width=device-width, initial-scale=1"/>
-<!--BEGIN PROJECT_NAME--><title>$projectname: $title</title><!--END PROJECT_NAME-->
-<!--BEGIN !PROJECT_NAME--><title>$title</title><!--END !PROJECT_NAME-->
-<link href="$relpath^tabs.css" rel="stylesheet" type="text/css"/>
-<script type="text/javascript" src="$relpath^jquery.js"></script>
-<script type="text/javascript" src="$relpath^dynsections.js"></script>
-$treeview
-$search
-$mathjax
-<link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
-<link REL="icon" HREF="https://sasjs.io/img/runningman.jpg">
-$extrastylesheet
-</head>
-<body>
-<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
-
-<!--BEGIN TITLEAREA-->
-<div id="titlearea">
-<table cellspacing="0" cellpadding="0">
- <tbody>
-  <tr style="height: 56px;">
-  <!--BEGIN PROJECT_LOGO-->
-  <td id="projectlogo">
-    <img alt="Logo" src="$relpath^$projectlogo"/></td>
-  <!--END PROJECT_LOGO-->
-<!--BEGIN PROJECT_NAME-->
-<td id="projectalign" style="padding-left: 0.5em;">
-  <div id="projectname">
-   <!--BEGIN PROJECT_NUMBER-->&#160;<span id="projectnumber">$projectnumber</span><!--END PROJECT_NUMBER-->
-   </div>
-   <!--BEGIN PROJECT_BRIEF--><div id="projectbrief">
-    Production Ready Macros for SAS Application Developers</br>
-    <a href="https://github.com/sasjs/core">
-        https://github.com/sasjs/core
-    </a>
-
-   </div>
-   <meta name="Description" content="$projectbrief">
-   <!--END PROJECT_BRIEF-->
-  </td>
-  <!--END PROJECT_NAME-->
-  <!--BEGIN !PROJECT_NAME-->
-   <!--BEGIN PROJECT_BRIEF-->
-   <td style="padding-left: 0.5em;">
-    <div id="projectbrief">$projectbrief</div>
-    <table style="padding-left: 2em;" cellspacing="0" cellpadding="0">
-      <tr><td> Production Ready Macros for SAS Application Developers</td></tr>
-      <tr><td><a href="https://github.com/sasjs/core">
-          https://github.com/sasjs/core
-      </a></td></tr>
-    </table>
-    </td>
-   <!--END PROJECT_BRIEF-->
-  <!--END !PROJECT_NAME-->
-  <!--BEGIN DISABLE_INDEX-->
-   <!--BEGIN SEARCHENGINE-->
-   <td>$searchbox</td>
-   <!--END SEARCHENGINE-->
-  <!--END DISABLE_INDEX-->
- </tr>
- </tbody>
-</table>
-</div>
-<!--END TITLEAREA-->
-<!-- end header part -->

meta/mm_deletelibrary.sas

@@ -0,0 +1,92 @@
+/**
+  @file
+  @brief Deletes a library by Name
+
+  @details  Used to delete a library.
+  Usage:
+
+      %* create a library in the home directory ;
+      %mm_createlibrary(
+        libname=My Temp Library,
+        libref=XXTEMPXX,
+        tree=/User Folders/&sysuserid,
+        directory=%sysfunc(pathname(work))
+      )
+
+      %* delete the library ;
+      %mm_deletelibrary(name=My Temp Library)
+
+  After running the above, the following will be shown in the log:
+
+  ![](https://i.imgur.com/Y4Tog24.png)
+
+  @param [in] name= the name (not libref) of the library to be deleted
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_abort.sas
+
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mm_deletelibrary(
+      name=
+)/*/STORE SOURCE*/;
+
+
+/**
+  * Check if library exists and get uri
+  */
+data _null_;
+  length type uri $256;
+  rc=metadata_resolve("omsobj:SASLibrary?@Name='&name'",type,uri);
+  call symputx('checktype',type,'l');
+  call symputx('liburi',uri,'l');
+  putlog (_all_)(=);
+run;
+%if &checktype ne SASLibrary %then %do;
+  %put &sysmacroname: Library (&name) was not found, and so will not be deleted;
+  %return;
+%end;
+
+%local fname1 fname2;
+%let fname1=%mf_getuniquefileref();
+%let fname2=%mf_getuniquefileref();
+
+filename &fname1 temp lrecl=10000;
+filename &fname2 temp lrecl=10000;
+data _null_ ;
+  file &fname1 ;
+  put "<DeleteMetadata><Metadata><SASLibrary Id='&liburi'/>";
+  put "</Metadata><NS>SAS</NS><Flags>268436480</Flags><Options/>";
+  put "</DeleteMetadata>";
+run ;
+proc metadata in=&fname1 out=&fname2 verbose;run;
+
+/* list the result */
+data _null_;infile &fname2; input; list; run;
+
+filename &fname1 clear;
+filename &fname2 clear;
+
+/**
+  * Check deletion
+  */
+%local isgone;
+data _null_;
+  length type uri $256;
+  rc=metadata_resolve("omsobj:SASLibrary?@Id='&liburi'",type,uri);
+  call symputx('isgone',type,'l');
+run;
+
+%mp_abort(iftrue=(&isgone = SASLibrary)
+  ,mac=&sysmacroname
+  ,msg=%str(Library (&name) NOT deleted)
+)
+
+%put &sysmacroname: Library &name (&liburi) was successfully deleted;
+
+%mend;

meta/mm_getfoldermembers.sas

@@ -0,0 +1,95 @@
+/**
+  @file
+  @brief Returns all direct child members of a particular folder
+  @details Displays the children for a particular folder, in a similar fashion
+  to the viya counterpart (mv_getfoldermembers.sas)
+
+  Usage:
+
+      %mm_getfoldermembers(root=/, outds=rootfolders)
+
+      %mm_getfoldermembers(root=/User Folders/&sysuserid, outds=usercontent)
+
+  @param [in] root= the parent folder under which to return all contents
+  @param [out] outds= the dataset to create that contains the list of directories
+  @param [in] mDebug= set to 1 to show debug messages in the log
+
+  <h4> Data Outputs </h4>
+
+  Example for `root=/`:
+
+  |metauri $17|metaname $256|metatype $32|
+  |---|---|---|
+  |A5XLSNXI.AA000001|Products	|Folder|
+  |A5XLSNXI.AA000002|Shared Data	|Folder|
+  |A5XLSNXI.AA000003|User Folders	|Folder|
+  |A5XLSNXI.AA000004|System	|Folder|
+  |A5XLSNXI.AA00003K|30.SASApps	|Folder|
+  |A5XLSNXI.AA00006A|Public|Folder|
+
+  <h4> SAS Macros </h4>
+  @li mm_getfoldertree.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+%macro mm_getfoldermembers(
+     root=
+    ,outds=work.mm_getfoldertree
+)/*/STORE SOURCE*/;
+
+%if "&root" = "/" %then %do;
+  %local fname1 fname2 fname3;
+  %let fname1=%mf_getuniquefileref();
+  %let fname2=%mf_getuniquefileref();
+  %let fname3=%mf_getuniquefileref();
+  data _null_ ;
+    file &fname1 ;
+    put '<GetMetadataObjects>' ;
+    put '<Reposid>$METAREPOSITORY</Reposid>' ;
+    put '<Type>Tree</Type>' ;
+    put '<NS>SAS</NS>' ;
+    put '<Flags>388</Flags>' ;
+    put '<Options>' ;
+    put '<XMLSelect search="Tree[SoftwareComponents/SoftwareComponent'@;
+    put '[@Name=''BIP Service'']]"/>';
+    put '</Options>' ;
+    put '</GetMetadataObjects>' ;
+  run ;
+  proc metadata in=&fname1 out=&fname2 verbose;run;
+
+  /* create an XML map to read the response */
+  data _null_;
+    file &fname3;
+    put '<SXLEMAP version="1.2" name="SASFolders">';
+    put '<TABLE name="SASFolders">';
+    put '<TABLE-PATH syntax="XPath">//Objects/Tree</TABLE-PATH>';
+    put '<COLUMN name="metauri">><LENGTH>17</LENGTH>';
+    put '<PATH syntax="XPath">//Objects/Tree/@Id</PATH></COLUMN>';
+    put '<COLUMN name="metaname"><LENGTH>256</LENGTH>>';
+    put '<PATH syntax="XPath">//Objects/Tree/@Name</PATH></COLUMN>';
+    put '</TABLE></SXLEMAP>';
+  run;
+  %local libref1;
+  %let libref1=%mf_getuniquelibref();
+  libname &libref1 xml xmlfileref=&fname2 xmlmap=&fname3;
+
+  data &outds;
+    length metatype $32;
+    retain metatype 'Folder';
+    set &libref1..sasfolders;
+  run;
+
+%end;
+%else %do;
+  %mm_getfoldertree(root=&root, outds=&outds,depth=1)
+  data &outds;
+    set &outds(rename=(name=metaname publictype=metatype));
+    keep metaname metauri metatype;
+  run;
+%end;
+
+%mend;

meta/mm_getfoldertree.sas

@@ -1,18 +1,20 @@
 /**
-  @file mm_getfoldertree.sas
+  @file
   @brief Returns all folders / subfolder content for a particular root
   @details Shows all members and SubTrees recursively for a particular root.
   Note - for big sites, this returns a lot of data!  So you may wish to reduce
-  the logging to speed up the process (see example below)
+  the logging to speed up the process (see example below), OR - use mm_tree.sas
+  which uses proc metadata and is far more efficient.
+
   Usage:
 
     options ps=max nonotes nosource;
     %mm_getfoldertree(root=/My/Meta/Path, outds=iwantthisdataset)
     options notes source;
-    
+
-  @param root= the parent folder under which to return all contents
+  @param [in] root= the parent folder under which to return all contents
-  @param outds= the dataset to create that contains the list of directories
+  @param [out] outds= the dataset to create that contains the list of directories
-  @param mDebug= set to 1 to show debug messages in the log
+  @param [in] mDebug= set to 1 to show debug messages in the log
 
   <h4> SAS Macros </h4>
 
@@ -60,7 +62,7 @@ data &outds.TMP/view=&outds.TMP;
     __n1+1;
     /* Walk through all possible associations of this object. */
     __n2=1;
-    if assoctype in ('Members','SubTrees') then 
+    if assoctype in ('Members','SubTrees') then
     do while(metadata_getnasn(pathuri,assoctype,__n2,metauri)>0);
       __n2+1;
       call missing(name,publictype,MetadataUpdated,MetadataCreated);

meta/mm_updatestpservertype.sas

@@ -1,14 +1,13 @@
 /**
-  @file mm_updatestpservertype.sas
+  @file
   @brief Updates a type 2 stored process to run on STP or WKS context
   @details Only works on Type 2 (9.3 compatible) STPs
 
   Usage:
 
-    %mm_updatestpservertype(target=/some/meta/path/myStoredProcess
+      %mm_updatestpservertype(target=/some/meta/path/myStoredProcess
-      ,type=WKS)
+        ,type=WKS)
 
-  <h4> SAS Macros </h4>
 
   @param target= full path to the STP being deleted
   @param type= Either WKS or STP depending on whether Workspace or Stored Process

meta/mm_updatestpsourcecode.sas

@@ -6,31 +6,40 @@
 
   Usage:
 
-    %mm_updatestpsourcecode(stp=/my/metadata/path/mystpname
+      %mm_updatestpsourcecode(stp=/my/metadata/path/mystpname
-      ,stpcode="/file/system/source.sas")
+        ,stpcode="/file/system/source.sas")
 
-
+  @param [in] stp= the BIP Tree folder path plus Stored Process Name
-  @param stp= the BIP Tree folder path plus Stored Process Name
+  @param [in] stpcode= the source file (or fileref) containing the SAS code to load
-  @param stpcode= the source file (or fileref) containing the SAS code to load
     into the stp.  For multiple files, they should simply be concatenated first.
-  @param minify= set to YES in order to strip comments, blank lines, and CRLFs.
+  @param [in] minify= set to YES in order to strip comments, blank lines, and CRLFs.
 
-  @param frefin= change default inref if it clashes with an existing one
+  @param frefin= deprecated - a unique fileref is now always used
-  @param frefout= change default outref if it clashes with an existing one
+  @param frefout= deprecated - a unique fileref is now always used
   @param mDebug= set to 1 to show debug messages in the log
 
   @version 9.3
   @author Allan Bowe
 
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+
 **/
 
 %macro mm_updatestpsourcecode(stp=
   ,stpcode=
   ,minify=NO
+  ,mdebug=0
+  /* deprecated */
   ,frefin=inmeta
   ,frefout=outmeta
-  ,mdebug=0
 );
+
+%if &frefin ne inmeta or &frefout ne outmeta %then %do;
+  %put %str(WARN)ING: the frefin and frefout parameters will be deprecated in
+    an upcoming release.;
+%end;
+
 /* first, check if STP exists */
 %local tsuri;
 %let tsuri=stopifempty ;
@@ -68,7 +77,9 @@ run;
   %return;
 %end;
 
-filename &frefin temp lrecl=32767;
+%local frefin frefout;
+%let frefin=%mf_getuniquefileref();
+%let frefout=%mf_getuniquefileref();
 
 /* write header XML */
 data _null_;
@@ -81,7 +92,7 @@ run;
 /* write contents */
 %if %length(&stpcode)>2 %then %do;
   data _null_;
-    file &frefin mod;
+    file &frefin lrecl=32767 mod;
     infile &stpcode lrecl=32767;
     length outstr $32767;
     input outstr ;
@@ -110,9 +121,6 @@ data _null_;
     </UpdateMetadata>";
 run;
 
-
-filename &frefout temp;
-
 proc metadata in= &frefin out=&frefout;
 run;
 
@@ -124,5 +132,9 @@ run;
     put _infile_;
   run;
 %end;
+%else %do;
+  filename &frefin clear;
+  filename &frefout clear;
+%end;
 
 %mend;

package-lock.json

@@ -1,4 +1,35 @@
 {
-  "name": "macrocore",
+  "name": "@sasjs/core",
-  "lockfileVersion": 1
+  "version": "1.0.0",
+  "lockfileVersion": 2,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "@sasjs/core",
+      "version": "1.0.0",
+      "hasInstallScript": true,
+      "license": "MIT",
+      "dependencies": {
+        "node-git-hooks": "^1.0.5"
+      }
+    },
+    "node_modules/node-git-hooks": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/node-git-hooks/-/node-git-hooks-1.0.5.tgz",
+      "integrity": "sha512-05rULsJy8z2OvXTQFZv9fN20uo186EYgJYQjkL1OjnXj53QivOAGUzZilZ/MX8OmPRaN+deJBtlvMRydpdfnqA==",
+      "bin": {
+        "node-git-hooks": "bin/install.js"
+      },
+      "engines": {
+        "node": ">=4.0.0"
+      }
+    }
+  },
+  "dependencies": {
+    "node-git-hooks": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/node-git-hooks/-/node-git-hooks-1.0.5.tgz",
+      "integrity": "sha512-05rULsJy8z2OvXTQFZv9fN20uo186EYgJYQjkL1OjnXj53QivOAGUzZilZ/MX8OmPRaN+deJBtlvMRydpdfnqA=="
+    }
+  }
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sasjs/core",
-  "description": "SAS Macro Library for Application Development",
+  "description": "Production Ready Macros for SAS Application Developers",
   "license": "MIT",
   "keywords": [
     "SAS",
@@ -10,13 +10,28 @@
   "author": "Allan Bowe <support@macropeople.com>",
   "repository": {
     "type": "git",
-    "url": "https://github.com/sasjs/core"
+    "url": "git+https://github.com/sasjs/core.git"
   },
   "release": {
-    "branches": ["main"]
+    "branches": [
+      "main"
+    ]
   },
   "publishConfig": {
     "access": "public"
   },
-  "devDependencies": {}
+  "bugs": {
+    "url": "https://github.com/sasjs/core/issues"
+  },
+  "homepage": "https://github.com/sasjs/core#readme",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "docs": "sasjs doc && ./sasjs/utils/build.sh",
+    "postinstall": "node-git-hooks"
+  },
+  "dependencies": {
+    "node-git-hooks": "^1.0.5"
+  }
 }

sasjs/doxy/Doxyfile

@@ -1,5 +1,5 @@
 ALPHABETICAL_INDEX     = NO
-DISABLE_INDEX          = YES
+DISABLE_INDEX          = NO
 ENABLE_PREPROCESSING   = NO
 EXTENSION_MAPPING      = sas=Java ddl=Java
 EXTRACT_LOCAL_CLASSES  = NO
@@ -13,18 +13,21 @@ HIDE_IN_BODY_DOCS      = YES
 HIDE_SCOPE_NAMES       = YES
 HIDE_UNDOC_CLASSES     = YES
 HIDE_UNDOC_MEMBERS     = YES
-HTML_OUTPUT            = doxy
+HTML_OUTPUT            = $(DOXY_HTML_OUTPUT)
-HTML_HEADER            = ./doxy/new_header.html
+HTML_HEADER            = $(HTML_HEADER)
-HTML_FOOTER            = ./doxy/new_footer.html
+HTML_EXTRA_FILES       = $(HTML_EXTRA_FILES)
-HTML_EXTRA_STYLESHEET  = ./doxy/new_stylesheet.css
+HTML_FOOTER            = $(HTML_FOOTER)
+HTML_EXTRA_STYLESHEET  = $(HTML_EXTRA_STYLESHEET)
 INHERIT_DOCS           = NO
 INLINE_INFO            = NO
-INPUT                  = base meta metax viya lua
+INPUT                  = $(DOXY_INPUT)
-LAYOUT_FILE            = ./doxy/DoxygenLayout.xml
+INPUT                 += main.dox
+LAYOUT_FILE            = $(LAYOUT_FILE)
+USE_MDFILE_AS_MAINPAGE = README.md
 MAX_INITIALIZER_LINES  = 0
-PROJECT_NAME           = Macro Core
+PROJECT_NAME           = $(PROJECT_NAME)
-PROJECT_LOGO           = doxy/Macro_core_website_1.png
+PROJECT_LOGO           = $(PROJECT_LOGO)
-PROJECT_BRIEF          = "Production Ready Macros for SAS Application Developers"
+PROJECT_BRIEF          = $(PROJECT_BRIEF)
 RECURSIVE              = YES
 REPEAT_BRIEF           = NO
 SHOW_NAMESPACES        = NO
@@ -35,4 +38,4 @@ STRICT_PROTO_MATCHING  = YES
 STRIP_CODE_COMMENTS    = NO
 SUBGROUPING            = NO
 TAB_SIZE               = 2
-VERBATIM_HEADERS       = NO
+VERBATIM_HEADERS       = NO

sasjs/doxy/DoxygenLayout.xml

@@ -2,7 +2,7 @@
   <!-- Generated by doxygen 1.8.14 -->
   <!-- Navigation index tabs for HTML output -->
   <navindex>
-    <tab type="mainpage" visible="no" title=""/>
+    <tab type="mainpage" visible="yes" title="Home"/>
     <tab type="pages" visible="no" title="" intro=""/>
     <tab type="modules" visible="no" title="" intro=""/>
     <tab type="namespaces" visible="no" title="">
@@ -108,4 +108,4 @@
       <files visible="yes"/>
     </memberdecl>
   </directory>
-</doxygenlayout>
+</doxygenlayout>

sasjs/doxy/new_footer.html

@@ -0,0 +1,33 @@
+<!-- HTML footer for doxygen 1.8.17-->
+<!-- start footer part -->
+<!--BEGIN GENERATE_TREEVIEW-->
+<div id="nav-path" class="navpath">
+  <!-- id is needed for treeview function! -->
+  <ul>
+    $navpath
+    <li class="footer">
+      $generatedby
+      <a href="https://www.doxygen.org/index.html">
+        <img class="footer" src="$relpath^doxygen.svg" alt="doxygen"
+      /></a>
+      $doxygenversion
+    </li>
+    <li>
+      <i> For more information visit the </i>
+      <a href="https://github.com/sasjs/core">Macro Core library</a>.
+    </li>
+  </ul>
+</div>
+<!--END GENERATE_TREEVIEW-->
+<!--BEGIN !GENERATE_TREEVIEW-->
+<hr class="footer" />
+<address class="footer">
+  <small>
+    $generatedby &#160;<a href="http://www.doxygen.org/index.html">
+      <img class="footer" src="$relpath^doxygen.svg" alt="doxygen" />
+    </a>
+    $doxygenversion
+  </small>
+</address>
+
+<!--END !GENERATE_TREEVIEW-->

sasjs/doxy/new_header.html

@@ -0,0 +1,67 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!-- HTML header for doxygen 1.8.17-->
+<html xmlns="https://www.w3.org/1999/xhtml" lang="en">
+  <head>
+    <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=9" />
+    <meta property="og:type" content="website">
+    <meta name="author" content="Allan Bowe">
+    <meta name="generator" content="Doxygen $doxygenversion" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <!--BEGIN PROJECT_NAME-->
+    <meta name="description" content="$projectbrief" />
+    <!--END PROJECT_NAME-->
+    <!--BEGIN !PROJECT_NAME-->
+    <title>$title</title>
+    <!--END !PROJECT_NAME-->
+    <link href="$relpath^tabs.css" rel="stylesheet" type="text/css" />
+    <script type="text/javascript" src="$relpath^jquery.js"></script>
+    <script type="text/javascript" src="$relpath^dynsections.js"></script>
+    $treeview $search $mathjax
+    <link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
+    <link rel="shortcut icon" href="$relpath^favicon.ico" type="image/x-icon" />
+    $extrastylesheet
+  </head>
+  <body>
+    <div id="top">
+      <!-- do not remove this div, it is closed by doxygen! -->
+
+      <!--BEGIN TITLEAREA-->
+      <div id="titlearea">
+        <table cellspacing="0" cellpadding="0">
+          <tbody>
+            <tr style="height: 56px">
+              <!--BEGIN PROJECT_LOGO-->
+              <td id="projectlogo">
+                <a href="$relpath^"
+                  ><img alt="Logo" src="$relpath^$projectlogo"
+                /></a>
+              </td>
+              <!--END PROJECT_LOGO-->
+              <td id="projectalign" style="padding-left: 0.5em">
+                <div id="projectbrief">
+                  Production Ready Macros for SAS Application Developers<br />
+                  <a href="https://github.com/sasjs/core">
+                    https://github.com/sasjs/core
+                  </a>
+                </div>
+              </td>
+                  </tr>
+                </table>
+              </td>
+
+
+              <!--BEGIN DISABLE_INDEX-->
+              <!--BEGIN SEARCHENGINE-->
+              <td>$searchbox</td>
+              <!--END SEARCHENGINE-->
+              <!--END DISABLE_INDEX-->
+            </tr>
+          </tbody>
+        </table>
+      </div>
+      <!--END TITLEAREA-->
+      <!-- end header part -->
+    </div>
+  </body>
+</html>

sasjs/doxy/new_stylesheet.css

@@ -1,5 +1,5 @@
-#projectlogo img
+#projectlogo img
-{
+{
-	border: 0px none;
+	border: 0px none;
-	max-height:70px
+	max-height:70px
 }

sasjs/sasjsconfig.json

@@ -0,0 +1,13 @@
+{
+  "$schema": "https://cli.sasjs.io/sasjsconfig-schema.json",
+  "macroFolders": ["base", "meta", "metax", "viya", "lua"],
+  "docConfig":{
+    "displayMacroCore": false,
+    "enableLineage": false,
+    "doxyContent": {
+      "favIcon": "runningman.jpg",
+      "logo": "Macro_core_website_1.png",
+      "readMe": "../../README.md"
+    }
+  }
+}

sasjs/utils/build.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+####################################################################
+# PROJECT: Macro Core Docs Build                                   #
+# To execute, use the npm command (npm run docs)                   #
+####################################################################
+
+# refresh github pages site
+rm -rf sasjsbuild/docsite
+git clone git@github.com:sasjs/core.github.io.git sasjsbuild/docsite
+rm -rf sasjsbuild/docsite/*
+mv sasjsbuild/docs/* sasjsbuild/docsite/
+cd sasjsbuild/docsite/
+echo 'core.sasjs.io' > CNAME
+git add .
+git commit -m "build.sh build on $(date +%F:%H:%M:%S)"
+git push
+npx sitemap-generator-cli https://core.sasjs.io
+git add .
+git commit -m "adding sitemap"
+git push
+
+echo "check it out:  https://sasjs.github.io/core.github.io/files.html"

viya/mv_createfolder.sas

@@ -4,7 +4,6 @@
   @details Expects oauth token in a global macro variable (default
   ACCESS_TOKEN).
 
-      options mprint;
       %mv_createfolder(path=/Public)
 
 

viya/mv_createjob.sas

@@ -0,0 +1,307 @@
+/**
+  @file
+  @brief Creates a Viya Job
+  @details
+  Code is passed in as one or more filerefs.
+
+      %* Step 1 - compile macros ;
+      filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+      %* Step 2 - Create some SAS code and add it to a job;
+      filename ft15f001 temp;
+      parmcards4;
+          data some_code;
+            set sashelp.class;
+          run;
+      ;;;;
+      %mv_createjob(path=/Public/app/sasjstemp/jobs/myjobs,name=myjob)
+
+  The path to the job will then be shown in the log, eg as follows:
+
+  ![viya job location](https://i.imgur.com/XRUDHgA.png)
+
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mv_createfolder.sas
+  @li mf_getuniquelibref.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getplatform.sas
+  @li mf_isblank.sas
+  @li mv_deletejes.sas
+
+  @param path= The full path (on SAS Drive) where the job will be created
+  @param name= The name of the job
+  @param desc= The description of the job
+  @param precode= Space separated list of filerefs, pointing to the code that
+    needs to be attached to the beginning of the job
+  @param code= Fileref(s) of the actual code to be added
+  @param access_token_var= The global macro variable to contain the access token
+  @param grant_type= valid values are "password" or "authorization_code" (unquoted).
+    The default is authorization_code.
+  @param replace= select NO to avoid replacing any existing job in that location
+  @param contextname= Choose a specific context on which to run the Job.  Leave
+    blank to use the default context.  From Viya 3.5 it is possible to configure
+    a shared context - see
+    https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5pyhn1wg3ja0drdl6h.htm&docsetVersion=3.5&locale=en
+
+  @version VIYA V.03.04
+  @author [Allan Bowe](https://www.linkedin.com/in/allanbowe)
+
+**/
+
+%macro mv_createjob(path=
+    ,name=
+    ,desc=Created by the mv_createjob.sas macro
+    ,precode=
+    ,code=ft15f001
+    ,access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+    ,replace=YES
+    ,debug=0
+    ,contextname=
+  );
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+%put &sysmacroname: grant_type=&grant_type;
+
+/* initial validation checking */
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+%mp_abort(iftrue=(%mf_isblank(&path)=1)
+  ,mac=&sysmacroname
+  ,msg=%str(path value must be provided)
+)
+%mp_abort(iftrue=(%length(&path)=1)
+  ,mac=&sysmacroname
+  ,msg=%str(path value must be provided)
+)
+%mp_abort(iftrue=(%mf_isblank(&name)=1)
+  ,mac=&sysmacroname
+  ,msg=%str(name value must be provided)
+)
+
+options noquotelenmax;
+
+* remove any trailing slash ;
+%if "%substr(&path,%length(&path),1)" = "/" %then
+  %let path=%substr(&path,1,%length(&path)-1);
+
+/* ensure folder exists */
+%put &sysmacroname: Path &path being checked / created;
+%mv_createfolder(path=&path)
+
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+/* fetching folder details for provided path */
+%local fname1;
+%let fname1=%mf_getuniquefileref();
+proc http method='GET' out=&fname1 &oauth_bearer
+  url="&base_uri/folders/folders/@item?path=&path";
+%if &grant_type=authorization_code %then %do;
+  headers "Authorization"="Bearer &&&access_token_var";
+%end;
+run;
+%if &debug %then %do;
+  data _null_;
+    infile &fname1;
+    input;
+    putlog _infile_;
+  run;
+%end;
+%mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200)
+  ,mac=&sysmacroname
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+/* path exists. Grab follow on link to check members */
+%local libref1;
+%let libref1=%mf_getuniquelibref();
+libname &libref1 JSON fileref=&fname1;
+
+data _null_;
+  set &libref1..links;
+  if rel='members' then call symputx('membercheck',quote("&base_uri"!!trim(href)),'l');
+  else if rel='self' then call symputx('parentFolderUri',href,'l');
+run;
+data _null_;
+  set &libref1..root;
+  call symputx('folderid',id,'l');
+run;
+%local fname2;
+%let fname2=%mf_getuniquefileref();
+proc http method='GET'
+    out=&fname2
+    &oauth_bearer
+    url=%unquote(%superq(membercheck));
+    headers
+  %if &grant_type=authorization_code %then %do;
+            "Authorization"="Bearer &&&access_token_var"
+  %end;
+            'Accept'='application/vnd.sas.collection+json'
+            'Accept-Language'='string';
+%if &debug=1 %then %do;
+   debug level = 3;
+%end;
+run;
+/*data _null_;infile &fname2;input;putlog _infile_;run;*/
+%mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200)
+  ,mac=&sysmacroname
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+%if %upcase(&replace)=YES %then %do;
+  %mv_deletejes(path=&path, name=&name)
+%end;
+%else %do;
+  /* check that job does not already exist in that folder */
+  %local libref2;
+  %let libref2=%mf_getuniquelibref();
+  libname &libref2 JSON fileref=&fname2;
+  %local exists; %let exists=0;
+  data _null_;
+    set &libref2..items;
+    if contenttype='jobDefinition' and upcase(name)="%upcase(&name)" then
+      call symputx('exists',1,'l');
+  run;
+  %mp_abort(iftrue=(&exists=1)
+    ,mac=&sysmacroname
+    ,msg=%str(Job &name already exists in &path)
+  )
+  libname &libref2 clear;
+%end;
+
+/* set up the body of the request to create the service */
+%local fname3;
+%let fname3=%mf_getuniquefileref();
+data _null_;
+  file &fname3 TERMSTR=' ';
+  length string $32767;
+  string=cats('{"version": 0,"name":"'
+  	,"&name"
+  	,'","type":"Compute","parameters":[{"name":"_addjesbeginendmacros"'
+    ,',"type":"CHARACTER","defaultValue":"false"}');
+  context=quote(cats(symget('contextname')));
+  if context ne '""' then do;
+    string=cats(string,',{"version": 1,"name": "_contextName","defaultValue":'
+     ,context,',"type":"CHARACTER","label":"Context Name","required": false}');
+  end;
+  string=cats(string,'],"code":"');
+  put string;
+run;
+
+
+/* insert the code, escaping double quotes and carriage returns */
+%local x fref freflist;
+%let freflist= &precode &code ;
+%do x=1 %to %sysfunc(countw(&freflist));
+  %let fref=%scan(&freflist,&x);
+  %put &sysmacroname: adding &fref;
+  data _null_;
+    length filein 8 fileid 8;
+    filein = fopen("&fref","I",1,"B");
+    fileid = fopen("&fname3","A",1,"B");
+    rec = "20"x;
+    do while(fread(filein)=0);
+      rc = fget(filein,rec,1);
+      if rec='"' then do;  /* DOUBLE QUOTE */
+        rc =fput(fileid,'\');rc =fwrite(fileid);
+        rc =fput(fileid,'"');rc =fwrite(fileid);
+      end;
+      else if rec='0A'x then do; /* LF */
+        rc =fput(fileid,'\');rc =fwrite(fileid);
+        rc =fput(fileid,'n');rc =fwrite(fileid);
+      end;
+      else if rec='0D'x then do; /* CR */
+        rc =fput(fileid,'\');rc =fwrite(fileid);
+        rc =fput(fileid,'r');rc =fwrite(fileid);
+      end;
+      else if rec='09'x then do; /* TAB */
+        rc =fput(fileid,'\');rc =fwrite(fileid);
+        rc =fput(fileid,'t');rc =fwrite(fileid);
+      end;
+      else if rec='5C'x then do; /* BACKSLASH */
+        rc =fput(fileid,'\');rc =fwrite(fileid);
+        rc =fput(fileid,'\');rc =fwrite(fileid);
+      end;
+      else do;
+        rc =fput(fileid,rec);
+        rc =fwrite(fileid);
+      end;
+    end;
+    rc=fclose(filein);
+    rc=fclose(fileid);
+  run;
+%end;
+
+/* finish off the body of the code file loaded to JES */
+data _null_;
+  file &fname3 mod TERMSTR=' ';
+  put '"}';
+run;
+
+/* now we can create the job!! */
+%local fname4;
+%let fname4=%mf_getuniquefileref();
+proc http method='POST'
+    in=&fname3
+    out=&fname4
+    &oauth_bearer
+    url="&base_uri/jobDefinitions/definitions?parentFolderUri=&parentFolderUri";
+    headers 'Content-Type'='application/vnd.sas.job.definition+json'
+  %if &grant_type=authorization_code %then %do;
+            "Authorization"="Bearer &&&access_token_var"
+  %end;
+            "Accept"="application/vnd.sas.job.definition+json";
+%if &debug=1 %then %do;
+   debug level = 3;
+%end;
+run;
+/*data _null_;infile &fname4;input;putlog _infile_;run;*/
+%mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 201)
+  ,mac=&sysmacroname
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+/* clear refs */
+filename &fname1 clear;
+filename &fname2 clear;
+filename &fname3 clear;
+filename &fname4 clear;
+libname &libref1 clear;
+
+/* get the url so we can give a helpful log message */
+%local url;
+data _null_;
+  if symexist('_baseurl') then do;
+    url=symget('_baseurl');
+    if subpad(url,length(url)-9,9)='SASStudio'
+      then url=substr(url,1,length(url)-11);
+    else url="&systcpiphostname";
+  end;
+  else url="&systcpiphostname";
+  call symputx('url',url);
+run;
+
+
+%put &sysmacroname: Job &name successfully created in &path;
+%put &sysmacroname:;
+%put &sysmacroname: Check it out here:;
+%put &sysmacroname:;%put;
+%put    &url/SASJobExecution?_PROGRAM=&path/&name;%put;
+%put &sysmacroname:;
+%put &sysmacroname:;
+
+%mend;

viya/mv_createwebservice.sas

@@ -1,5 +1,5 @@
 /**
-  @file mv_createwebservice.sas
+  @file
   @brief Creates a JobExecution web service if it doesn't already exist
   @details
   Code is passed in as one or more filerefs.
@@ -566,23 +566,23 @@ run;
     rec = "20"x;
     do while(fread(filein)=0);
       rc = fget(filein,rec,1);
-      if rec='"' then do;
+      if rec='"' then do;  /* DOUBLE QUOTE */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'"');rc =fwrite(fileid);
       end;
-      else if rec='0A'x then do;
+      else if rec='0A'x then do; /* LF */
-        rc =fput(fileid,'\');rc =fwrite(fileid);
-        rc =fput(fileid,'r');rc =fwrite(fileid);
-      end;
-      else if rec='0D'x then do;
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'n');rc =fwrite(fileid);
       end;
-      else if rec='09'x then do;
+      else if rec='0D'x then do; /* CR */
+        rc =fput(fileid,'\');rc =fwrite(fileid);
+        rc =fput(fileid,'r');rc =fwrite(fileid);
+      end;
+      else if rec='09'x then do; /* TAB */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'t');rc =fwrite(fileid);
       end;
-      else if rec='5C'x then do;
+      else if rec='5C'x then do; /* BACKSLASH */
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'\');rc =fwrite(fileid);
       end;

viya/mv_deletefoldermember.sas

@@ -46,7 +46,7 @@
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )

viya/mv_deleteviyafolder.sas

@@ -4,7 +4,6 @@
   @details If not running in Studo 5 +, will expect an oauth token in a global
   macro variable (default ACCESS_TOKEN).
 
-      options mprint;
       %mv_createfolder(path=/Public/test/blah)
       %mv_deleteviyafolder(path=/Public/test)
 

viya/mv_getfoldermembers.sas

@@ -40,7 +40,7 @@
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )

viya/mv_getgroups.sas

@@ -1,31 +1,20 @@
 /**
   @file mv_getgroups.sas
   @brief Creates a dataset with a list of viya groups
-  @details First, be sure you have an access token (which requires an app token).
+  @details First, load the macros:
-
-  Using the macros here:
 
       filename mc url
         "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
 
-  An administrator needs to set you up with an access code:
+  Next, execute:
 
-      %mv_registerclient(outds=client)
+      %mv_getgroups(outds=work.groups)
 
-  Navigate to the url from the log (opting in to the groups) and paste the
+  @param [in] access_token_var= The global macro variable to contain the access token
-  access code below:
+  @param [in] grant_type= valid values are "password" or "authorization_code" (unquoted).
-
-      %mv_tokenauth(inds=client,code=wKDZYTEPK6)
-
-  Now we can run the macro!
-
-      %mv_getgroups()
-
-  @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values are "password" or "authorization_code" (unquoted).
     The default is authorization_code.
-  @param outds= The library.dataset to be created that contains the list of groups
+  @param [out] outds= The library.dataset to be created that contains the list of groups
 
 
   @version VIYA V.03.04
@@ -52,7 +41,7 @@
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )

viya/mv_getjobcode.sas

@@ -49,7 +49,6 @@
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
@@ -114,10 +113,10 @@ run;
 %let fname3=%mf_getuniquefileref();
 %let fpath1=%sysfunc(pathname(&fname1));
 %let fpath2=%sysfunc(pathname(&fname2));
-%let fpath3=%sysfunc(pathname(&fname2));
+%let fpath3=%sysfunc(pathname(&fname3));
 
 /* compile the lua JSON module */
-%ml_json() 
+%ml_json()
 /* read using LUA - this allows the code to be of any length */
 data _null_;
   file "&fpath3..lua";
@@ -147,4 +146,5 @@ data _null_;
 run;
 filename &fname1 clear;
 filename &fname2 clear;
+filename &fname3 clear;
 %mend;

viya/mv_getjoblog.sas

@@ -0,0 +1,267 @@
+/**
+  @file
+  @brief Extract the log from a completed SAS Viya Job
+  @details Extracts log from a Viya job and writes it out to a fileref
+
+  To query the job, you need the URI.  Sample code for achieving this
+  is provided below.
+
+  ## Example
+
+  First, compile the macros:
+
+      filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+  Next, create a job (in this case, a web service):
+
+      filename ft15f001 temp;
+      parmcards4;
+        data ;
+          rand=ranuni(0)*1000;
+          do x=1 to rand;
+            y=rand*4;
+            output;
+          end;
+        run;
+        proc sort data=&syslast
+          by descending y;
+        run;
+      ;;;;
+      %mv_createwebservice(path=/Public/temp,name=demo)
+
+  Execute it:
+
+      %mv_jobexecute(path=/Public/temp
+        ,name=demo
+        ,outds=work.info
+      )
+
+  Wait for it to finish, and grab the uri:
+
+      data _null_;
+        set work.info;
+        if method='GET' and rel='self';
+        call symputx('uri',uri);
+      run;
+
+  Finally, fetch the log:
+
+      %mv_getjoblog(uri=&uri,outref=mylog)
+
+  This macro is used by the mv_jobwaitfor.sas macro, which is generally a more
+  convenient way to wait for the job to finish before fetching the log.
+
+
+  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] mdebug= set to 1 to enable DEBUG messages
+  @param [in] grant_type= valid values:
+    @li password
+    @li authorization_code
+    @li detect - will check if access_token exists, if not will use sas_services if
+        a SASStudioV session else authorization_code.  Default option.
+    @li sas_services - will use oauth_bearer=sas_services.
+  @param [in] uri= The uri of the running job for which to fetch the status,
+    in the format `/jobExecution/jobs/$UUID/state` (unquoted).
+  @param [out] outref= The output fileref to which to APPEND the log (is always
+  appended).
+
+
+  @version VIYA V.03.04
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_getplatform.sas
+  @li mf_existfileref.sas
+  @li ml_json.sas
+
+**/
+
+%macro mv_getjoblog(uri=0,outref=0
+    ,contextName=SAS Job Execution compute context
+    ,access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+    ,mdebug=0
+  );
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+
+/* validation in datastep for better character safety */
+%local errmsg errflg;
+data _null_;
+  uri=symget('uri');
+  if length(uri)<12 then do;
+    call symputx('errflg',1);
+    call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
+  end;
+  if scan(uri,-1)='state' or scan(uri,1) ne 'jobExecution' then do;
+    call symputx('errflg',1);
+    call symputx('errmsg',
+      "URI should be in format /jobExecution/jobs/$$$$UUID$$$$"
+      !!" but is actually like: &uri",'l');
+  end;
+run;
+
+%mp_abort(iftrue=(&errflg=1)
+  ,mac=&sysmacroname
+  ,msg=%str(&errmsg)
+)
+
+%mp_abort(iftrue=(&outref=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Output fileref should be provided)
+)
+
+%if %mf_existfileref(&outref) ne 1 %then %do;
+  filename &outref temp;
+%end;
+
+options noquotelenmax;
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+/* prepare request*/
+%local  fname1;
+%let fname1=%mf_getuniquefileref();
+proc http method='GET' out=&fname1 &oauth_bearer
+  url="&base_uri&uri";
+  headers
+  %if &grant_type=authorization_code %then %do;
+          "Authorization"="Bearer &&&access_token_var"
+  %end;
+  ;
+run;
+%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+%do;
+  data _null_;infile &fname1;input;putlog _infile_;run;
+  %mp_abort(mac=&sysmacroname
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+%end;
+%local  fname2 fname3 fpath1 fpath2 fpath3;
+%let fname2=%mf_getuniquefileref();
+%let fname3=%mf_getuniquefileref();
+%let fpath1=%sysfunc(pathname(&fname1));
+%let fpath2=%sysfunc(pathname(&fname2));
+%let fpath3=%sysfunc(pathname(&fname3));
+
+/* compile the lua JSON module */
+%ml_json()
+/* read using LUA - this allows the code to be of any length */
+data _null_;
+  file "&fpath3..lua";
+  put '
+    infile = io.open (sas.symget("fpath1"), "r")
+    outfile = io.open (sas.symget("fpath2"), "w")
+    io.input(infile)
+    local resp=json.decode(io.read())
+    local logloc=resp["logLocation"]
+    outfile:write(logloc)
+    io.close(infile)
+    io.close(outfile)
+   ';
+run;
+%inc "&fpath3..lua";
+/* get log path*/
+%let errflg=1;
+%let errmsg=No entry in &fname2 fileref;
+data _null_;
+  infile &fname2;
+  input;
+  uri=_infile_;
+  if length(uri)<12 then do;
+    call symputx('errflg',1);
+    call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
+  end;
+  if scan(uri,1) ne 'files' or scan(uri,2) ne 'files' then do;
+    call symputx('errflg',1);
+    call symputx('errmsg',
+      "URI should be in format /files/files/$$$$UUID$$$$"
+      !!" but is actually like: &uri",'l');
+  end;
+  call symputx('errflg',0,'l');
+  call symputx('logloc',uri,'l');
+run;
+
+%mp_abort(iftrue=(&errflg=1)
+  ,mac=&sysmacroname
+  ,msg=%str(&errmsg)
+)
+
+/* we have a log uri - now fetch the log */
+proc http method='GET' out=&fname1 &oauth_bearer
+  url="&base_uri&logloc/content";
+  headers
+  %if &grant_type=authorization_code %then %do;
+          "Authorization"="Bearer &&&access_token_var"
+  %end;
+  ;
+run;
+%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+%do;
+  data _null_;infile &fname1;input;putlog _infile_;run;
+  %mp_abort(mac=&sysmacroname
+    ,msg=%str(logfetch: &SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+%end;
+
+data _null_;
+  file "&fpath3..lua";
+  put '
+    infile = io.open (sas.symget("fpath1"), "r")
+    outfile = io.open (sas.symget("fpath2"), "w")
+    io.input(infile)
+    local resp=json.decode(io.read())
+    for i, v in pairs(resp["items"]) do
+	    outfile:write(v.line,"\n")
+    end
+    io.close(infile)
+    io.close(outfile)
+  ';
+run;
+%inc "&fpath3..lua";
+
+/* write log out to the specified fileref */
+data _null_;
+  infile &fname2 end=last;
+  file &outref mod;
+  if _n_=1 then do;
+    put "/** SASJS Viya Job Log Extract start: &uri **/";
+  end;
+  input;
+  put _infile_;
+  %if &mdebug=1 %then %do;
+    putlog _infile_;
+  %end;
+  if last then do;
+    put "/** SASJS Viya Job Log Extract end: &uri **/";
+  end;
+run;
+
+%if &mdebug=0 %then %do;
+  filename &fname1 clear;
+  filename &fname2 clear;
+  filename &fname3 clear;
+%end;
+%else %do;
+  %put _local_;
+%end;
+%mend;
+
+
+

viya/mv_getjobstate.sas

@@ -0,0 +1,169 @@
+/**
+  @file
+  @brief Extract the status from a running SAS Viya job
+  @details Extracts the status from a running job and appends it to an output
+  dataset with the following structure:
+
+      | uri                                                           | state   | timestamp          |
+      |---------------------------------------------------------------|---------|--------------------|
+      | /jobExecution/jobs/5cebd840-2063-42c1-be0c-421ec3e1c175/state | running | 15JAN2021:12:35:08 |
+
+  To query the running job, you need the URI.  Sample code for achieving this
+  is provided below.
+
+  ## Example
+
+  First, compile the macros:
+
+      filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+  Next, create a long running job (in this case, a web service):
+
+      filename ft15f001 temp;
+      parmcards4;
+        data ;
+          rand=ranuni(0)*1000;
+          do x=1 to rand;
+            y=rand*4;
+            output;
+          end;
+        run;
+        data _null_;
+          call sleep(5,1);
+        run;
+      ;;;;
+      %mv_createwebservice(path=/Public/temp,name=demo)
+
+  Execute it, grab the uri, and finally, check the job status:
+
+      %mv_jobexecute(path=/Public/temp
+        ,name=demo
+        ,outds=work.info
+      )
+
+      data _null_;
+        set work.info;
+        if method='GET' and rel='state';
+        call symputx('uri',uri);
+      run;
+
+      %mv_getjobstate(uri=&uri,outds=results)
+
+  You can run this macro as part of a loop to await the final 'completed' status.
+  The full list of status values is:
+
+  @li idle
+  @li pending
+  @li running
+  @li canceled
+  @li completed
+  @li failed
+
+  If you have one or more jobs that you'd like to wait for completion you can
+  also use the [mv_jobwaitfor](/mv__jobwaitfor_8sas.html) macro.
+
+  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] grant_type= valid values:
+    @li password
+    @li authorization_code
+    @li detect - will check if access_token exists, if not will use sas_services if
+        a SASStudioV session else authorization_code.  Default option.
+    @li sas_services - will use oauth_bearer=sas_services.
+  @param [in] uri= The uri of the running job for which to fetch the status,
+    in the format `/jobExecution/jobs/$UUID/state` (unquoted).
+  @param [out] outds= The output dataset in which to APPEND the status. Three
+    fields are appended:  `CHECK_TM`, `URI` and `STATE`. If the dataset does not
+    exist, it is created.
+
+
+  @version VIYA V.03.04
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+
+**/
+
+%macro mv_getjobstate(uri=0,outds=work.mv_getjobstate
+    ,contextName=SAS Job Execution compute context
+    ,access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+  );
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+
+/* validation in datastep for better character safety */
+%local errmsg errflg;
+data _null_;
+  uri=symget('uri');
+  if length(uri)<12 then do;
+    call symputx('errflg',1);
+    call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
+  end;
+  if scan(uri,-1) ne 'state' or scan(uri,1) ne 'jobExecution' then do;
+
+    call symputx('errflg',1);
+    call symputx('errmsg',
+      "URI should be in format /jobExecution/jobs/$$$$UUID$$$$/state"
+      !!" but is actually like: &uri",'l');
+  end;
+run;
+
+%mp_abort(iftrue=(&errflg=1)
+  ,mac=&sysmacroname
+  ,msg=%str(&errmsg)
+)
+
+options noquotelenmax;
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+%local fname0;
+%let fname0=%mf_getuniquefileref();
+
+proc http method='GET' out=&fname0 &oauth_bearer url="&base_uri/&uri";
+  headers "Accept"="text/plain"
+  %if &grant_type=authorization_code %then %do;
+          "Authorization"="Bearer &&&access_token_var"
+  %end;  ;
+run;
+%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+%do;
+  data _null_;infile &fname0;input;putlog _infile_;run;
+  %mp_abort(mac=&sysmacroname
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+%end;
+
+data;
+  format uri $128. state $32. timestamp datetime19.;
+  infile &fname0;
+  uri="&uri";
+  timestamp=datetime();
+  input;
+  state=_infile_;
+run;
+
+proc append base=&outds data=&syslast;
+run;
+
+filename &fname0 clear;
+
+%mend;

viya/mv_jobexecute.sas

@@ -72,7 +72,7 @@
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
@@ -126,9 +126,11 @@ data _null_;
   length joburi contextname $128 paramstring $32765;
   joburi=quote(trim(symget('joburi')));
   contextname=quote(trim(symget('contextname')));
+  _program=quote("&path/&name");
   paramstring=symget('paramstring');
   put '{"jobDefinitionUri":' joburi ;
   put '  ,"arguments":{"_contextName":' contextname;
+  put '    ,"_program":' _program;
   if paramstring ne "0" then do;
     put '    ,' paramstring;
   end;

viya/mv_jobflow.sas

@@ -0,0 +1,348 @@
+/**
+  @file
+  @brief Execute a series of job flows
+  @details Very (very) simple flow manager.  Jobs execute in sequential waves,
+  all previous waves must finish successfully.
+
+  The input table is formed as per below.  Each observation represents one job.
+  Each variable is converted into a macro variable with the same name.
+
+  ## Input table (minimum variables needed)
+
+  @li _PROGRAM - Provides the path to the job itself
+  @li FLOW_ID - Numeric value, provides sequential ordering capability. Is
+    optional, will default to 0 if not provided.
+  @li _CONTEXTNAME - Dictates which context should be used to run the job. If
+    blank (or not provided), will default to `SAS Job Execution compute context`.
+
+  Any additional variables provided in this table are converted into macro
+  variables and passed into the relevant job.
+
+  |_PROGRAM| FLOW_ID (optional)| _CONTEXTNAME (optional) |
+  |---|---|---|
+  |/Public/jobs/somejob1|0|SAS Job Execution compute context|
+  |/Public/jobs/somejob2|0|SAS Job Execution compute context|
+
+  ## Output table (minimum variables produced)
+
+  @li _PROGRAM - the SAS Drive path of the job
+  @li URI - the URI of the executed job
+  @li STATE - the completed state of the job
+  @li TIMESTAMP - the datetime that the job completed
+  @li JOBPARAMS - the parameters that were passed to the job
+  @li FLOW_ID - the id of the flow in which the job was executed
+
+  ![https://i.imgur.com/nZE9PvT.png](https://i.imgur.com/nZE9PvT.png)
+
+  To avoid hammering the box with many hits in rapid succession, a one
+  second pause is made between every request.
+
+
+  ## Example
+
+  First, compile the macros:
+
+      filename mc url
+      "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+  Next, create some jobs (in this case, as web services):
+
+      filename ft15f001 temp;
+      parmcards4;
+        %put this is job: &_program;
+        %put this was run in flow &flow_id;
+        data ;
+          rand=ranuni(0)*&macrovar1;
+          do x=1 to rand;
+            y=rand*&macrovar2;
+            if y=100 then abort;
+            output;
+          end;
+        run;
+      ;;;;
+      %mv_createwebservice(path=/Public/temp,name=demo1)
+      %mv_createwebservice(path=/Public/temp,name=demo2)
+
+  Prepare an input table with 60 executions:
+
+      data work.inputjobs;
+        _contextName='SAS Job Execution compute context';
+        do flow_id=1 to 3;
+          do i=1 to 20;
+            _program='/Public/temp/demo1';
+            macrovar1=10*i;
+            macrovar2=4*i;
+            output;
+            i+1;
+            _program='/Public/temp/demo2';
+            macrovar1=40*i;
+            macrovar2=44*i;
+            output;
+          end;
+        end;
+      run;
+
+  Trigger the flow
+
+      %mv_jobflow(inds=work.inputjobs
+        ,maxconcurrency=4
+        ,outds=work.results
+        ,outref=myjoblog
+      )
+
+      data _null_;
+        infile myjoblog;
+        input; put _infile_;
+      run;
+
+
+  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] grant_type= valid values:
+      @li password
+      @li authorization_code
+      @li detect - will check if access_token exists, if not will use sas_services if
+        a SASStudioV session else authorization_code.  Default option.
+      @li sas_services - will use oauth_bearer=sas_services
+  @param [in] inds= The input dataset containing a list of jobs and parameters
+  @param [in] maxconcurrency= The max number of parallel jobs to run.  Default=8.
+  @param [in] raise_err=0 Set to 1 to raise SYSCC when a job does not complete
+            succcessfully
+  @param [in] mdebug= set to 1 to enable DEBUG messages
+  @param [out] outds= The output dataset containing the results
+  @param [out] outref= The output fileref to which to append the log file(s).
+
+  @version VIYA V.03.05
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mf_existvarlist.sas
+  @li mv_jobwaitfor.sas
+  @li mv_jobexecute.sas
+
+**/
+
+%macro mv_jobflow(inds=0,outds=work.mv_jobflow
+    ,maxconcurrency=8
+    ,access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+    ,outref=0
+    ,raise_err=0
+    ,mdebug=0
+  );
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+
+%mp_abort(iftrue=("&inds"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(Input dataset was not provided)
+)
+%mp_abort(iftrue=(%mf_existVarList(&inds,_PROGRAM)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(The _PROGRAM column must exist on input dataset &inds)
+)
+%mp_abort(iftrue=(&maxconcurrency<1)
+  ,mac=&sysmacroname
+  ,msg=%str(The maxconcurrency variable should be a positive integer)
+)
+
+/* set defaults if not provided */
+%if %mf_existVarList(&inds,_CONTEXTNAME FLOW_ID)=0 %then %do;
+  data &inds;
+    %if %mf_existvarList(&inds,_CONTEXTNAME)=0 %then %do;
+      length _CONTEXTNAME $128;
+      retain _CONTEXTNAME "SAS Job Execution compute context";
+    %end;
+    %if %mf_existvarList(&inds,FLOW_ID)=0 %then %do;
+      retain FLOW_ID 0;
+    %end;
+    set &inds;
+  run;
+%end;
+
+%local missings;
+proc sql noprint;
+select count(*) into: missings
+  from &inds
+  where flow_id is null or _program is null;
+%mp_abort(iftrue=(&missings>0)
+  ,mac=&sysmacroname
+  ,msg=%str(input dataset contains &missings missing values for FLOW_ID or _PROGRAM)
+)
+
+%if %mf_nobs(&inds)=0 %then %do;
+  %put No observations in &inds!  Leaving macro &sysmacroname;
+  %return;
+%end;
+
+/* ensure output table is available */
+data &outds;run;
+proc sql;
+drop table &outds;
+
+options noquotelenmax;
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+
+/* get flows */
+proc sort data=&inds;
+  by flow_id;
+run;
+data _null_;
+  set &inds (keep=flow_id) end=last;
+  by flow_id;
+  if last.flow_id then do;
+    cnt+1;
+    call symputx(cats('flow',cnt),flow_id,'l');
+  end;
+  if last then call symputx('flowcnt',cnt,'l');
+run;
+
+/* prepare temporary datasets and frefs */
+%local fid jid jds jjson jdsapp jdsrunning jdswaitfor jfref;
+data;run;%let jds=&syslast;
+data;run;%let jjson=&syslast;
+data;run;%let jdsapp=&syslast;
+data;run;%let jdsrunning=&syslast;
+data;run;%let jdswaitfor=&syslast;
+%let jfref=%mf_getuniquefileref();
+
+/* start loop */
+%do fid=1 %to &flowcnt;
+  %put preparing job attributes for flow &&flow&fid;
+  %local jds jcnt;
+  data &jds(drop=_contextName _program);
+    set &inds(where=(flow_id=&&flow&fid));
+    if _contextName='' then _contextName="SAS Job Execution compute context";
+    call symputx(cats('job',_n_),_program,'l');
+    call symputx(cats('context',_n_),_contextName,'l');
+    call symputx('jcnt',_n_,'l');
+  run;
+  %put exporting job variables in json format;
+  %do jid=1 %to &jcnt;
+    data &jjson;
+      set &jds;
+      if _n_=&jid then do;
+        output;
+        stop;
+      end;
+    run;
+    proc json out=&jfref;
+      export &jjson / nosastags fmtnumeric;
+    run;
+    data _null_;
+      infile &jfref lrecl=32767;
+      input;
+      jparams='jparams'!!left(symget('jid'));
+      call symputx(jparams,substr(_infile_,3,length(_infile_)-4));
+    run;
+    %local jobuid&jid;
+    %let jobuid&jid=0; /* used in next loop */
+  %end;
+  %local concurrency completed;
+  %let concurrency=0;
+  %let completed=0;
+  proc sql; drop table &jdsrunning;
+  %do jid=1 %to &jcnt;
+    /**
+      * now we can execute the jobs up to the maxconcurrency setting
+      */
+    %if "&&job&jid" ne "0" %then %do; /* this var is zero if job finished */
+
+      /* check to see if the job finished in the previous round */
+      %if %sysfunc(exist(&outds))=1 %then %do;
+        %local jobcheck;  %let jobcheck=0;
+        proc sql noprint;
+        select count(*) into: jobcheck
+          from &outds where uuid="&&jobuid&jid";
+        %if &jobcheck>0 %then %do;
+          %put &&job&jid in flow &fid with uid &&jobuid&jid completed!;
+          %let job&jid=0;
+        %end;
+      %end;
+
+      /* check if job was triggered and if so, if we have enough slots to run */
+      %if "&&jobuid&jid"="0" and &concurrency<&maxconcurrency %then %do;
+        %local jobname jobpath;
+        %let jobname=%scan(&&job&jid,-1,/);
+        %let jobpath=%substr(&&job&jid,1,%length(&&job&jid)-%length(&jobname)-1);
+        %put executing &jobpath/&jobname with paramstring &&jparams&jid;
+        %mv_jobexecute(path=&jobpath
+          ,name=&jobname
+          ,paramstring=%superq(jparams&jid)
+          ,outds=&jdsapp
+        )
+        data &jdsapp;
+          format jobparams $32767.;
+          set &jdsapp(where=(method='GET' and rel='state'));
+          jobparams=symget("jparams&jid");
+          /* uri here has the /state suffix */
+          uuid=scan(uri,-2,'/');
+          call symputx("jobuid&jid",uuid,'l');
+        run;
+        proc append base=&jdsrunning data=&jdsapp;
+        run;
+        %let concurrency=%eval(&concurrency+1);
+        /* sleep one second after every request to smooth the impact */
+        data _null_;
+          call sleep(1,1);
+        run;
+      %end;
+    %end;
+    %if &jid=&jcnt %then %do;
+      /* we are at the end of the loop - time to see which jobs have finished */
+      %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref
+                    ,raise_err=&raise_err)
+      %local done;
+      %let done=%mf_nobs(&jdswaitfor);
+      %if &done>0 %then %do;
+        %let completed=%eval(&completed+&done);
+        %let concurrency=%eval(&concurrency-&done);
+        data &jdsapp;
+          set &jdswaitfor;
+          flow_id=&&flow&fid;
+          uuid=scan(uri,-1,'/');
+        run;
+        proc append base=&outds data=&jdsapp;
+        run;
+      %end;
+      proc sql;
+      delete from &jdsrunning
+        where uuid in (select uuid from &outds
+          where state in ('canceled','completed','failed')
+        );
+
+      /* loop again if jobs are left */
+      %if &completed < &jcnt %then %do;
+        %let jid=0;
+        %put looping flow &fid again - &completed of &jcnt jobs completed, &concurrency jobs running;
+      %end;
+    %end;
+  %end;
+  /* back up and execute the next flow */
+%end;
+
+%if &mdebug=1 %then %do;
+  %put _local_;
+%end;
+
+%mend;

viya/mv_jobwaitfor.sas

@@ -1,9 +1,9 @@
 /**
   @file
-  @brief Takes a dataset of running jobs and waits for them to complete
+  @brief Takes a dataset of running jobs and waits for ANY or ALL of them to complete
-  @details Will poll `/jobs/{jobId}/state` at set intervals until they are all
+  @details Will poll `/jobs/{jobId}/state` at set intervals until ANY or ALL
-  completed.  Completion is determined by reference to the returned _state_, as
+  jobs are completed.  Completion is determined by reference to the returned
-  per the following table:
+  _state_, as per the following table:
 
   | state     | Wait? | Notes|
   |-----------|-------|------|
@@ -49,8 +49,7 @@
         where method='GET' and rel='state';
       run;
 
-      %mv_jobwaitfor(inds=work.jobs,outds=work.jobstates)
+      %mv_jobwaitfor(ALL,inds=work.jobs,outds=work.jobstates)
-
 
   Delete the job:
 
@@ -65,13 +64,17 @@
         a SASStudioV session else authorization_code.  Default option.
       - sas_services - will use oauth_bearer=sas_services
 
+  @param [in] action=Either ALL (to wait for every job) or ANY (if one job
+    completes, processing will continue).  Default=ALL.
   @param [in] inds= The input dataset containing the list of job uris, in the
     following format:  `/jobExecution/jobs/&JOBID./state` and the corresponding
     job name.  The uri should be in a `uri` variable, and the job path/name
     should be in a `_program` variable.
+  @param [in] raise_err=0 Set to 1 to raise SYSCC when a job does not complete
+              succcessfully
   @param [out] outds= The output dataset containing the list of states by job
     (default=work.mv_jobexecute)
-
+  @param [out] outref= A fileref to which the spawned job logs should be appended.
 
   @version VIYA V.03.04
   @author Allan Bowe, source: https://github.com/sasjs/core
@@ -80,16 +83,20 @@
   @li mp_abort.sas
   @li mf_getplatform.sas
   @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
   @li mf_existvar.sas
   @li mf_nobs.sas
+  @li mv_getjoblog.sas
 
 **/
 
-%macro mv_jobwaitfor(
+%macro mv_jobwaitfor(action
-     access_token_var=ACCESS_TOKEN
+    ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
     ,inds=0
     ,outds=work.mv_jobwaitfor
+    ,outref=0
+    ,raise_err=0
   );
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
@@ -100,7 +107,7 @@
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
@@ -131,17 +138,24 @@ options noquotelenmax;
 %let base_uri=%mf_getplatform(VIYARESTAPI);
 
 data _null_;
+  length jobparams $32767;
   set &inds end=last;
-  call symputx(cats('joburi',_n_),uri,'l');
+  call symputx(cats('joburi',_n_),substr(uri,1,55),'l');
   call symputx(cats('jobname',_n_),_program,'l');
+  call symputx(cats('jobparams',_n_),jobparams,'l');
   if last then call symputx('uricnt',_n_,'l');
 run;
 
+%local runcnt;
+%if &action=ALL %then %let runcnt=&uricnt;
+%else %if &action=ANY %then %let runcnt=1;
+%else %let runcnt=&uricnt;
+
 %local fname0 ;
 %let fname0=%mf_getuniquefileref();
 
 data &outds;
-  format _program uri $128. state $32. timestamp datetime19.;
+  format _program uri $128. state $32. stateDetails $32. timestamp datetime19. jobparams $32767.;
   stop;
 run;
 
@@ -149,7 +163,7 @@ run;
 %do i=1 %to &uricnt;
   %if "&&joburi&i" ne "0" %then %do;
     proc http method='GET' out=&fname0 &oauth_bearer url="&base_uri/&&joburi&i";
-      headers "Accept"="text/plain"
+      headers "Accept"="application/json"
       %if &grant_type=authorization_code %then %do;
               "Authorization"="Bearer &&&access_token_var"
       %end;  ;
@@ -163,20 +177,36 @@ run;
     %end;
 
     %let status=notset;
+
+    %local libref1;
+    %let libref1=%mf_getuniquelibref();
+    libname &libref1 json fileref=&fname0;
+
     data _null_;
-      infile &fname0;
+      length state stateDetails $32;
-      input;
+      set &libref1..root;
-      call symputx('status',_infile_,'l');
+      call symputx('status',state,'l');
+      call symputx('stateDetails',stateDetails,'l');
     run;
 
+    libname &libref1 clear;
+
     %if &status=completed or &status=failed or &status=canceled %then %do;
+      %local plainuri;
+      %let plainuri=%substr(&&joburi&i,1,55);
       proc sql;
       insert into &outds set
         _program="&&jobname&i",
-        uri="&&joburi&i",
+        uri="&plainuri",
         state="&status",
-        timestamp=datetime();
+        stateDetails=symget("stateDetails"),
-      %let joburi&i=0;
+        timestamp=datetime(),
+        jobparams=symget("jobparams&i");
+      %let joburi&i=0; /* do not re-check */
+      /* fetch log */
+      %if %str(&outref) ne 0 %then %do;
+        %mv_getjoblog(uri=&plainuri,outref=&outref)
+      %end;
     %end;
     %else %if &status=idle or &status=pending or &status=running %then %do;
       data _null_;
@@ -188,13 +218,23 @@ run;
         ,msg=%str(status &status not expected!!)
       )
     %end;
+
+    %if (&raise_err) %then %do;
+      %if (&status = canceled or &status = failed or %length(&stateDetails)>0) %then %do;
+        %if ("&stateDetails" = "%str(war)ning") %then %let SYSCC=4;
+        %else %let SYSCC=5;
+        %put %str(ERR)OR: Job &&jobname&i. did not complete successfully. &stateDetails;
+        %return;
+      %end; 
+    %end;
+
   %end;
   %if &i=&uricnt %then %do;
     %local goback;
     %let goback=0;
     proc sql noprint;
     select count(*) into:goback from &outds;
-    %if &goback ne &uricnt %then %let i=0;
+    %if &goback lt &runcnt %then %let i=0;
   %end;
 %end;