fix: deal with dashes in sysencoding

closes #42 Issue41

.git-hooks/commit-msg

@@ -0,0 +1,18 @@
+#!/bin/sh
+RED="\033[1;31m"
+GREEN="\033[1;32m"
+
+# Get the commit message (the parameter we're given is just the path to the
+# temporary file which holds the message).
+commit_message=$(cat "$1")
+
+if (echo "$commit_message" | grep -Eq "^(build|chore|ci|docs|feat|fix|perf|refactor|revert|style|test)(\([a-z \-]+\))?!?: .+$") then
+   echo "${GREEN} ✔ Commit message meets Conventional Commit standards"
+   exit 0
+fi
+
+echo "${RED}❌ Commit message does not meet the Conventional Commit standard!"
+echo "An example of a valid message is:"
+echo "  feat(login): add the 'remember me' button"
+echo "ℹ More details at: https://www.conventionalcommits.org/en/v1.0.0/#summary"
+exit 1

.git-hooks/pre-commit

@@ -0,0 +1,2 @@
+#!/bin/sh
+sasjs lint

.github/workflows/run-tests.yml

@@ -0,0 +1,59 @@
+# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+
+name: Node.js CI
+
+on:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [12.x]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Install Doxygen
+        run: sudo apt-get install doxygen
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Check code style
+        run: npm run lint
+
+      - name: Add client
+        run:  echo "CLIENT=${{secrets.CLIENT}}"> .env.viya
+
+      - name: Add secret
+        run:  echo "SECRET=${{secrets.SECRET}}" >> .env.viya
+
+      - name: Add access token
+        run:  echo "ACCESS_TOKEN=${{secrets.ACCESS_TOKEN}}" >> .env.viya
+
+      - name: Add refresh token
+        run:  echo "REFRESH_TOKEN=${{secrets.REFRESH_TOKEN}}" >> .env.viya
+
+      - name: Build Project
+        run: npm run build
+
+      - name: Run SASjs  tests
+        run: npm run test
+        env:
+          CI: true
+          CLIENT: ${{secrets.CLIENT}}
+          SECRET: ${{secrets.SECRET}}
+          SAS_USERNAME: ${{secrets.SAS_USERNAME}}
+          SAS_PASSWORD: ${{secrets.SAS_PASSWORD}}
+          SERVER_URL: ${{secrets.SERVER_URL}}
+          SERVER_TYPE: ${{secrets.SERVER_TYPE}}
+          ACCESS_TOKEN: ${{secrets.ACCESS_TOKEN}}
+          REFRESH_TOKEN: ${{secrets.REFRESH_TOKEN}}

.gitpod.yml

@@ -1,5 +1,5 @@
 tasks:
-  - init: npm i -g @sasjs/cli
+  - init: nvm install --latest-npm && npm i -g @sasjs/cli
 
 image:
   file: .gitpod.dockerfile

.npmignore

@@ -0,0 +1,11 @@
+all.sas
+build.py
+.gitpod*
+tests/
+sasjs/
+.github/
+.git-hooks/
+.vscode/
+main.dox
+make_singlefile.sh
+*.md

.sasjslint

@@ -6,7 +6,7 @@
     "hasMacroParentheses": true,
     "noNestedMacros": false,
     "noSpacesInFileNames": true,
-    "maxLineLength": 135,
+    "maxLineLength": 230,
     "lowerCaseFileNames": true,
     "noTabIndentation": true,
     "indentationMultiple": 2

CHANGELOG.md

@@ -0,0 +1,5 @@
+# CHANGELOG
+
+As the changes are managed automatically in github, we don't generate an additional changelog.  To view the fixes/features in each release, check out the releases page below:
+
+[https://github.com/sasjs/core/releases](https://github.com/sasjs/core/releases)

CODE_OF_CONDUCT.md

@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+https://sasapps.io/contact-us.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

README.md

@@ -1,4 +1,25 @@
 # Macro Core
+[![npm package][npm-image]][npm-url]
+[![Github Workflow][githubworkflow-image]][githubworkflow-url]
+[![Dependency Status][dependency-image]][dependency-url]
+[![npm](https://img.shields.io/npm/dt/@sasjs/core)]()
+![Snyk Vulnerabilities for npm package](https://img.shields.io/snyk/vulnerabilities/npm/@sasjs/core)
+[![License](https://img.shields.io/apm/l/atomic-design-ui.svg)](/LICENSE)
+![GitHub top language](https://img.shields.io/github/languages/top/sasjs/core)
+[![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sasjs/core)](https://github.com/sasjs/core/issues?q=is%3Aissue+is%3Aclosed)
+[![GitHub issues](https://img.shields.io/github/issues-raw/sasjs/core)](https://github.com/sasjs/core/issues)
+![total lines](https://tokei.rs/b1/github/sasjs/core)
+[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-908a85?logo=gitpod)](https://gitpod.io/#https://github.com/sasjs/core)
+
+
+[npm-image]:https://img.shields.io/npm/v/@sasjs/core.svg
+[npm-url]:http://npmjs.org/package/@sasjs/core
+[githubworkflow-image]:https://github.com/sasjs/core/actions/workflows/main.yml/badge.svg
+[githubworkflow-url]:https://github.com/sasjs/core/blob/main/.github/workflows/main.yml
+[dependency-image]:https://david-dm.org/sasjs/core.svg
+[dependency-url]:https://github.com/sasjs/core/blob/main/package.json
+
+
 
 Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
 
@@ -116,19 +137,19 @@ The **Macro Core** documentation is created using [doxygen](http://www.doxygen.n
 All macros must be commented in the doxygen format, to enable the [online documentation](https://core.sasjs.io).
 
 ### Dependencies
-SAS code can contain one of two types of dependency - SAS Macros, and SAS Programs.  When compiling projects using the [SASjs CLI](https://cli.sasjs.io) the doxygen header is scanned for `  @li` items under the following headers:
+SAS code can contain one of two types of dependency - SAS Macros, and SAS Includes.  When compiling projects using the [SASjs CLI](https://cli.sasjs.io) the doxygen header is scanned for `  @li` items under the following headers:
 
 ```sas
   <h4> SAS Macros </h4>
   @li mf_nobs.sas
   @li mm_assignlib.sas
 
-  <h4> SAS Programs </h4>
+  <h4> SAS Includes </h4>
   @li somefile.ddl SOMEFREF
   @li someprogram.sas FREFTWO
 ```
 
-The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Programs) when creating SAS Jobs and Services.
+The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Includes) when creating SAS Jobs and Services.
 
 When contributing to this library, it is therefore important to ensure that all dependencies are listed in the header in this format.
 
@@ -149,3 +170,12 @@ When contributing to this library, it is therefore important to ensure that all
 # General Notes
 
 - All macros should be compatible with SAS versions from support level B and above (so currently 9.2 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully (eg `%if %sysevalf(&sysver<9.3) %then %return`).
+
+## Star Gazing
+
+If you find this library useful, please leave a [star](https://github.com/sasjs/core/stargazers) and help us grow our star graph!
+
+![](https://starchart.cc/sasjs/core.svg)
+
+
+

SECURITY.md

@@ -0,0 +1,18 @@
+# Security Policy
+
+Security is an extremely high priority when it comes to the SASjs product suite.  We take a number of steps across all repositories to minimise risk, such as:
+
+* Regular dependabot updates
+* Snyk reports
+* Minimising dependencies, especially production dependencies (sasjs/core has NONE)
+* Testing & Code review process
+
+## Supported Versions
+
+We support only the latest version
+
+## Reporting a Vulnerability
+
+We welcome disclosures of all kinds in relation to all the SASjs libraries.  You can submit them here: [https://sasapps.io/contact-us](https://sasapps.io/contact-us)
+
+

base/mf_abort.sas

@@ -1,30 +1,17 @@
 /**
   @file
-  @brief abort gracefully according to context
+  @brief Abort, ungracefully
-  @details Do not use directly!  See bottom of explanation for details.
+  @details Will abort with a straightforward %abort if the condition is true.
 
-  Configures an abort mechanism according to site specific policies or the
+  <h4> Related Macros </h4>
-    particulars of an environment.  For instance, can stream custom
+  @li mp_abort.sas
-    results back to the client in an STP Web App context, or completely stop
-    in the case of a batch run.
-
-  For the sharp eyed readers - this is no longer a macro function!! It became
-  a macro procedure during a project and now it's kinda stuck that way until
-  that project is updated (if it's ever updated).  In the meantime we created
-  `mp_abort` which is just a wrapper for this one, and so we recomend you use
-  that for forwards compatibility reasons.
-
-  @param mac= to contain the name of the calling macro
-  @param type= deprecated.  Not used.
-  @param msg= message to be returned
-  @param iftrue= supply a condition under which the macro should be executed.
 
   @version 9.2
   @author Allan Bowe
   @cond
 **/
 
-%macro mf_abort(mac=mf_abort.sas, type=, msg=, iftrue=%str(1=1)
+%macro mf_abort(mac=mf_abort.sas, type=deprecated, msg=, iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
   %if not(%eval(%unquote(&iftrue))) %then %return;
@@ -33,116 +20,8 @@
   %if %length(&mac)>0 %then %put NOTE- called by &mac;
   %put NOTE - &msg;
 
-  /* Stored Process Server web app context */
+  %abort;
-  %if %symexist(_metaperson) or "&SYSPROCESSNAME"="Compute Server" %then %do;
-    options obs=max replace nosyntaxcheck mprint;
-    /* extract log err / warn, if exist */
-    %local logloc logline;
-    %global logmsg; /* capture global messages */
-    %if %symexist(SYSPRINTTOLOG) %then %let logloc=&SYSPRINTTOLOG;
-    %else %let logloc=%qsysfunc(getoption(LOG));
-    proc printto log=log;run;
-    %if %length(&logloc)>0 %then %do;
-      %let logline=0;
-      data _null_;
-        infile &logloc lrecl=5000;
-        input; putlog _infile_;
-        i=1;
-        retain logonce 0;
-        if (
-            _infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR"
-          ) and logonce=0
-        then do;
-          call symputx('logline',_n_);
-          logonce+1;
-        end;
-      run;
-      /* capture log including lines BEFORE the err */
-      %if &logline>0 %then %do;
-        data _null_;
-          infile &logloc lrecl=5000;
-          input;
-          i=1;
-          stoploop=0;
-          if _n_ ge &logline-5 and stoploop=0 then do until (i>12);
-            call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
-            input;
-            i+1;
-            stoploop=1;
-          end;
-          if stoploop=1 then stop;
-        run;
-      %end;
-    %end;
 
-    /* send response in SASjs JSON format */
+%mend mf_abort;
-    data _null_;
-      file _webout mod lrecl=32000;
-      length msg $32767;
-      sasdatetime=datetime();
-      msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
-      /* escape the quotes */
-      msg=tranwrd(msg,'"','\"');
-      /* ditch the CRLFs as chrome complains */
-      msg=compress(msg,,'kw');
-      /* quote without quoting the quotes (which are escaped instead) */
-      msg=cats('"',msg,'"');
-      if symexist('_debug') then debug=symget('_debug');
-      if debug ge 131 then put '>>weboutBEGIN<<';
-      put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
-      put ',"sasjsAbort" : [{';
-      put ' "MSG":' msg ;
-      put ' ,"MAC": "' "&mac" '"}]';
-      put ",""SYSUSERID"" : ""&sysuserid"" ";
-      if symexist('_metauser') then do;
-        _METAUSER=quote(trim(symget('_METAUSER')));
-        put ",""_METAUSER"": " _METAUSER;
-        _METAPERSON=quote(trim(symget('_METAPERSON')));
-        put ',"_METAPERSON": ' _METAPERSON;
-      end;
-      _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
-      put ',"_PROGRAM" : ' _PROGRAM ;
-      put ",""SYSCC"" : ""&syscc"" ";
-      put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
-      put ",""SYSJOBID"" : ""&sysjobid"" ";
-      put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
-      put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
-      put "}" @;
-      %if &_debug ge 131 %then %do;
-        put '>>weboutEND<<';
-      %end;
-    run;
-    %let syscc=0;
-    %if %symexist(SYS_JES_JOB_URI) %then %do;
-      /* refer web service output to file service in one hit */
-      filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
-        name="_webout.json";
-      %let rc=%sysfunc(fcopy(_web,_webout));
-    %end;
-    %else %do;
-      data _null_;
-        if symexist('sysprocessmode')
-          then if symget("sysprocessmode")="SAS Stored Process Server"
-            then rc=stpsrvset('program error', 0);
-      run;
-    %end;
-    /**
-      * endsas is reliable but kills some deployments.
-      * Abort variants are ungraceful (non zero return code)
-      * This approach lets SAS run silently until the end :-)
-      */
-    %put _all_;
-    filename skip temp;
-    data _null_;
-      file skip;
-      put '%macro skip(); %macro skippy();';
-    run;
-    %inc skip;
-  %end;
-  %else %do;
-    %put _all_;
-    %abort cancel;
-  %end;
-%mend;
 
 /** @endcond */

base/mf_existds.sas

@@ -23,4 +23,4 @@
   %if %sysfunc(exist(&libds)) ne 1 & %sysfunc(exist(&libds,VIEW)) ne 1 %then 0;
   %else 1;
 
-%mend;
+%mend mf_existds;

base/mf_getuniquename.sas

@@ -18,5 +18,5 @@
 
 
 %macro mf_getuniquename(prefix=MC);
-  &prefix.%substr(%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32-%length(&prefix))
+&prefix.%substr(%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32-%length(&prefix))
-%mend;
+%mend mf_getuniquename;

base/mf_getvarlist.sas

@@ -66,7 +66,8 @@
     %let rc=%sysfunc(close(&dsid));
   %end;
   %else %do;
-    %put unable to open &libds (rc=&dsid);
+    %put &sysmacroname: Unable to open &libds (rc=&dsid);
+    %put &sysmacroname: SYSMSG= %sysfunc(sysmsg());
     %let rc=%sysfunc(close(&dsid));
   %end;
   &outvar

base/mp_abort.sas

@@ -4,13 +4,24 @@
   @details Configures an abort mechanism according to site specific policies or
     the particulars of an environment.  For instance, can stream custom
     results back to the client in an STP Web App context, or completely stop
-    in the case of a batch run.
+    in the case of a batch run.  For STP sessions
+
+  The method used varies according to the context.  Important points:
+
+  @li should not use endsas or abort cancel in 9.4m3 environments as this can
+    cause hung multibridge sessions and result in a frozen STP server
+  @li should not use endsas in viya 3.5 as this destroys the session and cannot
+    fetch results (although both mv_getjoblog.sas and the @sasjs/adapter will
+    recognise this and fetch the log of the parent session instead)
+  @li STP environments must finish cleanly to avoid the log being sent to
+    _webout.  To assist with this, we also run stpsrvset('program error', 0)
+    and set SYSCC=0.  For 9.4m3 we take a unique approach - we open a macro
+    but don't close it!  This provides a graceful abort, EXCEPT when called
+    called within a %include within a macro (and that macro contains additional
+    logic).  See mp_abort.test.nofix.sas for the example case.
+    If you know of another way to gracefully abort a 9.4m3 STP session, we'd
+    love to hear about it!
 
-  Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored
-  Process environments.  This macro takes a unique approach - we set the SAS
-  syscc to 0, run `stpsrvset('program error', 0)` (if SAS 9) and then - we open
-  a macro but don't close it!  This provides a graceful abort for SAS web
-  services in all web enabled environments.
 
   @param mac= to contain the name of the calling macro
   @param msg= message to be returned
@@ -24,6 +35,8 @@
 %macro mp_abort(mac=mp_abort.sas, type=, msg=, iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
+  %global sysprocessmode sysprocessname;
+
   %if not(%eval(%unquote(&iftrue))) %then %return;
 
   %put NOTE: ///  mp_abort macro executing //;
@@ -31,9 +44,7 @@
   %put NOTE - &msg;
 
   /* Stored Process Server web app context */
-  %if %symexist(_metaperson)
+  %if %symexist(_metaperson) or "&SYSPROCESSNAME "="Compute Server " %then %do;
-  or (%symexist(SYSPROCESSNAME) and "&SYSPROCESSNAME"="Compute Server" )
-  %then %do;
     options obs=max replace nosyntaxcheck mprint;
     /* extract log errs / warns, if exist */
     %local logloc logline;
@@ -63,7 +74,7 @@
           input;
           i=1;
           stoploop=0;
-          if _n_ ge &logline-5 and stoploop=0 then do until (i>12);
+          if _n_ ge &logline-15 and stoploop=0 then do until (i>22);
             call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
             input;
             i+1;
@@ -88,7 +99,7 @@
 
     /* send response in SASjs JSON format */
     data _null_;
-      file _webout mod lrecl=32000;
+      file _webout mod lrecl=32000 encoding='utf-8';
       length msg $32767 debug $8;
       sasdatetime=datetime();
       msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
@@ -122,37 +133,67 @@
       put ",""SYSCC"" : ""&syscc"" ";
       put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
       put ",""SYSJOBID"" : ""&sysjobid"" ";
+      sysvlong=quote(trim(symget('sysvlong')));
+      put ',"SYSVLONG" : ' sysvlong;
       put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
       put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
       put "}" @;
       if debug ge '"131"' then put '>>weboutEND<<';
     run;
 
-    %let syscc=0;
+    %put _all_;
-    %if %symexist(_metaport) %then %do;
+
+    %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
       data _null_;
-        if symexist('sysprocessmode')
+        putlog 'stpsrvset program error and syscc';
-          then if symget("sysprocessmode")="SAS Stored Process Server"
+        rc=stpsrvset('program error', 0);
-            then rc=stpsrvset('program error', 0);
+        call symputx("syscc",0,"g");
+      run;
+      %if "%substr(&sysvlong.xxxxxxxxx,1,9)" ne "9.04.01M3" %then %do;
+        %put NOTE: Ending SAS session due to:;
+        %put NOTE- &msg;
+        endsas;
+      %end;
+    %end;
+    %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
+      /* endsas kills the session making it harder to fetch results */
+      data _null_;
+        syswarningtext=symget('syswarningtext');
+        syserrortext=symget('syserrortext');
+        abort_msg=symget('msg');
+        syscc=symget('syscc');
+        sysuserid=symget('sysuserid');
+        iftrue=symget('iftrue');
+        put (_all_)(/=);
+        abort cancel nolist;
       run;
     %end;
-    /**
+    %else %if "%substr(&sysvlong.xxxxxxxxx,1,9)" = "9.04.01M3" %then %do;
-      * endsas is reliable but kills some deployments.
+      /**
-      * Abort variants are ungraceful (non zero return code)
+        * endsas kills 9.4m3 deployments by orphaning multibridges.
-      * This approach lets SAS run silently until the end :-)
+        * Abort variants are ungraceful (non zero return code)
-      */
+        * This approach lets SAS run silently until the end :-)
-    %put _all_;
+        * Caution - fails when called within a %include within a macro
-    filename skip temp;
+        * See tests/mp_abort.test.1 for an example case.
-    data _null_;
+        */
-      file skip;
+      filename skip temp;
-      put '%macro skip(); %macro skippy();';
+      data _null_;
-    run;
+        file skip;
-    %inc skip;
+        put '%macro skip();';
+        comment '%mend skip; -> fix lint ';
+        put '%macro skippy();';
+        comment '%mend skippy; -> fix lint ';
+      run;
+      %inc skip;
+    %end;
+    %else %do;
+      %abort cancel;
+    %end;
   %end;
   %else %do;
     %put _all_;
     %abort cancel;
   %end;
-%mend;
+%mend mp_abort;
 
 /** @endcond */

base/mp_assertdsobs.sas

@@ -18,7 +18,11 @@
   @param [in] test= (HASOBS) The test to apply.  Valid values are:
     @li HASOBS - Test is a PASS if the input dataset has any observations
     @li EMPTY - Test is a PASS if input dataset is empty
-    @li EQUALS [integer] - Test passes if obs count matches the provided integer
+    @li EQUALS [integer] - Test passes if row count matches the provided integer
+    @LI ATLEAST [integer] - Test passes if row count is more than or equal to
+      the provided integer
+    @LI ATMOST [integer] - Test passes if row count is less than or equal to
+      the provided integer
   @param [out] outds= (work.test_results) The output dataset to contain the
   results.  If it does not exist, it will be created, with the following format:
   |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
@@ -27,6 +31,8 @@
 
   <h4> Related Macros </h4>
   @li mp_assertcolvals.sas
+  @li mp_assert.sas
+  @li mp_assertcols.sas
 
   @version 9.2
   @author Allan Bowe
@@ -51,6 +57,22 @@
     )
     %let test=EQUALS;
   %end;
+  %else %if %substr(&test.xxxxxxx,1,7)=ATLEAST %then %do;
+    %let val=%scan(&test,2,%str( ));
+    %mp_abort(iftrue= (%DATATYP(&val)=CHAR)
+      ,mac=&sysmacroname
+      ,msg=%str(Invalid test - &test, expected ATLEAST [integer])
+    )
+    %let test=ATLEAST;
+  %end;
+  %else %if %substr(&test.xxxxxxx,1,7)=ATMOST %then %do;
+    %let val=%scan(&test,2,%str( ));
+    %mp_abort(iftrue= (%DATATYP(&val)=CHAR)
+      ,mac=&sysmacroname
+      ,msg=%str(Invalid test - &test, expected ATMOST [integer])
+    )
+    %let test=ATMOST;
+  %end;
   %else %if &test ne HASOBS and &test ne EMPTY %then %do;
     %mp_abort(
       mac=&sysmacroname,
@@ -62,7 +84,8 @@
     length test_description $256 test_result $4 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
-    test_comments="&sysmacroname: Dataset &inds has &nobs observations";
+    test_comments="&sysmacroname: Dataset &inds has &nobs observations.";
+    test_comments=test_comments!!" Test was "!!symget('test');
   %if &test=HASOBS %then %do;
     if &nobs>0 then test_result='PASS';
   %end;
@@ -72,6 +95,12 @@
   %else %if &test=EQUALS %then %do;
     if &nobs=&val then test_result='PASS';
   %end;
+  %else %if &test=ATLEAST %then %do;
+    if &nobs ge &val then test_result='PASS';
+  %end;
+  %else %if &test=ATMOST %then %do;
+    if &nobs le &val then test_result='PASS';
+  %end;
   %else %do;
     test_comments="&sysmacroname: Unsatisfied test condition - &test";
   %end;
@@ -86,4 +115,4 @@
   proc sql;
   drop table &ds;
 
-%mend;
+%mend mp_assertdsobs;

base/mp_dropmembers.sas

@@ -1,20 +1,24 @@
 /**
   @file
   @brief Drops tables / views (if they exist) without warnings in the log
-  @details
+  @details Useful for dropping tables when you're not sure they exist, or if
+  you are not sure whether they are a dataset or view.  Also efficient for
+  dropping multiple tables / views.
+
   Example usage:
 
       proc sql;
       create table data1 as select * from sashelp.class;
       create view view2 as select * from sashelp.class;
-      %mp_dropmembers(list=data1 view2)
+      %mp_dropmembers(data1 view2, libref=WORK)
+
 
   <h4> SAS Macros </h4>
   @li mf_isblank.sas
 
 
-  @param list space separated list of datasets / views
+  @param list space separated list of datasets / views, WITHOUT libref
-  @param libref= can only drop from a single library at a time
+  @param libref= (WORK) Note - you can only drop from a single library at a time
 
   @version 9.2
   @author Allan Bowe
@@ -35,4 +39,4 @@
     delete &list;
     delete &list /mtype=view;
   run;
-%mend;
+%mend mp_dropmembers;

base/mp_ds2fmtds.sas

@@ -0,0 +1,98 @@
+/**
+  @file
+  @brief Converts every value in a dataset to it's formatted value
+  @details Converts every value to it's formatted value.  All variables will
+  become character, and will be in the same order.
+
+  Usage:
+
+      %mp_ds2fmtds(sashelp.cars,work.cars)
+
+  @param [in] libds The library.dataset to be converted
+  @param [out] outds The dataset to create.
+
+  <h4> Related Macros <h4>
+  @li mp_jsonout.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_ds2fmtds(libds, outds
+)/*/STORE SOURCE*/;
+
+/* validations */
+%if not %sysfunc(exist(&libds)) %then %do;
+  %put %str(WARN)ING:  &libds does not exist;
+  %return;
+%end;
+%if %index(&libds,.)=0 %then %let libds=WORK.&libds;
+
+/* grab metadata */
+proc contents noprint data=&libds
+  out=_data_(keep=name type length format formatl formatd varnum);
+run;
+proc sort;
+  by varnum;
+run;
+
+/* prepare formats and varnames */
+data _null_;
+  set &syslast end=last;
+  name=upcase(name);
+  /* fix formats */
+  if type=2 or type=6 then do;
+    length fmt $49.;
+    if format='' then fmt=cats('$',length,'.');
+    else if formatl=0 then fmt=cats(format,'.');
+    else fmt=cats(format,formatl,'.');
+    newlen=max(formatl,length);
+  end;
+  else do;
+    if format='' then fmt='best.';
+    else if formatl=0 then fmt=cats(format,'.');
+    else if formatd=0 then fmt=cats(format,formatl,'.');
+    else fmt=cats(format,formatl,'.',formatd);
+    /* needs to be wide, for datetimes etc */
+    newlen=max(length,formatl,24);
+  end;
+  /* 32 char unique name */
+  newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
+
+  call symputx(cats('name',_n_),name,'l');
+  call symputx(cats('newname',_n_),newname,'l');
+  call symputx(cats('len',_n_),newlen,'l');
+  call symputx(cats('fmt',_n_),fmt,'l');
+  call symputx(cats('type',_n_),type,'l');
+  if last then call symputx('nobs',_n_,'l');
+run;
+
+/* clean up */
+proc sql;
+drop table &syslast;
+
+%if &nobs=0 %then %do;
+  %put Dataset &libds has no columns!
+  data &outds;
+    set &libds;
+  run;
+  %return;
+%end;
+
+data &outds;
+  /* rename on entry */
+  set &libds(rename=(
+%local i;
+%do i=1 %to &nobs;
+  &&name&i=&&newname&i
+%end;
+  ));
+%do i=1 %to &nobs;
+  length &&name&i $&&len&i;
+  &&name&i=left(put(&&newname&i,&&fmt&i));
+  drop &&newname&i;
+%end;
+  if _error_ then call symputx('syscc',1012);
+run;
+
+%mend mp_ds2fmtds;

base/mp_filtercheck.sas

@@ -33,19 +33,19 @@
   @returns The &outds table containing any bad rows, plus a REASON_CD column.
 
   @param [in] inds The table to be checked, with the format above
-  @param [in] targetds= The target dataset against which to verify VARIABLE_NM
+  @param [in] targetds= The target dataset against which to verify VARIABLE_NM.
+    This must be available (ie, the library must be assigned).
   @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
   @param [out] outds= The output table, which is a copy of the &inds. table
-  plus a REASON_CD column, containing only bad records.  If bad records found,
+    plus a REASON_CD column, containing only bad records.  If bad records found,
-  the SYSCC value will be set to 1008 (general data problem).  Downstream
+    the SYSCC value will be set to 1008 (general data problem).  Downstream
-  processes should check this table (and return code) before continuing.
+    processes should check this table (and return code) before continuing.
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas
   @li mf_getuniquefileref.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
-  @li mf_nobs.sas
   @li mp_filtergenerate.sas
   @li mp_filtervalidate.sas
 
@@ -85,41 +85,52 @@
   * quotes, commas, periods and spaces.
   * Only numeric values should remain
   */
-%local reason_cd;
+%local reason_cd nobs;
+%let nobs=0;
 data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
-  length reason_cd $32;
+  length reason_cd $4032;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
-    REASON_CD='GROUP_LOGIC should be either AND or OR';
+    REASON_CD='GROUP_LOGIC should be AND/OR, not:'!!cats(GROUP_LOGIC);
     putlog REASON_CD= GROUP_LOGIC=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if SUBGROUP_LOGIC not in ('AND','OR') then do;
-    REASON_CD='SUBGROUP_LOGIC should be either AND or OR';
+    REASON_CD='SUBGROUP_LOGIC should be AND/OR, not:'!!cats(SUBGROUP_LOGIC);
     putlog REASON_CD= SUBGROUP_LOGIC=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if mod(SUBGROUP_ID,1) ne 0 then do;
-    REASON_CD='SUBGROUP_ID should be integer';
+    REASON_CD='SUBGROUP_ID should be integer, not '!!left(subgroup_id);
     putlog REASON_CD= SUBGROUP_ID=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if upcase(VARIABLE_NM) not in
   (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
   then do;
-    REASON_CD="VARIABLE_NM not in &targetds";
+    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
     putlog REASON_CD= VARIABLE_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if OPERATOR_NM not in
   ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
   then do;
-    REASON_CD='Invalid OPERATOR_NM';
+    REASON_CD='Invalid OPERATOR_NM: '!!left(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
 
@@ -129,8 +140,10 @@ data &outds;
     if substr(raw_value,1,1) ne '('
     or substr(cats(reverse(raw_value)),1,1) ne ')'
     then do;
-      REASON_CD='Missing brackets in RAW_VALUE';
+      REASON_CD='Missing start/end bracket in RAW_VALUE';
       putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
       output;
     end;
     else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
@@ -151,25 +164,27 @@ data &outds;
   /* output records that contain values other than digits and spaces */
   if notdigit(compress(raw_value3,' '))>0 then do;
     putlog raw_value3= $hex32.;
-    REASON_CD='Invalid RAW_VALUE';
+    REASON_CD=cats('Invalid RAW_VALUE:',raw_value);
     putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
 
 run;
 
+
 data _null_;
-  set &outds;
+  set &outds end=last;
-  call symputx('REASON_CD',reason_cd,'l');
+  putlog (_all_)(=);
-  stop;
 run;
 
-%mp_abort(iftrue=(&abort=YES and %mf_nobs(&outds)>0),
+%mp_abort(iftrue=(&abort=YES and &nobs>0),
   mac=&sysmacroname,
-  msg=%str(Filter issues in &inds, reason: &reason_cd, details in &outds)
+  msg=%str(Data issue: %superq(reason_cd))
 )
 
-%if %mf_nobs(&outds)>0 %then %do;
+%if &nobs>0 %then %do;
   %let syscc=1008;
   %return;
 %end;

base/mp_filtervalidate.sas

@@ -78,7 +78,8 @@ run;
 
 data &outds;
   if &sqlrc or &syscc or &syserr then do;
-    REASON_CD=coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
+    REASON_CD='VALIDATION_ERROR: '!!
+      coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
     output;
   end;
   else stop;
@@ -95,10 +96,11 @@ filename &fref1 clear;
     run;
     %mp_abort(
       mac=&sysmacroname,
-      msg=%str(Filter issues in &inref: %quote(&reason_cd))
+      msg=%str(Filter validation issues. ERR=%superq(SYSERRORTEXT)
+        , WARN=%superq(SYSWARNINGTEXT) )
     )
   %end;
   %let syscc=1008;
 %end;
 
-%mend;
+%mend mp_filtervalidate;

base/mp_jsonout.sas

@@ -48,6 +48,8 @@
   @param dbg= DEPRECATED - was used to conditionally add PRETTY to
     proc json but this can cause line truncation in large files.
 
+  <h4> Related Macros <h4>
+  @li mp_ds2fmtds.sas
 
   @version 9.2
   @author Allan Bowe
@@ -55,10 +57,11 @@
 
 **/
 
-%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=PROCJSON,dbg=0
+%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0
 )/*/STORE SOURCE*/;
 %put output location=&jref;
 %if &action=OPEN %then %do;
+  options nobomfile;
   data _null_;file &jref encoding='utf-8';
     put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
   run;
@@ -86,7 +89,65 @@
       %put &sysmacroname:  &ds NOT FOUND!!!;
       %return;
     %end;
-    data _null_;file &jref mod ;
+    %if &fmt=Y %then %do;
+      %put converting every variable to a formatted variable;
+      /* see mp_ds2fmtds.sas for source */
+      proc contents noprint data=&ds
+        out=_data_(keep=name type length format formatl formatd varnum);
+      run;
+      proc sort;
+        by varnum;
+      run;
+      %local fmtds;
+      %let fmtds=%scan(&syslast,2,.);
+      /* prepare formats and varnames */
+      data _null_;
+        if _n_=1 then call symputx('nobs',nobs,'l');
+        set &fmtds end=last nobs=nobs;
+        name=upcase(name);
+        /* fix formats */
+        if type=2 or type=6 then do;
+          length fmt $49.;
+          if format='' then fmt=cats('$',length,'.');
+          else if formatl=0 then fmt=cats(format,'.');
+          else fmt=cats(format,formatl,'.');
+          newlen=max(formatl,length);
+        end;
+        else do;
+          if format='' then fmt='best.';
+          else if formatl=0 then fmt=cats(format,'.');
+          else if formatd=0 then fmt=cats(format,formatl,'.');
+          else fmt=cats(format,formatl,'.',formatd);
+          /* needs to be wide, for datetimes etc */
+          newlen=max(length,formatl,24);
+        end;
+        /* 32 char unique name */
+        newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
+
+        call symputx(cats('name',_n_),name,'l');
+        call symputx(cats('newname',_n_),newname,'l');
+        call symputx(cats('len',_n_),newlen,'l');
+        call symputx(cats('fmt',_n_),fmt,'l');
+        call symputx(cats('type',_n_),type,'l');
+      run;
+      data &fmtds;
+        /* rename on entry */
+        set &ds(rename=(
+      %local i;
+      %do i=1 %to &nobs;
+        &&name&i=&&newname&i
+      %end;
+        ));
+      %do i=1 %to &nobs;
+        length &&name&i $&&len&i;
+        &&name&i=left(put(&&newname&i,&&fmt&i));
+        drop &&newname&i;
+      %end;
+        if _error_ then call symputx('syscc',1012);
+      run;
+      %let ds=&fmtds;
+    %end; /* &fmt=Y */
+    data _null_;file &jref mod encoding='utf-8';
       put "["; call symputx('cols',0,'l');
     proc sort
       data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
@@ -169,4 +230,4 @@
     put "}";
   run;
 %end;
-%mend;
+%mend mp_jsonout;

base/mp_searchcols.sas

@@ -9,7 +9,15 @@
 
       %mp_searchcols(libs=sashelp work, cols=name sex age)
 
-  @param libs=
+  @param libs=(SASHELP) Space separated list of libraries to search for columns
+  @param cols= Space separated list of column names to search for (not case
+    sensitive)
+  @param outds=(mp_searchcols) the table to create with the results.  Will have
+    one line per table match.
+  @param match=(ANY) The match type. Valid values:
+    @li ANY - The table contains at least one of the columns
+    @li WILD - The table contains a column with a name that partially matches
+
   @version 9.2
   @author Allan Bowe
 **/
@@ -17,6 +25,7 @@
 %macro mp_searchcols(libs=sashelp
   ,cols=
   ,outds=mp_searchcols
+  ,match=ANY
 )/*/STORE SOURCE*/;
 
 %put &sysmacroname process began at %sysfunc(datetime(),datetime19.);
@@ -38,8 +47,10 @@ create table _data_ as
 %end;
   order by 1,2,3;
 
+%local tempds;
+%let tempds=&syslast;
 data &outds;
-  set &syslast;
+  set &tempds;
   length cols matchcols $32767;
   cols=upcase(symget('cols'));
   colcount=countw(cols);
@@ -53,10 +64,29 @@ data &outds;
     retain matchcols;
     matchcols='';
   end;
+%if &match=ANY %then %do;
   if findw(cols,name,,'spit') then do;
     sumcols+1;
     matchcols=cats(matchcols)!!' '!!cats(name);
   end;
+%end;
+%else %if &match=WILD %then %do;
+  if _n_=1 then do;
+    retain wcount;
+    wcount=countw(cols);
+    drop wcount;
+  end;
+  do i=1 to wcount;
+    length curword $32;
+    curword=scan(cols,i,' ');
+    drop curword;
+    if index(name,cats(curword)) then do;
+      sumcols+1;
+      matchcols=cats(matchcols)!!' '!!cats(curword);
+    end;
+  end;
+%end;
+
   if last.memname then do;
     if sumcols>0 then output;
     if sumcols=colcount then putlog "Full Match: " libname memname;
@@ -66,6 +96,8 @@ run;
 
 proc sort; by descending sumcols memname libname; run;
 
+proc sql;
+drop table &tempds;
 %put &sysmacroname process finished at %sysfunc(datetime(),datetime19.);
 
-%mend;
+%mend mp_searchcols;

lua/json.lua

@@ -1,3 +1,5 @@
+-- NOTE - THE COPYRIGHT BELOW IS IN RELATION TO THE JSON.LUA FILE ONLY
+-- THIS FILE STARTS ON THE NEXT LINE AND WILL FINISH WITH "JSON.LUA ENDS HERE"
 --
 -- json.lua
 --
@@ -369,3 +371,5 @@ function json.decode(str)
 end
 
 return json
+
+-- JSON.LUA ENDS HERE

lua/ml_json.sas

@@ -12,6 +12,8 @@
 %macro ml_json();
 data _null_;
   file "%sysfunc(pathname(work))/ml_json.lua";
+  put '-- NOTE - THE COPYRIGHT BELOW IS IN RELATION TO THE JSON.LUA FILE ONLY ';
+  put '-- THIS FILE STARTS ON THE NEXT LINE AND WILL FINISH WITH "JSON.LUA ENDS HERE" ';
   put '-- ';
   put '-- json.lua ';
   put '-- ';
@@ -383,6 +385,8 @@ data _null_;
   put 'end ';
   put ' ';
   put 'return json ';
+  put ' ';
+  put '-- JSON.LUA ENDS HERE ';
 run;
 
 %inc "%sysfunc(pathname(work))/ml_json.lua";

main.dox

@@ -42,6 +42,13 @@
 
  */
 
+ /*! \dir Tests
+ *  \brief SASjs Tests
+ *  \details These folders contain the macro tests.  They are first compiled
+   and deployed (sasjs cbd) then executed (sasjs test).
+
+ */
+
 /*! \dir viya
  *  \brief Viya macros
  *  \details These macros have the following attributes:

meta/mm_createwebservice.sas

@@ -22,7 +22,7 @@ Usage:
         %webout(OBJ,example2) * Object format, easier to work with ;
         %webout(CLOSE)
     ;;;;
-    %mm_createwebservice(path=/Public/app/common,name=appInit)
+    %mm_createwebservice(path=/Public/app/common,name=appInit,code=ft15f001)
 
   <h4> SAS Macros </h4>
   @li mm_createstp.sas
@@ -37,12 +37,15 @@ Usage:
   @param desc= The description of the service (optional)
   @param precode= Space separated list of filerefs, pointing to the code that
     needs to be attached to the beginning of the service (optional)
-  @param code= Space seperated fileref(s) of the actual code to be added
+  @param code=(ft15f001) Space seperated fileref(s) of the actual code to be
-  @param server= The server which will run the STP.  Server name or uri is fine.
+    added
-  @param mDebug= set to 1 to show debug messages in the log
+  @param server=(SASApp) The server which will run the STP.  Server name or uri
-  @param replace= select YES to replace any existing service in that location
+    is fine.
-  @param adapter= the macro uses the sasjs adapter by default.  To use another
+  @param mDebug=(0) set to 1 to show debug messages in the log
-    adapter, add a (different) fileref here.
+  @param replace=(YES) select NO to avoid replacing an existing service in that
+    location
+  @param adapter=(sasjs) the macro uses the sasjs adapter by default.  To use
+    another adapter, add a (different) fileref here.
 
   @version 9.2
   @author Allan Bowe
@@ -52,11 +55,11 @@ Usage:
 %macro mm_createwebservice(path=
     ,name=initService
     ,precode=
-    ,code=
+    ,code=ft15f001
     ,desc=This stp was created automagically by the mm_createwebservice macro
     ,mDebug=0
     ,server=SASApp
-    ,replace=NO
+    ,replace=YES
     ,adapter=sasjs
 )/*/STORE SOURCE*/;
 
@@ -86,10 +89,11 @@ data _null_;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
   put ' ';
-  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=PROCJSON,dbg=0 ';
+  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0 ';
   put ')/*/STORE SOURCE*/; ';
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
+  put '  options nobomfile; ';
   put '  data _null_;file &jref encoding=''utf-8''; ';
   put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
   put '  run; ';
@@ -117,7 +121,65 @@ data _null_;
   put '      %put &sysmacroname:  &ds NOT FOUND!!!; ';
   put '      %return; ';
   put '    %end; ';
-  put '    data _null_;file &jref mod ; ';
+  put '    %if &fmt=Y %then %do; ';
+  put '      %put converting every variable to a formatted variable; ';
+  put '      /* see mp_ds2fmtds.sas for source */ ';
+  put '      proc contents noprint data=&ds ';
+  put '        out=_data_(keep=name type length format formatl formatd varnum); ';
+  put '      run; ';
+  put '      proc sort; ';
+  put '        by varnum; ';
+  put '      run; ';
+  put '      %local fmtds; ';
+  put '      %let fmtds=%scan(&syslast,2,.); ';
+  put '      /* prepare formats and varnames */ ';
+  put '      data _null_; ';
+  put '        if _n_=1 then call symputx(''nobs'',nobs,''l''); ';
+  put '        set &fmtds end=last nobs=nobs; ';
+  put '        name=upcase(name); ';
+  put '        /* fix formats */ ';
+  put '        if type=2 or type=6 then do; ';
+  put '          length fmt $49.; ';
+  put '          if format='''' then fmt=cats(''$'',length,''.''); ';
+  put '          else if formatl=0 then fmt=cats(format,''.''); ';
+  put '          else fmt=cats(format,formatl,''.''); ';
+  put '          newlen=max(formatl,length); ';
+  put '        end; ';
+  put '        else do; ';
+  put '          if format='''' then fmt=''best.''; ';
+  put '          else if formatl=0 then fmt=cats(format,''.''); ';
+  put '          else if formatd=0 then fmt=cats(format,formatl,''.''); ';
+  put '          else fmt=cats(format,formatl,''.'',formatd); ';
+  put '          /* needs to be wide, for datetimes etc */ ';
+  put '          newlen=max(length,formatl,24); ';
+  put '        end; ';
+  put '        /* 32 char unique name */ ';
+  put '        newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put ' ';
+  put '        call symputx(cats(''name'',_n_),name,''l''); ';
+  put '        call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '        call symputx(cats(''len'',_n_),newlen,''l''); ';
+  put '        call symputx(cats(''fmt'',_n_),fmt,''l''); ';
+  put '        call symputx(cats(''type'',_n_),type,''l''); ';
+  put '      run; ';
+  put '      data &fmtds; ';
+  put '        /* rename on entry */ ';
+  put '        set &ds(rename=( ';
+  put '      %local i; ';
+  put '      %do i=1 %to &nobs; ';
+  put '        &&name&i=&&newname&i ';
+  put '      %end; ';
+  put '        )); ';
+  put '      %do i=1 %to &nobs; ';
+  put '        length &&name&i $&&len&i; ';
+  put '        &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '        drop &&newname&i; ';
+  put '      %end; ';
+  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '      run; ';
+  put '      %let ds=&fmtds; ';
+  put '    %end; /* &fmt=Y */ ';
+  put '    data _null_;file &jref mod encoding=''utf-8''; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
   put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
@@ -200,11 +262,16 @@ data _null_;
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
-  put '%mend; ';
+  put '%mend mp_jsonout; ';
   put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
-  put '%local i tempds; ';
+  put '%local i tempds jsonengine; ';
+  put ' ';
+  put '/* see https://github.com/sasjs/core/issues/41 */ ';
+  put '%if "%upcase(&SYSENCODING)"="WLATIN1" %then %let jsonengine=PROCJSON; ';
+  put '%else %let jsonengine=DATASTEP; ';
+  put ' ';
   put ' ';
   put '%if &action=FETCH %then %do; ';
   put '  %if %str(&_debug) ge 131 %then %do; ';
@@ -239,7 +306,7 @@ data _null_;
   put '  OPTIONS NOBOMFILE; ';
   put ' ';
   put '  /** ';
-  put '    * check engine type to avoid the below err message: ';
+  put '    * check xengine type to avoid the below err message: ';
   put '    * > Function is only valid for filerefs using the CACHE access method. ';
   put '    */ ';
   put '  data _null_; ';
@@ -260,8 +327,8 @@ data _null_;
   put '%end; ';
   put ' ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
-  put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
+  put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '    ,engine=DATASTEP,dbg=%str(&_debug) ';
+  put '    ,engine=&jsonengine,dbg=%str(&_debug) ';
   put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
@@ -274,7 +341,7 @@ data _null_;
   put '    %local wtcnt;%let wtcnt=0; ';
   put '    data _null_; ';
   put '      set &tempds; ';
-  put '      if not (name =:"DATA"); ';
+  put '      if not (upcase(name) =:"DATA"); /* ignore temp datasets */ ';
   put '      i+1; ';
   put '      call symputx(''wt''!!left(i),name,''l''); ';
   put '      call symputx(''wtcnt'',i,''l''); ';
@@ -294,8 +361,8 @@ data _null_;
   put '        put " ""&wt"" : {"; ';
   put '        put ''"nlobs":'' nlobs; ';
   put '        put '',"nvars":'' nvars; ';
-  put '      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP) ';
+  put '      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=&jsonengine) ';
-  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP) ';
+  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=&jsonengine) ';
   put '      data _null_; file &fref mod encoding=''utf-8''; ';
   put '        put "}"; ';
   put '    %end; ';
@@ -319,6 +386,8 @@ data _null_;
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
+  put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
+  put '    put '',"SYSVLONG" : '' sysvlong; ';
   put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''" ''; ';
   put '    put "}" @; ';
@@ -328,7 +397,7 @@ data _null_;
   put '  run; ';
   put '%end; ';
   put ' ';
-  put '%mend; ';
+  put '%mend mm_webout; ';
   put ' ';
   put '%macro mf_getuser(type=META ';
   put ')/*/STORE SOURCE*/; ';
@@ -407,4 +476,4 @@ run;
 %put &url?_PROGRAM=&path/&name;
 %put ;%put ;%put ;%put ;%put ;%put ;
 
-%mend;
+%mend mm_createwebservice;

meta/mm_getlibmetadiffs.sas

@@ -48,7 +48,7 @@
   #### &prefix.added
   |name:$32.|metaID:$17.|SAStabName:$32.|
   |---|---|---|
-  |||DATA1|
+  | | |DATA1|
 
   #### &prefix.deleted
   |name:$32.|metaID:$17.|SAStabName:$32.|
@@ -59,7 +59,7 @@
   |tabName:$32.|tabMetaID:$17.|SAStabName:$32.|metaName:$32.|metaID:$17.|sasname:$32.|metaType:$16.|change:$64.|
   |---|---|---|---|---|---|---|---|
   |TABLE2|A5XLSNXI.BK0001HN|TABLE2|c|A5XLSNXI.BM000MA9|c|Column|Deleted|
-  ||||d||d|Column|Added|
+  | | | |d| |d|Column|Added|
 
   #### &prefix.meta
   |Label1:$28.|cValue1:$1.|nValue1:D12.3|
@@ -80,9 +80,8 @@
   such as dangling metadata, embedded passwords, security issues and more.
 
   @param [in] libname= the metadata name of the library to be compared
-  @param [out] outlib= The output library in which to store the output tables.
+  @param [out] outlib=(work) The library in which to store the output tables.
-  Default=WORK.
+  @param [out] prefix=(metadiff) The prefix for the four tables created.
-  @param [out] prefix The prefix for the four tables created. Default=metadiff.
 
   @version 9.3
   @author Allan Bowe

meta/mm_getusers.sas

@@ -6,7 +6,7 @@
 
   Usage:
 
-    %mm_getusers()
+      %mm_getusers()
 
   @param outds the dataset to create that contains the list of libraries
 
@@ -68,4 +68,4 @@ filename sxlemap clear;
 filename response clear;
 libname _XML_ clear;
 
-%mend;
+%mend mm_getusers;

meta/mm_webout.sas

@@ -26,7 +26,8 @@
   @param action Either FETCH, OPEN, ARR, OBJ or CLOSE
   @param ds The dataset to send back to the frontend
   @param dslabel= value to use instead of the real name for sending to JSON
-  @param fmt= set to N to send back unformatted values
+  @param fmt=(Y) Set to N to send back unformatted values
+  @param fref=(_webout) The fileref to which to write the JSON
 
   @version 9.3
   @author Allan Bowe
@@ -35,7 +36,12 @@
 %macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y);
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
-%local i tempds;
+%local i tempds jsonengine;
+
+/* see https://github.com/sasjs/core/issues/41 */
+%if "%upcase(&SYSENCODING)"="WLATIN1" %then %let jsonengine=PROCJSON;
+%else %let jsonengine=DATASTEP;
+
 
 %if &action=FETCH %then %do;
   %if %str(&_debug) ge 131 %then %do;
@@ -70,7 +76,7 @@
   OPTIONS NOBOMFILE;
 
   /**
-    * check engine type to avoid the below err message:
+    * check xengine type to avoid the below err message:
     * > Function is only valid for filerefs using the CACHE access method.
     */
   data _null_;
@@ -91,8 +97,8 @@
 %end;
 
 %else %if &action=ARR or &action=OBJ %then %do;
-  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt
+  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=DATASTEP,dbg=%str(&_debug)
+    ,engine=&jsonengine,dbg=%str(&_debug)
   )
 %end;
 %else %if &action=CLOSE %then %do;
@@ -105,7 +111,7 @@
     %local wtcnt;%let wtcnt=0;
     data _null_;
       set &tempds;
-      if not (name =:"DATA");
+      if not (upcase(name) =:"DATA"); /* ignore temp datasets */
       i+1;
       call symputx('wt'!!left(i),name,'l');
       call symputx('wtcnt',i,'l');
@@ -125,8 +131,8 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
+      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=&jsonengine)
-      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=&jsonengine)
       data _null_; file &fref mod encoding='utf-8';
         put "}";
     %end;
@@ -150,6 +156,8 @@
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
+    sysvlong=quote(trim(symget('sysvlong')));
+    put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
     put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
     put "}" @;
@@ -159,4 +167,4 @@
   run;
 %end;
 
-%mend;
+%mend mm_webout;

package-lock.json

@@ -1,96 +1,101 @@
 {
   "name": "@sasjs/core",
-  "version": "1.0.0",
-  "lockfileVersion": 1,
   "requires": true,
+  "lockfileVersion": 1,
   "dependencies": {
     "@sasjs/adapter": {
-      "version": "2.2.13",
+      "version": "2.2.19",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.2.13.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.2.19.tgz",
-      "integrity": "sha512-4tgpythcv+o1WhCwnxk1kt5a/xJc5mnJt7F2ie3vKyAP7bCuYokAP8RS9mwfUw55cFcJtLhxbr+5+3+hA7BTtg==",
+      "integrity": "sha512-M5G9V7GmhNmUOCEEiiPq8WeR9BTcvE1fmyEKFQDfJYuuJF7fG0OC+bZjZ/++qWkW8r3IFteu5y+Id9KswxkNCw==",
       "dev": true,
       "requires": {
-        "@sasjs/utils": "^2.6.3",
+        "@sasjs/utils": "^2.10.2",
         "axios": "^0.21.1",
-        "form-data": "^3.0.0",
+        "form-data": "^4.0.0",
         "https": "^1.0.0"
       }
     },
     "@sasjs/cli": {
-      "version": "2.14.2",
+      "version": "2.22.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.14.2.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.22.2.tgz",
-      "integrity": "sha512-Wnn2VtEqMtQ6zK+wwoRBWBQNzyD7nXS3p5TGx/f03QXTXYnLYBY2nqrjLs1NOWmi5tikQhnbMzdtD65eackDoQ==",
+      "integrity": "sha512-FdQUWVJ0bAj1HOwfMtChMD8v9I0r49BBcYarJ4lSMMbFWwvVgnUmuD2s5sf6quKdyPP12XM06kz3gMYO++NlPg==",
       "dev": true,
       "requires": {
-        "@sasjs/adapter": "^2.2.13",
+        "@sasjs/adapter": "2.2.19",
-        "@sasjs/core": "^2.13.2",
+        "@sasjs/core": "^2.30.6",
-        "@sasjs/lint": "^1.2.0",
+        "@sasjs/lint": "1.8.0",
-        "@sasjs/utils": "^2.8.0",
+        "@sasjs/utils": "2.12.1",
-        "btoa": "^1.2.1",
+        "@types/url-parse": "1.4.3",
-        "chalk": "^4.1.0",
+        "btoa": "1.2.1",
-        "cli-table": "^0.3.6",
+        "chalk": "4.1.1",
-        "csv-stringify": "^5.6.1",
+        "csv-stringify": "5.6.2",
-        "dotenv": "^8.2.0",
+        "dotenv": "8.6.0",
-        "esm": "^3.2.25",
+        "esm": "3.2.25",
-        "find": "^0.3.0",
+        "find": "0.3.0",
         "fs": "0.0.1-security",
-        "fs-extra": "^9.0.1",
+        "fs-extra": "9.1.0",
-        "get-installed-path": "^4.0.8",
+        "get-installed-path": "4.0.8",
-        "jsdom": "^16.5.1",
+        "jsdom": "16.5.3",
-        "jwt-decode": "^3.1.2",
+        "jwt-decode": "3.1.2",
-        "lodash.groupby": "^4.6.0",
+        "lodash.groupby": "4.6.0",
-        "lodash.uniqby": "^4.7.0",
+        "lodash.uniqby": "4.7.0",
-        "node-graphviz": "^0.1.0",
+        "node-graphviz": "0.1.0",
-        "ora": "^5.2.0",
+        "ora": "5.4.0",
-        "rimraf": "^3.0.2",
+        "rimraf": "3.0.2",
-        "shelljs": "^0.8.4"
+        "shelljs": "0.8.4",
+        "url-parse": "1.5.1"
       }
     },
     "@sasjs/core": {
-      "version": "2.15.0",
+      "version": "2.30.6",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.15.0.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.30.6.tgz",
-      "integrity": "sha512-n6ru6QBe9TTnNmrmP5VPPARvUQenQ2K5eEfu0TGofInjTrhx2wcMiffG1Po0fQRNFyA3rHFxrHUISoloZYx8hw==",
+      "integrity": "sha512-OqXEGQ58Ex/6O3OHx9IVcxPAkP17H9NpX6L1/ccAKQ3alBeLKf6xXayUSuM7NtgxumxA9rKYf5R7Ijnfn4H2ow==",
-      "dev": true,
+      "dev": true
-      "requires": {
-        "node-git-hooks": "^1.0.5"
-      }
     },
     "@sasjs/lint": {
-      "version": "1.4.1",
+      "version": "1.8.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.4.1.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.8.0.tgz",
-      "integrity": "sha512-865n0mVb6tQnu25X6iKGcSpadkbu7i4zXzY4zfSGx1B3ifCt+C8BLCeqd1EqvClsPfvpIYQWFPkDtWT9zofFTQ==",
+      "integrity": "sha512-fXijw+viTo7JPtM/1UsAARtoeBaQkDw7qDak+Zq0gje7uIDH1o6aH6DRHI2sfc3+Od9z6UXfPBE9sUhcmVfxBQ==",
       "dev": true,
       "requires": {
-        "@sasjs/utils": "^2.10.1"
+        "@sasjs/utils": "^2.12.0"
       }
     },
     "@sasjs/utils": {
-      "version": "2.10.2",
+      "version": "2.12.1",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.10.2.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.12.1.tgz",
-      "integrity": "sha512-N5nIsz7NUs1Yt0Am0QGs9UXDkN396ialCIfIRsNR9h4VtQRzvOwjXrsLnr3AUAAV9Z8h9CtadkC3W6MAzrQaOg==",
+      "integrity": "sha512-6gZS5zW0J70P7XaVuEczyfHVaVa8Ks/aWr4PIlpJcxWD0enJtCEmos2DdnezdSoNvODkPq/8rzMPqko5jaXK1Q==",
       "dev": true,
       "requires": {
-        "@types/prompts": "^2.0.10",
+        "@types/prompts": "^2.0.11",
+        "chalk": "^4.1.1",
+        "cli-table": "^0.3.6",
         "consola": "^2.15.0",
         "prompts": "^2.4.1",
         "valid-url": "^1.0.9"
       }
     },
     "@types/node": {
-      "version": "14.14.37",
+      "version": "15.3.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.37.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-15.3.0.tgz",
-      "integrity": "sha512-XYmBiy+ohOR4Lh5jE379fV2IU+6Jn4g5qASinhitfyO71b/sCo6MKsMLF5tc7Zf2CE8hViVQyYSobJNke8OvUw==",
+      "integrity": "sha512-8/bnjSZD86ZfpBsDlCIkNXIvm+h6wi9g7IqL+kmFkQ+Wvu3JrasgLElfiPgoo8V8vVfnEi0QVS12gbl94h9YsQ==",
       "dev": true
     },
     "@types/prompts": {
-      "version": "2.0.10",
+      "version": "2.0.12",
-      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.10.tgz",
+      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.12.tgz",
-      "integrity": "sha512-W3PEl3l4vmxdgfY6LUG7ysh+mLJOTOFYmSpiLe6MCo1OdEm8b5s6ZJfuTQgEpYNwcMiiaRzJespPS5Py2tqLlQ==",
+      "integrity": "sha512-Hr6osqfNg3IcQT3pJDXCsSnb0KnldY/hXeJCKJriwbZLnedN9n1e8kcZwLc25GIWULDb6h5aEyOBbf33XpZBXQ==",
       "dev": true,
       "requires": {
         "@types/node": "*"
       }
     },
+    "@types/url-parse": {
+      "version": "1.4.3",
+      "resolved": "https://registry.npmjs.org/@types/url-parse/-/url-parse-1.4.3.tgz",
+      "integrity": "sha512-4kHAkbV/OfW2kb5BLVUuUMoumB3CP8rHqlw48aHvFy5tf9ER0AfOonBlX29l/DD68G70DmyhRlSYfQPSYpC5Vw==",
+      "dev": true
+    },
     "abab": {
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.5.tgz",
@@ -98,9 +103,9 @@
       "dev": true
     },
     "acorn": {
-      "version": "8.1.0",
+      "version": "8.2.4",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.2.4.tgz",
-      "integrity": "sha512-LWCF/Wn0nfHOmJ9rzQApGnxnvgfROzGilS8936rqN/lfcYkY9MYZzdMqN+2NJ4SlTc+m5HiSa+kNfDtI64dwUA==",
+      "integrity": "sha512-Ibt84YwBDDA890eDiDCEqcbwvHlBvzzDkU2cGBBDDI1QWT12jTiXIOn2CIw5KK4i6N5Z2HUxwYjzriDyqaqqZg==",
       "dev": true
     },
     "acorn-globals": {
@@ -203,9 +208,9 @@
       }
     },
     "balanced-match": {
-      "version": "1.0.0",
+      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
-      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
+      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
       "dev": true
     },
     "base64-js": {
@@ -273,9 +278,9 @@
       "dev": true
     },
     "chalk": {
-      "version": "4.1.0",
+      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz",
-      "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+      "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==",
       "dev": true,
       "requires": {
         "ansi-styles": "^4.1.0",
@@ -304,14 +309,6 @@
       "dev": true,
       "requires": {
         "colors": "1.0.3"
-      },
-      "dependencies": {
-        "colors": {
-          "version": "1.0.3",
-          "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz",
-          "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=",
-          "dev": true
-        }
       }
     },
     "clone": {
@@ -336,9 +333,9 @@
       "dev": true
     },
     "colors": {
-      "version": "0.6.2",
+      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/colors/-/colors-0.6.2.tgz",
+      "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz",
-      "integrity": "sha1-JCP+ZnisDF2uiFLl0OW+CMmXq8w=",
+      "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=",
       "dev": true
     },
     "combined-stream": {
@@ -350,12 +347,6 @@
         "delayed-stream": "~1.0.0"
       }
     },
-    "commander": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-2.1.0.tgz",
-      "integrity": "sha1-0SG7roYNmZKj1Re6lvVliOR8Z4E=",
-      "dev": true
-    },
     "concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -468,9 +459,9 @@
       }
     },
     "dotenv": {
-      "version": "8.2.0",
+      "version": "8.6.0",
-      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.6.0.tgz",
-      "integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==",
+      "integrity": "sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==",
       "dev": true
     },
     "ecc-jsbn": {
@@ -568,20 +559,10 @@
         "traverse-chain": "~0.1.0"
       }
     },
-    "findup": {
-      "version": "0.1.5",
-      "resolved": "https://registry.npmjs.org/findup/-/findup-0.1.5.tgz",
-      "integrity": "sha1-itkpozk7rGJ5V6fl3kYjsGsOLOs=",
-      "dev": true,
-      "requires": {
-        "colors": "~0.6.0-1",
-        "commander": "~2.1.0"
-      }
-    },
     "follow-redirects": {
-      "version": "1.13.3",
+      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.3.tgz",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz",
-      "integrity": "sha512-DUgl6+HDzB0iEptNQEXLx/KhTmDb8tZUHSeLqpnjpknR70H0nC2t9N73BK6fN4hOvJ84pKlIQVQ4k5FFlBedKA==",
+      "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg==",
       "dev": true
     },
     "forever-agent": {
@@ -591,9 +572,9 @@
       "dev": true
     },
     "form-data": {
-      "version": "3.0.1",
+      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
-      "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
+      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
       "dev": true,
       "requires": {
         "asynckit": "^0.4.0",
@@ -649,24 +630,10 @@
         "assert-plus": "^1.0.0"
       }
     },
-    "ghooks": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/ghooks/-/ghooks-2.0.4.tgz",
-      "integrity": "sha1-/VDgQP9UiQauQstReToBv+JFZ7k=",
-      "dev": true,
-      "requires": {
-        "findup": "0.1.5",
-        "lodash.clone": "4.5.0",
-        "manage-path": "2.0.0",
-        "opt-cli": "1.5.1",
-        "path-exists": "3.0.0",
-        "spawn-command": "0.0.2"
-      }
-    },
     "glob": {
-      "version": "7.1.6",
+      "version": "7.1.7",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz",
-      "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
+      "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==",
       "dev": true,
       "requires": {
         "fs.realpath": "^1.0.0",
@@ -707,12 +674,6 @@
       "integrity": "sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ==",
       "dev": true
     },
-    "graceful-readlink": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz",
-      "integrity": "sha1-TK+tdrxi8C+gObL5Tpo906ORpyU=",
-      "dev": true
-    },
     "har-schema": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
@@ -823,9 +784,9 @@
       "dev": true
     },
     "is-core-module": {
-      "version": "2.2.0",
+      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.4.0.tgz",
-      "integrity": "sha512-XRAfAdyyY5F5cOXn7hYQDqh2Xmii+DEfIcQGxK/uNwMHhIkPWO0g8msXcbzLe+MpGoR951MlqM/2iIlU4vKDdQ==",
+      "integrity": "sha512-6A2fkfq1rfeQZjxrZJGerpLCTHRNEBiSgnu0+obeJpEPZRUooHgsizvzv0ZjJwOz3iWIHdJtVWJ/tmPr3D21/A==",
       "dev": true,
       "requires": {
         "has": "^1.0.3"
@@ -838,9 +799,9 @@
       "dev": true
     },
     "is-potential-custom-element-name": {
-      "version": "1.0.0",
+      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
-      "integrity": "sha1-DFLlS8yjkbssSUsh6GJtczbG45c=",
+      "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==",
       "dev": true
     },
     "is-typedarray": {
@@ -880,9 +841,9 @@
       "dev": true
     },
     "jsdom": {
-      "version": "16.5.2",
+      "version": "16.5.3",
-      "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.5.2.tgz",
+      "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.5.3.tgz",
-      "integrity": "sha512-JxNtPt9C1ut85boCbJmffaQ06NBnzkQY/MWO3YxPW8IWS38A26z+B1oBvA9LwKrytewdfymnhi4UNH3/RAgZrg==",
+      "integrity": "sha512-Qj1H+PEvUsOtdPJ056ewXM4UJPCi4hhLA8wpiz9F2YvsRBhuFsXxtrIFAgGBDynQA9isAMGE91PfUYbdMPXuTA==",
       "dev": true,
       "requires": {
         "abab": "^2.0.5",
@@ -981,18 +942,6 @@
       "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==",
       "dev": true
     },
-    "lodash._baseclone": {
-      "version": "4.5.7",
-      "resolved": "https://registry.npmjs.org/lodash._baseclone/-/lodash._baseclone-4.5.7.tgz",
-      "integrity": "sha1-zkKt4IOE711i+nfDD2GkbmhvhDQ=",
-      "dev": true
-    },
-    "lodash.clone": {
-      "version": "4.5.0",
-      "resolved": "https://registry.npmjs.org/lodash.clone/-/lodash.clone-4.5.0.tgz",
-      "integrity": "sha1-GVhwRQ9aExkkeN9Lw9I9LeoZB7Y=",
-      "dev": true
-    },
     "lodash.groupby": {
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz",
@@ -1015,12 +964,6 @@
         "is-unicode-supported": "^0.1.0"
       }
     },
-    "manage-path": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/manage-path/-/manage-path-2.0.0.tgz",
-      "integrity": "sha1-9M+EV7km7u4qg7FzUBQUvHbrlZc=",
-      "dev": true
-    },
     "mime-db": {
       "version": "1.47.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz",
@@ -1051,12 +994,6 @@
         "brace-expansion": "^1.1.7"
       }
     },
-    "node-git-hooks": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/node-git-hooks/-/node-git-hooks-1.0.5.tgz",
-      "integrity": "sha512-05rULsJy8z2OvXTQFZv9fN20uo186EYgJYQjkL1OjnXj53QivOAGUzZilZ/MX8OmPRaN+deJBtlvMRydpdfnqA==",
-      "dev": true
-    },
     "node-graphviz": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/node-graphviz/-/node-graphviz-0.1.0.tgz",
@@ -1093,44 +1030,6 @@
         "mimic-fn": "^2.1.0"
       }
     },
-    "opt-cli": {
-      "version": "1.5.1",
-      "resolved": "https://registry.npmjs.org/opt-cli/-/opt-cli-1.5.1.tgz",
-      "integrity": "sha1-BNtEexPJa5kusxaFJm9O0NlzbcI=",
-      "dev": true,
-      "requires": {
-        "commander": "2.9.0",
-        "lodash.clone": "4.3.2",
-        "manage-path": "2.0.0",
-        "spawn-command": "0.0.2-1"
-      },
-      "dependencies": {
-        "commander": {
-          "version": "2.9.0",
-          "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz",
-          "integrity": "sha1-nJkJQXbhIkDLItbFFGCYQA/g99Q=",
-          "dev": true,
-          "requires": {
-            "graceful-readlink": ">= 1.0.0"
-          }
-        },
-        "lodash.clone": {
-          "version": "4.3.2",
-          "resolved": "https://registry.npmjs.org/lodash.clone/-/lodash.clone-4.3.2.tgz",
-          "integrity": "sha1-5WsXa2gjp93jj38r9Y3n1ZcSAOk=",
-          "dev": true,
-          "requires": {
-            "lodash._baseclone": "~4.5.0"
-          }
-        },
-        "spawn-command": {
-          "version": "0.0.2-1",
-          "resolved": "https://registry.npmjs.org/spawn-command/-/spawn-command-0.0.2-1.tgz",
-          "integrity": "sha1-YvXpRmmBwbeW3Fkpk34RycaSG9A=",
-          "dev": true
-        }
-      }
-    },
     "optionator": {
       "version": "0.8.3",
       "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz",
@@ -1174,12 +1073,6 @@
       "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==",
       "dev": true
     },
-    "path-exists": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
-      "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
-      "dev": true
-    },
     "path-is-absolute": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
@@ -1232,6 +1125,12 @@
       "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==",
       "dev": true
     },
+    "querystringify": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
+      "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==",
+      "dev": true
+    },
     "readable-stream": {
       "version": "3.6.0",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@@ -1335,6 +1234,12 @@
         }
       }
     },
+    "requires-port": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
+      "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=",
+      "dev": true
+    },
     "resolve": {
       "version": "1.20.0",
       "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz",
@@ -1425,12 +1330,6 @@
       "dev": true,
       "optional": true
     },
-    "spawn-command": {
-      "version": "0.0.2",
-      "resolved": "https://registry.npmjs.org/spawn-command/-/spawn-command-0.0.2.tgz",
-      "integrity": "sha1-lUThpDygRfhTGqwaSMspva5iM44=",
-      "dev": true
-    },
     "sshpk": {
       "version": "1.16.1",
       "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",
@@ -1560,6 +1459,16 @@
         "punycode": "^2.1.0"
       }
     },
+    "url-parse": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.1.tgz",
+      "integrity": "sha512-HOfCOUJt7iSYzEx/UqgtwKRMC6EU91NFhsCHMv9oM03VJcVo2Qrp8T8kI9D7amFf1cu+/3CEhgb3rF9zL7k85Q==",
+      "dev": true,
+      "requires": {
+        "querystringify": "^2.1.1",
+        "requires-port": "^1.0.0"
+      }
+    },
     "util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
@@ -1670,9 +1579,9 @@
       "dev": true
     },
     "ws": {
-      "version": "7.4.4",
+      "version": "7.4.6",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.4.tgz",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz",
-      "integrity": "sha512-Qm8k8ojNQIMx7S+Zp8u/uHOx7Qazv3Yv4q68MiWWWOJhiwG5W3x7iqmRtJo8xxrciZUY4vRxUTJCKuRnF28ZZw==",
+      "integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==",
       "dev": true
     },
     "xml-name-validator": {

package.json

@@ -23,20 +23,17 @@
   "bugs": {
     "url": "https://github.com/sasjs/core/issues"
   },
-  "homepage": "https://github.com/sasjs/core#readme",
+  "homepage": "https://core.sasjs.io",
-  "version": "1.0.0",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "build": "sasjs cbd -t viya",
-    "docs": "sasjs doc && ./sasjs/utils/build.sh"
+    "docs": "sasjs doc && ./sasjs/utils/build.sh",
+    "test": "sasjs test -t viya",
+    "lint": "sasjs lint",
+    "postinstall": "[ -d .git ] && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "^2.14.2",
+    "@sasjs/cli": "^2.22.2"
-    "ghooks": "^2.0.4"
   },
-  "config": {
+  "dependencies": {}
-    "ghooks": {
-      "pre-commit": "sasjs lint"
-    }
-  }
 }

sasjs/sasjsconfig.json

@@ -5,7 +5,8 @@
     "meta",
     "metax",
     "viya",
-    "lua"
+    "lua",
+    "tests/base"
   ],
   "docConfig": {
     "displayMacroCore": false,
@@ -16,6 +17,13 @@
       "readMe": "../../README.md"
     }
   },
+  "testConfig": {
+    "initProgram": "tests/testinit.sas",
+    "termProgram": "tests/testterm.sas",
+    "macroVars": {
+      "mcTestAppLoc": "/Public/temp/macrocore"
+    }
+  },
   "defaultTarget": "viya",
   "targets": [
     {
@@ -27,22 +35,18 @@
         "deployServicePack": true
       },
       "macroFolders": [
-        "base",
-        "meta",
-        "metax",
-        "viya",
-        "lua",
-        "tests/base",
         "tests/viya"
       ],
-      "contextName": "SAS Job Execution compute context",
+      "contextName": "SAS Job Execution compute context"
-      "testConfig": {
+    },
-        "initProgram": "tests/testinit.sas",
+    {
-        "termProgram": "tests/testterm.sas",
+      "name": "sas9",
-        "macroVars": {
+      "serverUrl": "https://sas.analytium.co.uk:8343",
-          "mcTestAppLoc": "/Public/temp/macrocore"
+      "serverType": "SAS9",
-        }
+      "appLoc": "/Shared Data/temp/macrocore",
-      }
+      "macroFolders": [
+        "tests/meta"
+      ]
     }
   ]
 }

tests/base/mp_abort.test.nofix.sas

@@ -0,0 +1,36 @@
+/**
+  @file
+  @brief Testing mp_abort macro
+  @details This is an unfixed problem with mp_abort when using the
+  'unclosed macro' technique.  This is only relevant for 9.4m3 environments,
+  which can suffer from hung multibridge sessions from %abort and endsas.
+
+  The issue is that when called within a macro, within a %include, AND that
+  macro contains subsequent logic, the service does not end cleanly - rather,
+  we see:
+
+      ERROR: %EVAL function has no expression to evaluate, or %IF statement has no condition.
+      ERROR: The macro TEST will stop executing.
+
+  We are not able to test this without a 9.4m3 environment, it is marked as
+  nofix.
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_assert.sas
+
+**/
+
+%macro test();
+
+filename blah temp;
+data _null_;
+  file blah;
+  put '%mp_abort();';
+run;
+%inc blah;
+
+%if 1=1 %then %put Houston - we have a problem here;
+%mend test;
+
+%test()

tests/base/mp_assertdsobs.test.sas

@@ -0,0 +1,60 @@
+/**
+  @file
+  @brief Testing mp_assertdsobs.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assertdsobs.sas
+  @li mp_assertcolvals.sas
+
+**/
+
+
+data work.somedata;
+  do x=1 to 15;
+    output;
+  end;
+run;
+
+%mp_assertdsobs(work.somedata,
+  test=ATLEAST 15,
+  outds=work.test_the_test
+)
+
+%mp_assertdsobs(work.somedata,
+  test=ATMOST 15,
+  outds=work.test_the_test
+)
+
+data work.check;
+  val='PASS';
+run;
+%mp_assertcolvals(work.test_the_test.test_result,
+  checkvals=work.check.val,
+  desc=Testing ATLEAST / ATMOST for passes,
+  test=ALLVALS
+)
+
+%mp_assertdsobs(work.somedata,
+  test=ATLEAST 16,
+  outds=work.test_the_test2
+)
+%mp_assertdsobs(work.somedata,
+  test=ATMOST 14,
+  outds=work.test_the_test2
+)
+
+data _null_;
+  set work.test_the_test2;
+  putlog (_all_)(=);
+run;
+
+data work.check2;
+  val='FAIL';
+run;
+%mp_assertcolvals(work.test_the_test2.test_result,
+  checkvals=work.check2.val,
+  desc=Testing ATLEAST / ATMOST for failures,
+  test=ALLVALS
+)
+
+

tests/base/mp_ds2fmtds.test.sas

@@ -0,0 +1,28 @@
+/**
+  @file
+  @brief Testing mp_ds2fmtds.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2fmtds.sas
+  @li mp_assert.sas
+
+**/
+
+proc sql;
+create table test as select * from dictionary.tables where libname='SASHELP';
+
+filename inc temp;
+data _null_;
+  set work.test;
+  file inc;
+  line=cats('%mp_ds2fmtds(sashelp.',memname,',',memname,')');
+  put line;
+run;
+
+%inc inc;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking tables were created successfully,
+  outds=work.test_results
+)

tests/base/mp_filtervalidate.test.sas

@@ -6,6 +6,7 @@
   @li mp_filtergenerate.sas
   @li mp_filtervalidate.sas
   @li mp_assertdsobs.sas
+  @li mp_assert.sas
 
 **/
 
@@ -41,14 +42,14 @@ run;
 %mp_filtergenerate(work.inds,outref=myfilter)
 %mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
 %mp_assertdsobs(work.results,
-  desc=Valid filter,
+  desc=Empty filter,
   test=EMPTY,
   outds=work.test_results
 )
 
 
 
-/* invalid filter*/
+/* invalid filter - char var, num val */
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
@@ -60,9 +61,40 @@ run;
 %mp_filtergenerate(work.inds,outref=myfilter)
 %mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
 %let syscc=0;
-%mp_assertdsobs(work.results,
+%let test3=0;
-  desc=Valid filter,
+data _null_;
-  test=EQUALS 1,
+  set work.results;
+  if REASON_CD=:'VALIDATION_ERROR' then call symputx('test3',1);
+  putlog (_all_)(=);
+  stop;
+run;
+%mp_assert(
+  iftrue=(&test3=1),
+  desc=Checking char var could not receive num val,
   outds=work.test_results
 )
 
+/* invalid filter - num var, char val */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,age,NE,"'M'"
+;;;;
+run;
+%mp_filtergenerate(work.inds,outref=myfilter)
+%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%let syscc=0;
+%let test4=0;
+data _null_;
+  set work.results;
+  if REASON_CD=:'VALIDATION_ERROR' then call symputx('test4',1);
+  putlog (_all_)(=);
+  stop;
+run;
+%mp_assert(
+  iftrue=(&test4=1),
+  desc=Checking num var could not receive char val,
+  outds=work.test_results
+)

tests/base/mp_jsonout.test.sas

@@ -0,0 +1,42 @@
+/**
+  @file
+  @brief Testing mp_jsonout.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_jsonout.sas
+  @li mp_assert.sas
+
+**/
+
+filename webref temp;
+
+data demo;
+  dtval=date();
+  format dtval date9.;
+  compare=put(date(),date9.);
+  call symputx('compare',compare);
+run;
+
+%mp_jsonout(OPEN,jref=webref)
+%mp_jsonout(OBJ,demo,jref=webref,fmt=Y)
+%mp_jsonout(CLOSE,jref=webref)
+
+data _null_;
+  infile webref;
+  input;
+  putlog _infile_;
+run;
+
+libname web JSON fileref=webref;
+%let dtval=0;
+data work.test;
+  set web.demo;
+  call symputx('dtval',dtval);
+run;
+
+
+%mp_assert(
+  iftrue=(&dtval=&compare),
+  desc=Checking tables were created successfully,
+  outds=work.test_results
+)

tests/base/mp_searchcols.test.sas

@@ -0,0 +1,50 @@
+/**
+  @file
+  @brief Testing mp_searchcols.sas
+
+  <h4> SAS Macros </h4>
+  @li mp_searchcols.sas
+  @li mp_assertdsobs.sas
+
+
+**/
+
+
+/** Test 1 - full col match  */
+data example1;
+  var1=1;
+  var2=2;
+  var3=3;
+data example2;
+  var1=1;
+  var2=2;
+data example3;
+  var2=2;
+  var3=3;
+data example4;
+  matchmehere=1;
+data example5;
+  hereyoucan_matchme_also=1;
+data example6;
+  do_not_forget_me=1;
+data example7;
+  we_shall_not_forget=1;
+run;
+
+%mp_searchcols(libs=work,cols=var1 var2,outds=testme)
+
+%mp_assertdsobs(work.testme,
+  desc=Test1 - check exact variables are found,
+  test=EQUALS 3,
+  outds=work.test_results
+)
+
+/* test 2 - wildcard match */
+
+%mp_searchcols(libs=work,cols=matchme forget,match=WILD, outds=testme2)
+
+%mp_assertdsobs(work.testme2,
+  desc=Test1 - check fuzzy matches are found,
+  test=EQUALS 4,
+  outds=work.test_results
+)

tests/meta/mm_webout.test.sas

@@ -0,0 +1,35 @@
+/**
+  @file
+  @brief Testing mm_webout macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mm_webout.sas
+  @li mp_assert.sas
+
+**/
+
+
+%let fref=%mf_getuniquefileref();
+%global _metaperson;
+data some datasets;
+  x=1;
+run;
+%mm_webout(OPEN,fref=&fref)
+%mm_webout(ARR,some,fref=&fref)
+%mm_webout(OBJ,datasets,fref=&fref)
+%mm_webout(CLOSE,fref=&fref)
+
+libname test JSON (&fref);
+data root;
+  set test.root;
+  call symputx('checkval',sysvlong);
+run;
+data alldata;
+  set test.alldata;
+run;
+
+%mp_assert(
+  iftrue=(%str(&checkval)=%str(&sysvlong)),
+  desc=Check if the sysvlong value was created
+)

tests/viya/mfv_existfile.test.sas

@@ -0,0 +1,34 @@
+/**
+  @file
+  @brief Testing mfv_existfile macro function
+
+  <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mfv_existfile.sas
+  @li mp_assert.sas
+  @li mv_createfile.sas
+
+**/
+
+options mprint sgen;
+
+%let file=%mf_uid();
+
+/* create a folder */
+filename somefile temp;
+data _null_;
+  file somefile;
+  put 'hello testings';
+run;
+%mv_createfile(path=&mcTestAppLoc/temp, name=&file..txt,inref=somefile)
+
+
+%mp_assert(
+  iftrue=(%mfv_existfile(&mcTestAppLoc/temp/&file..txt)=1),
+  desc=Check if created file exists
+)
+
+%mp_assert(
+  iftrue=(%mfv_existfile(&mcTestAppLoc/temp/%mf_uid().txt)=0),
+  desc=Check if non created file does not exist
+)

tests/viya/mfv_existfolder.test.sas

@@ -0,0 +1,29 @@
+/**
+  @file
+  @brief Testing mfv_existfolder macro function
+
+  <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mfv_existfolder.sas
+  @li mp_assert.sas
+  @li mv_createfolder.sas
+
+**/
+
+options mprint sgen;
+
+%let folder=%mf_uid();
+
+/* create a folder */
+%mv_createfolder(path=&mcTestAppLoc/temp/&folder)
+
+
+%mp_assert(
+  iftrue=(%mfv_existfolder(&mcTestAppLoc/temp/&folder)=1),
+  desc=Check if created folder exists
+)
+
+%mp_assert(
+  iftrue=(%mfv_existfolder(&mcTestAppLoc/temp/&folder/%mf_uid()/noway)=0),
+  desc=Check if non created folder does not exist
+)

tests/viya/mv_createfile.test.sas

@@ -0,0 +1,42 @@
+/**
+  @file
+  @brief Testing mv_createfile macro
+
+  <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mfv_existfile.sas
+  @li mp_assert.sas
+  @li mv_createfile.sas
+
+
+**/
+
+options mprint;
+
+%let file=%mf_uid();
+
+%put TEST 1 - basic file upload ;
+filename somefile temp;
+data _null_;
+  file somefile;
+  put 'hello testings';
+run;
+%mv_createfile(path=&mcTestAppLoc/temp, name=&file..txt,inref=somefile)
+
+%mp_assert(
+  iftrue=(%mfv_existfile(&mcTestAppLoc/temp/&file..txt)=1),
+  desc=Check if created file exists
+)
+
+%put TEST 2 - dataset upload ;
+data temp;
+x=1;
+run;
+filename ds "%sysfunc(pathname(work))/temp.sas7bdat";
+
+%mv_createfile(path=&mcTestAppLoc/temp, name=&file..sas7bdat,inref=ds)
+
+%mp_assert(
+  iftrue=(%mfv_existfile(&mcTestAppLoc/temp/&file..sas7bdat)=1),
+  desc=Check if created dataset exists
+)

tests/viya/mv_createfolder.test.sas

@@ -0,0 +1,32 @@
+/**
+  @file
+  @brief Testing mv_createfolder macro
+
+  <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mp_assert.sas
+  @li mv_createfolder.sas
+  @li mv_deleteviyafolder.sas
+  @li mv_getfoldermembers.sas
+
+**/
+
+
+%let folder=%mf_uid();
+
+/* create a folder */
+%mv_createfolder(path=&mcTestAppLoc/temp/&folder/&folder)
+
+%mv_getfoldermembers(root=&mcTestAppLoc/temp/&folder, outds=work.folders)
+
+%let test=0;
+data _null_;
+  set work.folders;
+  putlog (_all_)(=);
+  if name="&folder" then call symputx('test',1);
+run;
+
+%mp_assert(
+  iftrue=(&test=1),
+  desc=Check if temp folder can be successfully created
+)

tests/viya/mv_getjoblog.test.sas

@@ -0,0 +1,70 @@
+/**
+  @file
+  @brief Testing mv_createwebservice macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mv_createjob.sas
+  @li mv_jobexecute.sas
+  @li mv_jobwaitfor.sas
+  @li mv_getjoblog.sas
+
+**/
+
+/**
+  * Test Case 1
+  */
+
+/* create a service */
+filename testref temp;
+data _null_;
+  file testref;
+  put 'data;run;';
+  put 'endsas;';
+run;
+%mv_createjob(
+  path=&mcTestAppLoc/jobs/temp,
+  code=testref,
+  name=testjob
+)
+
+%* Execute it;
+%mv_jobexecute(
+  path=&mcTestAppLoc/jobs/temp,
+  name=testjob,
+  outds=work.info
+)
+
+%* Wait for it to finish;
+data work.info;
+  set work.info;
+  where method='GET' and rel='state';
+run;
+%mv_jobwaitfor(ALL,inds=work.info,outds=work.jobstates)
+
+%* and grab the uri;
+data _null_;
+  set work.jobstates;
+  call symputx('uri',uri);
+run;
+
+%* Finally, fetch the log;
+%mv_getjoblog(uri=%str(&uri),outref=mylog)
+
+
+data _null_;
+  infile mylog end=eof;
+  input;
+  putlog _infile_;
+  retain found 0;
+  if index(_infile_,'endsas;') then do;
+    found=1;
+    call symputx('found',found);
+  end;
+  else if eof and found ne 1 then call symputx('found',0);
+run;
+
+%mp_assert(
+  iftrue=(%str(&found)=1),
+  desc=Check if the log was still fetched even though endsas was submitted
+)

tests/viya/mv_getjobresult.test.sas

@@ -10,6 +10,8 @@
 
 **/
 
+options mprint notes;
+
 /**
   * Test Case 1
   */

tests/viya/mv_jobflow.test.1.sas

@@ -0,0 +1,78 @@
+/**
+
+  @file
+  @brief Testing mv_jobflow macro
+  @details One of the remote jobs aborts with syscc>0 - test to
+  make sure this comes back to the calling session
+​
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mv_createjob.sas
+  @li mv_jobflow.sas
+
+**/
+
+/**
+  * Test Case 1
+  */
+
+filename testprog temp;
+data _null_;
+  file testprog;
+  put '%put this is job: &_program;'
+  /   '%put this was run in flow &flow_id;'
+  /   'data ;'
+  /   '  rval=rand("uniform");'
+  /   '  rand=rval*&macrovar1;'
+  /   '  do x=1 to rand;'
+  /   '    y=rand*&macrovar2;'
+  /   '    if (rval>0.50) then abort;'
+  /   '    else output;'
+  /   '  end;'
+  /   'run;'
+  ;
+run;
+
+%mv_createjob(path=/Public/temp,name=demo1,code=testprog)
+%mv_createjob(path=/Public/temp,name=demo2,code=testprog)
+
+data work.inputjobs;
+  _contextName='SAS Job Execution compute context';
+  do flow_id=1 to 2;
+    do i=1 to 4;
+      _program='/Public/temp/demo1';
+      macrovar1=10*i;
+      macrovar2=4*i;
+      output;
+      i+1;
+      _program='/Public/temp/demo2';
+      macrovar1=40*i;
+      macrovar2=44*i;
+      output;
+    end;
+  end;
+run;
+
+* Trigger the flow ;
+
+%put NOTE: &=syscc;
+
+%mv_jobflow(inds=work.inputjobs
+  ,maxconcurrency=2
+  ,outds=work.results
+  ,outref=myjoblog
+  ,raise_err=1
+  ,mdebug=1
+)
+
+%put NOTE: &=syscc;
+
+data _null_;
+  infile myjoblog;
+  input; put _infile_;
+run;
+
+%mp_assert(
+  iftrue=(&syscc ne 0),
+  desc=Check that non zero return code is returned if called job fails
+)

tests/viya/mv_jobflow.test.2.sas

@@ -0,0 +1,74 @@
+/**
+  @file
+  @brief Testing mv_jobflow macro
+  @details All jobs complete successfully with syscc = 0 - test to
+  make sure this comes back to the calling session
+​
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mv_createjob.sas
+  @li mv_jobflow.sas
+**/
+
+/**
+  * Test Case 1
+  */
+filename testprog temp;
+data _null_;
+  file testprog;
+  put '%put this is job: &_program;'
+  /   '%put this was run in flow &flow_id;'
+  /   'data ;'
+  /   '  rval=rand("uniform");'
+  /   '  rand=rval*&macrovar1;'
+  /   '  do x=1 to rand;'
+  /   '    y=rand*&macrovar2;'
+  /   '    output;'
+  /   '  end;'
+  /   'run;'
+  ;
+run;
+
+%mv_createjob(path=/Public/temp,name=demo1,code=testprog)
+%mv_createjob(path=/Public/temp,name=demo2,code=testprog)
+
+data work.inputjobs;
+  _contextName='SAS Job Execution compute context';
+  do flow_id=1 to 2;
+    do i=1 to 4;
+      _program='/Public/temp/demo1';
+      macrovar1=10*i;
+      macrovar2=4*i;
+      output;
+      i+1;
+      _program='/Public/temp/demo2';
+      macrovar1=40*i;
+      macrovar2=44*i;
+      output;
+    end;
+  end;
+run;
+
+* Trigger the flow ;
+
+%put NOTE: &=syscc;
+
+%mv_jobflow(inds=work.inputjobs
+  ,maxconcurrency=2
+  ,outds=work.results
+  ,outref=myjoblog
+  ,raise_err=1
+  ,mdebug=1
+)
+
+%put NOTE: &=syscc;
+
+data _null_;
+  infile myjoblog;
+  input; put _infile_;
+run;
+
+%mp_assert(
+  iftrue=(&syscc eq 0),
+  desc=Check that a zero return code is returned if no called job fails
+)

tests/viya/mv_registerclient.test.1.sas

@@ -0,0 +1,36 @@
+/**
+
+  @file
+  @brief Testing mv_registerclient.sas macro
+  @details Tests for successful registration.  For this to work, the test
+  account must be an admin.
+​
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mp_assertcolvals.sas
+  @li mv_registerclient.sas
+
+**/
+
+/**
+  * Test Case 1
+  */
+
+%let id=%mf_getuniquename();
+%let sec=%mf_getuniquename();
+%mv_registerclient(client_id=&id,client_secret=&sec, outds=testds)
+
+data work.checkds;
+  id="&id";
+  sec="&sec";
+run;
+%mp_assertcolvals(work.testds.client_id,
+  checkvals=work.checkds.id,
+  desc=Checking client id was created
+  test=ALLVALS
+)
+%mp_assertcolvals(work.testds.client_secret,
+  checkvals=work.checkds.sec,
+  desc=Checking client secret was created
+  test=ALLVALS
+)

tests/viya/mv_registerclient.test.2.sas

@@ -0,0 +1,43 @@
+/**
+
+  @file
+  @brief Testing mv_registerclient.sas macro
+  @details Tests for unsuccessful registration.  To do this, overrides are
+  applied for the mf_loc.sas and mp_abort.sas macros.
+​
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mv_registerclient.sas
+
+**/
+
+/**
+  * Test Case
+  */
+
+%macro mf_loc(param);
+  /does/not/exist
+%mend mf_loc;
+
+%macro mp_abort(iftrue=,mac=mp_abort.sas, type=, msg=);
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+  %put %substr(&msg,1,16);
+  %mp_assert(
+    iftrue=("%substr(&msg,1,16)"="Unable to access"),
+    desc=Check that abort happens when consul token is unavailable
+  )
+  %webout(OPEN)
+  %webout(OBJ, TEST_RESULTS)
+  %webout(CLOSE)
+  %let syscc=0;
+  data _null_;
+    abort cancel nolist;
+  run;
+%mend mp_abort;
+
+%mv_registerclient( outds=testds)
+
+%mp_assert(
+  iftrue=(0=1),
+  desc=Check that abort happens when consul token is unavailable
+)

tests/viya/mv_webout.test.sas

@@ -0,0 +1,41 @@
+/**
+  @file
+  @brief Testing mm_webout macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mv_webout.sas
+  @li mp_assert.sas
+
+**/
+
+
+%let fref=%mf_getuniquefileref();
+%global _metaperson;
+data some datasets;
+  x=1;
+run;
+%mv_webout(OPEN,fref=&fref,stream=N)
+%mv_webout(ARR,some,fref=&fref,stream=N)
+%mv_webout(OBJ,datasets,fref=&fref,stream=N)
+%mv_webout(CLOSE,fref=&fref,stream=N)
+
+data _null_;
+  infile &fref;
+  input;
+  putlog _infile_;
+run;
+
+libname test JSON (&fref);
+data root;
+  set test.root;
+  call symputx('checkval',sysvlong);
+run;
+data alldata;
+  set test.alldata;
+run;
+
+%mp_assert(
+  iftrue=(%str(&checkval)=%str(&sysvlong)),
+  desc=Check if the sysvlong value was created
+)

viya/mfv_existfile.sas

@@ -0,0 +1,51 @@
+/**
+  @file
+  @brief Checks whether a file exists in SAS Drive
+  @details Returns 1 if the file exists, and 0 if it doesn't.  Works by
+  attempting to assign a fileref with the filesrvc engine.  If not found, the
+  syscc is automatically set to a non zero value - so in this case it is reset.
+  To avoid hiding issues, there is therefore a test at the start to ensure the
+  syscc is zero.
+
+  Usage:
+
+      %put %mfv_existfile(/does/exist.txt);
+      %put %mfv_existfile(/does/not/exist.txt);
+
+  @param filepath The full path to the file on SAS drive (eg /Public/myfile.txt)
+
+  <h4> SAS Macros </h4>
+  @li mf_abort.sas
+  @li mf_getuniquefileref.sas
+
+  <h4> Related Macros </h4>
+  @li mfv_existfolder.sas
+
+  @version 3.5
+  @author [Allan Bowe](https://www.linkedin.com/in/allanbowe/)
+**/
+
+%macro mfv_existfile(filepath
+)/*/STORE SOURCE*/;
+
+  %mf_abort(
+    iftrue=(&syscc ne 0),
+    msg=Cannot enter mfv_existfile.sas with syscc=&syscc
+  )
+
+  %local fref rc path name;
+  %let fref=%mf_getuniquefileref();
+  %let name=%scan(&filepath,-1,/);
+  %let path=%substr(&filepath,1,%length(&filepath)-%length(&name)-1);
+
+  %if %sysfunc(filename(fref,,filesrvc,folderPath="&path" filename="&name"))=0
+  %then %do;
+    %sysfunc(fexist(&fref))
+    %let rc=%sysfunc(filename(fref));
+  %end;
+  %else %do;
+    0
+    %let syscc=0;
+  %end;
+
+%mend mfv_existfile;

viya/mfv_existfolder.sas

@@ -0,0 +1,48 @@
+/**
+  @file
+  @brief Checks whether a folder exists in SAS Drive
+  @details Returns 1 if the folder exists, and 0 if it doesn't.  Works by
+  attempting to assign a fileref with the filesrvc engine.  If not found, the
+  syscc is automatically set to a non zero value - so in this case it is reset.
+  To avoid hiding issues, there is therefore a test at the start to ensure the
+  syscc is zero.
+
+  Usage:
+
+      %put %mfv_existfolder(/does/exist);
+      %put %mfv_existfolder(/does/not/exist);
+
+  @param path The path to the folder on SAS drive
+
+  <h4> SAS Macros </h4>
+  @li mf_abort.sas
+  @li mf_getuniquefileref.sas
+
+  <h4> Related Macros </h4>
+  @li mfv_existfile.sas
+
+  @version 3.5
+  @author [Allan Bowe](https://www.linkedin.com/in/allanbowe/)
+**/
+
+%macro mfv_existfolder(path
+)/*/STORE SOURCE*/;
+
+  %mf_abort(
+    iftrue=(&syscc ne 0),
+    msg=Cannot enter mfv_existfolder.sas with syscc=&syscc
+  )
+
+  %local fref rc;
+  %let fref=%mf_getuniquefileref();
+
+  %if %sysfunc(filename(fref,,filesrvc,folderPath="&path"))=0 %then %do;
+    1
+    %let rc=%sysfunc(filename(fref));
+  %end;
+  %else %do;
+    0
+    %let syscc=0;
+  %end;
+
+%mend mfv_existfolder;

viya/mv_createfile.sas

@@ -0,0 +1,113 @@
+/**
+  @file
+  @brief Creates a file in SAS Drive
+  @details Creates a file in SAS Drive and adds the appropriate content type.
+  If the parent folder does not exist, it is created.
+
+  Usage:
+
+      filename myfile temp;
+      data _null_;
+        file myfile;
+        put 'something';
+      run;
+      %mv_createfile(path=/Public/temp,name=newfile.txt,inref=myfile)
+
+
+  @param [in] path= The parent folder in which to create the file
+  @param [in] name= The name of the file to be created
+  @param [in] inref= The fileref pointing to the file to be uploaded
+  @param [in] contentdisp= (inline) Content Disposition. Example values:
+      @li inline
+      @li attachment
+
+  @param [in] access_token_var= The global macro variable to contain the access
+    token, if using authorization_code grant type.
+  @param [in] grant_type= (sas_services) Valid values are:
+    @li password
+    @li authorization_code
+    @li sas_services
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  @version VIYA V.03.05
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_isblank.sas
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+  @li mv_createfolder.sas
+
+**/
+
+%macro mv_createfile(path=
+    ,name=
+    ,inref=
+    ,contentdisp=inline
+    ,access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+    ,mdebug=0
+  );
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+  %let oauth_bearer=oauth_bearer=sas_services;
+  %let &access_token_var=;
+%end;
+
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+
+%mp_abort(iftrue=(%mf_isblank(&path)=1 or %length(&path)=1)
+  ,mac=&sysmacroname
+  ,msg=%str(path value must be provided)
+)
+%mp_abort(iftrue=(%mf_isblank(&name)=1 or %length(&name)=1)
+  ,mac=&sysmacroname
+  ,msg=%str(name value with length >1 must be provided)
+)
+
+/* create folder if it does not already exist */
+%mv_createfolder(path=&path
+  ,access_token_var=&access_token_var
+  ,grant_type=&grant_type
+  ,mdebug=&mdebug
+)
+
+/* create file with relevant options */
+%local fref;
+%let fref=%mf_getuniquefileref();
+filename &fref filesrvc
+  folderPath="&path"
+  filename="&name"
+  cdisp="&contentdisp";
+
+%mp_binarycopy(inref=&inref, outref=&fref)
+
+filename &fref clear;
+
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+%put &sysmacroname: File &name successfully created in &path;
+%put &sysmacroname:;%put;
+%put    &base_uri/SASJobExecution?_file=&path/&name;%put;
+%put &sysmacroname:;
+
+%mend mv_createfile;

viya/mv_createfolder.sas

@@ -1,17 +1,22 @@
 /**
   @file mv_createfolder.sas
   @brief Creates a viya folder if that folder does not already exist
-  @details Expects oauth token in a global macro variable (default
+  @details Creates a viya folder by checking if each parent folder exists, and
-  ACCESS_TOKEN).
+  recursively creating children if needed.
+  Usage:
 
       %mv_createfolder(path=/Public)
 
 
-  @param path= The full path of the folder to be created
+  @param [in] path= The full path of the folder to be created
-  @param access_token_var= The global macro variable to contain the access token
+  @param [in] access_token_var= The global macro variable to contain the access
-  @param grant_type= (authorization_code) Valid values are "password" or
+    token, if using authorization_code grant type.
-    "authorization_code" (unquoted).
+  @param [in] grant_type= (sas_services) Valid values are:
+    @li password
+    @li authorization_code
+    @li sas_services
 
+  @param [in] mdebug=(0) set to 1 to enable DEBUG messages
 
   @version VIYA V.03.04
   @author Allan Bowe, source: https://github.com/sasjs/core
@@ -22,21 +27,36 @@
   @li mf_getuniquelibref.sas
   @li mf_isblank.sas
   @li mf_getplatform.sas
+  @li mfv_existfolder.sas
+
 
 **/
 
 %macro mv_createfolder(path=
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
+    ,mdebug=0
   );
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+%if %mfv_existfolder(&path)=1 %then %do;
+  %put &sysmacroname: &path already exists;
+  %return;
+%end;
+
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
   %else %let grant_type=sas_services;
 %end;
 %if &grant_type=sas_services %then %do;
-    %let oauth_bearer=oauth_bearer=sas_services;
+  %let oauth_bearer=oauth_bearer=sas_services;
-    %let &access_token_var=;
+  %let &access_token_var=;
 %end;
 
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
@@ -91,12 +111,21 @@ options noquotelenmax;
     ,mac=&sysmacroname
     ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
   )
+  %if &mdebug=1 %then %do;
+    %put &sysmacroname following check to see if &newpath exists:;
+    %put _local_;
+    data _null_;
+      set &fname1;
+      input;
+      putlog _infile_;
+    run;
+  %end;
   %if &SYS_PROCHTTP_STATUS_CODE=200 %then %do;
     %*put &sysmacroname &newpath exists so grab the follow on link ;
     data _null_;
       set &libref1..links;
       if rel='createChild' then
-        call symputx('href',quote("&base_uri"!!trim(href)),'l');
+        call symputx('href',quote(cats("&base_uri",href)),'l');
     run;
   %end;
   %else %if &SYS_PROCHTTP_STATUS_CODE=404 %then %do;
@@ -140,8 +169,10 @@ options noquotelenmax;
     %put &sysmacroname &newpath now created. Grabbing the follow on link ;
     data _null_;
       set &libref2..links;
-      if rel='createChild' then
+      if rel='createChild' then do;
-        call symputx('href',quote(trim(href)),'l');
+        call symputx('href',quote(cats("&base_uri",href)),'l');
+        &dbg put (_all_)(=);
+      end;
     run;
 
     libname &libref2 clear;
@@ -150,4 +181,4 @@ options noquotelenmax;
   filename &fname1 clear;
   libname &libref1 clear;
 %end;
-%mend;
+%mend mv_createfolder;

viya/mv_createwebservice.sas

@@ -237,10 +237,11 @@ data _null_;
   put "/* Created on %sysfunc(datetime(),datetime19.) by &sysuserid */";
 /* WEBOUT BEGIN */
   put ' ';
-  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=PROCJSON,dbg=0 ';
+  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0 ';
   put ')/*/STORE SOURCE*/; ';
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
+  put '  options nobomfile; ';
   put '  data _null_;file &jref encoding=''utf-8''; ';
   put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
   put '  run; ';
@@ -268,7 +269,65 @@ data _null_;
   put '      %put &sysmacroname:  &ds NOT FOUND!!!; ';
   put '      %return; ';
   put '    %end; ';
-  put '    data _null_;file &jref mod ; ';
+  put '    %if &fmt=Y %then %do; ';
+  put '      %put converting every variable to a formatted variable; ';
+  put '      /* see mp_ds2fmtds.sas for source */ ';
+  put '      proc contents noprint data=&ds ';
+  put '        out=_data_(keep=name type length format formatl formatd varnum); ';
+  put '      run; ';
+  put '      proc sort; ';
+  put '        by varnum; ';
+  put '      run; ';
+  put '      %local fmtds; ';
+  put '      %let fmtds=%scan(&syslast,2,.); ';
+  put '      /* prepare formats and varnames */ ';
+  put '      data _null_; ';
+  put '        if _n_=1 then call symputx(''nobs'',nobs,''l''); ';
+  put '        set &fmtds end=last nobs=nobs; ';
+  put '        name=upcase(name); ';
+  put '        /* fix formats */ ';
+  put '        if type=2 or type=6 then do; ';
+  put '          length fmt $49.; ';
+  put '          if format='''' then fmt=cats(''$'',length,''.''); ';
+  put '          else if formatl=0 then fmt=cats(format,''.''); ';
+  put '          else fmt=cats(format,formatl,''.''); ';
+  put '          newlen=max(formatl,length); ';
+  put '        end; ';
+  put '        else do; ';
+  put '          if format='''' then fmt=''best.''; ';
+  put '          else if formatl=0 then fmt=cats(format,''.''); ';
+  put '          else if formatd=0 then fmt=cats(format,formatl,''.''); ';
+  put '          else fmt=cats(format,formatl,''.'',formatd); ';
+  put '          /* needs to be wide, for datetimes etc */ ';
+  put '          newlen=max(length,formatl,24); ';
+  put '        end; ';
+  put '        /* 32 char unique name */ ';
+  put '        newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put ' ';
+  put '        call symputx(cats(''name'',_n_),name,''l''); ';
+  put '        call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '        call symputx(cats(''len'',_n_),newlen,''l''); ';
+  put '        call symputx(cats(''fmt'',_n_),fmt,''l''); ';
+  put '        call symputx(cats(''type'',_n_),type,''l''); ';
+  put '      run; ';
+  put '      data &fmtds; ';
+  put '        /* rename on entry */ ';
+  put '        set &ds(rename=( ';
+  put '      %local i; ';
+  put '      %do i=1 %to &nobs; ';
+  put '        &&name&i=&&newname&i ';
+  put '      %end; ';
+  put '        )); ';
+  put '      %do i=1 %to &nobs; ';
+  put '        length &&name&i $&&len&i; ';
+  put '        &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '        drop &&newname&i; ';
+  put '      %end; ';
+  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '      run; ';
+  put '      %let ds=&fmtds; ';
+  put '    %end; /* &fmt=Y */ ';
+  put '    data _null_;file &jref mod encoding=''utf-8''; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
   put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
@@ -351,8 +410,8 @@ data _null_;
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
-  put '%mend; ';
+  put '%mend mp_jsonout; ';
-  put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y); ';
+  put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y); ';
   put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
   put '  sasjs_tables SYS_JES_JOB_URI; ';
   put '%if %index("&_debug",log) %then %let _debug=131; ';
@@ -490,8 +549,9 @@ data _null_;
   put '    ods output Members=&tempds; ';
   put '    proc datasets library=WORK memtype=data; ';
   put '    %local wtcnt;%let wtcnt=0; ';
-  put '    data _null_; set &tempds; ';
+  put '    data _null_; ';
-  put '      if not (name =:"DATA"); ';
+  put '      set &tempds; ';
+  put '      if not (upcase(name) =:"DATA"); /* ignore temp datasets */ ';
   put '      i+1; ';
   put '      call symputx(''wt''!!left(i),name); ';
   put '      call symputx(''wtcnt'',i); ';
@@ -531,17 +591,19 @@ data _null_;
   put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
+  put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
+  put '    put '',"SYSVLONG" : '' sysvlong; ';
   put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''" ''; ';
   put '    put "}"; ';
   put ' ';
-  put '  %if %upcase(&fref) ne _WEBOUT %then %do; ';
+  put '  %if %upcase(&fref) ne _WEBOUT and &stream=Y %then %do; ';
   put '    data _null_; rc=fcopy("&fref","_webout");run; ';
   put '  %end; ';
   put ' ';
   put '%end; ';
   put ' ';
-  put '%mend; ';
+  put '%mend mv_webout; ';
   put ' ';
   put '%macro mf_getuser(type=META ';
   put ')/*/STORE SOURCE*/; ';

viya/mv_getfoldermembers.sas

@@ -8,7 +8,11 @@
 
 
   @param root= The path for which to return the list of folders
-  @param outds= The output dataset to create (default is work.mv_getfolders)
+  @param outds= The output dataset to create (default is work.mv_getfolders).  Format:
+  |ordinal_root|ordinal_items|creationTimeStamp| modifiedTimeStamp|createdBy|modifiedBy|id| uri|added| type|name|description|
+  |---|---|---|---|---|---|---|---|---|---|---|---|
+  |1|1|2021-05-25T11:15:04.204Z|2021-05-25T11:15:04.204Z|allbow|allbow|4f1e3945-9655-462b-90f2-c31534b3ca47|/folders/folders/ed701ff3-77e8-468d-a4f5-8c43dec0fd9e|2021-05-25T11:15:04.212Z|child|my_folder_name|My folder Description|
+
   @param access_token_var= The global macro variable to contain the access token
   @param grant_type= valid values are "password" or "authorization_code" (unquoted).
     The default is authorization_code.
@@ -116,4 +120,4 @@ options noquotelenmax;
 filename &fname1 clear;
 libname &libref1 clear;
 
-%mend;
+%mend mv_getfoldermembers;

viya/mv_getjoblog.sas

@@ -1,21 +1,19 @@
 /**
   @file
   @brief Extract the log from a completed SAS Viya Job
-  @details Extracts log from a Viya job and writes it out to a fileref
+  @details Extracts log from a Viya job and writes it out to a fileref.
 
   To query the job, you need the URI.  Sample code for achieving this
   is provided below.
 
   ## Example
 
-  First, compile the macros:
+      %* First, compile the macros;
-
       filename mc url
         "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
 
-  Next, create a job (in this case, a web service):
+      %* Next, create a job (in this case, a web service);
-
       filename ft15f001 temp;
       parmcards4;
         data ;
@@ -31,28 +29,40 @@
       ;;;;
       %mv_createwebservice(path=/Public/temp,name=demo)
 
-  Execute it:
+      %* Execute it;
-
       %mv_jobexecute(path=/Public/temp
         ,name=demo
         ,outds=work.info
       )
 
-  Wait for it to finish, and grab the uri:
+      %* Wait for it to finish;
-
+      data work.info;
-      data _null_;
         set work.info;
-        if method='GET' and rel='self';
+        where method='GET' and rel='state';
+      run;
+      %mv_jobwaitfor(ALL,inds=work.info,outds=work.jobstates)
+
+      %* and grab the uri;
+      data _null_;
+        set work.jobstates;
         call symputx('uri',uri);
       run;
 
-  Finally, fetch the log:
+      %* Finally, fetch the log;
-
       %mv_getjoblog(uri=&uri,outref=mylog)
 
   This macro is used by the mv_jobwaitfor.sas macro, which is generally a more
   convenient way to wait for the job to finish before fetching the log.
 
+  If the remote session calls `endsas` then it is not possible to get the log
+  from the provided uri, and so the log from the parent session is fetched
+  instead.  This happens for a 400 response, eg below:
+
+      ErrorResponse[version=2,status=400,err=5113,id=,message=The session
+      requested is currently in a failed or stopped state.,detail=[path:
+      /compute/sessions/LONGURI-ses0006/jobs/LONGURI/log/content, traceId: 63
+      51aa617d01fd2b],remediation=Correct the errors in the session request,
+      and create a new session.,targetUri=<null>,errors=[],links=[]]
 
   @param [in] access_token_var= The global macro variable to contain the access
     token
@@ -64,7 +74,7 @@
       if a SASStudioV session else authorization_code.  Default option.
     @li sas_services - will use oauth_bearer=sas_services.
   @param [in] uri= The uri of the running job for which to fetch the status,
-    in the format `/jobExecution/jobs/$UUID/state` (unquoted).
+    in the format `/jobExecution/jobs/$UUID` (unquoted).
   @param [out] outref= The output fileref to which to APPEND the log (is always
   appended).
 
@@ -122,7 +132,7 @@ data _null_;
     call symputx('errflg',1);
     call symputx('errmsg',
       "URI should be in format /jobExecution/jobs/$$$$UUID$$$$"
-      !!" but is actually like: &uri",'l');
+      !!" but is actually like:"!!uri,'l');
   end;
 run;
 
@@ -155,6 +165,10 @@ proc http method='GET' out=&fname1 &oauth_bearer
   %end;
   ;
 run;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname: fetching log loc from &uri;
+  data _null_;infile &fname1;input;putlog _infile_;run;
+%end;
 %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
 %do;
   data _null_;infile &fname1;input;putlog _infile_;run;
@@ -192,43 +206,74 @@ run;
 data _null_;
   infile &fname2;
   input;
-  uri=_infile_;
+  uri=cats(_infile_);
   if length(uri)<12 then do;
     call symputx('errflg',1);
     call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
   end;
-  if scan(uri,1) ne 'files' or scan(uri,2) ne 'files' then do;
+  else if (scan(uri,1,'/') ne 'compute' or scan(uri,2,'/') ne 'sessions')
+    and (scan(uri,1,'/') ne 'files' or scan(uri,2,'/') ne 'files')
+  then do;
     call symputx('errflg',1);
     call symputx('errmsg',
-      "URI should be in format /files/files/$$$$UUID$$$$"
+      "URI should be in format /compute/sessions/$$$$UUID$$$$/jobs/$$$$UUID$$$$"
-      !!" but is actually like: &uri",'l');
+      !!" or /files/files/$$$$UUID$$$$"
+      !!" but is actually like:"!!uri,'l');
+  end;
+  else do;
+    call symputx('errflg',0,'l');
+    call symputx('logloc',uri,'l');
   end;
-  call symputx('errflg',0,'l');
-  call symputx('logloc',uri,'l');
 run;
 
-%mp_abort(iftrue=(&errflg=1)
+%mp_abort(iftrue=(%str(&errflg)=1)
   ,mac=&sysmacroname
   ,msg=%str(&errmsg)
 )
 
 /* we have a log uri - now fetch the log */
+%&dbg.put &sysmacroname: querying &base_uri&logloc/content;
 proc http method='GET' out=&fname1 &oauth_bearer
-  url="&base_uri&logloc/content";
+  url="&base_uri&logloc/content?limit=10000";
   headers
   %if &grant_type=authorization_code %then %do;
           "Authorization"="Bearer &&&access_token_var"
   %end;
   ;
 run;
-%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+
-%do;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname: fetching log content from &base_uri&logloc/content;
   data _null_;infile &fname1;input;putlog _infile_;run;
+%end;
+
+%if &SYS_PROCHTTP_STATUS_CODE=400 %then %do;
+  /* fetch log from parent session */
+  %let logloc=%substr(&logloc,1,%index(&logloc,%str(/jobs/))-1);
+  %&dbg.put &sysmacroname: Now querying &base_uri&logloc/log/content;
+  proc http method='GET' out=&fname1 &oauth_bearer
+    url="&base_uri&logloc/log/content?limit=10000";
+    headers
+    %if &grant_type=authorization_code %then %do;
+            "Authorization"="Bearer &&&access_token_var"
+    %end;
+    ;
+  run;
+  %if &mdebug=1 %then %do;
+    %put &sysmacroname: fetching log content from &base_uri&logloc/log/content;
+    data _null_;infile &fname1;input;putlog _infile_;run;
+  %end;
+%end;
+
+%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201
+%then %do;
+  %if &mdebug ne 1 %then %do; /* have already output above */
+    data _null_;infile &fname1;input;putlog _infile_;run;
+  %end;
   %mp_abort(mac=&sysmacroname
     ,msg=%str(logfetch: &SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
   )
 %end;
-
 data _null_;
   file "&fpath3..lua";
   put '

viya/mv_jobflow.sas

@@ -97,8 +97,8 @@
       run;
 
 
-  @param [in] access_token_var= The global macro variable to contain the access
+  @param [in] access_token_var= The global macro variable to contain the
-    token
+              access token
   @param [in] grant_type= valid values:
       @li password
       @li authorization_code
@@ -237,123 +237,143 @@ data;run;%let jdswaitfor=&syslast;
 
 /* start loop */
 %do fid=1 %to &flowcnt;
-  %put preparing job attributes for flow &&flow&fid;
-  %local jds jcnt;
-  data &jds(drop=_contextName _program);
-    set &inds(where=(flow_id=&&flow&fid));
-    if _contextName='' then _contextName="SAS Job Execution compute context";
-    call symputx(cats('job',_n_),_program,'l');
-    call symputx(cats('context',_n_),_contextName,'l');
-    call symputx('jcnt',_n_,'l');
-  run;
-  %put exporting job variables in json format;
-  %do jid=1 %to &jcnt;
-    data &jjson;
-      set &jds;
-      if _n_=&jid then do;
-        output;
-        stop;
-      end;
-    run;
-    proc json out=&jfref;
-      export &jjson / nosastags fmtnumeric;
-    run;
-    data _null_;
-      infile &jfref lrecl=32767;
-      input;
-      jparams='jparams'!!left(symget('jid'));
-      call symputx(jparams,substr(_infile_,3,length(_infile_)-4));
-    run;
-    %local jobuid&jid;
-    %let jobuid&jid=0; /* used in next loop */
-  %end;
-  %local concurrency completed;
-  %let concurrency=0;
-  %let completed=0;
-  proc sql; drop table &jdsrunning;
-  %do jid=1 %to &jcnt;
-    /**
-      * now we can execute the jobs up to the maxconcurrency setting
-      */
-    %if "&&job&jid" ne "0" %then %do; /* this var is zero if job finished */
 
-      /* check to see if the job finished in the previous round */
+  %if not ( &raise_err and &syscc ) %then %do;
-      %if %sysfunc(exist(&outds))=1 %then %do;
+
-        %local jobcheck;  %let jobcheck=0;
+    %put preparing job attributes for flow &&flow&fid;
-        proc sql noprint;
+    %local jds jcnt;
-        select count(*) into: jobcheck
+    data &jds(drop=_contextName _program);
-          from &outds where uuid="&&jobuid&jid";
+      set &inds(where=(flow_id=&&flow&fid));
-        %if &jobcheck>0 %then %do;
+      if _contextName='' then _contextName="SAS Job Execution compute context";
-          %put &&job&jid in flow &fid with uid &&jobuid&jid completed!;
+      call symputx(cats('job',_n_),_program,'l');
-          %let job&jid=0;
+      call symputx(cats('context',_n_),_contextName,'l');
+      call symputx('jcnt',_n_,'l');
+    run;
+    %put exporting job variables in json format;
+    %do jid=1 %to &jcnt;
+      data &jjson;
+        set &jds;
+        if _n_=&jid then do;
+          output;
+          stop;
+        end;
+      run;
+      proc json out=&jfref;
+        export &jjson / nosastags fmtnumeric;
+      run;
+      data _null_;
+        infile &jfref lrecl=32767;
+        input;
+        jparams='jparams'!!left(symget('jid'));
+        call symputx(jparams,substr(_infile_,3,length(_infile_)-4));
+      run;
+      %local jobuid&jid;
+      %let jobuid&jid=0; /* used in next loop */
+    %end;
+    %local concurrency completed;
+    %let concurrency=0;
+    %let completed=0;
+    proc sql; drop table &jdsrunning;
+    %do jid=1 %to &jcnt;
+      /**
+        * now we can execute the jobs up to the maxconcurrency setting
+        */
+      %if "&&job&jid" ne "0" %then %do; /* this var is zero if job finished */
+
+        /* check to see if the job finished in the previous round */
+        %if %sysfunc(exist(&outds))=1 %then %do;
+          %local jobcheck;  %let jobcheck=0;
+          proc sql noprint;
+          select count(*) into: jobcheck
+            from &outds where uuid="&&jobuid&jid";
+          %if &jobcheck>0 %then %do;
+            %put &&job&jid in flow &fid with uid &&jobuid&jid completed!;
+            %let job&jid=0;
+          %end;
+        %end;
+
+        /* check if job was triggered and, if
+            so, if we have enough slots to run? */
+        %if ("&&jobuid&jid"="0") and (&concurrency<&maxconcurrency) %then %do;
+
+          /* But only start if no issues detected so far */
+          %if not ( &raise_err and &syscc ) %then %do;
+
+            %local jobname jobpath;
+            %let jobname=%scan(&&job&jid,-1,/);
+            %let jobpath=
+                  %substr(&&job&jid,1,%length(&&job&jid)-%length(&jobname)-1);
+
+            %put executing &jobpath/&jobname with paramstring &&jparams&jid;
+            %mv_jobexecute(path=&jobpath
+              ,name=&jobname
+              ,paramstring=%superq(jparams&jid)
+              ,outds=&jdsapp
+            )
+            data &jdsapp;
+              format jobparams $32767.;
+              set &jdsapp(where=(method='GET' and rel='state'));
+              jobparams=symget("jparams&jid");
+              /* uri here has the /state suffix */
+              uuid=scan(uri,-2,'/');
+              call symputx("jobuid&jid",uuid,'l');
+            run;
+            proc append base=&jdsrunning data=&jdsapp;
+            run;
+            %let concurrency=%eval(&concurrency+1);
+            /* sleep one second after every request to smooth the impact */
+            data _null_;
+              call sleep(1,1);
+            run;
+
+          %end;
+          %else %do; /* Job was skipped due to problems */
+
+            %put jobid &&job&jid in flow &fid skipped due to SYSCC (&syscc);
+            %let completed = %eval(&completed+1);
+            %let job&jid=0; /* Indicate job has finished */
+
+          %end;
+
         %end;
       %end;
+      %if &jid=&jcnt %then %do;
+        /* we are at the end of the loop - check which jobs have finished */
+        %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref
+                      ,raise_err=&raise_err,mdebug=&mdebug)
+        %local done;
+        %let done=%mf_nobs(&jdswaitfor);
+        %if &done>0 %then %do;
+          %let completed=%eval(&completed+&done);
+          %let concurrency=%eval(&concurrency-&done);
+          data &jdsapp;
+            set &jdswaitfor;
+            flow_id=&&flow&fid;
+            uuid=scan(uri,-1,'/');
+          run;
+          proc append base=&outds data=&jdsapp;
+          run;
+        %end;
+        proc sql;
+        delete from &jdsrunning
+          where uuid in (select uuid from &outds
+            where state in ('canceled','completed','failed')
+          );
 
-      /* check if job was triggered and if so, if we have enough slots to run */
+        /* loop again if jobs are left */
-      %if "&&jobuid&jid"="0" and &concurrency<&maxconcurrency %then %do;
+        %if &completed < &jcnt %then %do;
-        %local jobname jobpath;
+          %let jid=0;
-        %let jobname=%scan(&&job&jid,-1,/);
+          %put looping flow &fid again;
-        %let jobpath=
+          %put &completed of &jcnt jobs completed, &concurrency jobs running;
-          %substr(&&job&jid,1,%length(&&job&jid)-%length(&jobname)-1);
+        %end;
-        %put executing &jobpath/&jobname with paramstring &&jparams&jid;
-        %mv_jobexecute(path=&jobpath
-          ,name=&jobname
-          ,paramstring=%superq(jparams&jid)
-          ,outds=&jdsapp
-          ,mdebug=&mdebug
-        )
-        data &jdsapp;
-          format jobparams $32767.;
-          set &jdsapp(where=(method='GET' and rel='state'));
-          jobparams=symget("jparams&jid");
-          /* uri here has the /state suffix */
-          uuid=scan(uri,-2,'/');
-          call symputx("jobuid&jid",uuid,'l');
-        run;
-        proc append base=&jdsrunning data=&jdsapp;
-        run;
-        %let concurrency=%eval(&concurrency+1);
-        /* sleep one second after every request to smooth the impact */
-        data _null_;
-          call sleep(1,1);
-        run;
       %end;
     %end;
-    %if &jid=&jcnt %then %do;
-      /* we are at the end of the loop - time to see which jobs have finished */
-      %mv_jobwaitfor(ANY
-        ,inds=&jdsrunning
-        ,outds=&jdswaitfor
-        ,outref=&outref
-        ,raise_err=&raise_err
-        ,mdebug=&mdebug
-      )
-      %local done;
-      %let done=%mf_nobs(&jdswaitfor);
-      %if &done>0 %then %do;
-        %let completed=%eval(&completed+&done);
-        %let concurrency=%eval(&concurrency-&done);
-        data &jdsapp;
-          set &jdswaitfor;
-          flow_id=&&flow&fid;
-          uuid=scan(uri,-1,'/');
-        run;
-        proc append base=&outds data=&jdsapp;
-        run;
-      %end;
-      proc sql;
-      delete from &jdsrunning
-        where uuid in (select uuid from &outds
-          where state in ('canceled','completed','failed')
-        );
 
-      /* loop again if jobs are left */
+  %end;
-      %if &completed < &jcnt %then %do;
+  %else %do;
-        %let jid=0;
+
-        %put looping flow &fid again - &completed of &jcnt jobs completed,
+    %put Flow &&flow&fid skipped due to SYSCC (&syscc);
-          &concurrency jobs running;
+
-      %end;
-    %end;
   %end;
   /* back up and execute the next flow */
 %end;

viya/mv_registerclient.sas

@@ -1,14 +1,22 @@
 /**
   @file mv_registerclient.sas
   @brief Register Client and Secret (admin task)
-  @details When building apps on SAS Viya, an client id and secret is required.
+  @details When building apps on SAS Viya, an client id and secret are sometimes
-  This macro will obtain the Consul Token and use that to call the Web Service.
+  required.  In order to generate them, filesystem access to the Consul Token
+  is needed (it is not enough to be in the SASAdministrator group in SAS
+  Environment Manager).
 
-  more info: https://developer.sas.com/reference/auth/#register
+  If you are registering a lot of clients / secrets, you may find it more
-  and:
+  convenient to use the [Viya Token Generator]
-  http://proc-x.com/2019/01/authentication-to-sas-viya-a-couple-of-approaches
+  (https://sasjs.io/apps/#viya-client-token-generator) (a SASjs Web App to
+  automate the generation of clients & secrets with various settings).
 
-  The default viyaroot location is /opt/sas/viya/config
+  For further information on clients / secrets, see;
+  @li https://developer.sas.com/reference/auth/#register
+  @li https://proc-x.com/2019/01/authentication-to-sas-viya-a-couple-of-approaches
+  @li https://cli.sasjs.io/faq/#how-can-i-obtain-a-viya-client-and-secret
+
+  The default viyaroot location is: `/opt/sas/viya/config`
 
   Usage:
 
@@ -33,16 +41,19 @@
       )
 
   @param client_id= The client name.  Auto generated if blank.
-  @param client_secret= Client secret  Auto generated if client is blank.
+  @param client_secret= Client secret.  Auto generated if client is blank.
-  @param scopes= list of space-seperated unquoted scopes (default is openid)
+  @param scopes=(openid) List of space-seperated unquoted scopes
-  @param grant_type= valid values are "password" or "authorization_code"
+  @param grant_type=(authorization_code|refresh_token) Valid values are
-    (unquoted)
+    "password" or "authorization_code" (unquoted)
-  @param outds= the dataset to contain the registered client id and secret
+  @param outds=(mv_registerclient) The dataset to contain the registered client
-  @param access_token_validity= The duration of validity of the access token
+    id and secret
-    in seconds.  A value of DEFAULT will omit the entry (and use system default)
+  @param access_token_validity=(DEFAULT) The duration of validity of the access
-  @param refresh_token_validity= The duration of validity of the refresh token
+    token in seconds.  A value of DEFAULT will omit the entry (and use system
-    in seconds.  A value of DEFAULT will omit the entry (and use system default)
+    default)
-  @param name= A human readable name for the client
+  @param refresh_token_validity=(DEFAULT)  The duration of validity of the
+    refresh token in seconds.  A value of DEFAULT will omit the entry (and use
+    system default)
+  @param name= An optional, human readable name for the client
   @param required_user_groups= A list of group names. If a user does not belong
     to all the required groups, the user will not be authenticated and no tokens
     are issued to this client for that user. If this field is not specified,
@@ -51,8 +62,8 @@
     apply.  Setting this to true will autoapprove all the client scopes.
   @param use_session= If true, access tokens issued to this client will be
     associated with an HTTP session and revoked upon logout or time-out.
-  @param outjson= A dataset containing the lines of JSON submitted.  Useful
+  @param outjson= (_null_) A dataset containing the lines of JSON submitted.
-    for debugging.  Default= _null_.
+    Useful for debugging.
 
   @version VIYA V.03.04
   @author Allan Bowe, source: https://github.com/sasjs/core
@@ -89,12 +100,26 @@
 options noquotelenmax;
 /* first, get consul token needed to get client id / secret */
 %let tokloc=/etc/SASSecurityCertificateFramework/tokens/consul/default;
+%let tokloc=%mf_loc(VIYACONFIG)&tokloc/client.token;
+
+
+%mp_abort(iftrue=(%sysfunc(fileexist(&tokloc))=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Unable to access the consul token at &tokloc)
+)
+
+%let consul_token=0;
 data _null_;
-  infile "%mf_loc(VIYACONFIG)&tokloc/client.token";
+  infile "&tokloc";
   input token:$64.;
   call symputx('consul_token',token);
 run;
 
+%mp_abort(iftrue=("&consul_token"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(Unable to source the consul token from &tokloc)
+)
+
 %local base_uri; /* location of rest apis */
 %let base_uri=%mf_getplatform(VIYARESTAPI);
 
@@ -241,4 +266,4 @@ filename &fname2 clear;
 filename &fname3 clear;
 libname &libref clear;
 
-%mend;
+%mend mv_registerclient;

viya/mv_webout.sas

@@ -23,9 +23,10 @@
   @param action Either OPEN, ARR, OBJ or CLOSE
   @param ds The dataset to send back to the frontend
   @param _webout= fileref for returning the json
-  @param fref= temp fref
+  @param fref=(_mvwtemp) Temp fileref to which to write the output
   @param dslabel= value to use instead of the real name for sending to JSON
-  @param fmt= change to N to strip formats from output
+  @param fmt=(Y) change to N to strip formats from output
+  @param stream=(Y) Change to N if not streaming to _webout
 
   <h4> SAS Macros </h4>
   @li mp_jsonout.sas
@@ -35,7 +36,7 @@
   @author Allan Bowe, source: https://github.com/sasjs/core
 
 **/
-%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y);
+%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y);
 %global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name
   sasjs_tables SYS_JES_JOB_URI;
 %if %index("&_debug",log) %then %let _debug=131;
@@ -173,8 +174,9 @@
     ods output Members=&tempds;
     proc datasets library=WORK memtype=data;
     %local wtcnt;%let wtcnt=0;
-    data _null_; set &tempds;
+    data _null_;
-      if not (name =:"DATA");
+      set &tempds;
+      if not (upcase(name) =:"DATA"); /* ignore temp datasets */
       i+1;
       call symputx('wt'!!left(i),name);
       call symputx('wtcnt',i);
@@ -214,14 +216,16 @@
     put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
+    sysvlong=quote(trim(symget('sysvlong')));
+    put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
     put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
     put "}";
 
-  %if %upcase(&fref) ne _WEBOUT %then %do;
+  %if %upcase(&fref) ne _WEBOUT and &stream=Y %then %do;
     data _null_; rc=fcopy("&fref","_webout");run;
   %end;
 
 %end;
 
-%mend;
+%mend mv_webout;