Merge pull request #326 from sasjs/325-error-the-keyword-parameter-maxobs-was-not-defined-with-the-macro

fix: closes #325 by including maxobs param

.github/vpn/config.ovpn

@@ -1,30 +0,0 @@
-cipher AES-256-CBC
-setenv FORWARD_COMPATIBLE 1
-client
-server-poll-timeout 4
-nobind
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 443 tcp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-dev tun
-dev-type tun
-ns-cert-type server
-setenv opt tls-version-min 1.0 or-highest
-reneg-sec 604800
-sndbuf 0
-rcvbuf 0
-# NOTE: LZO commands are pushed by the Access Server at connect time.
-# NOTE: The below line doesn't disable LZO.
-comp-lzo no
-verb 3
-setenv PUSH_PEER_INFO
-
-ca ca.crt
-cert user.crt
-key user.key
-tls-auth tls.key 1

.github/workflows/main.yml

@@ -15,7 +15,14 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v2
+        uses: cycjimmy/semantic-release-action@v3
         env:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+      - name: SAS Packages Release
+        run: |
+          npx @sasjs/cli compile job -s sasjs/utils/create_sas_package.sas -o sasjsbuild
+          # this part depends on https://github.com/sasjs/server/issues/307
+          # sasjs run sasjsbuild/jobs/utils/create_sas_package.sas -t sas9
+
+

.github/workflows/run-tests.yml

@@ -21,31 +21,6 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
 
-      - name: Write VPN Files
-        run: |
-          echo "$CA_CRT" > .github/vpn/ca.crt
-          echo "$USER_CRT" > .github/vpn/user.crt
-          echo "$USER_KEY" > .github/vpn/user.key
-          echo "$TLS_KEY" > .github/vpn/tls.key
-        shell: bash
-        env:
-          CA_CRT: ${{ secrets.CA_CRT}}
-          USER_CRT: ${{ secrets.USER_CRT }}
-          USER_KEY: ${{ secrets.USER_KEY }}
-          TLS_KEY: ${{ secrets.TLS_KEY }}
-
-      - name: Install Open VPN
-        run: |
-          sudo apt install apt-transport-https
-          sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
-          sudo apt-key add openvpn-repo-pkg-key.pub
-          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-focal.list
-          sudo apt update
-          sudo apt install openvpn3
-
-      - name: Start Open VPN 3
-        run: openvpn3 session-start --config .github/vpn/config.ovpn
-
       - name: Install Doxygen
         run: sudo apt-get install doxygen
 

.gitignore

@@ -10,4 +10,6 @@ sasjsresults/
 mc_*
 
 # ignore .env files as they can contain sasjs access tokens
-*.env*
+*.env*
+
+~

.sasjslint

@@ -1,13 +1,14 @@
 {
-    "noTrailingSpaces": true,
-    "noEncodedPasswords": true,
-    "hasDoxygenHeader": true,
-    "hasMacroNameInMend": true,
-    "hasMacroParentheses": true,
-    "noNestedMacros": false,
-    "noSpacesInFileNames": true,
-    "maxLineLength": 300,
-    "lowerCaseFileNames": true,
-    "noTabIndentation": true,
-    "indentationMultiple": 2
-}
+  "noTrailingSpaces": true,
+  "noEncodedPasswords": true,
+  "hasDoxygenHeader": true,
+  "hasMacroNameInMend": true,
+  "hasMacroParentheses": true,
+  "noGremlins": true,
+  "noNestedMacros": false,
+  "noSpacesInFileNames": true,
+  "maxLineLength": 300,
+  "lowerCaseFileNames": true,
+  "noTabs": true,
+  "indentationMultiple": 2
+}

.vscode/settings.json

@@ -6,5 +6,7 @@
   "editor.rulers": [
     80
   ],
-  "files.trimTrailingWhitespace": true
+  "files.trimTrailingWhitespace": true,
+  "sasjs-for-vscode.target": "docsonly",
+  "sasjs-for-vscode.isLocal": true
 }

README.md

@@ -2,8 +2,6 @@
 [![npm package][npm-image]][npm-url]
 [![Github Workflow][githubworkflow-image]][githubworkflow-url]
 ![npm](https://img.shields.io/npm/dt/@sasjs/core)
-![Snyk Vulnerabilities for npm package](https://img.shields.io/snyk/vulnerabilities/npm/@sasjs/core)
-[![License](https://img.shields.io/apm/l/atomic-design-ui.svg)](/LICENSE)
 ![GitHub top language](https://img.shields.io/github/languages/top/sasjs/core)
 [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sasjs/core)](https://github.com/sasjs/core/issues?q=is%3Aissue+is%3Aclosed)
 [![GitHub issues](https://img.shields.io/github/issues-raw/sasjs/core)](https://github.com/sasjs/core/issues)
@@ -237,6 +235,7 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 The following repositories are also worth checking out:
 
+* [xieliaing/SAS](https://github.com/xieliaing/SAS)
 * [SASJedi/sas-macros](https://github.com/SASJedi/sas-macros)
 * [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
 * [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)

base/mf_abort.sas

@@ -12,7 +12,7 @@
 **/
 
 %macro mf_abort(mac=mf_abort.sas, msg=, iftrue=%str(1=1)
-)/*/STORE SOURCE*/;
+)/des='ungraceful abort' /*STORE SOURCE*/;
 
   %if not(%eval(%unquote(&iftrue))) %then %return;
 
@@ -24,4 +24,4 @@
 
 %mend mf_abort;
 
-/** @endcond */
+/** @endcond */

base/mf_existfileref.sas

@@ -30,4 +30,4 @@
     0
   %end;
 
-%mend mf_existfileref;
+%mend mf_existfileref;

base/mf_getgitbranch.sas

@@ -0,0 +1,37 @@
+/**
+  @file
+  @brief Retrieves the current branch from a local GIT repo
+  @details In a local git repository, the current branch is always available in
+  the `.git/HEAD` file in a format like this:  `ref: refs/heads/master`
+
+  This macro simply reads the file and returns the last word (eg `master`).
+
+  Example usage:
+
+      %let gitdir=%sysfunc(pathname(work))/core;
+      %let repo=https://github.com/sasjs/core;
+      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&gitdir));
+
+      %put The current branch is %mf_getgitbranch(&gitdir);
+
+  @param [in] gitdir The directory containing the GIT repository
+
+  <h4> SAS Macros </h4>
+  @li mf_readfile.sas
+
+  <h4> Related Macros </h4>
+  @li mp_gitadd.sas
+  @li mp_gitlog.sas
+  @li mp_gitreleaseinfo.sas
+  @li mp_gitstatus.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_getgitbranch(gitdir
+)/*/STORE SOURCE*/;
+
+  %scan(%mf_readfile(&gitdir/.git/HEAD),-1)
+
+%mend mf_getgitbranch;

base/mf_readfile.sas

@@ -0,0 +1,63 @@
+/**
+  @file
+  @brief Reads the first line of a file using pure macro
+  @details Reads the first line of a file and returns it.  Future versions may
+  read each line into a macro variable array.
+
+  Generally, reading data into macro variables is not great as certain
+  nonprintable characters (such as CR, LF) may be dropped in the conversion.
+
+    Usage:
+
+        %mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=more content)
+
+        %put %mf_readfile(&sasjswork/myfile.txt);
+
+
+  @param [in] fpath Full path to file to be read
+
+  <h4> Related Macros </h4>
+  @li mf_deletefile.sas
+  @li mf_writefile.sas
+  @li mf_readfile.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mf_readfile(fpath
+)/*/STORE SOURCE*/;
+%local fref rc fid fcontent;
+
+/* check file exists */
+%if %sysfunc(filename(fref,&fpath)) ne 0 %then %do;
+  %put &=fref &=fpath;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let fid=%sysfunc(fopen(&fref,I));
+
+%if &fid=0 %then %do;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%if %sysfunc(fread(&fid)) = 0 %then %do;
+  %let rc=%sysfunc(fget(&fid,fcontent,65534));
+  &fcontent
+%end;
+
+/*
+%do %while(%sysfunc(fread(&fid)) = 0);
+  %let rc=%sysfunc(fget(&fid,fcontent,65534));
+  &fcontent
+%end;
+*/
+
+%let rc=%sysfunc(fclose(&fid));
+%let rc=%sysfunc(filename(&fref));
+
+%mend mf_readfile;
+/** @endcond */

base/mp_abort.sas

@@ -225,15 +225,51 @@ and %superq(SYSPROCESSNAME) ne %str(Compute Server)
     _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
-    syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
-    put ",""SYSERRORTEXT"" : " syserrortext;
+    syserrortext=cats(symget('syserrortext'));
+    if findc(syserrortext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syserrortext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syserrortext)
+      )))))))))))))!!'"';
+    end;
+    else syserrortext=cats('"',syserrortext,'"');
+    put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');
+    syswarningtext=cats(symget('syswarningtext'));
+    if findc(syswarningtext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syswarningtext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syswarningtext)
+      )))))))))))))!!'"';
+    end;
+    else syswarningtext=cats('"',syswarningtext,'"');
     put ",""SYSWARNINGTEXT"" : " syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     put "}" ;

base/mp_createconstraints.sas

@@ -18,11 +18,14 @@
       %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
       %mp_createconstraints(inds=work.constraints,outds=created,execute=YES)
 
-  @param inds= The input table containing the constraint info
-  @param outds= a table containing the create statements (create_statement column)
-  @param execute= `YES|NO` - default is NO. To actually create, use YES.
+  @param inds= (work.mp_getconstraints) The input table containing the
+    constraint info
+  @param outds= (work.mp_createconstraints) A table containing the create
+    statements (create_statement column)
+  @param execute= (NO) To actually create, use YES.
 
-  <h4> SAS Macros </h4>
+  <h4> Related Files </h4>
+  @li mp_getconstraints.sas
 
   @version 9.2
   @author Allan Bowe
@@ -30,7 +33,7 @@
 **/
 
 %macro mp_createconstraints(inds=mp_getconstraints
-  ,outds=mp_createconstraints
+  ,outds=work.mp_createconstraints
   ,execute=NO
 )/*/STORE SOURCE*/;
 
@@ -64,4 +67,4 @@ data &outds;
   output;
 run;
 
-%mend mp_createconstraints;
+%mend mp_createconstraints;

base/mp_dictionary.sas

@@ -0,0 +1,52 @@
+/**
+  @file mp_dictionary.sas
+  @brief Creates a portal (libref) into the SQL Dictionary Views
+  @details Provide a libref and the macro will create a series of views against
+  each view in the special PROC SQL dictionary libref.
+
+  This is useful if you would like to visualise (navigate) the views in a SAS
+  client such as Base SAS, Enterprise Guide, or Studio (or [Data Controller](
+  https://datacontroller.io)).
+
+  It works by extracting the dictionary.dictionaries view into
+  YOURLIB.dictionaries, then uses that to create a YOURLIB.{viewName} for every
+  other dictionary.view, eg:
+
+      proc sql;
+      create view YOURLIB.columns as select * from dictionary.columns;
+
+  Usage:
+
+      libname demo "/lib/directory";
+      %mp_dictionary(lib=demo)
+
+  Or, to just create them in WORK:
+
+      %mp_dictionary()
+
+  If you'd just like to browse the dictionary data model, you can also check
+  out [this article](https://rawsas.com/dictionary-of-dictionaries/).
+
+  ![](https://user-images.githubusercontent.com/4420615/188278365-2987db97-0594-4a39-ac81-dbacdef5cdc8.png)
+
+  @param lib= (WORK) The libref in which to create the views
+
+  <h4> Related Files </h4>
+  @li mp_dictionary.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_dictionary(lib=WORK)/*/STORE SOURCE*/;
+  %local list i mem;
+  proc sql noprint;
+  create view &lib..dictionaries as select * from dictionary.dictionaries;
+  select distinct memname into: list separated by ' '  from &lib..dictionaries;
+  %do i=1 %to %sysfunc(countw(&list,%str( )));
+    %let mem=%scan(&list,&i,%str( ));
+    create view &lib..&mem as select * from dictionary.&mem;
+  %end;
+  quit;
+%mend mp_dictionary;

base/mp_dirlist.sas

@@ -27,6 +27,9 @@
   @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
     subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
     recursion, set to MAX.
+  @param [in] showparent= (NO) By default, the initial parent directory is not
+    part of the results.  Set to YES to include it.  For this record only,
+    directory=filepath.
   @param [out] outds= (work.mp_dirlist) The output dataset to create
   @param [out] getattrs= (NO)  If getattrs=YES then the doptname / foptname
     functions are used to scan all properties - any characters that are not
@@ -63,6 +66,7 @@
     , fref=0
     , outds=work.mp_dirlist
     , getattrs=NO
+    , showparent=NO
     , maxdepth=0
     , level=0 /* The level of recursion to perform.  For internal use only. */
 )/*/STORE SOURCE*/;
@@ -97,8 +101,7 @@ data &out_ds(compress=no
     if did=0 then do;
       putlog "NOTE: This directory is empty, or does not exist - &path";
       msg=sysmsg();
-      put msg;
-      put _all_;
+      put (_all_)(=);
       stop;
     end;
     /* attribute is OS-dependent - could be "Directory" or "Directory Name" */
@@ -145,6 +148,15 @@ data &out_ds(compress=no
     output;
   end;
   rc = dclose(did);
+  %if &showparent=YES and &level=0 %then %do;
+    filepath=directory;
+    file_or_folder='folder';
+    ext='';
+    filename=scan(directory,-1,'/\');
+    msg='';
+    level=&level;
+    output;
+  %end;
   stop;
 run;
 
@@ -232,6 +244,9 @@ run;
   data _null_;
     set &out_ds;
     where file_or_folder='folder';
+  %if &showparent=YES and &level=0 %then %do;
+    if filepath ne directory;
+  %end;
     length code $10000;
     code=cats('%nrstr(%mp_dirlist(path=',filepath,",outds=&outds"
       ,",getattrs=&getattrs,level=%eval(&level+1),maxdepth=&maxdepth))");

base/mp_ds2md.sas

@@ -92,7 +92,7 @@ data _null_;
 run;
 
 %if %upcase(&showlog)=YES %then %do;
-  options ps=max;
+  options ps=max lrecl=max;
   data _null_;
     infile &outref;
     input;
@@ -100,4 +100,4 @@ run;
   run;
 %end;
 
-%mend mp_ds2md;
+%mend mp_ds2md;

base/mp_gitadd.sas

@@ -0,0 +1,45 @@
+/**
+  @file
+  @brief Stages files in a GIT repo
+  @details Uses the output dataset from mp_gitstatus.sas to determine the files
+  that should be staged.
+
+  If `STAGED ne "TRUE"` then the file is staged.
+
+  Usage:
+
+      %let dir=%sysfunc(pathname(work))/core;
+      %let repo=https://github.com/sasjs/core;
+      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir));
+      %mf_writefile(&dir/somefile.txt,l1=some content)
+      %mf_deletefile(&dir/package.json)
+      %mp_gitstatus(&dir,outds=work.gitstatus)
+
+      %mp_gitadd(&dir,inds=work.gitstatus)
+
+  @param [in] gitdir The directory containing the GIT repository
+  @param [in] inds= (work.mp_gitadd) The input dataset with the list of files
+  to stage.  Will accept the output from mp_gitstatus(), else just use a table
+  with the following columns:
+    @li path $1024 - relative path to the file in the repo
+    @li staged $32 - whether the file is staged (TRUE or FALSE)
+    @li status $64 - either new, deleted, or modified
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> Related Files </h4>
+  @li mp_gitadd.test.sas
+  @li mp_gitstatus.sas
+
+**/
+
+%macro mp_gitadd(gitdir,inds=work.mp_gitadd,mdebug=0);
+
+data _null_;
+  set &inds;
+  if STAGED ne "TRUE";
+  rc=git_index_add("&gitdir",cats(path),status);
+  if rc ne 0 or &mdebug=1 then put rc=;
+run;
+
+%mend mp_gitadd;

base/mp_gitlog.sas

@@ -0,0 +1,104 @@
+/**
+  @file
+  @brief Creates a dataset with the commit history of a local repository
+  @details Returns the commit history from a local repository. The name of the
+  branch is also returned.
+
+  More details here:
+https://documentation.sas.com/doc/ko/pgmsascdc/v_033/lefunctionsref/n1qo5miyvry1nen111js203hlwrh.htm
+
+  Usage:
+
+      %let gitdir=%sysfunc(pathname(work))/core;
+      %let repo=https://github.com/sasjs/core;
+      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir));
+
+      %mp_gitlog(&gitdir,outds=work.mp_gitlog)
+
+  @param [in] gitdir The directory containing the GIT repository
+  @param [in] filter= (BRANCHONLY) To return only the commits for the current
+  branch, use BRANCHONLY (the default).  Anything else will return the entire
+  commit history.
+  @param [out] outds= (work.mp_gitlog) The output dataset to create.
+  All vars are $128 except `message` which is $4000.
+    @li author returns the author who submitted the commit.
+    @li children_ids returns a list of the children commit IDs
+    @li committer returns the name of the committer.
+    @li committer_email returns the email of the committer.
+    @li email returns the email of the commit author.
+    @li id returns the commit ID of the commit object.
+    @li in_current_branch returns "TRUE" or "FALSE" to indicate if the commit is
+      in the current branch.
+    @li message returns the commit message.
+    @li parent_ids returns a list of the parent commit IDs.
+    @li stash returns "TRUE" or "FALSE" to indicate if the commit is a stash
+      commit.
+    @li time returns the time of the commit as numeric string
+    @li commit_time_num time of the commit as numeric SAS datetime
+    @li commit_time_str the commit_time_num variable cast as string
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] nobs= (0) Set to an integer greater than 0 to restrict the number
+  of rows returned
+
+  <h4> SAS Macros </h4>
+  @li mf_getgitbranch.sas
+
+  <h4> Related Files </h4>
+  @li mp_gitadd.sas
+  @li mp_gitreleaseinfo.sas
+  @li mp_gitstatus.sas
+
+**/
+
+%macro mp_gitlog(gitdir,outds=work.mp_gitlog,mdebug=0,filter=BRANCHONLY,nobs=0);
+
+%local varlist i var;
+%let varlist=author children_ids committer committer_email email id
+  in_current_branch parent_ids stash time ;
+
+data &outds;
+  LENGTH gitdir branch $ 1024 message $4000 &varlist $128 commit_time_num 8.
+    commit_time_str $32;
+  call missing (of _all_);
+  branch="%mf_getgitbranch(&gitdir)";
+  gitdir=symget('gitdir');
+  rc=git_status_free(trim(gitdir));
+  if rc=-1 then do;
+    put "The libgit2 library is unavailable and no Git operations can be used.";
+    put "See: https://stackoverflow.com/questions/74082874";
+    stop;
+  end;
+  else if rc=-2 then do;
+    put "The libgit2 library is available, but the status function failed.";
+    put "See the log for details.";
+    stop;
+  end;
+  entries=git_commit_log(trim(gitdir));
+  do n=1 to entries;
+
+  %do i=1 %to %sysfunc(countw(&varlist message));
+    %let var=%scan(&varlist message,&i,%str( ));
+    rc=git_commit_get(n,trim(gitdir),"&var",&var);
+  %end;
+    /* convert unix time to SAS time - https://4gl.uk/corelink0 */
+    /* Number of seconds between 01JAN1960 and 01JAN1970: 315619200 */
+    format commit_time_num datetime19.;
+    commit_time_num=sum(input(cats(time),best.),315619200);
+    commit_time_str=put(commit_time_num,datetime19.);
+  %if &mdebug=1 %then %do;
+    putlog (_all_)(=);
+  %end;
+    if "&filter"="BRANCHONLY" then do;
+      if cats(in_current_branch)='TRUE' then output;
+    end;
+    else output;
+  %if &nobs>0 %then %do;
+    if n ge &nobs then stop;
+  %end;
+  end;
+  rc=git_commit_free(trim(gitdir));
+  keep gitdir branch &varlist message time commit_time_num commit_time_str;
+run;
+
+%mend mp_gitlog;

base/mp_gitreleaseinfo.sas

@@ -0,0 +1,74 @@
+/**
+  @file
+  @brief Pulls latest release info from a GIT repository
+  @details Useful for grabbing the latest version number or other attributes
+  from a GIT server.  Supported providers are GitLab and GitHub. Pull requests
+  are welcome if you'd like to see additional providers!
+
+  Note that each provider provides slightly different JSON output.  Therefore
+  the macro simply extracts the JSON and assigns the libname (using the JSON
+  engine).
+
+  Example usage (eg, to grab latest release version from github):
+
+      %mp_gitreleaseinfo(GITHUB,sasjs/core,outlib=mylibref)
+
+      data _null_;
+        set mylibref.root;
+        putlog TAG_NAME=;
+      run;
+
+  @param [in] provider The GIT provider for the release info.  Accepted values:
+    @li GITLAB
+    @li GITHUB - Tables include root, assets, author, alldata
+  @param [in] project The link to the repository.  This has different formats
+    depending on the vendor:
+    @li GITHUB - org/repo, eg sasjs/core
+    @li GITLAB - project, eg 1343223
+  @param [in] server= (0) If your repo is self-hosted, then provide the domain
+    here.  Otherwise it will default to the provider domain (eg gitlab.com).
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outlib= (GITREL) The JSON-engine libref to be created, which will
+    point at the returned JSON
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+
+  <h4> Related Files </h4>
+  @li mp_gitreleaseinfo.test.sas
+
+**/
+
+%macro mp_gitreleaseinfo(provider,project,server=0,outlib=GITREL,mdebug=0);
+%local url fref;
+
+%let provider=%upcase(&provider);
+
+%if &provider=GITHUB %then %do;
+  %if "&server"="0" %then %let server=https://api.github.com;
+  %let url=&server/repos/&project/releases/latest;
+%end;
+%else %if &provider=GITLAB %then %do;
+  %if "&server"="0" %then %let server=https://gitlab.com;
+  %let url=&server/api/v4/projects/&project/releases;
+%end;
+
+%let fref=%mf_getuniquefileref();
+
+proc http method='GET' out=&fref url="&url";
+%if &mdebug=1 %then %do;
+  debug level = 3;
+%end;
+run;
+
+libname &outlib JSON fileref=&fref;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fref;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_gitreleaseinfo;

base/mp_gitstatus.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Creates a dataset with the output from `GIT_STATUS()`
+  @details Uses `git_status()` to fetch the number of changed files, then
+  iterates with `git_status_get()`, inserting all attributes into an output
+  dataset.
+
+  Usage:
+
+      %let dir=%sysfunc(pathname(work))/core;
+      %let repo=https://github.com/sasjs/core;
+      %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir));
+      %mf_writefile(&dir/somefile.txt,l1=some content)
+      %mf_deletefile(&dir/package.json)
+
+      %mp_gitstatus(&dir,outds=work.gitstatus)
+
+  More info on these functions is in this [helpful paper]
+(https://www.sas.com/content/dam/SAS/support/en/sas-global-forum-proceedings/2019/3057-2019.pdf)
+  by Danny Zimmerman.
+
+  @param [in] gitdir The directory containing the GIT repository
+  @param [out] outds= (work.git_status) The output dataset to create.  Vars:
+    @li gitdir $1024 - directory of repo
+    @li path $1024 - relative path to the file in the repo
+    @li staged $32 - whether the file is staged (TRUE or FALSE)
+    @li status $64 - either new, deleted, or modified
+    @li cnt - number of files
+    @li n - the "nth" file in the list from git_status()
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> Related Files </h4>
+  @li mp_gitstatus.test.sas
+  @li mp_gitadd.sas
+
+**/
+
+%macro mp_gitstatus(gitdir,outds=work.mp_gitstatus,mdebug=0);
+
+data &outds;
+  LENGTH gitdir path $ 1024 STATUS $ 64 STAGED $ 32;
+  call missing (of _all_);
+  gitdir=symget('gitdir');
+  cnt=git_status(trim(gitdir));
+  if cnt=-1 then do;
+    put "The libgit2 library is unavailable and no Git operations can be used.";
+    put "See: https://stackoverflow.com/questions/74082874";
+  end;
+  else if cnt=-2 then do;
+    put "The libgit2 library is available, but the status function failed.";
+    put "See the log for details.";
+  end;
+  else do n=1 to cnt;
+    rc=GIT_STATUS_GET(n,gitdir,'PATH',path);
+    rc=GIT_STATUS_GET(n,gitdir,'STAGED',staged);
+    rc=GIT_STATUS_GET(n,gitdir,'STATUS',status);
+    output;
+  %if &mdebug=1 %then %do;
+    putlog (_all_)(=);
+  %end;
+  end;
+  rc=git_status_free(trim(gitdir));
+  drop rc cnt;
+run;
+
+%mend mp_gitstatus;

base/mp_hashdataset.sas

@@ -11,7 +11,7 @@
         put hashkey=;
       run;
 
-  ![sas md5 hash dataset log results](https://i.imgur.com/MqF98vk.png)
+  ![sas md5 hash dataset log results](https://i.4gl.io/1/KorUKoyE05.png/raw)
 
   <h4> SAS Macros </h4>
   @li mf_getattrn.sas
@@ -21,11 +21,12 @@
 
   <h4> Related Files </h4>
   @li mp_hashdataset.test.sas
+  @li mp_hashdirectory.sas
 
   @param [in] libds dataset to hash
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
-  @param [in] iftrue= A condition under which the macro should be executed.
-  @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [out] outds= (work._data_) The output dataset to create. This
   will contain one column (hashkey) with one observation (a $hex32.
   representation of the input hash)
   |hashkey:$32.|

base/mp_hashdirectory.sas

@@ -0,0 +1,164 @@
+/**
+  @file
+  @brief Returns a unique hash for each file in a directory
+  @details Hashes each file in each directory, and then hashes the hashes to
+  create a hash for each directory also.
+
+  This makes use of the new `hashing_file()` and `hashing` functions, available
+  since 9.4m6. Interestingly, those functions can be used in pure macro, eg:
+
+      %put %sysfunc(hashing_file(md5,/path/to/file.blob,0));
+
+  Actual usage:
+
+      %let fpath=/some/directory;
+
+      %mp_hashdirectory(&fpath,outds=myhash,maxdepth=2)
+
+      data _null_;
+        set work.myhash;
+        put (_all_)(=);
+      run;
+
+  Whilst files are hashed in their entirety, the logic for creating a folder
+  hash is as follows:
+
+  @li Sort the files by filename (case sensitive, uppercase then lower)
+  @li Take the first 100 hashes, concatenate and hash
+  @li Concatenate this hash with another 100 hashes and hash again
+  @li Continue until the end of the folder.  This is the folder hash
+  @li If a folder contains other folders, start from the bottom of the tree -
+    the folder hashes cascade upwards so you know immediately if there is a
+    change in a sub/sub directory
+  @li If a subfolder has no content (empty) then it is ignored. No hash created.
+  @li If the file is empty, it is also ignored / no hash created.
+  @li If the target directory (&inloc) is empty, &outds will also be empty
+
+  <h4> SAS Macros </h4>
+  @li mp_dirlist.sas
+
+  <h4> Related Files </h4>
+  @li mp_hashdataset.sas
+  @li mp_hashdirectory.test.sas
+  @li mp_md5.sas
+
+  @param [in] inloc Full filepath of the file to be hashed (unquoted)
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
+    subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
+  @param [in] method= (MD5) the hashing method to use.  Available options:
+    @li MD5
+    @li SH1
+    @li SHA256
+    @li SHA384
+    @li SHA512
+    @li CRC32
+  @param [out] outds= (work.mp_hashdirectory) The output dataset.  Contains:
+    @li directory - the parent folder
+    @li file_hash - the hash output
+    @li hash_duration - how long the hash took (first hash always takes longer)
+    @li file_path - /full/path/to/each/file.ext
+    @li file_or_folder - contains either "file" or "folder"
+    @li level - the depth of the directory (top level is 0)
+
+  @version 9.4m6
+  @author Allan Bowe
+**/
+
+%macro mp_hashdirectory(inloc,
+  outds=work.mp_hashdirectory,
+  method=MD5,
+  maxdepth=0,
+  iftrue=%str(1=1)
+)/*/STORE SOURCE*/;
+
+%local curlevel tempds maxlevel;
+
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+/* get the directory listing */
+%mp_dirlist(path=&inloc, outds=&outds, maxdepth=&maxdepth, showparent=YES)
+
+/* create the hashes */
+data &outds;
+  set &outds (rename=(filepath=file_path));
+  length FILE_HASH $32 HASH_DURATION 8;
+  keep directory file_hash hash_duration file_path file_or_folder level;
+
+  ts=datetime();
+  if file_or_folder='file' then do;
+    /* if file is empty, hashing_file will break - so ignore / delete */
+    length fname val $8;
+    drop fname val fid is_empty;
+    rc=filename(fname,file_path);
+    fid=fopen(fname);
+    if fid > 0 then do;
+      rc=fread(fid);
+      is_empty=fget(fid,val);
+    end;
+    rc=fclose(fid);
+    rc=filename(fname);
+    if is_empty ne 0 then delete;
+    else file_hash=hashing_file("&method",cats(file_path),0);
+  end;
+  hash_duration=datetime()-ts;
+run;
+
+proc sort data=&outds ;
+  by descending level directory file_path;
+run;
+
+%let maxlevel=0;
+data _null_;
+  set &outds;
+  call symputx('maxlevel',level,'l');
+  stop;
+run;
+
+/* now hash the hashes to populate folder hashes, starting from the bottom */
+%do curlevel=&maxlevel %to 0 %by -1;
+  data work._data_ (keep=directory file_hash);
+    set &outds;
+    where level=&curlevel;
+    by descending level directory file_path;
+    length str $32767 tmp_hash $32;
+    retain str tmp_hash ;
+    /* reset vars when starting a new directory */
+    if first.directory then do;
+      str='';
+      tmp_hash='';
+      i=0;
+    end;
+    /* hash each chunk of 100 file paths */
+    i+1;
+    str=cats(str,file_hash);
+    if mod(i,100)=0 or last.directory then do;
+      tmp_hash=hashing("&method",cats(tmp_hash,str));
+      str='';
+    end;
+    /* output the hash at directory level */
+    if last.directory then do;
+      file_hash=tmp_hash;
+      output;
+    end;
+    if last.level then stop;
+  run;
+  %let tempds=&syslast;
+  /* join the hash back into the main table */
+  proc sql undo_policy=none;
+  create table &outds as
+    select a.directory
+      ,coalesce(b.file_hash,a.file_hash) as file_hash
+      ,a.hash_duration
+      ,a.file_path
+      ,a.file_or_folder
+      ,a.level
+    from &outds a
+    left join &tempds b
+    on a.file_path=b.directory
+    order by level desc, directory, file_path;
+  drop table &tempds;
+%end;
+
+%mend mp_hashdirectory;

meta/mm_adduser2group.sas

@@ -81,7 +81,8 @@ run;
 filename __us2grp temp;
 
 proc metadata in= "<UpdateMetadata><Reposid>$METAREPOSITORY</Reposid><Metadata>
-    <Person Id='&uuri'><IdentityGroups><IdentityGroup ObjRef='&guri' />
+    <Person Id='%nrstr(&uuri)'>
+    <IdentityGroups><IdentityGroup ObjRef='%nrstr(&guri)' />
     </IdentityGroups></Person></Metadata>
     <NS>SAS</NS><Flags>268435456</Flags></UpdateMetadata>"
   out=__us2grp verbose;
@@ -98,4 +99,4 @@ run;
 
 filename __us2grp clear;
 
-%mend mm_adduser2group;
+%mend mm_adduser2group;

meta/mm_createwebservice.sas

@@ -546,7 +546,7 @@ data _null_;
   put '        put " ""&wt"" : {"; ';
   put '        put ''"nlobs":'' nlobs; ';
   put '        put '',"nvars":'' nvars; ';
-  put '      %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=Y,maxobs=10 ';
+  put '      %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=Y ';
   put '        ,maxobs=&workobs ';
   put '      ) ';
   put '      data _null_; file _sjsref mod encoding=''utf-8''; ';
@@ -571,7 +571,25 @@ data _null_;
   put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
   put '    put ",""SYSENCODING"" : ""&sysencoding"" "; ';
-  put '    syserrortext=cats(''"'',tranwrd(symget(''syserrortext''),''"'',''\"''),''"''); ';
+  put '    syserrortext=cats(symget(''syserrortext'')); ';
+  put '    if findc(syserrortext,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
+  put '      syserrortext=''"''!!trim( ';
+  put '        prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '        prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '        prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '        prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '        prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '        prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '        prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '        prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '        prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '        prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '        prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '        prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
+  put '        prxchange(''s/\\/\\\\/'',-1,syserrortext) ';
+  put '      )))))))))))))!!''"''; ';
+  put '    end; ';
+  put '    else syserrortext=cats(''"'',syserrortext,''"''); ';
   put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
@@ -584,7 +602,25 @@ data _null_;
   put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    syswarningtext=cats(''"'',tranwrd(symget(''syswarningtext''),''"'',''\"''),''"''); ';
+  put '    syswarningtext=cats(symget(''syswarningtext'')); ';
+  put '    if findc(syswarningtext,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
+  put '      syswarningtext=''"''!!trim( ';
+  put '        prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '        prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '        prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '        prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '        prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '        prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '        prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '        prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '        prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '        prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '        prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '        prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
+  put '        prxchange(''s/\\/\\\\/'',-1,syswarningtext) ';
+  put '      )))))))))))))!!''"''; ';
+  put '    end; ';
+  put '    else syswarningtext=cats(''"'',syswarningtext,''"''); ';
   put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';
@@ -610,9 +646,11 @@ data _null_;
   put ' ';
   put '%mend mm_webout; ';
 /* WEBOUT END */
-  put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO);';
+  put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO';
+  put '    ,maxobs=MAX';
+  put ');';
   put '  %mm_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt,missing=&missing';
-  put '    ,showmeta=&showmeta';
+  put '    ,showmeta=&showmeta,maxobs=&maxobs';
   put '  )';
   put '%mend;';
 run;

meta/mm_getstpinfo.sas

@@ -58,7 +58,7 @@ data &outds;
     rc5=metadata_getattr(tsuri,"Name",servercontext);
   end;
   else do;
-    put "%str(ERR)OR: could not find " pgm;
+    put "%str(ERR)OR: could not find " path;
     put (_all_)(=);
   end;
   &md.put (_all_)(=);

meta/mm_webout.sas

@@ -150,7 +150,7 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=Y,maxobs=10
+      %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=Y
         ,maxobs=&workobs
       )
       data _null_; file _sjsref mod encoding='utf-8';
@@ -175,7 +175,25 @@
     put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""SYSCC"" : ""&syscc"" ";
     put ",""SYSENCODING"" : ""&sysencoding"" ";
-    syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
+    syserrortext=cats(symget('syserrortext'));
+    if findc(syserrortext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syserrortext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syserrortext)
+      )))))))))))))!!'"';
+    end;
+    else syserrortext=cats('"',syserrortext,'"');
     put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
@@ -188,7 +206,25 @@
     put ",""SYSUSERID"" : ""&sysuserid"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');
+    syswarningtext=cats(symget('syswarningtext'));
+    if findc(syswarningtext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syswarningtext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syswarningtext)
+      )))))))))))))!!'"';
+    end;
+    else syswarningtext=cats('"',syswarningtext,'"');
     put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;

package.json

@@ -33,6 +33,6 @@
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "3.13.0"
+    "@sasjs/cli": "3.24.0"
   }
 }

sasjs/sasjsconfig.json

@@ -67,12 +67,16 @@
     },
     {
       "name": "server",
-      "serverUrl": "https://sas.4gl.io",
+      "serverUrl": "https://sas9.4gl.io",
       "serverType": "SASJS",
       "httpsAgentOptions": {
         "allowInsecureRequests": false
       },
       "appLoc": "/sasjs/core",
+      "deployConfig": {
+        "deployServicePack": true,
+        "deployScripts": []
+      },
       "macroFolders": [
         "server",
         "tests/serveronly"
@@ -81,7 +85,7 @@
     {
       "name": "docsonly",
       "serverType": "SASJS",
-      "appLoc": "dummy",
+      "appLoc": "/dummy",
       "macroFolders": [
         "meta",
         "metax",
@@ -105,6 +109,16 @@
         "deployServicePack": true
       },
       "contextName": "SAS Job Execution compute context"
+    },
+    {
+      "name": "sasjs9",
+      "serverUrl": "https://sas9.4gl.io",
+      "serverType": "SASJS",
+      "appLoc": "/Public/app/sasjs9",
+      "deployConfig": {
+        "deployServicePack": true,
+        "deployScripts": []
+      }
     }
   ]
 }

sasjs/utils/create_sas_package.sas

@@ -0,0 +1,224 @@
+/**
+  @file
+  @brief Deploy repo as a SAS PACKAGES module
+  @details After every release, this program is executed to update the SASPAC
+    repo with the latest macros (and same version number).
+    The program is first compiled using sasjs compile, then executed using
+    sasjs run.
+
+    Requires the server to have SSH keys.
+
+  <h4> SAS Macros </h4>
+  @li mp_gitadd.sas
+  @li mp_gitreleaseinfo.sas
+  @li mp_gitstatus.sas
+
+**/
+
+
+/* get package version */
+%mp_gitreleaseinfo(GITHUB,sasjs/core,outlib=splib)
+data _null_;
+  set splib.root;
+  call symputx('version',substr(TAG_NAME,2));
+run;
+
+/* clone the source repo */
+%let dir = %sysfunc(pathname(work))/core;
+%put source clone rc=%sysfunc(GITFN_CLONE(https://github.com/sasjs/core,&dir));
+
+/*
+  clone the target repo.
+  If you have issues, see: https://stackoverflow.com/questions/74082874
+*/
+options dlcreatedir;
+%let dirOut = %sysfunc(pathname(work))/package;
+libname _ "&dirOut.";
+
+%put tgt clone rc=%sysfunc(GITFN_CLONE(
+  git@github.com:SASPAC/sasjscore.git,
+  &dirOut,
+  git,
+  %str( ),
+  /home/sasjssrv/.ssh/id_ecdsa.pub,
+  /home/sasjssrv/.ssh/id_ecdsa
+));
+
+
+/*
+  Prepare Package Metadata
+*/
+data _null_;
+  infile CARDS4;
+  file  "&dirOut./description.sas";
+  input;
+  if _infile_ =: 'Version:' then put "Version: &version.";
+                            else put _infile_;
+CARDS4;
+Type: Package
+Package: SASjsCore
+Title: SAS Macros for Application Development
+Version: $(PLACEHOLDER)
+Author: Allan Bowe
+Maintainer: 4GL Ltd
+License: MIT
+Encoding: UTF8
+
+DESCRIPTION START:
+
+The SASjs Macro Core library is a component of the SASjs framework, the
+source for which is avaible here: https://github.com/sasjs
+
+Macros are divided by:
+
+* Macro Functions (prefix mf_)
+* Macro Procedures (prefix mp_)
+* Macros for Metadata (prefix mm_)
+* Macros for SASjs Server (prefix ms_)
+* Macros for Viya (prefix mv_)
+
+DESCRIPTION END:
+;;;;
+run;
+
+/*
+  Prepare Package License
+*/
+data _null_;
+  file  "&dirOut./license.sas";
+  infile "&dir/LICENSE";
+  input;
+  put _infile_;
+run;
+
+/*
+  Extract Core files into MacroCore Package location
+*/
+data members(compress=char);
+  length dref dref2 $ 8 name name2 $ 32 path $ 2048;
+  rc = filename(dref, "&dir.");
+  put dref=;
+  did = dopen(dref);
+  if did then
+    do i = 1 to dnum(did);
+      name = dread(did, i);
+      if name in
+        ("base" "ddl" "fcmp" "lua" "meta" "metax" "server" "viya" "xplatform")
+      then do;
+          rc = filename(dref2,catx("/", "&dir.", name));
+          put dref2= name;
+          did2 = dopen(dref2);
+
+          if did2 then
+            do j = 1 to dnum(did2);
+              name2 = dread(did2, j);
+              path = catx("/", "&dir.", name, name2);
+              if "sas" = scan(name2, -1, ".") then output;
+            end;
+          rc = dclose(did2);
+          rc = filename(dref2);
+        end;
+    end;
+  rc = dclose(did);
+  rc = filename(dref);
+  keep name name2 path;
+run;
+
+%let temp_options = %sysfunc(getoption(source)) %sysfunc(getoption(notes));
+options nosource nonotes;
+data _null_;
+  set members;
+  by name notsorted;
+
+  ord + first.name;
+
+  if first.name then
+    do;
+      call execute('libname _ '
+        !! quote(catx("/", "&dirOut.", put(ord, z3.)!!"_macros"))
+        !! ";"
+      );
+      put @1 "./" ord z3. "_macros/";
+    end;
+
+  put @10 name2;
+  call execute("
+  data _null_;
+    infile " !! quote(strip(path)) !! ";
+    file " !! quote(catx("/", "&dirOut.", put(ord, z3.)!!"_macros", name2)) !!";
+    input;
+    select;
+      when (2 = trigger) put _infile_;
+      when (_infile_ = '/**') do; put '/*** HELP START ***//**'; trigger+1; end;
+      when (_infile_ = '**/') do; put '**//*** HELP END ***/';   trigger+1; end;
+      otherwise put _infile_;
+    end;
+  run;");
+
+run;
+options &temp_options.;
+
+/*
+  Generate SASjsCore Package
+*/
+%GeneratePackage(
+  filesLocation=&dirOut
+)
+
+/**
+  * apply new version in a github action
+  * 1. create folder
+  * 2. create template yaml
+  * 3. replace version number
+  */
+
+%mf_mkdir(&dirout/.github/workflows)
+
+%let desc=Version &version of sasjs/core is now on SAS PACKAGES :ok_hand:;
+data _null_;
+  file "&dirout/.github/workflows/release.yml";
+  put "name: SASjs Core Package Publish Tag";
+  put "on:";
+  put "  push:";
+  put "    branches:";
+  put "      - main";
+  put "jobs:";
+  put "  update:";
+  put "    runs-on: ubuntu-latest";
+  put "    steps:";
+  put "      - uses: actions/checkout@master";
+  put "      - name: Make Release";
+  put "        uses: alice-biometrics/release-creator/@v1.0.5";
+  put "        with:";
+  put "          github_token: ${{ secrets.GH_TOKEN }}";
+  put "          branch: main";
+  put "          draft: false";
+  put "          version: &version";
+  put "          description: '&desc'";
+run;
+
+
+/**
+  * Add, Commit & Push!
+  */
+%mp_gitstatus(&dirout,outds=work.gitstatus,mdebug=1)
+%mp_gitadd(&dirout,inds=work.gitstatus,mdebug=1)
+
+data _null_;
+  rc=gitfn_commit("&dirout"
+    ,"HEAD","&sysuserid","sasjs@core"
+    ,"FEAT: Releasing &version"
+  );
+  put rc=;
+  rc=git_push(
+    "&dirout"
+    ,"git"
+    ,""
+    ,"/home/sasjssrv/.ssh/id_ecdsa.pub"
+    ,"/home/sasjssrv/.ssh/id_ecdsa"
+  );
+run;
+
+
+
+

server/ms_createwebservice.sas

@@ -539,7 +539,7 @@ data _null_;
   put '        put " ""&wt"" : {"; ';
   put '        put ''"nlobs":'' nlobs; ';
   put '        put '',"nvars":'' nvars; ';
-  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=Y,maxobs=10 ';
+  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=Y ';
   put '        ,maxobs=&workobs ';
   put '      ) ';
   put '      data _null_; file &fref mod encoding=''utf-8'' termstr=lf; ';
@@ -560,7 +560,25 @@ data _null_;
   put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
   put '    put ",""SYSENCODING"" : ""&sysencoding"" "; ';
-  put '    syserrortext=cats(''"'',tranwrd(symget(''syserrortext''),''"'',''\"''),''"''); ';
+  put '    syserrortext=cats(symget(''syserrortext'')); ';
+  put '    if findc(syserrortext,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
+  put '      syserrortext=''"''!!trim( ';
+  put '        prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '        prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '        prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '        prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '        prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '        prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '        prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '        prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '        prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '        prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '        prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '        prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
+  put '        prxchange(''s/\\/\\\\/'',-1,syserrortext) ';
+  put '      )))))))))))))!!''"''; ';
+  put '    end; ';
+  put '    else syserrortext=cats(''"'',syserrortext,''"''); ';
   put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    SYSHOSTINFOLONG=quote(trim(symget(''SYSHOSTINFOLONG''))); ';
   put '    put '',"SYSHOSTINFOLONG" : '' SYSHOSTINFOLONG; ';
@@ -576,7 +594,25 @@ data _null_;
   put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    syswarningtext=cats(''"'',tranwrd(symget(''syswarningtext''),''"'',''\"''),''"''); ';
+  put '    syswarningtext=cats(symget(''syswarningtext'')); ';
+  put '    if findc(syswarningtext,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
+  put '      syswarningtext=''"''!!trim( ';
+  put '        prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '        prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '        prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '        prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '        prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '        prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '        prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '        prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '        prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '        prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '        prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '        prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
+  put '        prxchange(''s/\\/\\\\/'',-1,syswarningtext) ';
+  put '      )))))))))))))!!''"''; ';
+  put '    end; ';
+  put '    else syswarningtext=cats(''"'',syswarningtext,''"''); ';
   put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';

server/ms_getgroups.sas

@@ -22,7 +22,7 @@
   @param [in] uid= (0) Provide the userid on which to filter
   @param [out] outds= (work.ms_getgroups) This output dataset will contain the
     list of groups. Format:
-|NAME:$32.|DESCRIPTION:$64.|GROUPID:best.|
+|NAME:$32.|DESCRIPTION:$256.|GROUPID:best.|
 |---|---|---|
 |`SomeGroup `|`A group `|`1`|
 |`Another Group`|`this is a different group`|`2`|
@@ -58,7 +58,7 @@
 %if %sysget(MODE)=desktop %then %do;
   /* groups api does not exist in desktop mode */
   data &outds;
-    length NAME $32 DESCRIPTION $64. GROUPID 8;
+    length NAME $32 DESCRIPTION $256. GROUPID 8;
     name="&sysuserid";
     description="&sysuserid (group - desktop mode)";
     groupid=1;
@@ -114,7 +114,7 @@ libname &libref JSON fileref=&fref1;
 
 %if "&user"="0" and "&uid"="0" %then %do;
   data &outds;
-    length NAME $32 DESCRIPTION $64. GROUPID 8;
+    length NAME $32 DESCRIPTION $256. GROUPID 8;
     if _n_=1 then call missing(of _all_);
     set &libref..root;
     drop ordinal_root;
@@ -122,7 +122,7 @@ libname &libref JSON fileref=&fref1;
 %end;
 %else %do;
   data &outds;
-    length NAME $32 DESCRIPTION $64. GROUPID 8;
+    length NAME $32 DESCRIPTION $256. GROUPID 8;
     if _n_=1 then call missing(of _all_);
     set &libref..groups;
     drop ordinal_:;

server/ms_runstp.sas

@@ -39,8 +39,9 @@
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
-  @li mf_getuniquelibref.sas
+  @li mf_getuniquename.sas
   @li mp_abort.sas
+  @li mp_chop.sas
 
 **/
 
@@ -153,7 +154,10 @@ run;
   run;
 %end;
 
-filename &outref temp lrecl=32767;
+%local resp_path;
+%let resp_path=%sysfunc(pathname(work))/%mf_getuniquename();
+filename &outref "&resp_path" lrecl=32767;
+
 /* prepare request*/
 proc http method='POST' headerin=&authref in=&mainref out=&outref
   url="&_sasjs_apiserverurl.&_sasjs_apipath?_program=&pgm%str(&)_debug=131";
@@ -161,6 +165,7 @@ proc http method='POST' headerin=&authref in=&mainref out=&outref
   debug level=2;
 %end;
 run;
+
 %if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
 or &mdebug=1
 %then %do;
@@ -176,11 +181,22 @@ or &mdebug=1
 options &optval;
 
 %if &outlogds ne _null_ or &mdebug=1 %then %do;
-  %local dumplib;
-  %let dumplib=%mf_getuniquelibref();
-  libname &dumplib json fileref=&outref;
+  %local matchstr chopout;
+  %let matchstr=SASJS_LOGS_SEPARATOR_163ee17b6ff24f028928972d80a26784;
+  %let chopout=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop);
+
+  %mp_chop("&resp_path"
+    ,matchvar=matchstr
+    ,keep=LAST
+    ,matchpoint=END
+    ,outfile="&chopout"
+    ,mdebug=&mdebug
+  )
+
   data &outlogds;
-    set &dumplib..log;
+    infile "&chopout" lrecl=2000;
+    length line $2000;
+    line=_infile_;
   %if &mdebug=1 %then %do;
     putlog line=;
   %end;

server/ms_testservice.sas

@@ -108,47 +108,34 @@ run;
 )
 
 
-/* SASjs services have the _webout embedded in wrapper JSON */
-/* Files can also be very large - so use a dedicated macro to chop it out */
-%local matchstr1 matchstr2 ;
-%let matchstr1={"status":"success","_webout":{;
-%let matchstr2=},"log":[{;
-%let chopout1=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop1);
-%let chopout2=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop2);
+/* chop out JSON section */
+%local matchstr chopout;
+%let matchstr=SASJS_LOGS_SEPARATOR_163ee17b6ff24f028928972d80a26784;
+%let chopout=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop);
 
 %mp_chop("%sysfunc(pathname(&fref1,F))"
-  ,matchvar=matchstr1
-  ,keep=LAST
-  ,matchpoint=END
-  ,offset=-1
-  ,outfile="&chopout1"
-  ,mdebug=&mdebug
-)
-
-%mp_chop("&chopout1"
-  ,matchvar=matchstr2
+  ,matchvar=matchstr
   ,keep=FIRST
   ,matchpoint=START
-  ,offset=1
-  ,outfile="&chopout2"
+  ,offset=-1
+  ,outfile="&chopout"
   ,mdebug=&mdebug
 )
 
 %if &outlib ne 0 %then %do;
-  libname &outlib json "&chopout2";
+  libname &outlib json "&chopout";
 %end;
 %if &outref ne 0 %then %do;
-  filename &outref "&chopout2";
+  filename &outref "&chopout";
 %end;
 
 %if &mdebug=0 %then %do;
   filename &webref clear;
   filename &fref1 clear;
-  filename &fref2 clear;
 %end;
 %else %do;
   %put &sysmacroname exit vars:;
   %put _local_;
 %end;
 
-%mend ms_testservice;
+%mend ms_testservice;

server/ms_webout.sas

@@ -141,7 +141,7 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=Y,maxobs=10
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=Y
         ,maxobs=&workobs
       )
       data _null_; file &fref mod encoding='utf-8' termstr=lf;
@@ -162,7 +162,25 @@
     put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""SYSCC"" : ""&syscc"" ";
     put ",""SYSENCODING"" : ""&sysencoding"" ";
-    syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
+    syserrortext=cats(symget('syserrortext'));
+    if findc(syserrortext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syserrortext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syserrortext)
+      )))))))))))))!!'"';
+    end;
+    else syserrortext=cats('"',syserrortext,'"');
     put ',"SYSERRORTEXT" : ' syserrortext;
     SYSHOSTINFOLONG=quote(trim(symget('SYSHOSTINFOLONG')));
     put ',"SYSHOSTINFOLONG" : ' SYSHOSTINFOLONG;
@@ -178,7 +196,25 @@
     put ",""SYSUSERID"" : ""&sysuserid"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');
+    syswarningtext=cats(symget('syswarningtext'));
+    if findc(syswarningtext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syswarningtext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syswarningtext)
+      )))))))))))))!!'"';
+    end;
+    else syswarningtext=cats('"',syswarningtext,'"');
     put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;

tests/base/mf_getgitbranch.test.sas

@@ -0,0 +1,20 @@
+/**
+  @file
+  @brief Testing mf_getgitbranch.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getgitbranch.sas
+  @li mp_assert.sas
+
+**/
+
+/* grab core repo */
+%let gitdir=%sysfunc(pathname(work))/core;
+%let repo=https://github.com/sasjs/core;
+%put source clone rc=%sysfunc(GITFN_CLONE(&repo,&gitdir));
+
+%mp_assert(
+  iftrue=(%mf_getgitbranch(&gitdir)=main),
+  desc=Checking correct branch was obtained,
+  outds=work.test_results
+)

tests/base/mf_readfile.test.sas

@@ -0,0 +1,40 @@
+/**
+  @file
+  @brief Testing mf_readfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_readfile.sas
+  @li mf_writefile.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%let f=&sasjswork/myfile.txt;
+
+%mf_writefile(&f,l1=some content,l2=more content)
+data _null_;
+  infile "&f";
+  input;
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check code ran without errors,
+  outds=work.test_results
+)
+
+/* test for scope leakage */
+%global result;
+%mp_assertscope(SNAPSHOT)
+%put %mf_readfile(&f);
+%mp_assertscope(COMPARE)
+
+/* test result */
+%mp_assert(
+  iftrue=(%mf_readfile(&f)=some content),
+  desc=Checking first line was ingested successfully,
+  outds=work.test_results
+)
+

tests/base/mp_dictionary.test.sas

@@ -0,0 +1,26 @@
+/**
+  @file
+  @brief Testing mp_dictionary.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_dictionary.sas
+  @li mp_assert.sas
+
+**/
+
+libname test (work);
+%mp_dictionary(lib=test)
+
+proc sql;
+create table work.compare1 as select * from test.styles;
+create table work.compare2 as select * from dictionary.styles;
+
+proc compare base=compare1 compare=compare2;
+run;
+%put _all_;
+
+%mp_assert(
+  iftrue=(%mf_existds(&sysinfo)=0),
+  desc=Compare was exact,
+  outds=work.test_results
+)

tests/base/mp_gitadd.test.sas

@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Testing mp_gitadd.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_deletefile.sas
+  @li mf_writefile.sas
+  @li mp_gitadd.sas
+  @li mp_gitstatus.sas
+  @li mp_assert.sas
+
+**/
+
+/* clone the source repo */
+%let dir = %sysfunc(pathname(work))/core;
+%put source clone rc=%sysfunc(GITFN_CLONE(https://github.com/sasjs/core,&dir));
+
+/* add a file */
+%mf_writefile(&dir/somefile.txt,l1=some content)
+/* change a file */
+%mf_writefile(&dir/readme.md,l1=new readme)
+/* delete a file */
+%mf_deletefile(&dir/package.json)
+
+/* Run git status */
+%mp_gitstatus(&dir,outds=work.gitstatus)
+
+%let test1=0;
+proc sql noprint;
+select count(*) into: test1 from work.gitstatus where staged='FALSE';
+
+/* should be three unstaged changes now */
+%mp_assert(
+  iftrue=(&test1=3),
+  desc=3 changes are ready to add,
+  outds=work.test_results
+)
+
+/* add them */
+%mp_gitadd(&dir,inds=work.gitstatus,mdebug=&sasjs_mdebug)
+
+/* check status */
+%mp_gitstatus(&dir,outds=work.gitstatus2)
+%let test2=0;
+proc sql noprint;
+select count(*) into: test2 from work.gitstatus2 where staged='TRUE';
+
+/* should be three staged changes now */
+%mp_assert(
+  iftrue=(&test2=3),
+  desc=3 changes were added,
+  outds=work.test_results
+)

tests/base/mp_gitlog.test.sas

@@ -0,0 +1,32 @@
+/**
+  @file
+  @brief Testing mp_gitlog.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_gitlog.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+/* grab core repo */
+%let gitdir=%sysfunc(pathname(work))/core;
+%let repo=https://github.com/sasjs/core;
+%put source clone rc=%sysfunc(GITFN_CLONE(&repo,&gitdir));
+
+%mp_assertscope(SNAPSHOT)
+%mp_gitlog(&gitdir,outds=work.test1)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Regular test works,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.test1)>1000),
+  desc=output has gt 1000 rows,
+  outds=work.test_results
+)

tests/base/mp_gitreleaseinfo.test.sas

@@ -0,0 +1,30 @@
+/**
+  @file
+  @brief Testing mp_gitreleaseinfo.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_gitreleaseinfo.sas
+  @li mp_assert.sas
+
+**/
+
+
+%mp_gitreleaseinfo(github,sasjs/core,outlib=mylibref,mdebug=1)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=mp_gitreleaseinfo runs without errors,
+  outds=work.test_results
+)
+
+data _null_;
+  set mylibref.author;
+  putlog (_all_)(=);
+  call symputx('author',login);
+run;
+
+%mp_assert(
+  iftrue=(&author=sasjsbot),
+  desc=release info extracted successfully,
+  outds=work.test_results
+)

tests/base/mp_gitstatus.test.sas

@@ -0,0 +1,39 @@
+/**
+  @file
+  @brief Testing mp_gitstatus.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_deletefile.sas
+  @li mf_writefile.sas
+  @li mp_gitstatus.sas
+  @li mp_assertdsobs.sas
+
+**/
+
+/* clone the source repo */
+%let dir = %sysfunc(pathname(work))/core;
+%put source clone rc=%sysfunc(GITFN_CLONE(https://github.com/sasjs/core,&dir));
+
+%mp_gitstatus(&dir,outds=work.gitstatus)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Initial mp_gitstatus runs without errors,
+  outds=work.test_results
+)
+
+/* should be empty as there are no changes yet */
+%mp_assertdsobs(work.gitstatus,test=EMPTY)
+
+/* add a file */
+%mf_writefile(&dir/somefile.txt,l1=some content)
+/* change a file */
+%mf_writefile(&dir/readme.md,l1=new readme)
+/* delete a file */
+%mf_deletefile(&dir/package.json)
+
+/* re-run git status */
+%mp_gitstatus(&dir,outds=work.gitstatus)
+
+/* should be three changes now */
+%mp_assertdsobs(work.gitstatus,test=EQUALS 3)

tests/base/mp_hashdirectory.test.sas

@@ -0,0 +1,149 @@
+/**
+  @file
+  @brief Testing mp_hashdirectory.sas macro
+
+
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mf_nobs.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mp_hashdirectory.sas
+
+**/
+
+/* set up a directory to hash */
+%let fpath=%sysfunc(pathname(work))/testdir;
+
+%mf_mkdir(&fpath)
+%mf_mkdir(&fpath/sub1)
+%mf_mkdir(&fpath/sub2)
+%mf_mkdir(&fpath/sub1/subsub)
+
+/* note - the path in the file means the hash is different in each run */
+%macro makefile(path,name);
+  data _null_;
+    file "&path/&name" termstr=lf;
+    put "This file is located at:";
+    put "&path";
+    put "and it is called:";
+    put "&name";
+  run;
+%mend makefile;
+
+%macro spawner(path);
+  %do x=1 %to 5;
+    %makefile(&path,file&x..txt)
+  %end;
+%mend spawner;
+
+%spawner(&fpath)
+%spawner(&fpath/sub1)
+%spawner(&fpath/sub1/subsub)
+
+
+%mp_assertscope(SNAPSHOT)
+%mp_hashdirectory(&fpath,outds=work.hashes,maxdepth=MAX)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errors,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.hashes)=19),
+  desc=record created for each entry,
+  outds=work.test_results
+)
+
+proc sql;
+select count(*) into: misscheck
+  from work.hashes
+  where file_hash is missing;
+
+%mp_assert(
+  iftrue=(&misscheck=1),
+  desc=Only one missing hash - the empty directory,
+  outds=work.test_results
+)
+
+data _null_;
+  set work.hashes;
+  if directory=file_path then call symputx('tophash',file_hash);
+run;
+
+%mp_assert(
+  iftrue=(%length(&tophash)=32),
+  desc=ensure valid top level hash created,
+  outds=work.test_results
+)
+
+/* now change a file and re-hash */
+data _null_;
+  file "&fpath/sub1/subsub/file1.txt" termstr=lf;
+  put "This file has changed!";
+run;
+
+%mp_hashdirectory(&fpath,outds=work.hashes2,maxdepth=MAX)
+
+data _null_;
+  set work.hashes2;
+  if directory=file_path then call symputx('tophash2',file_hash);
+run;
+
+%mp_assert(
+  iftrue=(&tophash ne &tophash2),
+  desc=ensure the changing of the hash results in a new value,
+  outds=work.test_results
+)
+
+/* now change it back and see if it matches */
+data _null_;
+  file "&fpath/sub1/subsub/file1.txt" termstr=lf;
+    put "This file is located at:";
+    put "&fpath/sub1/subsub";
+    put "and it is called:";
+    put "file1.txt";
+  run;
+run;
+
+%mp_hashdirectory(&fpath,outds=work.hashes3,maxdepth=MAX)
+
+data _null_;
+  set work.hashes3;
+  if directory=file_path then call symputx('tophash3',file_hash);
+run;
+
+%mp_assert(
+  iftrue=(&tophash=&tophash3),
+  desc=ensure the same files result in the same hash,
+  outds=work.test_results
+)
+
+/* dump contents for debugging */
+data _null_;
+  set work.hashes;
+  put file_hash file_path;
+run;
+data _null_;
+  set work.hashes2;
+  put file_hash file_path;
+run;
+
+/* check that it works when the target directory is missing */
+
+%mp_hashdirectory(&fpath/doesnotexist,outds=work.hashes3,maxdepth=MAX)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errors when directory is missing,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.hashes3)=0),
+  desc=no records created when directory is missing,
+  outds=work.test_results
+)

tests/viyaonly/mv_jobflow.test.1.sas

@@ -4,7 +4,7 @@
   @brief Testing mv_jobflow macro
   @details One of the remote jobs aborts with syscc>0 - test to
   make sure this comes back to the calling session
-​
+
   <h4> SAS Macros </h4>
   @li mp_assert.sas
   @li mv_createjob.sas

tests/viyaonly/mv_jobflow.test.2.sas

@@ -3,7 +3,7 @@
   @brief Testing mv_jobflow macro
   @details All jobs complete successfully with syscc = 0 - test to
   make sure this comes back to the calling session
-​
+
   <h4> SAS Macros </h4>
   @li mp_assert.sas
   @li mv_createjob.sas

tests/viyaonly/mv_registerclient.test.1.sas

@@ -4,7 +4,7 @@
   @brief Testing mv_registerclient.sas macro
   @details Tests for successful registration.  For this to work, the test
   account must be an admin.
-​
+
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas
   @li mp_assertcolvals.sas

tests/viyaonly/mv_registerclient.test.2.sas

@@ -1,10 +1,9 @@
 /**
-
   @file
   @brief Testing mv_registerclient.sas macro
   @details Tests for unsuccessful registration.  To do this, overrides are
   applied for the mf_loc.sas and mp_abort.sas macros.
-​
+
   <h4> SAS Macros </h4>
   @li mp_assert.sas
   @li mv_registerclient.sas

viya/mfv_existsashdat.sas

@@ -0,0 +1,60 @@
+/**
+  @file mfv_existsashdat.sas
+  @brief Checks whether a CAS sashdat dataset exists in persistent storage.
+  @details Can be used in open code, eg as follows:
+
+      %if %mfv_existsashdat(libds=casuser.sometable) %then %put  yes it does!;
+
+  The function uses `dosubl()` to run the `table.fileinfo` action, for the
+  specified library, filtering for `*.sashdat` tables.  The results are stored
+  in a WORK table (&outprefix._&lib). If that table already exists, it is
+  queried instead, to avoid the dosubl() performance hit.
+
+  To force a rescan, just use a new `&outprefix` value, or delete the table(s)
+  before running the function.
+
+  @param libds library.dataset
+  @param outprefix= (work.mfv_existsashdat) Used to store the current HDATA
+    tables to improve subsequent query performance.  This reference is a prefix
+    and is converted to `&prefix._{libref}`
+
+  @return output returns 1 or 0
+
+  @version 0.2
+  @author Mathieu Blauw
+**/
+
+%macro mfv_existsashdat(libds,outprefix=work.mfv_existsashdat
+);
+%local rc dsid name lib ds;
+%let lib=%upcase(%scan(&libds,1,'.'));
+%let ds=%upcase(%scan(&libds,-1,'.'));
+
+/* if table does not exist, create it */
+%if %sysfunc(exist(&outprefix._&lib)) ne 1 %then %do;
+  %let rc=%sysfunc(dosubl(%nrstr(
+    /* Read in table list (once per &lib per session) */
+    proc cas;
+      table.fileinfo result=source_list /caslib="&lib";
+      val=findtable(source_list);
+      saveresult val dataout=&outprefix._&lib;
+    quit;
+    /* Only keep name, without file extension */
+    data &outprefix._&lib;
+      set &outprefix._&lib(where=(Name like '%.sashdat') keep=Name);
+      Name=upcase(scan(Name,1,'.'));
+    run;
+  )));
+%end;
+
+/* Scan table for hdat existence */
+%let dsid=%sysfunc(open(&outprefix._&lib(where=(name="&ds"))));
+%syscall set(dsid);
+%let rc = %sysfunc(fetch(&dsid));
+%let rc = %sysfunc(close(&dsid));
+
+/* Return result */
+%if "%trim(&name)"="%trim(&ds)" %then 1;
+%else 0;
+
+%mend mfv_existsashdat;

viya/mv_createfile.sas

@@ -24,7 +24,8 @@
   @param [in] contentdisp= (inline) Content Disposition. Example values:
     @li inline
     @li attachment
-
+  @param [in] ctype= (0) Set a default HTTP Content-Type header to be returned
+    with the file when the content is retrieved from the Files service.
   @param [in] access_token_var= The global macro variable to contain the access
     token, if using authorization_code grant type.
   @param [in] grant_type= (sas_services) Valid values are:
@@ -52,6 +53,7 @@
     ,inref=
     ,intype=BINARY
     ,contentdisp=inline
+    ,ctype=0
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
     ,mdebug=0
@@ -103,8 +105,10 @@ filename &fref filesrvc
   folderPath="&path"
   filename="&name"
   cdisp="&contentdisp"
+%if "&ctype" ne "0" %then %do;
+  ctype="&ctype"
+%end;
   lrecl=1048544;
-
 %if &intype=BINARY %then %do;
   %mp_binarycopy(inref=&inref, outref=&fref)
 %end;

viya/mv_createjob.sas

@@ -34,14 +34,25 @@
 
   @param path= The full path (on SAS Drive) where the job will be created
   @param name= The name of the job
-  @param desc= The description of the job
+  @param desc= (Created by the mv_createjob.sas macro) The job description
   @param precode= Space separated list of filerefs, pointing to the code that
     needs to be attached to the beginning of the job
-  @param code= Fileref(s) of the actual code to be added
-  @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values are "password" or "authorization_code"
-    (unquoted). The default is authorization_code.
-  @param replace= select NO to avoid replacing any existing job in that location
+  @param code= (ft15f001) Fileref(s) of the actual code to be added
+  @param access_token_var= (ACCESS_TOKEN) Global macro variable containing the
+    access token
+  @param grant_type= (sas_services) Valid values:
+    @li sas_services
+    @li detect
+    @li authorization_code
+    @li password
+  @param replace= (YES) select NO to avoid replacing any existing job
+  @param addjesbeginendmacros= (false) Relates to the `_addjesbeginendmacros`
+    setting.  Normally this would always be false however due to a Viya bug
+    (https://github.com/sasjs/cli/issues/1229) this is now configurable.  Valid
+    values:
+    @li true
+    @li false
+    @li 0 - this will prevent the flag from being set (job will default to true)
   @param contextname= Choose a specific context on which to run the Job.  Leave
     blank to use the default context.  From Viya 3.5 it is possible to configure
     a shared context - see
@@ -62,6 +73,7 @@ https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5p
     ,replace=YES
     ,debug=0
     ,contextname=
+    ,addjesbeginendmacros=false
   );
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
@@ -185,19 +197,29 @@ run;
 %end;
 
 /* set up the body of the request to create the service */
-%local fname3;
+%local fname3 comma;
 %let fname3=%mf_getuniquefileref();
 data _null_;
   file &fname3 TERMSTR=' ';
   length string $32767;
   string=cats('{"version": 0,"name":"'
     ,"&name"
-    ,'","type":"Compute","parameters":[{"name":"_addjesbeginendmacros"'
-    ,',"type":"CHARACTER","defaultValue":"false"}');
+    ,'","type":"Compute","parameters":['
+%if &addjesbeginendmacros ne 0 %then %do;
+    ,'{"name":"_addjesbeginendmacros"'
+    ,',"type":"CHARACTER","defaultValue":"'
+    ,"&addjesbeginendmacros"
+    ,'"}'
+  %let comma=%str(,);
+%end;
+  );
   context=quote(cats(symget('contextname')));
   if context ne '""' then do;
-    string=cats(string,',{"version": 1,"name": "_contextName","defaultValue":'
-      ,context,',"type":"CHARACTER","label":"Context Name","required": false}');
+    string=cats(string
+      ,"&comma"
+      ,'{"version": 1,"name": "_contextName","defaultValue":'
+      ,context,',"type":"CHARACTER","label":"Context Name","required": false}'
+    );
   end;
   string=cats(string,'],"code":"');
   put string;

viya/mv_createwebservice.sas

@@ -743,7 +743,25 @@ data _null_;
   put '    put '',"SYS_JES_JOB_URI" : '' SYS_JES_JOB_URI ; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    syserrortext=cats(''"'',tranwrd(symget(''syserrortext''),''"'',''\"''),''"''); ';
+  put '    syserrortext=cats(symget(''syserrortext'')); ';
+  put '    if findc(syserrortext,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
+  put '      syserrortext=''"''!!trim( ';
+  put '        prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '        prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '        prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '        prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '        prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '        prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '        prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '        prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '        prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '        prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '        prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '        prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
+  put '        prxchange(''s/\\/\\\\/'',-1,syserrortext) ';
+  put '      )))))))))))))!!''"''; ';
+  put '    end; ';
+  put '    else syserrortext=cats(''"'',syserrortext,''"''); ';
   put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
@@ -756,7 +774,25 @@ data _null_;
   put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    syswarningtext=cats(''"'',tranwrd(symget(''syswarningtext''),''"'',''\"''),''"''); ';
+  put '    syswarningtext=cats(symget(''syswarningtext'')); ';
+  put '    if findc(syswarningtext,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
+  put '      syswarningtext=''"''!!trim( ';
+  put '        prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '        prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '        prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '        prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '        prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '        prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '        prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '        prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '        prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '        prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '        prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '        prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
+  put '        prxchange(''s/\\/\\\\/'',-1,syswarningtext) ';
+  put '      )))))))))))))!!''"''; ';
+  put '    end; ';
+  put '    else syswarningtext=cats(''"'',syswarningtext,''"''); ';
   put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';

viya/mv_webout.sas

@@ -204,7 +204,25 @@
     put ',"SYS_JES_JOB_URI" : ' SYS_JES_JOB_URI ;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSCC"" : ""&syscc"" ";
-    syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
+    syserrortext=cats(symget('syserrortext'));
+    if findc(syserrortext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syserrortext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syserrortext)
+      )))))))))))))!!'"';
+    end;
+    else syserrortext=cats('"',syserrortext,'"');
     put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
@@ -217,7 +235,25 @@
     put ",""SYSUSERID"" : ""&sysuserid"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');
+    syswarningtext=cats(symget('syswarningtext'));
+    if findc(syswarningtext,'"\'!!'0A0D09000E0F010210111A'x) then do;
+      syswarningtext='"'!!trim(
+        prxchange('s/"/\\"/',-1,        /* double quote */
+        prxchange('s/\x0A/\n/',-1,      /* new line */
+        prxchange('s/\x0D/\r/',-1,      /* carriage return */
+        prxchange('s/\x09/\\t/',-1,     /* tab */
+        prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+        prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+        prxchange('s/\x01/\\u0001/',-1, /* SOH */
+        prxchange('s/\x02/\\u0002/',-1, /* STX */
+        prxchange('s/\x10/\\u0010/',-1, /* DLE */
+        prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+        prxchange('s/\x1A/\\u001A/',-1, /* SUB */
+        prxchange('s/\\/\\\\/',-1,syswarningtext)
+      )))))))))))))!!'"';
+    end;
+    else syswarningtext=cats('"',syswarningtext,'"');
     put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;