chore: merge conflicts

fix: removing notes when running mp_zip, closes #88. .

.all-contributorsrc

@@ -98,7 +98,18 @@
       "contributions": [
         "ideas"
       ]
+    },
+    {
+      "login": "VladislavParhomchik",
+      "name": "Vladislav Parhomchik",
+      "avatar_url": "https://avatars.githubusercontent.com/u/83717836?v=4",
+      "profile": "https://github.com/VladislavParhomchik",
+      "contributions": [
+        "test",
+        "review"
+      ]
     }
   ],
-  "contributorsPerLine": 7
+  "contributorsPerLine": 7,
+  "skipCi": true
 }

.github/workflows/run-tests.yml

@@ -12,12 +12,12 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x]
+        node-version: [lts/fermium]
 
     steps:
       - uses: actions/checkout@v2
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node-version }}
 

.gitpod.yml

@@ -1,8 +1,25 @@
 tasks:
-  - init: nvm install --latest-npm && npm i -g @sasjs/cli
+  - init: nvm install --lts && npm i -g @sasjs/cli
 
 image:
   file: .gitpod.dockerfile
 vscode:
   extensions:
     - sasjs.sasjs-for-vscode
+
+github:
+  prebuilds:
+    # enable for the master/default branch (defaults to true)
+    master: true
+    # enable for all branches in this repo (defaults to false)
+    branches: false
+    # enable for pull requests coming from this repo (defaults to true)
+    pullRequests: true
+    # enable for pull requests coming from forks (defaults to false)
+    pullRequestsFromForks: true
+    # add a "Review in Gitpod" button as a comment to pull requests (defaults to true)
+    addComment: true
+    # add a "Review in Gitpod" button to pull requests (defaults to false)
+    addBadge: false
+    # add a label once the prebuild is ready to pull requests (defaults to false)
+    addLabel: prebuilt-in-gitpod

.npmignore

@@ -9,3 +9,4 @@ sasjs/
 main.dox
 make_singlefile.sh
 *.md
+.all-contributorsrc

README.md

@@ -189,7 +189,7 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-8-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-9-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -208,6 +208,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
   </tr>
   <tr>
     <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
+    <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
   </tr>
 </table>
 

all.sas

@@ -975,7 +975,8 @@ https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionex
     %end;
   %end;
   %else %do;
-    %put dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
     %return;
   %end;
 
@@ -1037,7 +1038,11 @@ https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionex
       %let vlen = %str( );
     %end;
   %end;
-  %else %put dataset &libds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));
@@ -1161,7 +1166,11 @@ returns:
       %let vnum = %str( );
     %end;
   %end;
-  %else %put dataset &ds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));
@@ -1210,7 +1219,11 @@ Usage:
       %let vtype = %str( );
     %end;
   %end;
-  %else %put dataset &libds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));
@@ -1806,7 +1819,7 @@ Usage:
     /* send response in SASjs JSON format */
     data _null_;
       file _webout mod lrecl=32000 encoding='utf-8';
-      length msg $32767 debug $8;
+      length msg $32767 ;
       sasdatetime=datetime();
       msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
       /* escape the quotes */
@@ -1817,7 +1830,7 @@ Usage:
       msg=cats('"',msg,'"');
       if symexist('_debug') then debug=quote(trim(symget('_debug')));
       else debug='""';
-      if debug ge '"131"' then put '>>weboutBEGIN<<';
+      put '>>weboutBEGIN<<';
       put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
       put ',"sasjsAbort" : [{';
       put ' "MSG":' msg ;
@@ -1837,23 +1850,26 @@ Usage:
       _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
       put ',"_PROGRAM" : ' _PROGRAM ;
       put ",""SYSCC"" : ""&syscc"" ";
-      put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+      syserrortext=quote(trim(symget('syserrortext')));
+      put ",""SYSERRORTEXT"" : " syserrortext;
       put ",""SYSHOSTNAME"" : ""&syshostname"" ";
       put ",""SYSJOBID"" : ""&sysjobid"" ";
+      put ",""SYSSCPL"" : ""&sysscpl"" ";
       put ",""SYSSITE"" : ""&syssite"" ";
       sysvlong=quote(trim(symget('sysvlong')));
       put ',"SYSVLONG" : ' sysvlong;
-      put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+      syswarningtext=quote(trim(symget('syswarningtext')));
+      put ",""SYSWARNINGTEXT"" : " syswarningtext;
       put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
       put "}" @;
-      if debug ge '"131"' then put '>>weboutEND<<';
+      put '>>weboutEND<<';
     run;
 
     %put _all_;
 
     %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
       data _null_;
-        putlog 'stpsrvset program error and syscc';
+        putlog 'stpsrvset program err and syscc';
         rc=stpsrvset('program error', 0);
         call symputx("syscc",0,"g");
       run;
@@ -1899,6 +1915,62 @@ Usage:
 %mend mp_abort;
 
 /** @endcond *//**
+  @file
+  @brief Append (concatenate) two or more files.
+  @details Will append one more more `appendrefs` (filerefs) to a `baseref`.
+  Uses a binary mechanism, so will work with any file type.  For that reason -
+  use with care!   And supply your own trailing carriage returns in each file..
+
+  Usage:
+
+        filename tmp1 temp;
+        filename tmp2 temp;
+        filename tmp3 temp;
+        data _null_; file tmp1; put 'base file';
+        data _null_; file tmp2; put 'append1';
+        data _null_; file tmp3; put 'append2';
+        run;
+        %mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3)
+
+
+  @param [in] baseref= Fileref of the base file (should exist)
+  @param [in] appendrefs= One or more filerefs to be appended to the base
+    fileref.  Space separated.
+
+  @version 9.2
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+
+
+**/
+
+%macro mp_appendfile(
+  baseref=0,
+  appendrefs=0
+)/*/STORE SOURCE*/;
+
+%mp_abort(iftrue= (&baseref=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Baseref NOT specified!)
+)
+%mp_abort(iftrue= (&appendrefs=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Appendrefs NOT specified!)
+)
+
+%local i;
+%do i=1 %to %sysfunc(countw(&appendrefs));
+  %mp_abort(iftrue= (&syscc>0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc)
+  )
+  %mp_binarycopy(inref=%scan(&appendrefs,&i), outref=&baseref, mode=APPEND)
+%end;
+
+%mend mp_appendfile;/**
   @file
   @brief Generic assertion
   @details Useful in the context of writing sasjs tests.  The results of the
@@ -2129,6 +2201,7 @@ Usage:
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
+  @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
 
@@ -2214,6 +2287,26 @@ Usage:
   select count(*) into: orig from &lib..&ds;
   quit;
 
+  %local notfound tmp1 tmp2;
+  %let tmp1=%mf_getuniquename();
+  %let tmp2=%mf_getuniquename();
+
+  /* this is a bit convoluted - but using sql outobs=10 throws warnings */
+  proc sql noprint;
+  create view &tmp1 as
+    select distinct &col
+    from &lib..&ds
+    where &col not in (
+      select &ccol from &clib..&cds
+    );
+  data &tmp2;
+    set &tmp1;
+    if _n_>10 then stop;
+  run;
+  proc sql;
+  select distinct &col  into: notfound separated by ' ' from &tmp2;
+
+
   %mp_abort(iftrue= (&syscc ne 0)
     ,mac=&sysmacroname
     ,msg=%str(syscc=&syscc after macro query)
@@ -2224,7 +2317,7 @@ Usage:
     test_description=symget('desc');
     test_result='FAIL';
     test_comments="&sysmacroname: &lib..&ds..&col has &result values "
-      !!"not in &clib..&cds..&ccol ";
+      !!"not in &clib..&cds..&ccol.. First 10 vals:"!!symget('notfound');
   %if &test=ANYVAL %then %do;
     if &result < &orig then test_result='PASS';
   %end;
@@ -3235,12 +3328,21 @@ run;
   @file
   @brief Create a CARDS file from a SAS dataset.
   @details Uses dataset attributes to convert all data into datalines.
-    Running the generated file will rebuild the original dataset.
+    Running the generated file will rebuild the original dataset.  Includes
+    support for large decimals, binary data, PROCESSED_DTTM columns, and
+    alternative encoding.  If the input dataset is empty, the cards file will
+    still be created.
+
+    Additional support to generate a random sample and max rows.
+
   Usage:
 
       %mp_ds2cards(base_ds=sashelp.class
+        , tgt_ds=work.class
         , cards_file= "C:\temp\class.sas"
-        , maxobs=5)
+        , showlog=NO
+        , maxobs=5
+      )
 
   TODO:
     - labelling the dataset
@@ -3251,15 +3353,24 @@ run;
                   that is converted to a cards file.
   @param [in] tgt_ds= Table that the generated cards file would create.
     Optional - if omitted, will be same as BASE_DS.
-  @param [out] cards_file= Location in which to write the (.sas) cards file
-  @param [in] maxobs= to limit output to the first <code>maxobs</code>
-    observations
-  @param [in] showlog= whether to show generated cards file in the SAS log
-    (YES/NO)
-  @param [in] outencoding= provide encoding value for file statement (eg utf-8)
-  @param [in] append= If NO then will rebuild the cards file if it already
+  @param [out] cards_file= ("%sysfunc(pathname(work))/cardgen.sas") Location in
+    which to write the (.sas) cards file
+  @param [in] maxobs= (max) To limit output to the first <code>maxobs</code>
+    observations, enter an integer here.
+  @param [in] random_sample= (NO) Set to YES to generate a random sample of
+    data.  Can be quite slow.
+  @param [in] showlog= (YES) Whether to show generated cards file in the SAS
+    log.  Valid values:
+    @li YES
+    @li NO
+  @param [in] outencoding= Provide encoding value for file statement (eg utf-8)
+  @param [in] append= (NO) If NO then will rebuild the cards file if it already
     exists, otherwise will append to it.  Used by the mp_lib2cards.sas macro.
 
+  <h4> Related Macros </h4>
+  @li mp_lib2cards.sas
+  @li mp_ds2inserts.sas
+  @li mp_mdtablewrite.sas
 
   @version 9.2
   @author Allan Bowe
@@ -3284,15 +3395,15 @@ run;
 %if (&tgt_ds = ) %then %let tgt_ds=&base_ds;
 %if %index(&tgt_ds,.)=0 %then %let tgt_ds=WORK.%scan(&base_ds,2,.);
 %if ("&outencoding" ne "") %then %let outencoding=encoding="&outencoding";
-%if ("&append" = "") %then %let append=;
+%if ("&append" = "" or "&append" = "NO") %then %let append=;
 %else %let append=mod;
 
 /* get varcount */
 %let nvars=0;
 proc sql noprint;
 select count(*) into: nvars from dictionary.columns
-  where libname="%scan(%upcase(&base_ds),1)"
-    and memname="%scan(%upcase(&base_ds),2)";
+  where upcase(libname)="%scan(%upcase(&base_ds),1)"
+    and upcase(memname)="%scan(%upcase(&base_ds),2)";
 %if &nvars=0 %then %do;
   %put %str(WARN)ING: Dataset &base_ds has no variables, will not be converted.;
   %return;
@@ -3348,8 +3459,8 @@ proc sql
 reset outobs=max;
 create table datalines1 as
   select name,type,length,varnum,format,label from dictionary.columns
-  where libname="%upcase(%scan(&base_ds,1))"
-    and memname="%upcase(%scan(&base_ds,2))";
+  where upcase(libname)="%upcase(%scan(&base_ds,1))"
+    and upcase(memname)="%upcase(%scan(&base_ds,2))";
 
 /**
   Due to long decimals cannot use best. format
@@ -3370,7 +3481,18 @@ data datalines_2;
       ,put(',name,',best32.-l)
       ,substrn(put(',name,',bestd32.-l),1
       ,findc(put(',name,',bestd32.-l),"0","TBK")))');
-  else dataline=name;
+  /**
+    * binary data must be converted, to store in text format.  It is identified
+    * by the presence of the $HEX keyword in the format.
+    */
+  else if upcase(format)=:'$HEX' then
+    dataline=cats('put(trim(',name,'),',format,')');
+  /**
+    * There is no easy way to store line breaks in a cards file.
+    * To discuss this, use: https://github.com/sasjs/core/issues/80
+    * Removing all nonprintables with kw (keep writeable)
+    */
+  else dataline=cats('compress(',name,', ,"kw")');
 run;
 
 proc sql noprint;
@@ -3395,7 +3517,8 @@ data _null_;
 
 
 /* Build input statement */
-  if type='char' then type3=':$char.';
+  if upcase(format)=:'$HEX' then type3=':'!!format;
+  else if type='char' then type3=':$char.';
   str2=put(name,$33.)||type3;
 
 
@@ -3417,11 +3540,12 @@ data _null_;
   file &cards_file. &outencoding lrecl=32767 termstr=nl &append;
   length __attrib $32767;
   if _n_=1 then do;
-    put '/*******************************************************************';
-    put " Datalines for %upcase(%scan(&base_ds,2)) dataset ";
-    put " Generated by %nrstr(%%)mp_ds2cards()";
-    put " Available on github.com/sasjs/core";
-    put '********************************************************************/';
+    put '/**';
+    put '  @file';
+    put "  @brief Datalines for %upcase(%scan(&base_ds,2)) dataset";
+    put "  @details Generated by %nrstr(%%)mp_ds2cards()";
+    put "  Available on github.com/sasjs/core";
+    put '**/';
     put "data &tgt_ds &indexes;";
     put "attrib ";
     %do i = 1 %to &nvars;
@@ -3445,11 +3569,11 @@ data _null_;
       put 'run;';
     end;
     else do;
-      put "infile cards dsd delimiter=',';";
+      put "infile cards dsd;";
       put "input ";
       %do i = 1 %to &nvars.;
         %if(%length(&&input_stmt_&i..)) %then
-          put "   &&input_stmt_&i..";
+          put "  &&input_stmt_&i..";
         ;
       %end;
       put ";";
@@ -4228,6 +4352,70 @@ filename &fref1 clear;
 
 %mend mp_filtervalidate;
 /**
+  @file
+  @brief Creates a dataset with column metadata.
+  @details This macro takes the `proc contents` output and "tidies it up" in the
+  following ways:
+
+    @li Blank labels are filled in with column names
+    @li Formats are reconstructed with default values
+    @li Types such as DATE / TIME / DATETIME are inferred from the formats
+
+  Example usage:
+
+      %mp_getcols(sashelp.airline,outds=work.myds)
+
+  @param ds The dataset from which to obtain column metadata
+  @param outds= (work.cols) The output dataset to create. Sample data:
+  |NAME $|LENGTH 8|VARNUM 8|LABEL $|FORMAT $49|TYPE $1 |DDTYPE $|
+  |---|---|---|---|---|---|---|
+  |AIR|8|2|international airline travel (thousands)|8.|N|NUMERIC|
+  |DATE|8|1|DATE|MONYY.|N|DATE|
+  |REGION|3|3|REGION|$3.|C|CHARACTER|
+
+  <h4> Related Macros </h4>
+  @li mf_getvarlist.sas
+  @li mm_getcols.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_getcols(ds, outds=work.cols);
+
+proc contents noprint data=&ds
+  out=_data_ (keep=name type length label varnum format:);
+run;
+data &outds(keep=name type length varnum format label ddtype);
+  set &syslast(rename=(format=format2 type=type2));
+  name=upcase(name);
+  if type2=2 then do;
+    length format $49.;
+    if format2='' then format=cats('$',length,'.');
+    else if formatl=0 then format=cats(format2,'.');
+    else format=cats(format2,formatl,'.');
+    type='C';
+    ddtype='CHARACTER';
+  end;
+  else do;
+    if format2='' then format=cats(length,'.');
+    else if formatl=0 then format=cats(format2,'.');
+    else if formatd=0 then format=cats(format2,formatl,'.');
+    else format=cats(format2,formatl,'.',formatd);
+    type='N';
+    if format=:'DATETIME' then ddtype='DATETIME';
+    else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
+      or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
+      or format=:'MONYY'
+    then ddtype='DATE';
+    else if format=:'TIME' then ddtype='TIME';
+    else ddtype='NUMERIC';
+  end;
+  if label='' then label=name;
+run;
+
+%mend mp_getcols;/**
   @file mp_getconstraints.sas
   @brief Get constraint details at column level
   @details Useful for capturing constraints before they are dropped / reapplied
@@ -4268,21 +4456,24 @@ filename &fref1 clear;
 /* must use SQL as proc datasets does not support length changes */
 proc sql noprint;
 create table &outds as
-  select a.TABLE_CATALOG as libref
-    ,a.TABLE_NAME
+  select upcase(a.TABLE_CATALOG) as libref
+    ,upcase(a.TABLE_NAME) as TABLE_NAME
     ,a.constraint_type
     ,a.constraint_name
     ,b.column_name
   from dictionary.TABLE_CONSTRAINTS a
   left join dictionary.constraint_column_usage  b
-  on a.TABLE_CATALOG=b.TABLE_CATALOG
-    and a.TABLE_NAME=b.TABLE_NAME
+  on upcase(a.TABLE_CATALOG)=upcase(b.TABLE_CATALOG)
+    and upcase(a.TABLE_NAME)=upcase(b.TABLE_NAME)
     and a.constraint_name=b.constraint_name
-  where a.TABLE_CATALOG="&lib"
-    and b.TABLE_CATALOG="&lib"
+/**
+  * We cannot apply this clause to the underlying dictionary table.  See:
+  * https://communities.sas.com/t5/SAS-Programming/Unexpected-Where-Clause-behaviour-in-dictionary-TABLE/m-p/771554#M244867
+  */
+  where calculated libref="&lib"
   %if "&ds" ne "" %then %do;
-    and a.TABLE_NAME="&ds"
-    and b.TABLE_NAME="&ds"
+    and upcase(a.TABLE_NAME)="&ds"
+    and upcase(b.TABLE_NAME)="&ds"
   %end;
   ;
 
@@ -4833,7 +5024,7 @@ run;
   proc sql noprint;
   select sysvalue into: schemaactual
     from dictionary.libnames
-    where libname="&libref" and engine='SQLSVR';
+    where upcase(libname)="&libref" and engine='SQLSVR';
   %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
 
   %do x=1 %to %sysfunc(countw(&dsnlist));
@@ -4926,7 +5117,7 @@ run;
   proc sql noprint;
   select sysvalue into: schemaactual
     from dictionary.libnames
-    where libname="&libref" and engine='POSTGRES';
+    where upcase(libname)="&libref" and engine='POSTGRES';
   %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
   data _null_;
     file &fref mod;
@@ -5092,6 +5283,59 @@ create table &outds (rename=(
   run;
 
 %mend mp_getmaxvarlengths;/**
+  @file
+  @brief Performs a text substitution on a file
+  @details Makes use of the GSUB function in LUA to perform a text substitution
+  in a file - either in-place, or writing to a new location.  The benefit of
+  using LUA is that the entire file can be loaded into a single variable,
+  thereby side stepping the 32767 character limit in a data step.
+
+  Usage:
+
+      %let file=%sysfunc(pathname(work))/file.txt;
+      %let str=replace/me;
+      %let rep=with/this;
+      data _null_;
+        file "&file";
+        put "&str";
+      run;
+      %mp_gsubfile(file=&file, patternvar=str, replacevar=rep)
+      data _null_;
+        infile "&file";
+        input;
+        list;
+      run;
+
+  @param file= (0) The file to perform the substitution on
+  @param patternvar= A macro variable containing the Lua
+    [pattern](https://www.lua.org/pil/20.2.html) to search for.  Due to the use
+    of special (magic) characters in Lua patterns, it is safer to pass the NAME
+    of the macro variable containing the string, rather than the value itself.
+  @param replacevar= The name of the macro variable containing the replacement
+    _string_.
+  @param outfile= (0) The file to write the output to. If zero, then the file
+    is overwritten in-place.
+
+  <h4> SAS Macros </h4>
+  @li ml_gsubfile.sas
+
+  <h4> Related Macros </h4>
+  @li mp_gsubfile.test.sas
+
+  @version 9.4
+  @author Allan Bowe
+**/
+
+%macro mp_gsubfile(file=0,
+  patternvar=,
+  replacevar=,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+  %ml_gsubfile()
+
+%mend mp_gsubfile;
+/**
   @file mp_guesspk.sas
   @brief Guess the primary key of a table
   @details Tries to guess the primary key of a table based on the following logic:
@@ -5975,14 +6219,14 @@ select distinct lowcase(memname)
   We take the standard definition one step further by embedding the informat
   in the table header row, like so:
 
-      |var1:$|var2:best.|var3:date9.|
+      |var1:$32|var2:best.|var3:date9.|
       |---|---|---|
       |some text|42|01JAN1960|
       |blah|1|31DEC1999|
 
   Which resolves to:
 
-  |var1:$|var2:best.|var3:date9.|
+  |var1:$32|var2:best.|var3:date9.|
   |---|---|---|
   |some text|42|01JAN1960|
   |blah|1|31DEC1999|
@@ -7323,10 +7567,10 @@ run;
   @file mp_unzip.sas
   @brief Unzips a zip file
   @details Opens the zip file and copies all the contents to another directory.
-    It is not possible to retain permissions / timestamps, also the BOF marker
-    is lost so it cannot extract binary files.
+  It is not possible to retain permissions / timestamps, also the BOF marker
+  is lost so it cannot extract binary files.
 
-    Usage:
+  Usage:
 
       filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
@@ -7337,8 +7581,9 @@ run;
   @li mf_mkdir.sas
   @li mf_getuniquefileref.sas
 
-  @param ziploc= fileref or quoted full path to zip file ("/path/to/file.zip")
-  @param outdir= directory in which to write the outputs (created if non existant)
+  @param ziploc= Fileref or quoted full path to zip file ("/path/to/file.zip")
+  @param outdir= (%sysfunc(pathname(work))) Directory in which to write the
+    outputs (created if non existant)
 
   @version 9.4
   @author Allan Bowe
@@ -7356,7 +7601,8 @@ run;
 %let fname2=%mf_getuniquefileref();
 %let fname3=%mf_getuniquefileref();
 
-filename &fname1 ZIP &ziploc; * Macro variable &datazip would be read from the file*;
+/* Macro variable &datazip would be read from the file */
+filename &fname1 ZIP &ziploc;
 
 /* Read the "members" (files) from the ZIP file */
 data _data_(keep=memname isFolder);
@@ -7544,6 +7790,42 @@ alter table &libds modify &var char(&len);
 
 %mend mp_validatecol;
 /**
+  @file
+  @brief Wait until a file arrives before continuing execution
+  @details Loops with a `sleep()` command until a file arrives or the max wait
+  period expires.
+
+  @example
+
+  Wait 3 minutes OR for /tmp/flag.txt to appear
+
+    %mp_wait4file(/tmp/flag.txt , maxwait=60*3)
+
+  @param [in] file The file to wait for.  Must be provided.
+  @param [in] maxwait= (0) Number of seconds to wait.  If set to zero, will
+    loop indefinitely (to a maximum of 46 days, per SAS [documentation](
+      https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a001418809.htm
+    )).  Otherwise, execution will proceed upon sleep expiry.
+  @param [in] interval= (1) The wait period between sleeps, in seconds
+
+
+**/
+
+%macro mp_wait4file(file, maxwait=0, interval=1);
+
+%if %str(&file)=%str() %then %do;
+  %put %str(ERR)OR: file not provided;
+%end;
+
+data _null_;
+  maxwait=&maxwait;
+  if maxwait=0 then maxwait=60*60*24*46;
+  do until (fileexist("&file") or slept>maxwait );
+    slept=sum(slept,sleep(&interval,1));
+  end;
+run;
+
+%mend mp_wait4file;/**
   @file
   @brief Fix the `_WEBIN` variables provided to SAS web services
   @details When uploading files to SAS Stored Processes or Viya Jobs a number
@@ -7571,6 +7853,7 @@ alter table &libds modify &var char(&len);
 
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
 
 **/
 
@@ -7618,11 +7901,18 @@ alter table &libds modify &var char(&len);
   @li mp_dirlist.sas
 
   @param in= unquoted filepath, dataset of files or directory to zip
-  @param type= FILE, DATASET, DIRECTORY. (FILE / DATASET not ready yet)
-  @param outname= output file to create, without .zip extension
-  @param outpath= location for output zip file
+  @param type= (FILE) Valid values:
+    @li FILE - /full/path/and/filename.extension to a particular file
+    @li DATASET - a dataset containing a list of files to zip (see `incol`)
+    @li DIRECTORY - a directory to zip
+  @param outname= (FILE) Output file to create, _without_ .zip extension
+  @param outpath= (%sysfunc(pathname(WORK))) Parent folder for output zip file
   @param incol= if DATASET input, say which column contains the filepath
 
+  <h4> Related Macros </h4>
+  @li mp_unzip.sas
+  @li mp_zip.test.sas
+
   @version 9.2
   @author Allan Bowe
   @source https://github.com/sasjs/core
@@ -7653,9 +7943,9 @@ ods package open nopf;
     set &ds;
     length __command $4000;
     if file_or_folder='file';
-    command=cats('ods package add file="',filepath
+    __command=cats('ods package add file="',filepath
       ,'" mimetype="application/x-compress";');
-    call execute(command);
+    call execute(__command);
   run;
   /* tidy up */
   %if &debug=NO %then %do;
@@ -7666,11 +7956,10 @@ ods package open nopf;
   data _null_;
     set &in;
     length __command $4000;
-    command=cats('ods package add file="',&incol
+    __command=cats('ods package add file="',&incol
       ,'" mimetype="application/x-compress";');
-    call execute(command);
+    call execute(__command);
   run;
-  ods package add file="&in" mimetype="application/x-compress";
 %end;
 
 
@@ -9930,6 +10219,7 @@ data _null_;
   put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
+  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
@@ -13425,6 +13715,7 @@ run;
     put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSJOBID"" : ""&sysjobid"" ";
+    put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
@@ -14896,6 +15187,7 @@ data _null_;
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
   put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
+  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
@@ -18735,6 +19027,7 @@ filename &fref1 clear;
     put ",""SYSCC"" : ""&syscc"" ";
     put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
@@ -18749,6 +19042,50 @@ filename &fref1 clear;
 %end;
 
 %mend mv_webout;
+/**
+  @file ml_gsubfile.sas
+  @brief Compiles the gsubfile.lua lua file
+  @details Writes gsubfile.lua to the work directory
+  and then includes it.
+  Usage:
+
+      %ml_gsubfile()
+
+**/
+
+%macro ml_gsubfile();
+data _null_;
+  file "%sysfunc(pathname(work))/ml_gsubfile.lua";
+  put 'local fpath, outpath, file, fcontent ';
+  put ' ';
+  put '-- configure in / out paths ';
+  put 'fpath = sas.symget("file") ';
+  put 'outpath = sas.symget("outfile") ';
+  put 'if ( outpath == 0 ) ';
+  put 'then ';
+  put '   outpath=fpath ';
+  put 'end ';
+  put ' ';
+  put '-- open file and perform the substitution ';
+  put 'file = io.open(fpath,"r") ';
+  put 'fcontent = file:read("*all") ';
+  put 'file:close() ';
+  put 'fcontent = string.gsub( ';
+  put '  fcontent, ';
+  put '  sas.symget(sas.symget("patternvar")), ';
+  put '  sas.symget(sas.symget("replacevar")) ';
+  put ') ';
+  put ' ';
+  put '-- write the file back out ';
+  put 'file = io.open(outpath, "w+") ';
+  put 'io.output(file) ';
+  put 'io.write(fcontent) ';
+  put 'io.close(file) ';
+run;
+
+%inc "%sysfunc(pathname(work))/ml_gsubfile.lua" /source2;
+
+%mend ml_gsubfile;
 /**
   @file ml_json.sas
   @brief Compiles the json.lua lua file
@@ -19140,7 +19477,7 @@ data _null_;
   put '-- JSON.LUA ENDS HERE ';
 run;
 
-%inc "%sysfunc(pathname(work))/ml_json.lua";
+%inc "%sysfunc(pathname(work))/ml_json.lua" /source2;
 
 %mend ml_json;
 /**

base/mf_getvarformat.sas

@@ -51,7 +51,8 @@
     %end;
   %end;
   %else %do;
-    %put dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
     %return;
   %end;
 

base/mf_getvarlen.sas

@@ -43,7 +43,11 @@
       %let vlen = %str( );
     %end;
   %end;
-  %else %put dataset &libds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));

base/mf_getvarnum.sas

@@ -43,7 +43,11 @@ returns:
       %let vnum = %str( );
     %end;
   %end;
-  %else %put dataset &ds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));

base/mf_getvartype.sas

@@ -39,7 +39,11 @@ Usage:
       %let vtype = %str( );
     %end;
   %end;
-  %else %put dataset &libds not opened! (rc=&dsid);
+  %else %do;
+    %put &sysmacroname: dataset &libds not opened! (rc=&dsid);
+    %put &sysmacroname: %sysfunc(sysmsg());
+    %return;
+  %end;
 
   /* Close dataset */
   %let rc = %sysfunc(close(&dsid));

base/mp_abort.sas

@@ -158,7 +158,7 @@
     /* send response in SASjs JSON format */
     data _null_;
       file _webout mod lrecl=32000 encoding='utf-8';
-      length msg $32767 debug $8;
+      length msg $32767 ;
       sasdatetime=datetime();
       msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
       /* escape the quotes */
@@ -169,7 +169,7 @@
       msg=cats('"',msg,'"');
       if symexist('_debug') then debug=quote(trim(symget('_debug')));
       else debug='""';
-      if debug ge '"131"' then put '>>weboutBEGIN<<';
+      put '>>weboutBEGIN<<';
       put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
       put ',"sasjsAbort" : [{';
       put ' "MSG":' msg ;
@@ -189,23 +189,26 @@
       _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
       put ',"_PROGRAM" : ' _PROGRAM ;
       put ",""SYSCC"" : ""&syscc"" ";
-      put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+      syserrortext=quote(trim(symget('syserrortext')));
+      put ",""SYSERRORTEXT"" : " syserrortext;
       put ",""SYSHOSTNAME"" : ""&syshostname"" ";
       put ",""SYSJOBID"" : ""&sysjobid"" ";
+      put ",""SYSSCPL"" : ""&sysscpl"" ";
       put ",""SYSSITE"" : ""&syssite"" ";
       sysvlong=quote(trim(symget('sysvlong')));
       put ',"SYSVLONG" : ' sysvlong;
-      put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+      syswarningtext=quote(trim(symget('syswarningtext')));
+      put ",""SYSWARNINGTEXT"" : " syswarningtext;
       put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
       put "}" @;
-      if debug ge '"131"' then put '>>weboutEND<<';
+      put '>>weboutEND<<';
     run;
 
     %put _all_;
 
     %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
       data _null_;
-        putlog 'stpsrvset program error and syscc';
+        putlog 'stpsrvset program err and syscc';
         rc=stpsrvset('program error', 0);
         call symputx("syscc",0,"g");
       run;

base/mp_appendfile.sas

@@ -0,0 +1,57 @@
+/**
+  @file
+  @brief Append (concatenate) two or more files.
+  @details Will append one more more `appendrefs` (filerefs) to a `baseref`.
+  Uses a binary mechanism, so will work with any file type.  For that reason -
+  use with care!   And supply your own trailing carriage returns in each file..
+
+  Usage:
+
+        filename tmp1 temp;
+        filename tmp2 temp;
+        filename tmp3 temp;
+        data _null_; file tmp1; put 'base file';
+        data _null_; file tmp2; put 'append1';
+        data _null_; file tmp3; put 'append2';
+        run;
+        %mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3)
+
+
+  @param [in] baseref= Fileref of the base file (should exist)
+  @param [in] appendrefs= One or more filerefs to be appended to the base
+    fileref.  Space separated.
+
+  @version 9.2
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+
+
+**/
+
+%macro mp_appendfile(
+  baseref=0,
+  appendrefs=0
+)/*/STORE SOURCE*/;
+
+%mp_abort(iftrue= (&baseref=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Baseref NOT specified!)
+)
+%mp_abort(iftrue= (&appendrefs=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Appendrefs NOT specified!)
+)
+
+%local i;
+%do i=1 %to %sysfunc(countw(&appendrefs));
+  %mp_abort(iftrue= (&syscc>0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc)
+  )
+  %mp_binarycopy(inref=%scan(&appendrefs,&i), outref=&baseref, mode=APPEND)
+%end;
+
+%mend mp_appendfile;

base/mp_assertcolvals.sas

@@ -30,6 +30,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
+  @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
 
@@ -115,6 +116,26 @@
   select count(*) into: orig from &lib..&ds;
   quit;
 
+  %local notfound tmp1 tmp2;
+  %let tmp1=%mf_getuniquename();
+  %let tmp2=%mf_getuniquename();
+
+  /* this is a bit convoluted - but using sql outobs=10 throws warnings */
+  proc sql noprint;
+  create view &tmp1 as
+    select distinct &col
+    from &lib..&ds
+    where &col not in (
+      select &ccol from &clib..&cds
+    );
+  data &tmp2;
+    set &tmp1;
+    if _n_>10 then stop;
+  run;
+  proc sql;
+  select distinct &col  into: notfound separated by ' ' from &tmp2;
+
+
   %mp_abort(iftrue= (&syscc ne 0)
     ,mac=&sysmacroname
     ,msg=%str(syscc=&syscc after macro query)
@@ -125,7 +146,7 @@
     test_description=symget('desc');
     test_result='FAIL';
     test_comments="&sysmacroname: &lib..&ds..&col has &result values "
-      !!"not in &clib..&cds..&ccol ";
+      !!"not in &clib..&cds..&ccol.. First 10 vals:"!!symget('notfound');
   %if &test=ANYVAL %then %do;
     if &result < &orig then test_result='PASS';
   %end;

base/mp_ds2cards.sas

@@ -2,12 +2,21 @@
   @file
   @brief Create a CARDS file from a SAS dataset.
   @details Uses dataset attributes to convert all data into datalines.
-    Running the generated file will rebuild the original dataset.
+    Running the generated file will rebuild the original dataset.  Includes
+    support for large decimals, binary data, PROCESSED_DTTM columns, and
+    alternative encoding.  If the input dataset is empty, the cards file will
+    still be created.
+
+    Additional support to generate a random sample and max rows.
+
   Usage:
 
       %mp_ds2cards(base_ds=sashelp.class
+        , tgt_ds=work.class
         , cards_file= "C:\temp\class.sas"
-        , maxobs=5)
+        , showlog=NO
+        , maxobs=5
+      )
 
   TODO:
     - labelling the dataset
@@ -18,15 +27,24 @@
                   that is converted to a cards file.
   @param [in] tgt_ds= Table that the generated cards file would create.
     Optional - if omitted, will be same as BASE_DS.
-  @param [out] cards_file= Location in which to write the (.sas) cards file
-  @param [in] maxobs= to limit output to the first <code>maxobs</code>
-    observations
-  @param [in] showlog= whether to show generated cards file in the SAS log
-    (YES/NO)
-  @param [in] outencoding= provide encoding value for file statement (eg utf-8)
-  @param [in] append= If NO then will rebuild the cards file if it already
+  @param [out] cards_file= ("%sysfunc(pathname(work))/cardgen.sas") Location in
+    which to write the (.sas) cards file
+  @param [in] maxobs= (max) To limit output to the first <code>maxobs</code>
+    observations, enter an integer here.
+  @param [in] random_sample= (NO) Set to YES to generate a random sample of
+    data.  Can be quite slow.
+  @param [in] showlog= (YES) Whether to show generated cards file in the SAS
+    log.  Valid values:
+    @li YES
+    @li NO
+  @param [in] outencoding= Provide encoding value for file statement (eg utf-8)
+  @param [in] append= (NO) If NO then will rebuild the cards file if it already
     exists, otherwise will append to it.  Used by the mp_lib2cards.sas macro.
 
+  <h4> Related Macros </h4>
+  @li mp_lib2cards.sas
+  @li mp_ds2inserts.sas
+  @li mp_mdtablewrite.sas
 
   @version 9.2
   @author Allan Bowe
@@ -51,15 +69,15 @@
 %if (&tgt_ds = ) %then %let tgt_ds=&base_ds;
 %if %index(&tgt_ds,.)=0 %then %let tgt_ds=WORK.%scan(&base_ds,2,.);
 %if ("&outencoding" ne "") %then %let outencoding=encoding="&outencoding";
-%if ("&append" = "") %then %let append=;
+%if ("&append" = "" or "&append" = "NO") %then %let append=;
 %else %let append=mod;
 
 /* get varcount */
 %let nvars=0;
 proc sql noprint;
 select count(*) into: nvars from dictionary.columns
-  where libname="%scan(%upcase(&base_ds),1)"
-    and memname="%scan(%upcase(&base_ds),2)";
+  where upcase(libname)="%scan(%upcase(&base_ds),1)"
+    and upcase(memname)="%scan(%upcase(&base_ds),2)";
 %if &nvars=0 %then %do;
   %put %str(WARN)ING: Dataset &base_ds has no variables, will not be converted.;
   %return;
@@ -115,8 +133,8 @@ proc sql
 reset outobs=max;
 create table datalines1 as
   select name,type,length,varnum,format,label from dictionary.columns
-  where libname="%upcase(%scan(&base_ds,1))"
-    and memname="%upcase(%scan(&base_ds,2))";
+  where upcase(libname)="%upcase(%scan(&base_ds,1))"
+    and upcase(memname)="%upcase(%scan(&base_ds,2))";
 
 /**
   Due to long decimals cannot use best. format
@@ -137,7 +155,18 @@ data datalines_2;
       ,put(',name,',best32.-l)
       ,substrn(put(',name,',bestd32.-l),1
       ,findc(put(',name,',bestd32.-l),"0","TBK")))');
-  else dataline=name;
+  /**
+    * binary data must be converted, to store in text format.  It is identified
+    * by the presence of the $HEX keyword in the format.
+    */
+  else if upcase(format)=:'$HEX' then
+    dataline=cats('put(trim(',name,'),',format,')');
+  /**
+    * There is no easy way to store line breaks in a cards file.
+    * To discuss this, use: https://github.com/sasjs/core/issues/80
+    * Removing all nonprintables with kw (keep writeable)
+    */
+  else dataline=cats('compress(',name,', ,"kw")');
 run;
 
 proc sql noprint;
@@ -162,7 +191,8 @@ data _null_;
 
 
 /* Build input statement */
-  if type='char' then type3=':$char.';
+  if upcase(format)=:'$HEX' then type3=':'!!format;
+  else if type='char' then type3=':$char.';
   str2=put(name,$33.)||type3;
 
 
@@ -184,11 +214,12 @@ data _null_;
   file &cards_file. &outencoding lrecl=32767 termstr=nl &append;
   length __attrib $32767;
   if _n_=1 then do;
-    put '/*******************************************************************';
-    put " Datalines for %upcase(%scan(&base_ds,2)) dataset ";
-    put " Generated by %nrstr(%%)mp_ds2cards()";
-    put " Available on github.com/sasjs/core";
-    put '********************************************************************/';
+    put '/**';
+    put '  @file';
+    put "  @brief Datalines for %upcase(%scan(&base_ds,2)) dataset";
+    put "  @details Generated by %nrstr(%%)mp_ds2cards()";
+    put "  Available on github.com/sasjs/core";
+    put '**/';
     put "data &tgt_ds &indexes;";
     put "attrib ";
     %do i = 1 %to &nvars;
@@ -212,11 +243,11 @@ data _null_;
       put 'run;';
     end;
     else do;
-      put "infile cards dsd delimiter=',';";
+      put "infile cards dsd;";
       put "input ";
       %do i = 1 %to &nvars.;
         %if(%length(&&input_stmt_&i..)) %then
-          put "   &&input_stmt_&i..";
+          put "  &&input_stmt_&i..";
         ;
       %end;
       put ";";

base/mp_getcols.sas

@@ -0,0 +1,65 @@
+/**
+  @file
+  @brief Creates a dataset with column metadata.
+  @details This macro takes the `proc contents` output and "tidies it up" in the
+  following ways:
+
+    @li Blank labels are filled in with column names
+    @li Formats are reconstructed with default values
+    @li Types such as DATE / TIME / DATETIME are inferred from the formats
+
+  Example usage:
+
+      %mp_getcols(sashelp.airline,outds=work.myds)
+
+  @param ds The dataset from which to obtain column metadata
+  @param outds= (work.cols) The output dataset to create. Sample data:
+  |NAME $|LENGTH 8|VARNUM 8|LABEL $|FORMAT $49|TYPE $1 |DDTYPE $|
+  |---|---|---|---|---|---|---|
+  |AIR|8|2|international airline travel (thousands)|8.|N|NUMERIC|
+  |DATE|8|1|DATE|MONYY.|N|DATE|
+  |REGION|3|3|REGION|$3.|C|CHARACTER|
+
+  <h4> Related Macros </h4>
+  @li mf_getvarlist.sas
+  @li mm_getcols.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_getcols(ds, outds=work.cols);
+
+proc contents noprint data=&ds
+  out=_data_ (keep=name type length label varnum format:);
+run;
+data &outds(keep=name type length varnum format label ddtype);
+  set &syslast(rename=(format=format2 type=type2));
+  name=upcase(name);
+  if type2=2 then do;
+    length format $49.;
+    if format2='' then format=cats('$',length,'.');
+    else if formatl=0 then format=cats(format2,'.');
+    else format=cats(format2,formatl,'.');
+    type='C';
+    ddtype='CHARACTER';
+  end;
+  else do;
+    if format2='' then format=cats(length,'.');
+    else if formatl=0 then format=cats(format2,'.');
+    else if formatd=0 then format=cats(format2,formatl,'.');
+    else format=cats(format2,formatl,'.',formatd);
+    type='N';
+    if format=:'DATETIME' then ddtype='DATETIME';
+    else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
+      or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
+      or format=:'MONYY'
+    then ddtype='DATE';
+    else if format=:'TIME' then ddtype='TIME';
+    else ddtype='NUMERIC';
+  end;
+  if label='' then label=name;
+run;
+
+%mend mp_getcols;

base/mp_getconstraints.sas

@@ -39,21 +39,24 @@
 /* must use SQL as proc datasets does not support length changes */
 proc sql noprint;
 create table &outds as
-  select a.TABLE_CATALOG as libref
-    ,a.TABLE_NAME
+  select upcase(a.TABLE_CATALOG) as libref
+    ,upcase(a.TABLE_NAME) as TABLE_NAME
     ,a.constraint_type
     ,a.constraint_name
     ,b.column_name
   from dictionary.TABLE_CONSTRAINTS a
   left join dictionary.constraint_column_usage  b
-  on a.TABLE_CATALOG=b.TABLE_CATALOG
-    and a.TABLE_NAME=b.TABLE_NAME
+  on upcase(a.TABLE_CATALOG)=upcase(b.TABLE_CATALOG)
+    and upcase(a.TABLE_NAME)=upcase(b.TABLE_NAME)
     and a.constraint_name=b.constraint_name
-  where a.TABLE_CATALOG="&lib"
-    and b.TABLE_CATALOG="&lib"
+/**
+  * We cannot apply this clause to the underlying dictionary table.  See:
+  * https://communities.sas.com/t5/SAS-Programming/Unexpected-Where-Clause-behaviour-in-dictionary-TABLE/m-p/771554#M244867
+  */
+  where calculated libref="&lib"
   %if "&ds" ne "" %then %do;
-    and a.TABLE_NAME="&ds"
-    and b.TABLE_NAME="&ds"
+    and upcase(a.TABLE_NAME)="&ds"
+    and upcase(b.TABLE_NAME)="&ds"
   %end;
   ;
 

base/mp_getddl.sas

@@ -211,7 +211,7 @@ run;
   proc sql noprint;
   select sysvalue into: schemaactual
     from dictionary.libnames
-    where libname="&libref" and engine='SQLSVR';
+    where upcase(libname)="&libref" and engine='SQLSVR';
   %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
 
   %do x=1 %to %sysfunc(countw(&dsnlist));
@@ -304,7 +304,7 @@ run;
   proc sql noprint;
   select sysvalue into: schemaactual
     from dictionary.libnames
-    where libname="&libref" and engine='POSTGRES';
+    where upcase(libname)="&libref" and engine='POSTGRES';
   %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
   data _null_;
     file &fref mod;

base/mp_gsubfile.sas

@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Performs a text substitution on a file
+  @details Makes use of the GSUB function in LUA to perform a text substitution
+  in a file - either in-place, or writing to a new location.  The benefit of
+  using LUA is that the entire file can be loaded into a single variable,
+  thereby side stepping the 32767 character limit in a data step.
+
+  Usage:
+
+      %let file=%sysfunc(pathname(work))/file.txt;
+      %let str=replace/me;
+      %let rep=with/this;
+      data _null_;
+        file "&file";
+        put "&str";
+      run;
+      %mp_gsubfile(file=&file, patternvar=str, replacevar=rep)
+      data _null_;
+        infile "&file";
+        input;
+        list;
+      run;
+
+  @param file= (0) The file to perform the substitution on
+  @param patternvar= A macro variable containing the Lua
+    [pattern](https://www.lua.org/pil/20.2.html) to search for.  Due to the use
+    of special (magic) characters in Lua patterns, it is safer to pass the NAME
+    of the macro variable containing the string, rather than the value itself.
+  @param replacevar= The name of the macro variable containing the replacement
+    _string_.
+  @param outfile= (0) The file to write the output to. If zero, then the file
+    is overwritten in-place.
+
+  <h4> SAS Macros </h4>
+  @li ml_gsubfile.sas
+
+  <h4> Related Macros </h4>
+  @li mp_gsubfile.test.sas
+
+  @version 9.4
+  @author Allan Bowe
+**/
+
+%macro mp_gsubfile(file=0,
+  patternvar=,
+  replacevar=,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+  %ml_gsubfile()
+
+%mend mp_gsubfile;

base/mp_mdtablewrite.sas

@@ -14,14 +14,14 @@
   We take the standard definition one step further by embedding the informat
   in the table header row, like so:
 
-      |var1:$|var2:best.|var3:date9.|
+      |var1:$32|var2:best.|var3:date9.|
       |---|---|---|
       |some text|42|01JAN1960|
       |blah|1|31DEC1999|
 
   Which resolves to:
 
-  |var1:$|var2:best.|var3:date9.|
+  |var1:$32|var2:best.|var3:date9.|
   |---|---|---|
   |some text|42|01JAN1960|
   |blah|1|31DEC1999|

base/mp_unzip.sas

@@ -2,10 +2,10 @@
   @file mp_unzip.sas
   @brief Unzips a zip file
   @details Opens the zip file and copies all the contents to another directory.
-    It is not possible to retain permissions / timestamps, also the BOF marker
-    is lost so it cannot extract binary files.
+  It is not possible to retain permissions / timestamps, also the BOF marker
+  is lost so it cannot extract binary files.
 
-    Usage:
+  Usage:
 
       filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
@@ -16,8 +16,9 @@
   @li mf_mkdir.sas
   @li mf_getuniquefileref.sas
 
-  @param ziploc= fileref or quoted full path to zip file ("/path/to/file.zip")
-  @param outdir= directory in which to write the outputs (created if non existant)
+  @param ziploc= Fileref or quoted full path to zip file ("/path/to/file.zip")
+  @param outdir= (%sysfunc(pathname(work))) Directory in which to write the
+    outputs (created if non existant)
 
   @version 9.4
   @author Allan Bowe
@@ -35,7 +36,8 @@
 %let fname2=%mf_getuniquefileref();
 %let fname3=%mf_getuniquefileref();
 
-filename &fname1 ZIP &ziploc; * Macro variable &datazip would be read from the file*;
+/* Macro variable &datazip would be read from the file */
+filename &fname1 ZIP &ziploc;
 
 /* Read the "members" (files) from the ZIP file */
 data _data_(keep=memname isFolder);

base/mp_wait4file.sas

@@ -0,0 +1,37 @@
+/**
+  @file
+  @brief Wait until a file arrives before continuing execution
+  @details Loops with a `sleep()` command until a file arrives or the max wait
+  period expires.
+
+  @example
+
+  Wait 3 minutes OR for /tmp/flag.txt to appear
+
+    %mp_wait4file(/tmp/flag.txt , maxwait=60*3)
+
+  @param [in] file The file to wait for.  Must be provided.
+  @param [in] maxwait= (0) Number of seconds to wait.  If set to zero, will
+    loop indefinitely (to a maximum of 46 days, per SAS [documentation](
+      https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a001418809.htm
+    )).  Otherwise, execution will proceed upon sleep expiry.
+  @param [in] interval= (1) The wait period between sleeps, in seconds
+
+
+**/
+
+%macro mp_wait4file(file, maxwait=0, interval=1);
+
+%if %str(&file)=%str() %then %do;
+  %put %str(ERR)OR: file not provided;
+%end;
+
+data _null_;
+  maxwait=&maxwait;
+  if maxwait=0 then maxwait=60*60*24*46;
+  do until (fileexist("&file") or slept>maxwait );
+    slept=sum(slept,sleep(&interval,1));
+  end;
+run;
+
+%mend mp_wait4file;

base/mp_zip.sas

@@ -16,11 +16,18 @@
   @li mp_dirlist.sas
 
   @param in= unquoted filepath, dataset of files or directory to zip
-  @param type= FILE, DATASET, DIRECTORY. (FILE / DATASET not ready yet)
-  @param outname= output file to create, without .zip extension
-  @param outpath= location for output zip file
+  @param type= (FILE) Valid values:
+    @li FILE - /full/path/and/filename.extension to a particular file
+    @li DATASET - a dataset containing a list of files to zip (see `incol`)
+    @li DIRECTORY - a directory to zip
+  @param outname= (FILE) Output file to create, _without_ .zip extension
+  @param outpath= (%sysfunc(pathname(WORK))) Parent folder for output zip file
   @param incol= if DATASET input, say which column contains the filepath
 
+  <h4> Related Macros </h4>
+  @li mp_unzip.sas
+  @li mp_zip.test.sas
+
   @version 9.2
   @author Allan Bowe
   @source https://github.com/sasjs/core
@@ -51,9 +58,9 @@ ods package open nopf;
     set &ds;
     length __command $4000;
     if file_or_folder='file';
-    command=cats('ods package add file="',filepath
+    __command=cats('ods package add file="',filepath
       ,'" mimetype="application/x-compress";');
-    call execute(command);
+    call execute(__command);
   run;
   /* tidy up */
   %if &debug=NO %then %do;
@@ -64,11 +71,10 @@ ods package open nopf;
   data _null_;
     set &in;
     length __command $4000;
-    command=cats('ods package add file="',&incol
+    __command=cats('ods package add file="',&incol
       ,'" mimetype="application/x-compress";');
-    call execute(command);
+    call execute(__command);
   run;
-  ods package add file="&in" mimetype="application/x-compress";
 %end;
 
 

build.py

@@ -22,7 +22,7 @@ for file in files:
         for line in infile:
           ml.write("  put '" + line.rstrip().replace("'","''") + " ';\n")
     ml.write("run;\n\n")
-    ml.write("%inc \"%sysfunc(pathname(work))/" + name + ".lua\";\n\n")
+    ml.write("%inc \"%sysfunc(pathname(work))/" + name + ".lua\" /source2;\n\n")
     ml.write("%mend " + name + ";\n")
 
 ml.close()

lua/gsubfile.lua

@@ -0,0 +1,25 @@
+local fpath, outpath, file, fcontent
+
+-- configure in / out paths
+fpath = sas.symget("file")
+outpath = sas.symget("outfile")
+if ( outpath == 0 )
+then
+   outpath=fpath
+end
+
+-- open file and perform the substitution
+file = io.open(fpath,"r")
+fcontent = file:read("*all")
+file:close()
+fcontent = string.gsub(
+  fcontent,
+  sas.symget(sas.symget("patternvar")),
+  sas.symget(sas.symget("replacevar"))
+)
+
+-- write the file back out
+file = io.open(outpath, "w+")
+io.output(file)
+io.write(fcontent)
+io.close(file)

lua/ml_gsubfile.sas

@@ -0,0 +1,44 @@
+/**
+  @file ml_gsubfile.sas
+  @brief Compiles the gsubfile.lua lua file
+  @details Writes gsubfile.lua to the work directory
+  and then includes it.
+  Usage:
+
+      %ml_gsubfile()
+
+**/
+
+%macro ml_gsubfile();
+data _null_;
+  file "%sysfunc(pathname(work))/ml_gsubfile.lua";
+  put 'local fpath, outpath, file, fcontent ';
+  put ' ';
+  put '-- configure in / out paths ';
+  put 'fpath = sas.symget("file") ';
+  put 'outpath = sas.symget("outfile") ';
+  put 'if ( outpath == 0 ) ';
+  put 'then ';
+  put '   outpath=fpath ';
+  put 'end ';
+  put ' ';
+  put '-- open file and perform the substitution ';
+  put 'file = io.open(fpath,"r") ';
+  put 'fcontent = file:read("*all") ';
+  put 'file:close() ';
+  put 'fcontent = string.gsub( ';
+  put '  fcontent, ';
+  put '  sas.symget(sas.symget("patternvar")), ';
+  put '  sas.symget(sas.symget("replacevar")) ';
+  put ') ';
+  put ' ';
+  put '-- write the file back out ';
+  put 'file = io.open(outpath, "w+") ';
+  put 'io.output(file) ';
+  put 'io.write(fcontent) ';
+  put 'io.close(file) ';
+run;
+
+%inc "%sysfunc(pathname(work))/ml_gsubfile.lua" /source2;
+
+%mend ml_gsubfile;

lua/ml_json.sas

@@ -389,6 +389,6 @@ data _null_;
   put '-- JSON.LUA ENDS HERE ';
 run;
 
-%inc "%sysfunc(pathname(work))/ml_json.lua";
+%inc "%sysfunc(pathname(work))/ml_json.lua" /source2;
 
 %mend ml_json;

meta/mm_createwebservice.sas

@@ -385,6 +385,7 @@ data _null_;
   put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
+  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';

meta/mm_webout.sas

@@ -155,6 +155,7 @@
     put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSJOBID"" : ""&sysjobid"" ";
+    put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;

package-lock.json

@@ -7,43 +7,42 @@
       "name": "@sasjs/core",
       "license": "MIT",
       "devDependencies": {
-        "@sasjs/cli": "^2.37.8"
+        "@sasjs/cli": "^2.39.0"
       }
     },
     "node_modules/@sasjs/adapter": {
-      "version": "2.10.4",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.10.4.tgz",
-      "integrity": "sha512-9b3WGZUzRgszZVyPfrabSx6TycL4JFFXQR/RQKsFCDDwT8UgLGfJ4JmgaGCSmGCcsqML/Q41QipdgWu1M/QA3g==",
+      "version": "2.12.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.12.0.tgz",
+      "integrity": "sha512-zzIuohhR8KUDl3DfIFOW38gv3LADPnOBCLOvLoKu4hH5R/UJDkjZ/Gdgc8B35vI7aOprYOLK/T5D/Z44OaTkqw==",
       "dev": true,
+      "hasInstallScript": true,
       "dependencies": {
-        "@sasjs/utils": "^2.27.1",
-        "axios": "^0.21.1",
+        "@sasjs/utils": "^2.32.0",
+        "axios": "^0.21.4",
         "axios-cookiejar-support": "^1.0.1",
         "form-data": "^4.0.0",
         "https": "^1.0.0",
         "tough-cookie": "^4.0.0"
-      },
-      "engines": {
-        "node": ">=15"
       }
     },
     "node_modules/@sasjs/cli": {
-      "version": "2.37.8",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.37.8.tgz",
-      "integrity": "sha512-V1FGK0ByS5v5+xOX5J1rlQDOXSA37YLL18etEUQOmWK5X9R5tbK4LPIx/pEjLyfZcCHddrm+0nGzRroC715Ilg==",
+      "version": "2.39.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.39.0.tgz",
+      "integrity": "sha512-n2LcU4n0QCEbUpXqZnBz/Ey5Td0nMJmgJpZRymMGfYEM0Y0x/CeXemd+kXHPjUvgQ+FX+SQzcvUQTEY/YlT4hA==",
       "dev": true,
+      "hasInstallScript": true,
       "dependencies": {
-        "@sasjs/adapter": "2.10.4",
-        "@sasjs/core": "2.42.0",
+        "@sasjs/adapter": "2.12.0",
+        "@sasjs/core": "2.45.2",
         "@sasjs/lint": "1.11.2",
-        "@sasjs/utils": "2.28.0",
+        "@sasjs/utils": "2.32.0",
         "chalk": "4.1.2",
-        "csv-stringify": "5.6.2",
+        "csv-stringify": "5.6.5",
         "dotenv": "10.0.0",
         "esm": "3.2.25",
         "find": "0.3.0",
         "get-installed-path": "4.0.8",
-        "js-base64": "3.6.1",
+        "js-base64": "3.7.2",
         "jsdom": "17.0.0",
         "jwt-decode": "3.1.2",
         "lodash.groupby": "4.6.0",
@@ -53,16 +52,16 @@
         "rimraf": "3.0.2",
         "shelljs": "0.8.4",
         "xml": "1.0.1",
-        "yargs": "17.1.1"
+        "yargs": "17.2.1"
       },
       "bin": {
         "sasjs": "build/index.js"
       }
     },
     "node_modules/@sasjs/core": {
-      "version": "2.42.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.42.0.tgz",
-      "integrity": "sha512-EISPPCEv+vB3Nqp03NUaIvMZwEnggzGJeEpVfm2Sb504ySN1I2xiJ8bOHXIzvvYP5N/V9X8bXe1raLYnnt8HGA==",
+      "version": "2.45.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.45.2.tgz",
+      "integrity": "sha512-tg+oZCD8GFMXsg+vDL66LMnyU+t151Hrqd7yl+pMXH2qwkA14N/j6QdkTBZOchskqOA/3PnpOlAZN/xxMW2gdg==",
       "dev": true
     },
     "node_modules/@sasjs/lint": {
@@ -75,24 +74,23 @@
       }
     },
     "node_modules/@sasjs/utils": {
-      "version": "2.28.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.28.0.tgz",
-      "integrity": "sha512-aZAbBDSjvW2TCY1lkuGqqiyWtR4W8GFpCNCUKa72VIkl4zwTE/pnSHQ+v8QilGDSHSPbPEsJGaBaldMGwoI12w==",
+      "version": "2.32.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.32.0.tgz",
+      "integrity": "sha512-xnvdEuI4PhTtulcdDEIMK7IxVj9bOMU1JTnxRuSEKWcsclY9P9Fw3cnMOOEgXCDffrOPn3f54DP7Wb1GXd+f8g==",
       "dev": true,
+      "hasInstallScript": true,
       "dependencies": {
         "@types/fs-extra": "^9.0.11",
         "@types/prompts": "^2.0.13",
         "chalk": "^4.1.1",
         "cli-table": "^0.3.6",
         "consola": "^2.15.0",
+        "csv-stringify": "^5.6.5",
         "fs-extra": "^10.0.0",
         "jwt-decode": "^3.1.2",
         "prompts": "^2.4.1",
         "rimraf": "^3.0.2",
         "valid-url": "^1.0.9"
-      },
-      "engines": {
-        "node": ">=15"
       }
     },
     "node_modules/@tootallnate/once": {
@@ -197,9 +195,9 @@
       }
     },
     "node_modules/ansi-regex": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
-      "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
       "dev": true,
       "engines": {
         "node": ">=8"
@@ -478,9 +476,9 @@
       "dev": true
     },
     "node_modules/csv-stringify": {
-      "version": "5.6.2",
-      "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-5.6.2.tgz",
-      "integrity": "sha512-n3rIVbX6ylm1YsX2NEug9IaPV8xRnT+9/NNZbrA/bcHgOSSeqtWla6XnI/xmyu57wIw+ASCAoX1oM6EZtqJV0A==",
+      "version": "5.6.5",
+      "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-5.6.5.tgz",
+      "integrity": "sha512-PjiQ659aQ+fUTQqSrd1XEDnOr52jh30RBurfzkscaE2tPaFsDH5wOAHJiw8XAHphRknCwMUE9KRayc4K/NbO8A==",
       "dev": true
     },
     "node_modules/data-urls": {
@@ -679,9 +677,9 @@
       }
     },
     "node_modules/follow-redirects": {
-      "version": "1.14.3",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.3.tgz",
-      "integrity": "sha512-3MkHxknWMUtb23apkgz/83fDoe+y+qr0TdgacGIA7bew+QLBo3vdgEN2xEsuXNivpFy4CyDhBBZnNZOtalmenw==",
+      "version": "1.14.5",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.5.tgz",
+      "integrity": "sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA==",
       "dev": true,
       "funding": [
         {
@@ -1031,9 +1029,9 @@
       "dev": true
     },
     "node_modules/js-base64": {
-      "version": "3.6.1",
-      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.6.1.tgz",
-      "integrity": "sha512-Frdq2+tRRGLQUIQOgsIGSCd1VePCS2fsddTG5dTCqR0JHgltXWfsxnY0gIXPoMeRmdom6Oyq+UMOFg5suduOjQ==",
+      "version": "3.7.2",
+      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.2.tgz",
+      "integrity": "sha512-NnRs6dsyqUXejqk/yv2aiXlAvOs56sLkX6nUdeaNezI5LFFLlsZjOThmwnrcwh5ZZRwZlCMnVAY3CvhIhoVEKQ==",
       "dev": true
     },
     "node_modules/jsdom": {
@@ -1806,9 +1804,9 @@
       }
     },
     "node_modules/yargs": {
-      "version": "17.1.1",
-      "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.1.1.tgz",
-      "integrity": "sha512-c2k48R0PwKIqKhPMWjeiF6y2xY/gPMUlro0sgxqXpbOIohWiLNXWslsootttv7E1e73QPAMQSg5FeySbVcpsPQ==",
+      "version": "17.2.1",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.2.1.tgz",
+      "integrity": "sha512-XfR8du6ua4K6uLGm5S6fA+FIJom/MdJcFNVY8geLlp2v8GYbOXD4EB1tPNZsRn4vBzKGMgb5DRZMeWuFc2GO8Q==",
       "dev": true,
       "dependencies": {
         "cliui": "^7.0.2",
@@ -1835,13 +1833,13 @@
   },
   "dependencies": {
     "@sasjs/adapter": {
-      "version": "2.10.4",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.10.4.tgz",
-      "integrity": "sha512-9b3WGZUzRgszZVyPfrabSx6TycL4JFFXQR/RQKsFCDDwT8UgLGfJ4JmgaGCSmGCcsqML/Q41QipdgWu1M/QA3g==",
+      "version": "2.12.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.12.0.tgz",
+      "integrity": "sha512-zzIuohhR8KUDl3DfIFOW38gv3LADPnOBCLOvLoKu4hH5R/UJDkjZ/Gdgc8B35vI7aOprYOLK/T5D/Z44OaTkqw==",
       "dev": true,
       "requires": {
-        "@sasjs/utils": "^2.27.1",
-        "axios": "^0.21.1",
+        "@sasjs/utils": "^2.32.0",
+        "axios": "^0.21.4",
         "axios-cookiejar-support": "^1.0.1",
         "form-data": "^4.0.0",
         "https": "^1.0.0",
@@ -1849,22 +1847,22 @@
       }
     },
     "@sasjs/cli": {
-      "version": "2.37.8",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.37.8.tgz",
-      "integrity": "sha512-V1FGK0ByS5v5+xOX5J1rlQDOXSA37YLL18etEUQOmWK5X9R5tbK4LPIx/pEjLyfZcCHddrm+0nGzRroC715Ilg==",
+      "version": "2.39.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.39.0.tgz",
+      "integrity": "sha512-n2LcU4n0QCEbUpXqZnBz/Ey5Td0nMJmgJpZRymMGfYEM0Y0x/CeXemd+kXHPjUvgQ+FX+SQzcvUQTEY/YlT4hA==",
       "dev": true,
       "requires": {
-        "@sasjs/adapter": "2.10.4",
-        "@sasjs/core": "2.42.0",
+        "@sasjs/adapter": "2.12.0",
+        "@sasjs/core": "2.45.2",
         "@sasjs/lint": "1.11.2",
-        "@sasjs/utils": "2.28.0",
+        "@sasjs/utils": "2.32.0",
         "chalk": "4.1.2",
-        "csv-stringify": "5.6.2",
+        "csv-stringify": "5.6.5",
         "dotenv": "10.0.0",
         "esm": "3.2.25",
         "find": "0.3.0",
         "get-installed-path": "4.0.8",
-        "js-base64": "3.6.1",
+        "js-base64": "3.7.2",
         "jsdom": "17.0.0",
         "jwt-decode": "3.1.2",
         "lodash.groupby": "4.6.0",
@@ -1874,13 +1872,13 @@
         "rimraf": "3.0.2",
         "shelljs": "0.8.4",
         "xml": "1.0.1",
-        "yargs": "17.1.1"
+        "yargs": "17.2.1"
       }
     },
     "@sasjs/core": {
-      "version": "2.42.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.42.0.tgz",
-      "integrity": "sha512-EISPPCEv+vB3Nqp03NUaIvMZwEnggzGJeEpVfm2Sb504ySN1I2xiJ8bOHXIzvvYP5N/V9X8bXe1raLYnnt8HGA==",
+      "version": "2.45.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.45.2.tgz",
+      "integrity": "sha512-tg+oZCD8GFMXsg+vDL66LMnyU+t151Hrqd7yl+pMXH2qwkA14N/j6QdkTBZOchskqOA/3PnpOlAZN/xxMW2gdg==",
       "dev": true
     },
     "@sasjs/lint": {
@@ -1893,9 +1891,9 @@
       }
     },
     "@sasjs/utils": {
-      "version": "2.28.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.28.0.tgz",
-      "integrity": "sha512-aZAbBDSjvW2TCY1lkuGqqiyWtR4W8GFpCNCUKa72VIkl4zwTE/pnSHQ+v8QilGDSHSPbPEsJGaBaldMGwoI12w==",
+      "version": "2.32.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.32.0.tgz",
+      "integrity": "sha512-xnvdEuI4PhTtulcdDEIMK7IxVj9bOMU1JTnxRuSEKWcsclY9P9Fw3cnMOOEgXCDffrOPn3f54DP7Wb1GXd+f8g==",
       "dev": true,
       "requires": {
         "@types/fs-extra": "^9.0.11",
@@ -1903,6 +1901,7 @@
         "chalk": "^4.1.1",
         "cli-table": "^0.3.6",
         "consola": "^2.15.0",
+        "csv-stringify": "^5.6.5",
         "fs-extra": "^10.0.0",
         "jwt-decode": "^3.1.2",
         "prompts": "^2.4.1",
@@ -1993,9 +1992,9 @@
       }
     },
     "ansi-regex": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
-      "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
       "dev": true
     },
     "ansi-styles": {
@@ -2198,9 +2197,9 @@
       }
     },
     "csv-stringify": {
-      "version": "5.6.2",
-      "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-5.6.2.tgz",
-      "integrity": "sha512-n3rIVbX6ylm1YsX2NEug9IaPV8xRnT+9/NNZbrA/bcHgOSSeqtWla6XnI/xmyu57wIw+ASCAoX1oM6EZtqJV0A==",
+      "version": "5.6.5",
+      "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-5.6.5.tgz",
+      "integrity": "sha512-PjiQ659aQ+fUTQqSrd1XEDnOr52jh30RBurfzkscaE2tPaFsDH5wOAHJiw8XAHphRknCwMUE9KRayc4K/NbO8A==",
       "dev": true
     },
     "data-urls": {
@@ -2347,9 +2346,9 @@
       }
     },
     "follow-redirects": {
-      "version": "1.14.3",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.3.tgz",
-      "integrity": "sha512-3MkHxknWMUtb23apkgz/83fDoe+y+qr0TdgacGIA7bew+QLBo3vdgEN2xEsuXNivpFy4CyDhBBZnNZOtalmenw==",
+      "version": "1.14.5",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.5.tgz",
+      "integrity": "sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA==",
       "dev": true
     },
     "form-data": {
@@ -2600,9 +2599,9 @@
       "dev": true
     },
     "js-base64": {
-      "version": "3.6.1",
-      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.6.1.tgz",
-      "integrity": "sha512-Frdq2+tRRGLQUIQOgsIGSCd1VePCS2fsddTG5dTCqR0JHgltXWfsxnY0gIXPoMeRmdom6Oyq+UMOFg5suduOjQ==",
+      "version": "3.7.2",
+      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.2.tgz",
+      "integrity": "sha512-NnRs6dsyqUXejqk/yv2aiXlAvOs56sLkX6nUdeaNezI5LFFLlsZjOThmwnrcwh5ZZRwZlCMnVAY3CvhIhoVEKQ==",
       "dev": true
     },
     "jsdom": {
@@ -3192,9 +3191,9 @@
       "dev": true
     },
     "yargs": {
-      "version": "17.1.1",
-      "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.1.1.tgz",
-      "integrity": "sha512-c2k48R0PwKIqKhPMWjeiF6y2xY/gPMUlro0sgxqXpbOIohWiLNXWslsootttv7E1e73QPAMQSg5FeySbVcpsPQ==",
+      "version": "17.2.1",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.2.1.tgz",
+      "integrity": "sha512-XfR8du6ua4K6uLGm5S6fA+FIJom/MdJcFNVY8geLlp2v8GYbOXD4EB1tPNZsRn4vBzKGMgb5DRZMeWuFc2GO8Q==",
       "dev": true,
       "requires": {
         "cliui": "^7.0.2",

package.json

@@ -33,6 +33,6 @@
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "^2.37.8"
+    "@sasjs/cli": "^2.39.0"
   }
 }

sasjs/sasjsconfig.json

@@ -50,7 +50,10 @@
       "appLoc": "/Shared Data/temp/macrocore",
       "macroFolders": [
         "tests/sas9only"
-      ]
+      ],
+      "deployConfig": {
+        "deployServicePack": true
+      }
     },
     {
       "name": "docsonly",

tests/crossplatform/mp_appendfile.test.sas

@@ -0,0 +1,41 @@
+/**
+  @file
+  @brief Testing mp_appendfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_appendfile.sas
+  @li mp_assert.sas
+
+**/
+
+
+filename tmp1 temp;
+filename tmp2 temp;
+filename tmp3 temp;
+data _null_; file tmp1; put 'base file';
+data _null_; file tmp2; put 'append1';
+data _null_; file tmp3; put 'append2';
+run;
+%mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3)
+data _null_;
+  infile tmp1;
+  input;
+  put _infile_;
+  call symputx(cats('check',_n_),_infile_);
+run;
+%global check1 check2 check3;
+%mp_assert(
+  iftrue=("&check1"="base file"),
+  desc=Line 1 of file tmp1 is correct,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&check2"="append1"),
+  desc=Line 2 of file tmp1 is correct,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&check3"="append2"),
+  desc=Line 3 of file tmp1 is correct,
+  outds=work.test_results
+)

tests/crossplatform/mp_ds2cards.test.sas

@@ -0,0 +1,60 @@
+/**
+  @file
+  @brief Testing mp_ds2cards.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2cards.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * test 1 - rebuild an existing dataset
+  * Cars is a great dataset - it contains leading spaces, and formatted numerics
+  */
+
+%mp_ds2cards(base_ds=sashelp.cars
+  , tgt_ds=work.test
+  , cards_file= "%sysfunc(pathname(work))/cars.sas"
+  , showlog=NO
+)
+%inc "%sysfunc(pathname(work))/cars.sas"/source2;
+
+proc compare base=sashelp.cars compare=work.test;
+quit;
+
+%mp_assert(
+  iftrue=(&sysinfo=1),
+  desc=sashelp.cars is identical except for ds label,
+  outds=work.test_results
+)
+
+/**
+  * test 2 - binary data compare
+  */
+data work.binarybase;
+  format bin $hex500. z $hex.;
+  do x=1 to 250;
+    z=byte(x);
+    bin=trim(bin)!!z;
+    output;
+  end;
+run;
+
+%mp_ds2cards(base_ds=work.binarybase
+  , showlog=YES
+  , cards_file="%sysfunc(pathname(work))/c2.sas"
+  , tgt_ds=work.binarycompare
+  , append=
+)
+
+%inc "%sysfunc(pathname(work))/c2.sas"/source2;
+
+proc compare base=work.binarybase compare=work.binarycompare;
+run;
+
+%mp_assert(
+  iftrue=(&sysinfo=0),
+  desc=work.binarybase dataset is identical,
+  outds=work.test_results
+)

tests/crossplatform/mp_getcols.test.sas

@@ -0,0 +1,33 @@
+/**
+  @file
+  @brief Testing mp_getcols macro
+
+  <h4> SAS Macros </h4>
+  @li mp_getcols.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
+
+**/
+
+
+/* valid filter */
+%mp_getcols(sashelp.airline,outds=work.info)
+
+
+%mp_assertdsobs(work.info,
+  desc=Has 3 records,
+  test=EQUALS 3,
+  outds=work.test_results
+)
+
+data work.check;
+  length val $10;
+  do val='NUMERIC','DATE','CHARACTER';
+    output;
+  end;
+run;
+%mp_assertcolvals(work.info.ddtype,
+  checkvals=work.check.val,
+  desc=All values have a match,
+  test=ALLVALS
+)

tests/crossplatform/mp_getconstraints.test.sas

@@ -0,0 +1,29 @@
+/**
+  @file
+  @brief Testing mp_getconstraints.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_getconstraints.sas
+  @li mp_assert.sas
+
+**/
+
+proc sql;
+create table work.example(
+  TX_FROM float format=datetime19.,
+  DD_TYPE char(16),
+  DD_SOURCE char(2048),
+  DD_SHORTDESC char(256),
+  constraint pk primary key(tx_from, dd_type,dd_source),
+  constraint unq unique(tx_from, dd_type),
+  constraint nnn not null(DD_SHORTDESC)
+);
+
+%mp_getconstraints(lib=work,ds=example,outds=work.constraints)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.constraints)=6),
+  desc=Output table work.constraints created with correct number of records,
+  outds=work.test_results
+)

tests/crossplatform/mp_gsubfile.test.sas

@@ -0,0 +1,66 @@
+/**
+  @file
+  @brief Testing mp_gsubfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_gsubfile.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * test 1 - simple replace
+  */
+%global str1;
+%let file=%sysfunc(pathname(work))/file.txt;
+%let pat=replace/me;
+%let str=with/this;
+data _null_;
+  file "&file";
+  put "&pat";
+run;
+%mp_gsubfile(file=&file, patternvar=pat, replacevar=str)
+data _null_;
+  infile "&file";
+  input;
+  call symputx('str1',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&str1"="&str"),
+  desc=Check that simple replacement was successful,
+  outds=work.test_results
+)
+
+/**
+  * test 2 - replace from additional line
+  */
+%global str2 strcheck2 strcheck2b;
+%let file2=%sysfunc(pathname(work))/file2.txt;
+%let pat2=replace/me;
+%let str2=with/this;
+data _null_;
+  file "&file2";
+  put 'line1';output;
+  put "&pat2";output;
+  put "&pat2";output;
+run;
+%mp_gsubfile(file=&file2, patternvar=pat2, replacevar=str2)
+data _null_;
+  infile "&file2";
+  input;
+  if _n_=2 then call symputx('strcheck2',_infile_);
+  if _n_=3 then call symputx('strcheck2b',_infile_);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=("&strcheck2"="&str2"),
+  desc=Check that multi line replacement was successful (line2),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&strcheck2b"="&str2"),
+  desc=Check that multi line replacement was successful (line3),
+  outds=work.test_results
+)

tests/crossplatform/mp_zip.test.sas

@@ -0,0 +1,115 @@
+/**
+  @file
+  @brief Testing mp_zip macro
+
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mp_assert.sas
+  @li mp_zip.sas
+  @li mp_unzip.sas
+
+**/
+
+%let work=%sysfunc(pathname(work));
+%let root=&work/zipme;
+
+/* TEST 1 - zip a file */
+%mf_mkdir(&root)
+
+data _null_;
+  file "&root/test.txt";
+  put "houston, this is a test";
+run;
+
+%mp_zip(in=&root/test.txt
+  ,type=FILE
+  ,outpath=&work
+  ,outname=myFile
+)
+
+%mp_unzip(ziploc="&work/myFile.zip",outdir=&work)
+
+data _null_;
+  infile "&work/test.txt";
+  input;
+  call symputx('content1',_infile_);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(
+    %str(&content1)=%str(houston, this is a test)
+  ),
+  desc=Checking if file zip / unzip works,
+  outds=work.test_results
+)
+
+/* TEST 2 - zip a dataset of files */
+data _null_;
+  file "&root/test2.txt";
+  put "houston, this is test2";
+run;
+libname tmp "&root";
+data tmp.test;
+  filepath="&root/test2.txt";
+run;
+
+%mp_zip(in=tmp.test
+  ,incol=filepath
+  ,type=DATASET
+  ,outpath=&work
+  ,outname=myFile2
+)
+
+%mp_unzip(ziploc="&work/myFile2.zip",outdir=&work)
+
+data _null_;
+  infile "&work/test2.txt";
+  input;
+  call symputx('content2',_infile_);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(
+    %str(&content2)=%str(houston, this is test2)
+  ),
+  desc=Checking if file zip / unzip from a dataset works,
+  outds=work.test_results
+)
+
+/* TEST 3 - zip a dataset of files */
+%mf_mkdir(&work/out3)
+
+%mp_zip(in=&root
+  ,type=DIRECTORY
+  ,outpath=&work
+  ,outname=myFile3
+)
+
+%mp_unzip(ziploc="&work/myFile3.zip",outdir=&work/out3)
+
+data _null_;
+  infile "&work/out3/test.txt";
+  input;
+  call symputx('content3a',_infile_);
+  putlog _infile_;
+run;
+data _null_;
+  infile "&work/out3/test2.txt";
+  input;
+  call symputx('content3b',_infile_);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(
+    %str(&content3a)=%str(houston, this is a test)
+    and
+    %str(&content3b)=%str(houston, this is test2)
+  ),
+  desc=Checking if file zip / unzip from a directory works,
+  outds=work.test_results
+)
+
+

viya/mv_createwebservice.sas

@@ -590,6 +590,7 @@ data _null_;
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
   put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
+  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';

viya/mv_webout.sas

@@ -215,6 +215,7 @@
     put ",""SYSCC"" : ""&syscc"" ";
     put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;