Merge pull request #150 from sasjs/mp_ds2squeeze

sasjs/core - v4

README.md

@@ -125,6 +125,7 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 - macro names must be lowercase
 - one macro per file
 - prefixes:
+  - _mcf_ for macro compiled functions (proc fcmp)
   - _mf_ for macro functions (can be used in open code).
   - _ml_ for macros that are used to compile LUA modules
   - _mm_ for metadata macros (interface with the metadata server).
@@ -166,7 +167,7 @@ SAS code can contain one of two types of dependency - SAS Macros, and SAS Includ
   @li someprogram.sas FREFTWO
 ```
 
-The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Includes) when creating SAS Jobs and Services.
+The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Includes) when creating SAS Jobs and Services (and Tests).
 
 When contributing to this library, it is therefore important to ensure that all dependencies are listed in the header in this format.
 
@@ -182,12 +183,20 @@ When contributing to this library, it is therefore important to ensure that all
 - Mandatory parameters should be positional, all optional parameters should be keyword (var=) style.
 - All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect.
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
+- Where global macro variables are absolutely necessary, they should make use of `&sasjs_prefix` - see mp_init.sas
 - The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
+- Use [sasjs lint](https://github.com/sasjs/lint)!
 
 ## General Notes
 
 - All macros should be compatible with SAS versions from support level B and above (so currently 9.2 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully (eg `%if %sysevalf(&sysver<9.3) %then %return`).
 
+## Breaking Changes
+
+We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major (breaking) release becomes necessary:
+
+* (None as yet)
+
 ## Star Gazing
 
 If you find this library useful, please leave a [star](https://github.com/sasjs/core/stargazers) and help us grow our star graph!

base/mf_abort.sas

@@ -11,7 +11,7 @@
   @cond
 **/
 
-%macro mf_abort(mac=mf_abort.sas, type=deprecated, msg=, iftrue=%str(1=1)
+%macro mf_abort(mac=mf_abort.sas, msg=, iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
   %if not(%eval(%unquote(&iftrue))) %then %return;

base/mp_abort.sas

@@ -110,8 +110,8 @@
     %if %symexist(SYSPRINTTOLOG) %then %let logloc=&SYSPRINTTOLOG;
     %else %let logloc=%qsysfunc(getoption(LOG));
     proc printto log=log;run;
-    %if %length(&logloc)>0 %then %do;
     %let logline=0;
+    %if %length(&logloc)>0 %then %do;
       data _null_;
         infile &logloc lrecl=5000;
         input; putlog _infile_;
@@ -160,7 +160,10 @@
       file _webout mod lrecl=32000 encoding='utf-8';
       length msg $32767 ;
       sasdatetime=datetime();
-      msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
+      msg=symget('msg');
+    %if &logline>0 %then %do;
+      msg=cats(msg,'\n\nLog Extract:\n',symget('logmsg'));
+    %end;
       /* escape the quotes */
       msg=tranwrd(msg,'"','\"');
       /* ditch the CRLFs as chrome complains */
@@ -170,7 +173,8 @@
       if symexist('_debug') then debug=quote(trim(symget('_debug')));
       else debug='""';
       put '>>weboutBEGIN<<';
-      put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
+      put '{"SYSDATE" : "' "&SYSDATE" '"';
+      put ',"SYSTIME" : "' "&SYSTIME" '"';
       put ',"sasjsAbort" : [{';
       put ' "MSG":' msg ;
       put ' ,"MAC": "' "&mac" '"}]';
@@ -199,7 +203,7 @@
       put ',"SYSVLONG" : ' sysvlong;
       syswarningtext=quote(trim(symget('syswarningtext')));
       put ",""SYSWARNINGTEXT"" : " syswarningtext;
-      put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+      put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
       put "}" @;
       put '>>weboutEND<<';
     run;

base/mp_applyformats.sas

@@ -142,16 +142,16 @@ run;
 proc sql noprint;
 select distinct lib into: liblist separated by ' ' from &inds;
 %put &=liblist;
-%do i=1 %to %sysfunc(countw(&liblist));
+%if %length(&liblist)>0 %then %do i=1 %to %sysfunc(countw(&liblist));
   %let lib=%scan(&liblist,1);
   %let engine=%mf_getengine(&lib);
   %if &engine ne V9 and &engine ne BASE %then %do;
     %let msg=&lib has &engine engine - formats cannot be applied;
-    proc sql;
     insert into &outds set lib="&lib",ds="_all_",var="_all", msg="&msg" ;
     %if &errds=0 %then %put %str(ERR)OR: &msg;
   %end;
 %end;
+quit;
 
 %if %mf_nobs(&outds)>0 %then %return;
 

base/mp_assertscope.sas

@@ -0,0 +1,119 @@
+/**
+  @file
+  @brief Used to capture scope leakage of macro variables
+  @details A common 'difficult to detect' bug in macros is where a nested
+    macro over-writes variables in a higher level macro.
+
+    This assertion takes a snapshot of the macro variables before and after
+    a macro invocation.  This makes it easy to detect whether any macro
+    variables were modified or changed.
+
+    Currently, the macro only checks for global scope variables.  In the future
+    it may be extended to work at multiple levels of nesting.
+
+    If you would like this feature, feel free to contribute / raise an issue /
+    engage the SASjs team directly.
+
+  Example usage:
+
+      %mp_assertscope(SNAPSHOT)
+
+      %let oops=I did it again;
+
+      %mp_assertscope(COMPARE,
+        desc=Checking macro variables against previous snapshot
+      )
+
+  @param [in] action (SNAPSHOT) The action to take.  Valid values:
+    @li SNAPSHOT - take a copy of the current macro variables
+    @li COMPARE - compare the current macro variables against previous values
+  @param [in] scope= (GLOBAL) The scope of the variables to be checked.  This
+    corresponds to the values in the SCOPE column in `sashelp.vmacro`.
+  @param [in] desc= (Testing scope leakage) The user provided test description
+  @param [in,out] scopeds= (work.mp_assertscope) The dataset to contain the
+    scope snapshot
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|No out of scope variables created or modified|
+
+  <h4> Related Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertcols.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertscope(action,
+  desc=Testing Scope Leakage,
+  scope=GLOBAL,
+  scopeds=work.mp_assertscope,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+%local ds test_result test_comments del add mod;
+
+/* get current variables */
+%if &action=SNAPSHOT %then %do;
+  proc sql;
+  create table &scopeds as
+    select name,offset,value
+    from dictionary.macros
+    where scope="&scope"
+    order by name,offset;
+%end;
+%else %if &action=COMPARE %then %do;
+
+  proc sql;
+  create table _data_ as
+    select name,offset,value
+    from dictionary.macros
+    where scope="&scope"
+    order by name,offset;
+
+  %let ds=&syslast;
+
+  proc compare base=&scopeds compare=&ds;
+  run;
+
+  %if &sysinfo=0 %then %do;
+    %let test_result=PASS;
+    %let test_comments=&scope Variables Unmodified;
+  %end;
+  %else %do;
+    proc sql noprint undo_policy=none;
+    select distinct name into: del separated by ' '  from &scopeds
+      where name not in (select name from &ds);
+    select distinct name into: add separated by ' '  from &ds
+      where name not in (select name from &scopeds);
+    select distinct a.name into: mod separated by ' '
+      from &scopeds a
+      inner join &ds b
+      on a.name=b.name
+        and a.offset=b.offset
+      where a.value ne b.value;
+    %let test_result=FAIL;
+    %let test_comments=%str(Mod:(&mod) Add:(&add) Del:(&del));
+  %end;
+
+
+  data ;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_comments=symget('test_comments');
+    test_result=symget('test_result');
+  run;
+
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+%end;
+
+%mend mp_assertscope;

base/mp_dirlist.sas

@@ -6,8 +6,7 @@
   Credit for the rename approach:
   https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003
 
-
+  Usage:
-  usage:
 
       %mp_dirlist(path=/some/location, outds=myTable, maxdepth=MAX)
 
@@ -23,12 +22,12 @@
   X CMD) do please raise an issue!
 
 
-  @param [in] path= for which to return contents
+  @param [in] path= (%sysfunc(pathname(work))) Path for which to return contents
-  @param [in] fref= Provide a DISK engine fileref as an alternative to PATH
+  @param [in] fref= (0) Provide a DISK engine fileref as an alternative to PATH
   @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
     subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
     recursion, set to MAX.
-  @param [out] outds= the output dataset to create
+  @param [out] outds= (work.mp_dirlist) The output dataset to create
   @param [out] getattrs= (NO)  If getattrs=YES then the doptname / foptname
     functions are used to scan all properties - any characters that are not
     valid in a SAS name (v7) are simply stripped, and the table is transposed
@@ -49,13 +48,15 @@
     - OS SPECIFIC variables, if <code>getattrs=</code> is used.
 
   <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_getvarlist.sas
+  @li mf_wordsinstr1butnotstr2.sas
   @li mp_dropmembers.sas
 
   <h4> Related Macros </h4>
   @li mp_dirlist.test.sas
 
   @version 9.2
-  @author Allan Bowe
 **/
 
 %macro mp_dirlist(path=%sysfunc(pathname(work))
@@ -193,9 +194,29 @@ data &out_ds;
   set &out_ds(where=(filepath ne ''));
 run;
 
-/* update main table */
+/**
+  * The above transpose can mean that some updates create additional columns.
+  * This necessitates the occasional use of datastep over proc append.
+  */
+%if %mf_existds(&outds) %then %do;
+  %local basevars appvars newvars;
+  %let basevars=%mf_getvarlist(&outds);
+  %let appvars=%mf_getvarlist(&out_ds);
+  %let newvars=%length(%mf_wordsinstr1butnotstr2(Str1=&appvars,Str2=&basevars));
+  %if &newvars>0 %then %do;
+    data &outds;
+      set &outds &out_ds;
+    run;
+  %end;
+  %else %do;
+    proc append base=&outds data=&out_ds force nowarn;
+    run;
+  %end;
+%end;
+%else %do;
   proc append base=&outds data=&out_ds;
   run;
+%end;
 
 /* recursive call */
 %if &maxdepth>&level or &maxdepth=MAX %then %do;

base/mp_ds2cards.sas

@@ -139,8 +139,9 @@ create table datalines1 as
 /**
   Due to long decimals cannot use best. format
   So - use bestd. format and then use character functions to strip trailing
-    zeros, if NOT an integer!!
+    zeros, if NOT an integer or missing!!  Cannot use int() as it upsets
-  resolved code = ifc(int(VARIABLE)=VARIABLE
+    note2err when there are missings.
+  resolved code = ifc( mod(coalesce(VARIABLE,0),1)=0
     ,put(VARIABLE,best32.)
     ,substrn(put(VARIABLE,bestd32.),1
     ,findc(put(VARIABLE,bestd32.),'0','TBK')));
@@ -151,7 +152,7 @@ data datalines_2;
   set datalines1 (where=(upcase(name) not in
     ('PROCESSED_DTTM','VALID_FROM_DTTM','VALID_TO_DTTM')));
   if type='num' then dataline=
-    cats('ifc(int(',name,')=',name,'
+    cats('ifc(mod(coalesce(',name,',0),1)=0
       ,put(',name,',best32.-l)
       ,substrn(put(',name,',bestd32.-l),1
       ,findc(put(',name,',bestd32.-l),"0","TBK")))');

base/mp_ds2csv.sas

@@ -41,7 +41,7 @@ data _null_;
   dsid=open("&ds.","i");
   num=attrn(dsid,"nvars");
   do i=1 to num;
-    header = trim(left(coalescec(varlabel(dsid,i),varname(dsid,i))));
+    header = cats(coalescec(varlabel(dsid,i),varname(dsid,i)));
     put header @;
   end;
   rc=close(dsid);

base/mp_ds2squeeze.sas

@@ -0,0 +1,119 @@
+/**
+  @file
+  @brief Create a smaller version of a dataset, without data loss
+  @details This macro will scan the input dataset and create a new one, that
+  has the minimum variable lengths needed to store the data without data loss.
+
+  Inspiration was taken from [How to Reduce the Disk Space Required by a
+  SAS® Data Set](https://www.lexjansen.com/nesug/nesug06/io/io18.pdf) by
+  Selvaratnam Sridharma.  The end of the referenced paper presents a macro named
+  "squeeze", hence the nomenclature.
+
+  Usage:
+
+      data big;
+        length my big $32000;
+        do i=1 to 1e4;
+          my=repeat('oh my',100);
+          big='dawg';
+          special=._;
+          output;
+        end;
+      run;
+
+      %mp_ds2squeeze(work.big,outds=work.smaller)
+
+  The following will also be printed to the log (exact values may differ
+  depending on your OS and COMPRESS settings):
+
+  > MP_DS2SQUEEZE: work.big was  625MB
+
+  > MP_DS2SQUEEZE: work.smaller is    5MB
+
+  @param [in] libds The library.dataset to be squeezed
+  @param [out] outds= (work.mp_ds2squeeze) The squeezed dataset to create
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> SAS Macros </h4>
+  @li mf_getfilesize.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_getmaxvarlengths.sas
+
+  <h4> Related Programs </h4>
+  @li mp_ds2squeeze.test.sas
+
+  @version 9.3
+  @author Allan Bowe
+**/
+
+%macro mp_ds2squeeze(
+  libds,
+  outds=work.work.mp_ds2squeeze,
+  mdebug=0
+)/*/STORE SOURCE*/;
+%local dbg source;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %do;
+  %let dbg=*;
+  %let source=/source2;
+%end;
+
+%local optval ds fref;
+%let ds=%mf_getuniquename();
+%let fref=%mf_getuniquefileref();
+
+%mp_getmaxvarlengths(&libds,outds=&ds)
+
+data _null_;
+  set &ds end=last;
+  file &fref;
+  /* grab the types */
+  retain dsid;
+  if _n_=1 then dsid=open("&libds",'is');
+  if dsid le 0 then do;
+    msg=sysmsg();
+    put msg=;
+    stop;
+  end;
+  type=vartype(dsid,varnum(dsid, name));
+  if last then rc=close(dsid);
+  /* write out the length statement */
+  if _n_=1 then put 'length ';
+  length len $6;
+  if type='C' then do;
+    if maxlen=0 then len='$1';
+    else len=cats('$',maxlen);
+  end;
+  else do;
+    if maxlen=0 then len='3';
+    else len=cats(maxlen);
+  end;
+  put '  ' name ' ' len;
+  if last then put ';';
+run;
+
+/* configure varlenchk - as we are explicitly shortening the variables */
+%let optval=%sysfunc(getoption(varlenchk));
+options varlenchk=NOWARN;
+
+data &outds;
+  %inc &fref &source;
+  set &libds;
+run;
+
+options varlenchk=&optval;
+
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &ds;
+  filename &fref clear;
+%end;
+
+%put &sysmacroname: &libds was %mf_getfilesize(libds=&libds,format=yes);
+%put &sysmacroname: &outds is %mf_getfilesize(libds=&outds,format=yes);
+
+%mend mp_ds2squeeze;

base/mp_filtercheck.sas

@@ -109,7 +109,7 @@ data &outds;
     output;
   end;
   if mod(SUBGROUP_ID,1) ne 0 then do;
-    REASON_CD='SUBGROUP_ID should be integer, not '!!left(subgroup_id);
+    REASON_CD='SUBGROUP_ID should be integer, not '!!cats(subgroup_id);
     putlog REASON_CD= SUBGROUP_ID=;
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');
@@ -127,7 +127,7 @@ data &outds;
   if OPERATOR_NM not in
   ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
   then do;
-    REASON_CD='Invalid OPERATOR_NM: '!!left(OPERATOR_NM);
+    REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');

base/mp_getcols.sas

@@ -14,11 +14,11 @@
 
   @param ds The dataset from which to obtain column metadata
   @param outds= (work.cols) The output dataset to create. Sample data:
-  |NAME $|LENGTH 8|VARNUM 8|LABEL $|FORMAT $49|TYPE $1 |DDTYPE $|
+|NAME:$32.|LENGTH:best.|VARNUM:best.|LABEL:$256.|FMTNAME:$32.|FORMAT:$49.|TYPE:$1.|DDTYPE:$9.|
-  |---|---|---|---|---|---|---|
+|---|---|---|---|---|---|---|---|
-  |AIR|8|2|international airline travel (thousands)|8.|N|NUMERIC|
+|`AIR `|`8 `|`2 `|`international airline travel (thousands) `|` `|`8. `|`N `|`NUMERIC `|
-  |DATE|8|1|DATE|MONYY.|N|DATE|
+|`DATE `|`8 `|`1 `|`DATE `|`MONYY `|`MONYY. `|`N `|`DATE `|
-  |REGION|3|3|REGION|$3.|C|CHARACTER|
+|`REGION `|`3 `|`3 `|`REGION `|` `|`$3. `|`C `|`CHARACTER `|
 
   <h4> Related Macros </h4>
   @li mf_getvarlist.sas
@@ -30,26 +30,27 @@
 **/
 
 %macro mp_getcols(ds, outds=work.cols);
-
+%local dropds;
 proc contents noprint data=&ds
   out=_data_ (keep=name type length label varnum format:);
 run;
-data &outds(keep=name type length varnum format label ddtype);
+%let dropds=&syslast;
-  set &syslast(rename=(format=format2 type=type2));
+data &outds(keep=name type length varnum format label ddtype fmtname);
+  set &dropds(rename=(format=fmtname type=type2));
   name=upcase(name);
   if type2=2 then do;
     length format $49.;
-    if format2='' then format=cats('$',length,'.');
+    if fmtname='' then format=cats('$',length,'.');
-    else if formatl=0 then format=cats(format2,'.');
+    else if formatl=0 then format=cats(fmtname,'.');
-    else format=cats(format2,formatl,'.');
+    else format=cats(fmtname,formatl,'.');
     type='C';
     ddtype='CHARACTER';
   end;
   else do;
-    if format2='' then format=cats(length,'.');
+    if fmtname='' then format=cats(length,'.');
-    else if formatl=0 then format=cats(format2,'.');
+    else if formatl=0 then format=cats(fmtname,'.');
-    else if formatd=0 then format=cats(format2,formatl,'.');
+    else if formatd=0 then format=cats(fmtname,formatl,'.');
-    else format=cats(format2,formatl,'.',formatd);
+    else format=cats(fmtname,formatl,'.',formatd);
     type='N';
     if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
     else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
@@ -61,5 +62,6 @@ data &outds(keep=name type length varnum format label ddtype);
   end;
   if label='' then label=name;
 run;
-
+proc sql;
+drop table &dropds;
 %mend mp_getcols;

base/mp_getddl.sas

@@ -158,7 +158,7 @@ run;
         lab=" label="!!cats("'",tranwrd(label,"'","''"),"'");
       if notnull='yes' then notnul=' not null';
       if type='char' then typ=cats('char(',length,')');
-      else if length ne 8 then typ='num length='!!left(length);
+      else if length ne 8 then typ='num length='!!cats(length);
       else typ='num';
       put name typ fmt notnul lab;
     run;

base/mp_getmaxvarlengths.sas

@@ -1,5 +1,5 @@
 /**
-  @file mp_getmaxvarlengths.sas
+  @file
   @brief Scans a dataset to find the max length of the variable values
   @details
   This macro will scan a base dataset and produce an output dataset with two
@@ -8,21 +8,39 @@
   - NAME    Name of the base dataset column
   - MAXLEN  Maximum length of the data contained therein.
 
-  Character fields may be allocated very large widths (eg 32000) of which the
+  Character fields are often allocated very large widths (eg 32000) of which the
-  maximum  value is likely to be much narrower.  This macro was designed to
+  maximum  value is likely to be much narrower.  Identifying such cases can be
-  enable a HTML table to be appropriately sized however this could be used as
+  helpful in the following scenarios:
-  part of a data audit to ensure we aren't over-sizing our tables in relation to
+
-  the data therein.
+  @li Enabling a HTML table to be appropriately sized (`num2char=YES`)
+  @li Reducing the size of a dataset to save on storage (mp_ds2squeeze.sas)
+  @li Identifying columns containing nothing but missing values (`MAXLEN=0` in
+    the output table)
+
+  If the entire column is made up of (non-special) missing values then a value
+  of 0 is returned.
 
-  Numeric fields are converted using the relevant format to determine the width.
   Usage:
 
       %mp_getmaxvarlengths(sashelp.class,outds=work.myds)
 
-  @param libds Two part dataset (or view) reference.
+  @param [in] libds Two part dataset (or view) reference.
-  @param outds= The output dataset to create
+  @param [in] num2char= (NO) When set to NO, numeric fields are sized according
+    to the number of bytes used (or set to zero in the case of non-special
+    missings). When YES, the numeric field is converted to character (using the
+    format, if available), and that is sized instead, using `lengthn()`.
+  @param [out] outds= The output dataset to create, eg:
+  |NAME:$8.|MAXLEN:best.|
+  |---|---|
+  |`Name `|`7 `|
+  |`Sex `|`1 `|
+  |`Age `|`3 `|
+  |`Height `|`8 `|
+  |`Weight `|`3 `|
 
   <h4> SAS Macros </h4>
+  @li mcf_length.sas
+  @li mf_getuniquename.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
   @li mf_getvarformat.sas
@@ -30,20 +48,32 @@
   @version 9.2
   @author Allan Bowe
 
+  <h4> Related Macros </h4>
+  @li mp_ds2squeeze.sas
+  @li mp_getmaxvarlengths.test.sas
+
 **/
 
 %macro mp_getmaxvarlengths(
-    libds      /* libref.dataset to analyse */
+  libds
-    ,outds=work.mp_getmaxvarlengths /* name of output dataset to create */
+  ,num2char=NO
+  ,outds=work.mp_getmaxvarlengths
 )/*/STORE SOURCE*/;
 
-%local vars x var fmt;
+%local vars prefix x var fmt;
 %let vars=%mf_getvarlist(libds=&libds);
+%let prefix=%substr(%mf_getuniquename(),1,25);
+%let num2char=%upcase(&num2char);
+
+%if &num2char=NO %then %do;
+  /* compile length function for numeric fields */
+  %mcf_length(wrap=YES, insert_cmplib=YES)
+%end;
 
 proc sql;
 create table &outds (rename=(
     %do x=1 %to %sysfunc(countw(&vars,%str( )));
-      ________&x=%scan(&vars,&x)
+      &prefix.&x=%scan(&vars,&x)
     %end;
     ))
   as select
@@ -51,18 +81,21 @@ create table &outds (rename=(
       %let var=%scan(&vars,&x);
       %if &x>1 %then ,;
       %if %mf_getvartype(&libds,&var)=C %then %do;
-        max(length(&var)) as ________&x
+        max(lengthn(&var)) as &prefix.&x
       %end;
-      %else %do;
+      %else %if &num2char=YES %then %do;
         %let fmt=%mf_getvarformat(&libds,&var);
         %put fmt=&fmt;
         %if %str(&fmt)=%str() %then %do;
-          max(length(cats(&var))) as ________&x
+          max(lengthn(cats(&var))) as &prefix.&x
         %end;
         %else %do;
-          max(length(put(&var,&fmt))) as ________&x
+          max(lengthn(put(&var,&fmt))) as &prefix.&x
         %end;
       %end;
+      %else %do;
+        max(mcf_length(&var)) as &prefix.&x
+      %end;
     %end;
   from &libds;
 

base/mp_init.sas

@@ -33,24 +33,29 @@
 %macro mp_init(prefix=SASJS
 )/*/STORE SOURCE*/;
 
+%if %symexist(SASJS_PREFIX) %then %return;  /* only run once */
+
 %global
+  SASJS_PREFIX       /* the ONLY hard-coded global macro variable in SASjs    */
   &prefix._INIT_NUM  /* initialisation time as numeric                        */
   &prefix._INIT_DTTM /* initialisation time in E8601DT26.6 format             */
   &prefix.WORK       /* avoid typing %sysfunc(pathname(work)) every time      */
 ;
-  %if %eval(&&&prefix._INIT_NUM>0) %then %return;  /* only run once */
+
+%let sasjs_prefix=&prefix;
 
 data _null_;
   dttm=datetime();
-    call symputx("&prefix._init_num",dttm,'g');
+  call symputx("&sasjs_prefix._init_num",dttm,'g');
-    call symputx("&prefix._init_dttm",put(dttm,E8601DT26.6),'g');
+  call symputx("&sasjs_prefix._init_dttm",put(dttm,E8601DT26.6),'g');
-    call symputx("&prefix.work",pathname('WORK'),'g');
+  call symputx("&sasjs_prefix.work",pathname('WORK'),'g');
 run;
 
 options
   noautocorrect           /* disallow misspelled procedure names            */
   compress=CHAR           /* default is none so ensure we have something!   */
   datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
+  dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
   %str(err)orcheck=STRICT /* catch errs in libname/filename statements      */
   fmterr                  /* ensure err when a format cannot be found       */
   mergenoby=%str(ERR)OR   /* throw err when a merge has no BY variables     */

base/mp_jsonout.sas

@@ -19,11 +19,12 @@
 
         %mp_jsonout(OPEN,jref=tmp)
         %mp_jsonout(OBJ,class,jref=tmp)
+        %mp_jsonout(OBJ,class,dslabel=class2,jref=tmp,showmeta=YES)
         %mp_jsonout(CLOSE,jref=tmp)
 
         data _null_;
         infile tmp;
-        input;list;
+        input;putlog _infile_;
         run;
 
   If you are building web apps with SAS then you are strongly encouraged to use
@@ -31,22 +32,24 @@
   [sasjs adapter](https://github.com/sasjs/adapter).
   For more information see https://sasjs.io
 
-  @param action Valid values:
+  @param [in] action Valid values:
     @li OPEN - opens the JSON
     @li OBJ - sends a table with each row as an object
     @li ARR - sends a table with each row in an array
     @li CLOSE - closes the JSON
-
+  @param [in] ds The dataset to send.  Must be a work table.
-  @param ds the dataset to send.  Must be a work table.
+  @param [out] jref= (_webout) The fileref to which to send the JSON
-  @param jref= the fileref to which to send the JSON
+  @param [out] dslabel= The name to give the table in the exported JSON
-  @param dslabel= the name to give the table in the exported JSON
+  @param [in] fmt= (Y) Whether to keep (Y) or strip (N) formats from the table
-  @param fmt= Whether to keep or strip formats from the table
+  @param [in] engine= (DATASTEP) Which engine to use to send the JSON. Options:
-  @param engine= Which engine to use to send the JSON, valid options are:
     @li PROCJSON (default)
     @li DATASTEP (more reliable when data has non standard characters)
-
+  @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
-  @param dbg= DEPRECATED - was used to conditionally add PRETTY to
+    (eg `null`) or as STRING values (eg `".a"` or `".b"`)
-    proc json but this can cause line truncation in large files.
+  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+    such as the column formats and types.  The metadata is contained inside an
+    object with the same name as the table but prefixed with a dollar sign - ie,
+    `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
 
   <h4> Related Macros <h4>
   @li mp_ds2fmtds.sas
@@ -57,9 +60,14 @@
 
 **/
 
-%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0
+%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y
+  ,engine=DATASTEP
+  ,missing=NULL
+  ,showmeta=NO
 )/*/STORE SOURCE*/;
-%put output location=&jref;
+%local tempds colinfo fmtds i numcols;
+%let numcols=0;
+
 %if &action=OPEN %then %do;
   options nobomfile;
   data _null_;file &jref encoding='utf-8' ;
@@ -68,45 +76,25 @@
 %end;
 %else %if (&action=ARR or &action=OBJ) %then %do;
   options validvarname=upcase;
-  data _null_;file &jref mod encoding='utf-8' ;
+  data _null_; file &jref encoding='utf-8' mod;
     put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":";
 
-  %if &engine=PROCJSON %then %do;
+  /* grab col defs */
-    data;run;%let tempds=&syslast;
-    proc sql;drop table &tempds;
-    data &tempds /view=&tempds;set &ds;
-    %if &fmt=N %then format _numeric_ best32.;;
-    proc json out=&jref pretty
-        %if &action=ARR %then nokeys ;
-        ;export &tempds / nosastags fmtnumeric;
-    run;
-    proc sql;drop view &tempds;
-  %end;
-  %else %if &engine=DATASTEP %then %do;
-    %local cols i tempds;
-    %let cols=0;
-    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 %then %do;
-      %put &sysmacroname:  &ds NOT FOUND!!!;
-      %return;
-    %end;
-    %if &fmt=Y %then %do;
-      %put converting every variable to a formatted variable;
-      /* see mp_ds2fmtds.sas for source */
   proc contents noprint data=&ds
-        out=_data_(keep=name type length format formatl formatd varnum);
+    out=_data_(keep=name type length format formatl formatd varnum label);
   run;
-      proc sort;
+  %let colinfo=%scan(&syslast,2,.);
+  proc sort data=&colinfo;
     by varnum;
   run;
-      %local fmtds;
+  /* move meta to mac vars */
-      %let fmtds=%scan(&syslast,2,.);
-      /* prepare formats and varnames */
   data _null_;
-        if _n_=1 then call symputx('nobs',nobs,'l');
+    if _n_=1 then call symputx('numcols',nobs,'l');
-        set &fmtds end=last nobs=nobs;
+    set &colinfo end=last nobs=nobs;
     name=upcase(name);
     /* fix formats */
     if type=2 or type=6 then do;
+      typelong='char';
       length fmt $49.;
       if format='' then fmt=cats('$',length,'.');
       else if formatl=0 then fmt=cats(format,'.');
@@ -114,6 +102,7 @@
       newlen=max(formatl,length);
     end;
     else do;
+      typelong='num';
       if format='' then fmt='best.';
       else if formatl=0 then fmt=cats(format,'.');
       else if formatd=0 then fmt=cats(format,formatl,'.');
@@ -127,59 +116,84 @@
     call symputx(cats('name',_n_),name,'l');
     call symputx(cats('newname',_n_),newname,'l');
     call symputx(cats('len',_n_),newlen,'l');
+    call symputx(cats('length',_n_),length,'l');
     call symputx(cats('fmt',_n_),fmt,'l');
     call symputx(cats('type',_n_),type,'l');
+    call symputx(cats('typelong',_n_),typelong,'l');
+    call symputx(cats('label',_n_),coalescec(label,name),'l');
   run;
-      data &fmtds;
+
+  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+
+  %if &engine=PROCJSON %then %do;
+    %if &missing=STRING %then %do;
+      %put &sysmacroname: Special Missings not supported in proc json.;
+      %put &sysmacroname: Switching to DATASTEP engine;
+      %goto datastep;
+    %end;
+    data &tempds /view=&tempds;set &ds;
+    %if &fmt=N %then format _numeric_ best32.;;
+    /* PRETTY is necessary to avoid line truncation in large files */
+    proc json out=&jref pretty
+        %if &action=ARR %then nokeys ;
+        ;export &tempds / nosastags fmtnumeric;
+    run;
+  %end;
+  %else %if &engine=DATASTEP %then %do;
+    %datastep:
+    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1
+    %then %do;
+      %put &sysmacroname:  &ds NOT FOUND!!!;
+      %return;
+    %end;
+
+    %if &fmt=Y %then %do;
+      data _data_;
         /* rename on entry */
         set &ds(rename=(
-      %local i;
+      %do i=1 %to &numcols;
-      %do i=1 %to &nobs;
         &&name&i=&&newname&i
       %end;
         ));
-      %do i=1 %to &nobs;
+      %do i=1 %to &numcols;
         length &&name&i $&&len&i;
         &&name&i=left(put(&&newname&i,&&fmt&i));
         drop &&newname&i;
       %end;
         if _error_ then call symputx('syscc',1012);
       run;
-      %let ds=&fmtds;
+      %let fmtds=&syslast;
-    %end; /* &fmt=Y */
+    %end;
-    data _null_;file &jref mod encoding='utf-8' ;
-      put "["; call symputx('cols',0,'l');
-    proc sort
-      data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
-      out=_data_;
-      by varnum;
-
-    data _null_;
-      set _last_ end=last;
-      call symputx(cats('name',_n_),name,'l');
-      call symputx(cats('type',_n_),type,'l');
-      call symputx(cats('len',_n_),length,'l');
-      if last then call symputx('cols',_n_,'l');
-    run;
 
     proc format; /* credit yabwon for special null removal */
-      value bart ._ - .z = null
+    value bart (default=40)
+    %if &missing=NULL %then %do;
+      ._ - .z = null
+    %end;
+    %else %do;
+      ._ = [quote()]
+      . = null
+      .a - .z = [quote()]
+    %end;
       other = [best.];
 
-    data;run; %let tempds=&syslast; /* temp table for spesh char management */
-    proc sql; drop table &tempds;
     data &tempds/view=&tempds;
       attrib _all_ label='';
-      %do i=1 %to &cols;
+      %do i=1 %to &numcols;
-        %if &&type&i=char %then %do;
+        %if &&typelong&i=char or &fmt=Y %then %do;
           length &&name&i $32767;
           format &&name&i $32767.;
         %end;
       %end;
+      %if &fmt=Y %then %do;
+        set &fmtds;
+      %end;
+      %else %do;
         set &ds;
+      %end;
       format _numeric_ bart.;
-    %do i=1 %to &cols;
+    %do i=1 %to &numcols;
-      %if &&type&i=char %then %do;
+      %if &&typelong&i=char or &fmt=Y %then %do;
         &&name&i='"'!!trim(prxchange('s/"/\"/',-1,
                     prxchange('s/'!!'0A'x!!'/\n/',-1,
                     prxchange('s/'!!'0D'x!!'/\r/',-1,
@@ -189,21 +203,21 @@
       %end;
     %end;
     run;
+
     /* write to temp loc to avoid _webout truncation
       - https://support.sas.com/kb/49/325.html */
     filename _sjs temp lrecl=131068 encoding='utf-8';
     data _null_; file _sjs lrecl=131068 encoding='utf-8' mod ;
+      if _n_=1 then put "[";
       set &tempds;
       if _n_>1 then put "," @; put
       %if &action=ARR %then "[" ; %else "{" ;
-      %do i=1 %to &cols;
+      %do i=1 %to &numcols;
         %if &i>1 %then  "," ;
         %if &action=OBJ %then """&&name&i"":" ;
         &&name&i
       %end;
       %if &action=ARR %then "]" ; %else "}" ; ;
-    proc sql;
-    drop view &tempds;
     /* now write the long strings to _webout 1 byte at a time */
     data _null_;
       length filein 8 fileid 8;
@@ -215,12 +229,33 @@
         rc=fput(fileid, rec);
         rc=fwrite(fileid);
       end;
+      /* close out the table */
+      rc=fput(fileid, "]");
+      rc=fwrite(fileid);
       rc=fclose(filein);
       rc=fclose(fileid);
     run;
     filename _sjs clear;
-    data _null_; file &jref mod encoding='utf-8' ;
+  %end;
-      put "]";
+
+  proc sql;
+  drop view &tempds;
+  drop table &colinfo;
+
+  %if &showmeta=YES %then %do;
+    data _null_; file &jref encoding='utf-8' mod;
+      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{";
+      do i=1 to &numcols;
+        name=quote(trim(symget(cats('name',i))));
+        format=quote(trim(symget(cats('fmt',i))));
+        label=quote(trim(symget(cats('label',i))));
+        length=quote(trim(symget(cats('length',i))));
+        type=quote(trim(symget(cats('typelong',i))));
+        if i>1 then put "," @@;
+        put name ':{"format":' format ',"label":' label
+          ',"length":' length ',"type":' type '}';
+      end;
+      put '}}';
     run;
   %end;
 %end;

base/mp_lockanytable.sas

@@ -15,7 +15,7 @@
     length is 200 characters.
   @param [out] ctl_ds= (0) The control table which controls the actual locking.
     Should already be assigned and available.  The definition is available by
-    running mp_coretable.sas as follows:  `mp_coretable(LOCKTABLE)`.
+    running mp_coretable.sas as follows:  `%mp_coretable(LOCKTABLE)`.
 
   @param [in] loops= (25) Number of times to check for a lock.
   @param [in] loop_secs= (1) Seconds to wait between each lock attempt

base/mp_testservice.sas

@@ -76,7 +76,7 @@
     else do;
       x+1;
       call symputx(name,quote(cats(value)),'l');
-      call symputx('pval'!!left(x),name,'l');
+      call symputx(cats('pval',x),name,'l');
       call symputx('pcnt',x,'l');
     end;
   run;

base/mp_validatecol.sas

@@ -48,9 +48,11 @@
 %let tempcol=%mf_getuniquename();
 
 %if &rule=ISINT %then %do;
-  &tempcol=input(&incol,?? best32.);
   &outcol=0;
-  if not missing(&tempcol) then if mod(&incol,1)=0 then &outcol=1;
+  if not missing(&incol) then do;
+    &tempcol=input(&incol,?? best32.);
+    if not missing(&tempcol) then if mod(&tempcol,1)=0 then &outcol=1;
+  end;
   drop &tempcol;
 %end;
 %else %if &rule=ISNUM %then %do;

fcmp/mcf_length.sas

@@ -0,0 +1,82 @@
+/**
+  @file
+  @brief Returns the length of a numeric value
+  @details
+  Returns the length, in bytes, of a numeric value.  If the value is
+  missing, then 0 is returned.
+
+  The function itself takes the following (positional) parameters:
+
+  | PARAMETER | DESCRIPTION |
+  |---|---|
+  | var | variable (or value) to be tested|
+
+  Usage:
+
+      %mcf_length(wrap=YES, insert_cmplib=YES)
+
+      data _null_;
+        ina=1;
+        inb=10000000;
+        inc=12345678;
+        ind=.;
+        outa=mcf_length(ina);
+        outb=mcf_length(inb);
+        outc=mcf_length(inc);
+        outd=mcf_length(ind);
+        put (out:)(=);
+      run;
+
+  Returns:
+
+  > outa=3 outb=4 outc=5 outd=0
+
+  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
+  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
+    CMPLIB reference.
+  @param [out] lib= (work) The output library in which to create the catalog.
+  @param [out] cat= (sasjs) The output catalog in which to create the package.
+  @param [out] pkg= (utils) The output package in which to create the function.
+    Uses a 3 part format:  libref.catalog.package
+
+  <h4> SAS Macros </h4>
+  @li mf_existfunction.sas
+
+  <h4> Related Macros </h4>
+  @li mcf_length.test.sas
+
+**/
+
+%macro mcf_length(wrap=NO
+  ,insert_cmplib=NO
+  ,lib=WORK
+  ,cat=SASJS
+  ,pkg=UTILS
+)/*/STORE SOURCE*/;
+
+%if %mf_existfunction(mcf_length)=1 %then %return;
+
+%if &wrap=YES  %then %do;
+  proc fcmp outlib=&lib..&cat..&pkg;
+%end;
+
+function mcf_length(var);
+  if var=. then len=0;
+  else if missing(var) or trunc(var,3)=var then len=3;
+  else if trunc(var,4)=var then len=4;
+  else if trunc(var,5)=var then len=5;
+  else if trunc(var,6)=var then len=6;
+  else if trunc(var,7)=var then len=7;
+  else len=8;
+  return(len);
+endsub;
+
+%if &wrap=YES %then %do;
+  quit;
+%end;
+
+%if &insert_cmplib=YES %then %do;
+  options insert=(CMPLIB=(&lib..&cat));
+%end;
+
+%mend mcf_length;

fcmp/mcf_stpsrv_header.sas

@@ -69,7 +69,7 @@
 %if %mf_existfunction(stpsrv_header)=1 %then %return;
 
 %if &wrap=YES  %then %do;
-  proc fcmp outcat=&lib..&cat..&pkg;
+  proc fcmp outlib=&lib..&cat..&pkg;
 %end;
 
 function stpsrv_header(name $, value $);

fcmp/mcf_string2file.sas

@@ -39,6 +39,9 @@
   @param [out] pkg= (utils) The output package in which to create the function.
     Uses a 3 part format:  libref.catalog.package
 
+  <h4> SAS Macros </h4>
+  @li mf_existfunction.sas
+
 **/
 
 %macro mcf_string2file(wrap=NO
@@ -48,8 +51,10 @@
   ,pkg=UTILS
 )/*/STORE SOURCE*/;
 
+%if %mf_existfunction(mcf_string2file)=1 %then %return;
+
 %if &wrap=YES  %then %do;
-  proc fcmp outcat=&lib..&cat..&pkg;
+  proc fcmp outlib=&lib..&cat..&pkg;
 %end;
 
 function mcf_string2file(filepath $, string $, mode $);

meta/mm_createstp.sas

@@ -39,14 +39,6 @@
         ,Server=SASApp
         ,stptype=2)
 
-  <h4> SAS Macros </h4>
-  @li mf_nobs.sas
-  @li mf_verifymacvars.sas
-  @li mm_getdirectories.sas
-  @li mm_updatestpsourcecode.sas
-  @li mp_dropmembers.sas
-  @li mm_getservercontexts.sas
-
   @param stpname= Stored Process name.  Avoid spaces - testing has shown that
     the check to avoid creating multiple STPs in the same folder with the same
     name does not work when the name contains spaces.
@@ -77,6 +69,17 @@
     - fileuri
     - texturi
 
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mf_verifymacvars.sas
+  @li mm_getdirectories.sas
+  @li mm_updatestpsourcecode.sas
+  @li mp_dropmembers.sas
+  @li mm_getservercontexts.sas
+
+  <h4> Related Macros </h4>
+  @li mm_createwebservice.sas
+
   @version 9.2
   @author Allan Bowe
 

meta/mm_createwebservice.sas

@@ -12,6 +12,7 @@ Usage:
     %* parmcards lets us write to a text file from open code ;
     filename ft15f001 temp;
     parmcards4;
+        %webout(FETCH)
         %* do some sas, any inputs are now already WORK tables;
         data example1 example2;
           set sashelp.class;
@@ -24,11 +25,8 @@ Usage:
     ;;;;
     %mm_createwebservice(path=/Public/app/common,name=appInit,code=ft15f001)
 
-  <h4> SAS Macros </h4>
+  For more examples of using these web services with the SASjs Adapter, see:
-  @li mm_createstp.sas
+  https://github.com/sasjs/adapter#readme
-  @li mf_getuser.sas
-  @li mm_createfolder.sas
-  @li mm_deletestp.sas
 
   @param path= The full path (in SAS Metadata) where the service will be created
   @param name= Stored Process name.  Avoid spaces - testing has shown that
@@ -47,6 +45,12 @@ Usage:
   @param adapter= (sasjs) the macro uses the sasjs adapter by default.  To use
     another adapter, add a (different) fileref here.
 
+  <h4> SAS Macros </h4>
+  @li mm_createstp.sas
+  @li mf_getuser.sas
+  @li mm_createfolder.sas
+  @li mm_deletestp.sas
+
   @version 9.2
   @author Allan Bowe
 
@@ -89,9 +93,14 @@ data _null_;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
   put ' ';
-  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0 ';
+  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
+  put '  ,engine=DATASTEP ';
+  put '  ,missing=NULL ';
+  put '  ,showmeta=NO ';
   put ')/*/STORE SOURCE*/; ';
-  put '%put output location=&jref; ';
+  put '%local tempds colinfo fmtds i numcols; ';
+  put '%let numcols=0; ';
+  put ' ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
   put '  data _null_;file &jref encoding=''utf-8'' ; ';
@@ -100,45 +109,25 @@ data _null_;
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
   put '  options validvarname=upcase; ';
-  put '  data _null_;file &jref mod encoding=''utf-8'' ; ';
+  put '  data _null_; file &jref encoding=''utf-8'' mod; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
   put ' ';
-  put '  %if &engine=PROCJSON %then %do; ';
+  put '  /* grab col defs */ ';
-  put '    data;run;%let tempds=&syslast; ';
-  put '    proc sql;drop table &tempds; ';
-  put '    data &tempds /view=&tempds;set &ds; ';
-  put '    %if &fmt=N %then format _numeric_ best32.;; ';
-  put '    proc json out=&jref pretty ';
-  put '        %if &action=ARR %then nokeys ; ';
-  put '        ;export &tempds / nosastags fmtnumeric; ';
-  put '    run; ';
-  put '    proc sql;drop view &tempds; ';
-  put '  %end; ';
-  put '  %else %if &engine=DATASTEP %then %do; ';
-  put '    %local cols i tempds; ';
-  put '    %let cols=0; ';
-  put '    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 %then %do; ';
-  put '      %put &sysmacroname:  &ds NOT FOUND!!!; ';
-  put '      %return; ';
-  put '    %end; ';
-  put '    %if &fmt=Y %then %do; ';
-  put '      %put converting every variable to a formatted variable; ';
-  put '      /* see mp_ds2fmtds.sas for source */ ';
   put '  proc contents noprint data=&ds ';
-  put '        out=_data_(keep=name type length format formatl formatd varnum); ';
+  put '    out=_data_(keep=name type length format formatl formatd varnum label); ';
   put '  run; ';
-  put '      proc sort; ';
+  put '  %let colinfo=%scan(&syslast,2,.); ';
+  put '  proc sort data=&colinfo; ';
   put '    by varnum; ';
   put '  run; ';
-  put '      %local fmtds; ';
+  put '  /* move meta to mac vars */ ';
-  put '      %let fmtds=%scan(&syslast,2,.); ';
-  put '      /* prepare formats and varnames */ ';
   put '  data _null_; ';
-  put '        if _n_=1 then call symputx(''nobs'',nobs,''l''); ';
+  put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
-  put '        set &fmtds end=last nobs=nobs; ';
+  put '    set &colinfo end=last nobs=nobs; ';
   put '    name=upcase(name); ';
   put '    /* fix formats */ ';
   put '    if type=2 or type=6 then do; ';
+  put '      typelong=''char''; ';
   put '      length fmt $49.; ';
   put '      if format='''' then fmt=cats(''$'',length,''.''); ';
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
@@ -146,6 +135,7 @@ data _null_;
   put '      newlen=max(formatl,length); ';
   put '    end; ';
   put '    else do; ';
+  put '      typelong=''num''; ';
   put '      if format='''' then fmt=''best.''; ';
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else if formatd=0 then fmt=cats(format,formatl,''.''); ';
@@ -159,59 +149,84 @@ data _null_;
   put '    call symputx(cats(''name'',_n_),name,''l''); ';
   put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
   put '    call symputx(cats(''len'',_n_),newlen,''l''); ';
+  put '    call symputx(cats(''length'',_n_),length,''l''); ';
   put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
+  put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
+  put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
   put '  run; ';
-  put '      data &fmtds; ';
+  put ' ';
+  put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put ' ';
+  put '  %if &engine=PROCJSON %then %do; ';
+  put '    %if &missing=STRING %then %do; ';
+  put '      %put &sysmacroname: Special Missings not supported in proc json.; ';
+  put '      %put &sysmacroname: Switching to DATASTEP engine; ';
+  put '      %goto datastep; ';
+  put '    %end; ';
+  put '    data &tempds /view=&tempds;set &ds; ';
+  put '    %if &fmt=N %then format _numeric_ best32.;; ';
+  put '    /* PRETTY is necessary to avoid line truncation in large files */ ';
+  put '    proc json out=&jref pretty ';
+  put '        %if &action=ARR %then nokeys ; ';
+  put '        ;export &tempds / nosastags fmtnumeric; ';
+  put '    run; ';
+  put '  %end; ';
+  put '  %else %if &engine=DATASTEP %then %do; ';
+  put '    %datastep: ';
+  put '    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 ';
+  put '    %then %do; ';
+  put '      %put &sysmacroname:  &ds NOT FOUND!!!; ';
+  put '      %return; ';
+  put '    %end; ';
+  put ' ';
+  put '    %if &fmt=Y %then %do; ';
+  put '      data _data_; ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
-  put '      %local i; ';
+  put '      %do i=1 %to &numcols; ';
-  put '      %do i=1 %to &nobs; ';
   put '        &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
-  put '      %do i=1 %to &nobs; ';
+  put '      %do i=1 %to &numcols; ';
   put '        length &&name&i $&&len&i; ';
   put '        &&name&i=left(put(&&newname&i,&&fmt&i)); ';
   put '        drop &&newname&i; ';
   put '      %end; ';
   put '        if _error_ then call symputx(''syscc'',1012); ';
   put '      run; ';
-  put '      %let ds=&fmtds; ';
+  put '      %let fmtds=&syslast; ';
-  put '    %end; /* &fmt=Y */ ';
+  put '    %end; ';
-  put '    data _null_;file &jref mod encoding=''utf-8'' ; ';
-  put '      put "["; call symputx(''cols'',0,''l''); ';
-  put '    proc sort ';
-  put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
-  put '      out=_data_; ';
-  put '      by varnum; ';
-  put ' ';
-  put '    data _null_; ';
-  put '      set _last_ end=last; ';
-  put '      call symputx(cats(''name'',_n_),name,''l''); ';
-  put '      call symputx(cats(''type'',_n_),type,''l''); ';
-  put '      call symputx(cats(''len'',_n_),length,''l''); ';
-  put '      if last then call symputx(''cols'',_n_,''l''); ';
-  put '    run; ';
   put ' ';
   put '    proc format; /* credit yabwon for special null removal */ ';
-  put '      value bart ._ - .z = null ';
+  put '    value bart (default=40) ';
+  put '    %if &missing=NULL %then %do; ';
+  put '      ._ - .z = null ';
+  put '    %end; ';
+  put '    %else %do; ';
+  put '      ._ = [quote()] ';
+  put '      . = null ';
+  put '      .a - .z = [quote()] ';
+  put '    %end; ';
   put '      other = [best.]; ';
   put ' ';
-  put '    data;run; %let tempds=&syslast; /* temp table for spesh char management */ ';
-  put '    proc sql; drop table &tempds; ';
   put '    data &tempds/view=&tempds; ';
   put '      attrib _all_ label=''''; ';
-  put '      %do i=1 %to &cols; ';
+  put '      %do i=1 %to &numcols; ';
-  put '        %if &&type&i=char %then %do; ';
+  put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
   put '          length &&name&i $32767; ';
   put '          format &&name&i $32767.; ';
   put '        %end; ';
   put '      %end; ';
+  put '      %if &fmt=Y %then %do; ';
+  put '        set &fmtds; ';
+  put '      %end; ';
+  put '      %else %do; ';
   put '        set &ds; ';
+  put '      %end; ';
   put '      format _numeric_ bart.; ';
-  put '    %do i=1 %to &cols; ';
+  put '    %do i=1 %to &numcols; ';
-  put '      %if &&type&i=char %then %do; ';
+  put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
   put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
   put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
   put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
@@ -221,21 +236,21 @@ data _null_;
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
+  put ' ';
   put '    /* write to temp loc to avoid _webout truncation ';
   put '      - https://support.sas.com/kb/49/325.html */ ';
   put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
   put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
+  put '      if _n_=1 then put "["; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
   put '      %if &action=ARR %then "[" ; %else "{" ; ';
-  put '      %do i=1 %to &cols; ';
+  put '      %do i=1 %to &numcols; ';
   put '        %if &i>1 %then  "," ; ';
   put '        %if &action=OBJ %then """&&name&i"":" ; ';
   put '        &&name&i ';
   put '      %end; ';
   put '      %if &action=ARR %then "]" ; %else "}" ; ; ';
-  put '    proc sql; ';
-  put '    drop view &tempds; ';
   put '    /* now write the long strings to _webout 1 byte at a time */ ';
   put '    data _null_; ';
   put '      length filein 8 fileid 8; ';
@@ -247,12 +262,33 @@ data _null_;
   put '        rc=fput(fileid, rec); ';
   put '        rc=fwrite(fileid); ';
   put '      end; ';
+  put '      /* close out the table */ ';
+  put '      rc=fput(fileid, "]"); ';
+  put '      rc=fwrite(fileid); ';
   put '      rc=fclose(filein); ';
   put '      rc=fclose(fileid); ';
   put '    run; ';
   put '    filename _sjs clear; ';
-  put '    data _null_; file &jref mod encoding=''utf-8'' ; ';
+  put '  %end; ';
-  put '      put "]"; ';
+  put ' ';
+  put '  proc sql; ';
+  put '  drop view &tempds; ';
+  put '  drop table &colinfo; ';
+  put ' ';
+  put '  %if &showmeta=YES %then %do; ';
+  put '    data _null_; file &jref encoding=''utf-8'' mod; ';
+  put '      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; ';
+  put '      do i=1 to &numcols; ';
+  put '        name=quote(trim(symget(cats(''name'',i)))); ';
+  put '        format=quote(trim(symget(cats(''fmt'',i)))); ';
+  put '        label=quote(trim(symget(cats(''label'',i)))); ';
+  put '        length=quote(trim(symget(cats(''length'',i)))); ';
+  put '        type=quote(trim(symget(cats(''typelong'',i)))); ';
+  put '        if i>1 then put "," @@; ';
+  put '        put name '':{"format":'' format '',"label":'' label ';
+  put '          '',"length":'' length '',"type":'' type ''}''; ';
+  put '      end; ';
+  put '      put ''}}''; ';
   put '    run; ';
   put '  %end; ';
   put '%end; ';
@@ -263,7 +299,9 @@ data _null_;
   put '  run; ';
   put '%end; ';
   put '%mend mp_jsonout; ';
-  put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y); ';
+  put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
+  put '  ,showmeta=NO ';
+  put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
   put '%local i tempds jsonengine; ';
@@ -321,14 +359,15 @@ data _null_;
   put '  %if %str(&_debug) ge 131 %then %do; ';
   put '    put ''>>weboutBEGIN<<''; ';
   put '  %end; ';
-  put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
+  put '    put ''{"SYSDATE" : "'' "&SYSDATE" ''"''; ';
+  put '    put '',"SYSTIME" : "'' "&SYSTIME" ''"''; ';
   put '  run; ';
   put ' ';
   put '%end; ';
   put ' ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '    ,engine=&jsonengine,dbg=%str(&_debug) ';
+  put '    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta ';
   put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
@@ -343,15 +382,12 @@ data _null_;
   put '      set &tempds; ';
   put '      if not (upcase(name) =:"DATA"); /* ignore temp datasets */ ';
   put '      i+1; ';
-  put '      call symputx(''wt''!!left(i),name,''l''); ';
+  put '      call symputx(cats(''wt'',i),name,''l''); ';
   put '      call symputx(''wtcnt'',i,''l''); ';
   put '    data _null_; file &fref mod encoding=''utf-8''; ';
   put '      put ",""WORK"":{"; ';
   put '    %do i=1 %to &wtcnt; ';
   put '      %let wt=&&wt&i; ';
-  put '      proc contents noprint data=&wt ';
-  put '        out=_data_ (keep=name type length format:); ';
-  put '      run;%let tempds=%scan(&syslast,2,.); ';
   put '      data _null_; file &fref mod encoding=''utf-8''; ';
   put '        dsid=open("WORK.&wt",''is''); ';
   put '        nlobs=attrn(dsid,''NLOBS''); ';
@@ -361,8 +397,7 @@ data _null_;
   put '        put " ""&wt"" : {"; ';
   put '        put ''"nlobs":'' nlobs; ';
   put '        put '',"nvars":'' nvars; ';
-  put '      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=&jsonengine) ';
+  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) ';
-  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=&jsonengine) ';
   put '      data _null_; file &fref mod encoding=''utf-8''; ';
   put '        put "}"; ';
   put '    %end; ';
@@ -390,7 +425,11 @@ data _null_;
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
   put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
-  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''" ''; ';
+  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
+  put '    length memsize $32; ';
+  put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
+  put '    memsize=quote(cats(memsize)); ';
+  put '    put '',"MEMSIZE" : '' memsize; ';
   put '    put "}" @; ';
   put '  %if %str(&_debug) ge 131 %then %do; ';
   put '    put ''>>weboutEND<<''; ';
@@ -418,8 +457,10 @@ data _null_;
   put ' ';
   put '%mend mf_getuser; ';
 /* WEBOUT END */
-  put '%macro webout(action,ds,dslabel=,fmt=);';
+  put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO);';
-  put '  %mm_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt)';
+  put '  %mm_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt,missing=&missing';
+  put '    ,showmeta=&showmeta';
+  put '  )';
   put '%mend;';
 run;
 

meta/mm_getservercontexts.sas

@@ -35,7 +35,7 @@ data _null_;
   set repos;
   where repositorytype in('CUSTOM','FOUNDATION');
   keep id name ;
-  call symputx('repo'!!left(_n_),name,'l');
+  call symputx(cats('repo',_n_),name,'l');
   call symputx('repocnt',_n_,'l');
 run;
 

meta/mm_updatestpsourcecode.sas

@@ -13,9 +13,6 @@
   @param [in] stpcode= the source file (or fileref) containing the SAS code to load
     into the stp.  For multiple files, they should simply be concatenated first.
   @param [in] minify= set to YES in order to strip comments, blank lines, and CRLFs.
-
-  @param frefin= deprecated - a unique fileref is now always used
-  @param frefout= deprecated - a unique fileref is now always used
   @param mDebug= set to 1 to show debug messages in the log
 
   @version 9.3
@@ -30,16 +27,8 @@
   ,stpcode=
   ,minify=NO
   ,mdebug=0
-  /* deprecated */
-  ,frefin=inmeta
-  ,frefout=outmeta
 );
 
-%if &frefin ne inmeta or &frefout ne outmeta %then %do;
-  %put %str(WARN)ING: the frefin and frefout parameters will be deprecated in
-    an upcoming release.;
-%end;
-
 /* first, check if STP exists */
 %local tsuri;
 %let tsuri=stopifempty ;

meta/mm_webout.sas

@@ -23,17 +23,25 @@
         %mm_webout(CLOSE)
 
 
-  @param action Either FETCH, OPEN, ARR, OBJ or CLOSE
+  @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE
-  @param ds The dataset to send back to the frontend
+  @param [in] ds The dataset to send back to the frontend
-  @param dslabel= value to use instead of the real name for sending to JSON
+  @param [out] dslabel= Value to use instead of table name for sending to JSON
-  @param fmt=(Y) Set to N to send back unformatted values
+  @param [in] fmt=(Y) Set to N to send back unformatted values
-  @param fref=(_webout) The fileref to which to write the JSON
+  @param [out] fref= (_webout) The fileref to which to write the JSON
+  @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
+    (eg `null`) or as STRING values (eg `".a"` or `".b"`)
+  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+    such as the column formats and types.  The metadata is contained inside an
+    object with the same name as the table but prefixed with a dollar sign - ie,
+    `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
 
   @version 9.3
   @author Allan Bowe
 
 **/
-%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y);
+%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
+  ,showmeta=NO
+);
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
 %local i tempds jsonengine;
@@ -91,14 +99,15 @@
   %if %str(&_debug) ge 131 %then %do;
     put '>>weboutBEGIN<<';
   %end;
-    put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
+    put '{"SYSDATE" : "' "&SYSDATE" '"';
+    put ',"SYSTIME" : "' "&SYSTIME" '"';
   run;
 
 %end;
 
 %else %if &action=ARR or &action=OBJ %then %do;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=&jsonengine,dbg=%str(&_debug)
+    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta
   )
 %end;
 %else %if &action=CLOSE %then %do;
@@ -113,15 +122,12 @@
       set &tempds;
       if not (upcase(name) =:"DATA"); /* ignore temp datasets */
       i+1;
-      call symputx('wt'!!left(i),name,'l');
+      call symputx(cats('wt',i),name,'l');
       call symputx('wtcnt',i,'l');
     data _null_; file &fref mod encoding='utf-8';
       put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
-      proc contents noprint data=&wt
-        out=_data_ (keep=name type length format:);
-      run;%let tempds=%scan(&syslast,2,.);
       data _null_; file &fref mod encoding='utf-8';
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
@@ -131,8 +137,7 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=&jsonengine)
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES)
-      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=&jsonengine)
       data _null_; file &fref mod encoding='utf-8';
         put "}";
     %end;
@@ -160,7 +165,11 @@
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
-    put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
+    length memsize $32;
+    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
+    memsize=quote(cats(memsize));
+    put ',"MEMSIZE" : ' memsize;
     put "}" @;
   %if %str(&_debug) ge 131 %then %do;
     put '>>weboutEND<<';

package-lock.json

@@ -10,73 +10,25 @@
         "ts-loader": "^9.2.6"
       },
       "devDependencies": {
-        "@sasjs/cli": "^2.39.0"
+        "@sasjs/cli": "3.6.0"
       }
     },
     "node_modules/@sasjs/adapter": {
-      "version": "2.12.0",
+      "version": "3.4.1",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.12.0.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-3.4.1.tgz",
-      "integrity": "sha512-zzIuohhR8KUDl3DfIFOW38gv3LADPnOBCLOvLoKu4hH5R/UJDkjZ/Gdgc8B35vI7aOprYOLK/T5D/Z44OaTkqw==",
+      "integrity": "sha512-FbsvYDaoJAuH8FMidXhX3Kh4Eb8qIcxy5iiCxHgcSRWM89W29W21WfCqCw0F28yFr+V5vZkvphwXMKFdOGGxlw==",
       "dev": true,
       "hasInstallScript": true,
       "dependencies": {
-        "@sasjs/utils": "^2.32.0",
-        "axios": "^0.21.4",
-        "axios-cookiejar-support": "^1.0.1",
-        "form-data": "^4.0.0",
-        "https": "^1.0.0",
-        "tough-cookie": "^4.0.0"
-      }
-    },
-    "node_modules/@sasjs/cli": {
-      "version": "2.39.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.39.0.tgz",
-      "integrity": "sha512-n2LcU4n0QCEbUpXqZnBz/Ey5Td0nMJmgJpZRymMGfYEM0Y0x/CeXemd+kXHPjUvgQ+FX+SQzcvUQTEY/YlT4hA==",
-      "dev": true,
-      "hasInstallScript": true,
-      "dependencies": {
-        "@sasjs/adapter": "2.12.0",
-        "@sasjs/core": "2.45.2",
-        "@sasjs/lint": "1.11.2",
         "@sasjs/utils": "2.32.0",
-        "chalk": "4.1.2",
+        "axios": "0.25.0",
-        "csv-stringify": "5.6.5",
+        "axios-cookiejar-support": "1.0.1",
-        "dotenv": "10.0.0",
+        "form-data": "4.0.0",
-        "esm": "3.2.25",
+        "https": "1.0.0",
-        "find": "0.3.0",
+        "tough-cookie": "4.0.0"
-        "get-installed-path": "4.0.8",
-        "js-base64": "3.7.2",
-        "jsdom": "17.0.0",
-        "jwt-decode": "3.1.2",
-        "lodash.groupby": "4.6.0",
-        "lodash.uniqby": "4.7.0",
-        "node-graphviz": "0.1.0",
-        "ora": "5.4.1",
-        "rimraf": "3.0.2",
-        "shelljs": "0.8.4",
-        "xml": "1.0.1",
-        "yargs": "17.2.1"
-      },
-      "bin": {
-        "sasjs": "build/index.js"
       }
     },
-    "node_modules/@sasjs/core": {
+    "node_modules/@sasjs/adapter/node_modules/@sasjs/utils": {
-      "version": "2.45.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.45.2.tgz",
-      "integrity": "sha512-tg+oZCD8GFMXsg+vDL66LMnyU+t151Hrqd7yl+pMXH2qwkA14N/j6QdkTBZOchskqOA/3PnpOlAZN/xxMW2gdg==",
-      "dev": true
-    },
-    "node_modules/@sasjs/lint": {
-      "version": "1.11.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.2.tgz",
-      "integrity": "sha512-zEonhvha9kwrD+hxhG0hEhtfqpXwffH4vRDIr6eDiXkC7S8M3yImpjyFBvX/THJO5+8iuY8TYkOXKl7+nK/wAg==",
-      "dev": true,
-      "dependencies": {
-        "@sasjs/utils": "^2.19.0"
-      }
-    },
-    "node_modules/@sasjs/utils": {
       "version": "2.32.0",
       "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.32.0.tgz",
       "integrity": "sha512-xnvdEuI4PhTtulcdDEIMK7IxVj9bOMU1JTnxRuSEKWcsclY9P9Fw3cnMOOEgXCDffrOPn3f54DP7Wb1GXd+f8g==",
@@ -96,6 +48,101 @@
         "valid-url": "^1.0.9"
       }
     },
+    "node_modules/@sasjs/cli": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-3.6.0.tgz",
+      "integrity": "sha512-px5aFXNoTlML7g4TvP92q/uqWnHGbxnwEzy+4D+sJbXgpzLlCyyuUU7rbd/9sXT+/Pi7vjfHfBCv0JhoGYr6gw==",
+      "dev": true,
+      "hasInstallScript": true,
+      "dependencies": {
+        "@sasjs/adapter": "3.4.1",
+        "@sasjs/core": "3.10.0",
+        "@sasjs/lint": "1.11.2",
+        "@sasjs/utils": "2.35.0",
+        "chalk": "4.1.2",
+        "csv-stringify": "5.6.5",
+        "dotenv": "10.0.0",
+        "esm": "3.2.25",
+        "find": "0.3.0",
+        "get-installed-path": "4.0.8",
+        "js-base64": "3.7.2",
+        "jsdom": "17.0.0",
+        "jwt-decode": "3.1.2",
+        "lodash.groupby": "4.6.0",
+        "lodash.uniqby": "4.7.0",
+        "node-graphviz": "0.1.0",
+        "ora": "5.4.1",
+        "rimraf": "3.0.2",
+        "shelljs": "0.8.5",
+        "xml": "1.0.1",
+        "yargs": "17.2.1"
+      },
+      "bin": {
+        "sasjs": "build/index.js"
+      }
+    },
+    "node_modules/@sasjs/cli/node_modules/@sasjs/utils": {
+      "version": "2.35.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.35.0.tgz",
+      "integrity": "sha512-q9ZKV+TXqwiaj+0z5U7/00eBpp2QpjKfC9BKx7A6rQjBl10WtoWd5C9Em+RQULWVEdRbVS2XcnNsWelbKq/Zsw==",
+      "dev": true,
+      "hasInstallScript": true,
+      "dependencies": {
+        "@types/fs-extra": "^9.0.13",
+        "@types/prompts": "^2.0.13",
+        "chalk": "^4.1.1",
+        "cli-table": "^0.3.6",
+        "consola": "^2.15.0",
+        "csv-stringify": "^5.6.5",
+        "find": "0.3.0",
+        "fs-extra": "^10.0.0",
+        "jwt-decode": "^3.1.2",
+        "prompts": "^2.4.1",
+        "rimraf": "^3.0.2",
+        "valid-url": "^1.0.9"
+      }
+    },
+    "node_modules/@sasjs/core": {
+      "version": "3.10.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-3.10.0.tgz",
+      "integrity": "sha512-lgLxDYpIvwSrXFaUaTFCR0KXHQEc5QIOL4DU87TvBHEUUAWNQHzuVQWkavLtW5hbvLGnPXnyvspzoSzmBojXzg==",
+      "dev": true,
+      "dependencies": {
+        "ts-loader": "^9.2.6"
+      }
+    },
+    "node_modules/@sasjs/lint": {
+      "version": "1.11.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.2.tgz",
+      "integrity": "sha512-zEonhvha9kwrD+hxhG0hEhtfqpXwffH4vRDIr6eDiXkC7S8M3yImpjyFBvX/THJO5+8iuY8TYkOXKl7+nK/wAg==",
+      "dev": true,
+      "dependencies": {
+        "@sasjs/utils": "^2.19.0"
+      }
+    },
+    "node_modules/@sasjs/utils": {
+      "version": "2.34.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.34.1.tgz",
+      "integrity": "sha512-hd1qieH3d7+xH96n5DpRGTEazeAhYyBBKCdnKhOXMgF2TZVoHFdRs5REfT88CKza6DHBGRVGnIVm5ORGP4cVLg==",
+      "dev": true,
+      "hasInstallScript": true,
+      "dependencies": {
+        "@types/fs-extra": "^9.0.11",
+        "@types/prompts": "^2.0.13",
+        "chalk": "^4.1.1",
+        "cli-table": "^0.3.6",
+        "consola": "^2.15.0",
+        "csv-stringify": "^5.6.5",
+        "find": "0.3.0",
+        "fs-extra": "^10.0.0",
+        "jwt-decode": "^3.1.2",
+        "lodash.groupby": "4.6.0",
+        "lodash.uniqby": "4.7.0",
+        "prompts": "^2.4.1",
+        "rimraf": "^3.0.2",
+        "valid-url": "^1.0.9"
+      }
+    },
     "node_modules/@tootallnate/once": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz",
@@ -132,9 +179,9 @@
       "peer": true
     },
     "node_modules/@types/fs-extra": {
-      "version": "9.0.12",
+      "version": "9.0.13",
-      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.12.tgz",
+      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.13.tgz",
-      "integrity": "sha512-I+bsBr67CurCGnSenZZ7v94gd3tc3+Aj2taxMT4yu4ABLuOgOjeFxX3dokG24ztSRg5tnT00sL8BszO7gSMoIw==",
+      "integrity": "sha512-nEnwB++1u5lVDM2UI4c1+5R+FYaKfaAzS4OococimjVm3nQw3TuzH5UNsocrcTBbhnerblyHj4A49qXbIiZdpA==",
       "dev": true,
       "dependencies": {
         "@types/node": "*"
@@ -449,12 +496,12 @@
       "dev": true
     },
     "node_modules/axios": {
-      "version": "0.21.4",
+      "version": "0.25.0",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.25.0.tgz",
-      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
+      "integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==",
       "dev": true,
       "dependencies": {
-        "follow-redirects": "^1.14.0"
+        "follow-redirects": "^1.14.7"
       }
     },
     "node_modules/axios-cookiejar-support": {
@@ -1051,9 +1098,9 @@
       }
     },
     "node_modules/follow-redirects": {
-      "version": "1.14.5",
+      "version": "1.14.7",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.5.tgz",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz",
-      "integrity": "sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA==",
+      "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==",
       "dev": true,
       "funding": [
         {
@@ -1335,9 +1382,9 @@
       }
     },
     "node_modules/is-core-module": {
-      "version": "2.4.0",
+      "version": "2.8.1",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.4.0.tgz",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.1.tgz",
-      "integrity": "sha512-6A2fkfq1rfeQZjxrZJGerpLCTHRNEBiSgnu0+obeJpEPZRUooHgsizvzv0ZjJwOz3iWIHdJtVWJ/tmPr3D21/A==",
+      "integrity": "sha512-SdNCUs284hr40hFTFP6l0IfZ/RSrMXF3qgoRHd3/79unUTvrFO/JoXwkGm+5J/Oe3E/b5GsnG330uUNgRpu1PA==",
       "dev": true,
       "dependencies": {
         "has": "^1.0.3"
@@ -1887,13 +1934,17 @@
       }
     },
     "node_modules/resolve": {
-      "version": "1.20.0",
+      "version": "1.21.0",
-      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.21.0.tgz",
-      "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==",
+      "integrity": "sha512-3wCbTpk5WJlyE4mSOtDLhqQmGFi0/TD9VPwmiolnk8U0wRgMEktqCXd3vy5buTO3tljvalNvKrjHEfrd2WpEKA==",
       "dev": true,
       "dependencies": {
-        "is-core-module": "^2.2.0",
+        "is-core-module": "^2.8.0",
-        "path-parse": "^1.0.6"
+        "path-parse": "^1.0.7",
+        "supports-preserve-symlinks-flag": "^1.0.0"
+      },
+      "bin": {
+        "resolve": "bin/resolve"
       },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
@@ -2019,9 +2070,9 @@
       }
     },
     "node_modules/shelljs": {
-      "version": "0.8.4",
+      "version": "0.8.5",
-      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.4.tgz",
+      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz",
-      "integrity": "sha512-7gk3UZ9kOfPLIAbslLzyWeGiEqx9e3rxwZM0KE6EL8GlGwjym9Mrlx5/p33bWTu9YG6vcS4MBxYZDHYr5lr8BQ==",
+      "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==",
       "dev": true,
       "dependencies": {
         "glob": "^7.0.0",
@@ -2111,6 +2162,18 @@
         "node": ">=8"
       }
     },
+    "node_modules/supports-preserve-symlinks-flag": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz",
+      "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/symbol-tree": {
       "version": "3.2.4",
       "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
@@ -2581,63 +2644,19 @@
   },
   "dependencies": {
     "@sasjs/adapter": {
-      "version": "2.12.0",
+      "version": "3.4.1",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.12.0.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-3.4.1.tgz",
-      "integrity": "sha512-zzIuohhR8KUDl3DfIFOW38gv3LADPnOBCLOvLoKu4hH5R/UJDkjZ/Gdgc8B35vI7aOprYOLK/T5D/Z44OaTkqw==",
+      "integrity": "sha512-FbsvYDaoJAuH8FMidXhX3Kh4Eb8qIcxy5iiCxHgcSRWM89W29W21WfCqCw0F28yFr+V5vZkvphwXMKFdOGGxlw==",
       "dev": true,
       "requires": {
-        "@sasjs/utils": "^2.32.0",
-        "axios": "^0.21.4",
-        "axios-cookiejar-support": "^1.0.1",
-        "form-data": "^4.0.0",
-        "https": "^1.0.0",
-        "tough-cookie": "^4.0.0"
-      }
-    },
-    "@sasjs/cli": {
-      "version": "2.39.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.39.0.tgz",
-      "integrity": "sha512-n2LcU4n0QCEbUpXqZnBz/Ey5Td0nMJmgJpZRymMGfYEM0Y0x/CeXemd+kXHPjUvgQ+FX+SQzcvUQTEY/YlT4hA==",
-      "dev": true,
-      "requires": {
-        "@sasjs/adapter": "2.12.0",
-        "@sasjs/core": "2.45.2",
-        "@sasjs/lint": "1.11.2",
         "@sasjs/utils": "2.32.0",
-        "chalk": "4.1.2",
+        "axios": "0.25.0",
-        "csv-stringify": "5.6.5",
+        "axios-cookiejar-support": "1.0.1",
-        "dotenv": "10.0.0",
+        "form-data": "4.0.0",
-        "esm": "3.2.25",
+        "https": "1.0.0",
-        "find": "0.3.0",
+        "tough-cookie": "4.0.0"
-        "get-installed-path": "4.0.8",
-        "js-base64": "3.7.2",
-        "jsdom": "17.0.0",
-        "jwt-decode": "3.1.2",
-        "lodash.groupby": "4.6.0",
-        "lodash.uniqby": "4.7.0",
-        "node-graphviz": "0.1.0",
-        "ora": "5.4.1",
-        "rimraf": "3.0.2",
-        "shelljs": "0.8.4",
-        "xml": "1.0.1",
-        "yargs": "17.2.1"
-      }
-    },
-    "@sasjs/core": {
-      "version": "2.45.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.45.2.tgz",
-      "integrity": "sha512-tg+oZCD8GFMXsg+vDL66LMnyU+t151Hrqd7yl+pMXH2qwkA14N/j6QdkTBZOchskqOA/3PnpOlAZN/xxMW2gdg==",
-      "dev": true
-    },
-    "@sasjs/lint": {
-      "version": "1.11.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.2.tgz",
-      "integrity": "sha512-zEonhvha9kwrD+hxhG0hEhtfqpXwffH4vRDIr6eDiXkC7S8M3yImpjyFBvX/THJO5+8iuY8TYkOXKl7+nK/wAg==",
-      "dev": true,
-      "requires": {
-        "@sasjs/utils": "^2.19.0"
-      }
       },
+      "dependencies": {
         "@sasjs/utils": {
           "version": "2.32.0",
           "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.32.0.tgz",
@@ -2656,6 +2675,99 @@
             "rimraf": "^3.0.2",
             "valid-url": "^1.0.9"
           }
+        }
+      }
+    },
+    "@sasjs/cli": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-3.6.0.tgz",
+      "integrity": "sha512-px5aFXNoTlML7g4TvP92q/uqWnHGbxnwEzy+4D+sJbXgpzLlCyyuUU7rbd/9sXT+/Pi7vjfHfBCv0JhoGYr6gw==",
+      "dev": true,
+      "requires": {
+        "@sasjs/adapter": "3.4.1",
+        "@sasjs/core": "3.10.0",
+        "@sasjs/lint": "1.11.2",
+        "@sasjs/utils": "2.35.0",
+        "chalk": "4.1.2",
+        "csv-stringify": "5.6.5",
+        "dotenv": "10.0.0",
+        "esm": "3.2.25",
+        "find": "0.3.0",
+        "get-installed-path": "4.0.8",
+        "js-base64": "3.7.2",
+        "jsdom": "17.0.0",
+        "jwt-decode": "3.1.2",
+        "lodash.groupby": "4.6.0",
+        "lodash.uniqby": "4.7.0",
+        "node-graphviz": "0.1.0",
+        "ora": "5.4.1",
+        "rimraf": "3.0.2",
+        "shelljs": "0.8.5",
+        "xml": "1.0.1",
+        "yargs": "17.2.1"
+      },
+      "dependencies": {
+        "@sasjs/utils": {
+          "version": "2.35.0",
+          "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.35.0.tgz",
+          "integrity": "sha512-q9ZKV+TXqwiaj+0z5U7/00eBpp2QpjKfC9BKx7A6rQjBl10WtoWd5C9Em+RQULWVEdRbVS2XcnNsWelbKq/Zsw==",
+          "dev": true,
+          "requires": {
+            "@types/fs-extra": "^9.0.13",
+            "@types/prompts": "^2.0.13",
+            "chalk": "^4.1.1",
+            "cli-table": "^0.3.6",
+            "consola": "^2.15.0",
+            "csv-stringify": "^5.6.5",
+            "find": "0.3.0",
+            "fs-extra": "^10.0.0",
+            "jwt-decode": "^3.1.2",
+            "prompts": "^2.4.1",
+            "rimraf": "^3.0.2",
+            "valid-url": "^1.0.9"
+          }
+        }
+      }
+    },
+    "@sasjs/core": {
+      "version": "3.10.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-3.10.0.tgz",
+      "integrity": "sha512-lgLxDYpIvwSrXFaUaTFCR0KXHQEc5QIOL4DU87TvBHEUUAWNQHzuVQWkavLtW5hbvLGnPXnyvspzoSzmBojXzg==",
+      "dev": true,
+      "requires": {
+        "ts-loader": "^9.2.6"
+      }
+    },
+    "@sasjs/lint": {
+      "version": "1.11.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.2.tgz",
+      "integrity": "sha512-zEonhvha9kwrD+hxhG0hEhtfqpXwffH4vRDIr6eDiXkC7S8M3yImpjyFBvX/THJO5+8iuY8TYkOXKl7+nK/wAg==",
+      "dev": true,
+      "requires": {
+        "@sasjs/utils": "^2.19.0"
+      }
+    },
+    "@sasjs/utils": {
+      "version": "2.34.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.34.1.tgz",
+      "integrity": "sha512-hd1qieH3d7+xH96n5DpRGTEazeAhYyBBKCdnKhOXMgF2TZVoHFdRs5REfT88CKza6DHBGRVGnIVm5ORGP4cVLg==",
+      "dev": true,
+      "requires": {
+        "@types/fs-extra": "^9.0.11",
+        "@types/prompts": "^2.0.13",
+        "chalk": "^4.1.1",
+        "cli-table": "^0.3.6",
+        "consola": "^2.15.0",
+        "csv-stringify": "^5.6.5",
+        "find": "0.3.0",
+        "fs-extra": "^10.0.0",
+        "jwt-decode": "^3.1.2",
+        "lodash.groupby": "4.6.0",
+        "lodash.uniqby": "4.7.0",
+        "prompts": "^2.4.1",
+        "rimraf": "^3.0.2",
+        "valid-url": "^1.0.9"
+      }
     },
     "@tootallnate/once": {
       "version": "1.1.2",
@@ -2690,9 +2802,9 @@
       "peer": true
     },
     "@types/fs-extra": {
-      "version": "9.0.12",
+      "version": "9.0.13",
-      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.12.tgz",
+      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.13.tgz",
-      "integrity": "sha512-I+bsBr67CurCGnSenZZ7v94gd3tc3+Aj2taxMT4yu4ABLuOgOjeFxX3dokG24ztSRg5tnT00sL8BszO7gSMoIw==",
+      "integrity": "sha512-nEnwB++1u5lVDM2UI4c1+5R+FYaKfaAzS4OococimjVm3nQw3TuzH5UNsocrcTBbhnerblyHj4A49qXbIiZdpA==",
       "dev": true,
       "requires": {
         "@types/node": "*"
@@ -2974,12 +3086,12 @@
       "dev": true
     },
     "axios": {
-      "version": "0.21.4",
+      "version": "0.25.0",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.25.0.tgz",
-      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
+      "integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==",
       "dev": true,
       "requires": {
-        "follow-redirects": "^1.14.0"
+        "follow-redirects": "^1.14.7"
       }
     },
     "axios-cookiejar-support": {
@@ -3421,9 +3533,9 @@
       }
     },
     "follow-redirects": {
-      "version": "1.14.5",
+      "version": "1.14.7",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.5.tgz",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz",
-      "integrity": "sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA==",
+      "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==",
       "dev": true
     },
     "form-data": {
@@ -3627,9 +3739,9 @@
       "dev": true
     },
     "is-core-module": {
-      "version": "2.4.0",
+      "version": "2.8.1",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.4.0.tgz",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.1.tgz",
-      "integrity": "sha512-6A2fkfq1rfeQZjxrZJGerpLCTHRNEBiSgnu0+obeJpEPZRUooHgsizvzv0ZjJwOz3iWIHdJtVWJ/tmPr3D21/A==",
+      "integrity": "sha512-SdNCUs284hr40hFTFP6l0IfZ/RSrMXF3qgoRHd3/79unUTvrFO/JoXwkGm+5J/Oe3E/b5GsnG330uUNgRpu1PA==",
       "dev": true,
       "requires": {
         "has": "^1.0.3"
@@ -4051,13 +4163,14 @@
       "dev": true
     },
     "resolve": {
-      "version": "1.20.0",
+      "version": "1.21.0",
-      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.21.0.tgz",
-      "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==",
+      "integrity": "sha512-3wCbTpk5WJlyE4mSOtDLhqQmGFi0/TD9VPwmiolnk8U0wRgMEktqCXd3vy5buTO3tljvalNvKrjHEfrd2WpEKA==",
       "dev": true,
       "requires": {
-        "is-core-module": "^2.2.0",
+        "is-core-module": "^2.8.0",
-        "path-parse": "^1.0.6"
+        "path-parse": "^1.0.7",
+        "supports-preserve-symlinks-flag": "^1.0.0"
       }
     },
     "resolve-dir": {
@@ -4138,9 +4251,9 @@
       }
     },
     "shelljs": {
-      "version": "0.8.4",
+      "version": "0.8.5",
-      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.4.tgz",
+      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz",
-      "integrity": "sha512-7gk3UZ9kOfPLIAbslLzyWeGiEqx9e3rxwZM0KE6EL8GlGwjym9Mrlx5/p33bWTu9YG6vcS4MBxYZDHYr5lr8BQ==",
+      "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==",
       "dev": true,
       "requires": {
         "glob": "^7.0.0",
@@ -4212,6 +4325,12 @@
         "has-flag": "^4.0.0"
       }
     },
+    "supports-preserve-symlinks-flag": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz",
+      "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==",
+      "dev": true
+    },
     "symbol-tree": {
       "version": "3.2.4",
       "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sasjs/core",
-  "description": "Production Ready Macros for SAS Application Developers",
+  "description": "Macros for SAS Application Developers",
   "license": "MIT",
   "keywords": [
     "SAS",
@@ -33,9 +33,6 @@
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "^2.39.0"
+    "@sasjs/cli": "3.6.0"
-  },
-  "dependencies": {
-    "ts-loader": "^9.2.6"
   }
 }

sasjs/doxy/new_header.html

@@ -1,15 +1,21 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!DOCTYPE html
+  PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 <!-- HTML header for doxygen 1.8.17-->
 <html xmlns="https://www.w3.org/1999/xhtml" lang="en">
+
 <head>
   <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8" />
   <meta http-equiv="X-UA-Compatible" content="IE=9" />
   <meta property="og:type" content="website">
+  <meta property="og:title" content="MacroCore" />
+  <meta property="og:url" content="https://core.sasjs.io" />
+  <meta property="og:image" content="https://core.sasjs.io/Macro_core_website_1.png" />
   <meta name="author" content="Allan Bowe">
   <meta name="generator" content="Doxygen $doxygenversion" />
   <meta name="viewport" content="width=device-width, initial-scale=1" />
   <!--BEGIN PROJECT_NAME-->
   <meta name="description" content="$projectbrief" />
+  <meta name="og:description" content="$projectbrief" />
   <!--END PROJECT_NAME-->
   <!--BEGIN !PROJECT_NAME-->
   <title>$title</title>
@@ -22,6 +28,7 @@
   <link rel="shortcut icon" href="$relpath^favicon.ico" type="image/x-icon" />
   $extrastylesheet
 </head>
+
 <body>
   <div id="top">
     <!-- do not remove this div, it is closed by doxygen! -->
@@ -33,14 +40,12 @@
           <tr style="height: 56px">
             <!--BEGIN PROJECT_LOGO-->
             <td id="projectlogo">
-                <a href="$relpath^"
+              <a href="$relpath^"><img alt="Logo" src="$relpath^$projectlogo" /></a>
-                  ><img alt="Logo" src="$relpath^$projectlogo"
-                /></a>
             </td>
             <!--END PROJECT_LOGO-->
             <td id="projectalign" style="padding-left: 0.5em">
               <div id="projectbrief">
-                  Production Ready Macros for SAS Application Developers<br />
+                Macros for SAS Application Developers<br />
                 <a href="https://github.com/sasjs/core">
                   https://github.com/sasjs/core
                 </a>
@@ -64,4 +69,5 @@
     <!-- end header part -->
   </div>
 </body>
+
 </html>

sasjs/sasjsconfig.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://cli.sasjs.io/sasjsconfig-schema.json",
+  "$schema": "https://raw.githubusercontent.com/sasjs/utils/main/src/types/sasjsconfig-schema.json",
   "macroFolders": [
     "base",
     "fcmp",

server/ms_webout.sas

@@ -20,11 +20,17 @@
         %ms_webout(CLOSE)
 
 
-  @param action Either FETCH, OPEN, ARR, OBJ or CLOSE
+  @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE
-  @param ds The dataset to send back to the frontend
+  @param [in] ds The dataset to send back to the frontend
-  @param dslabel= value to use instead of the real name for sending to JSON
+  @param [out] dslabel= value to use instead of table name for sending to JSON
-  @param fmt=(Y) Set to N to send back unformatted values
+  @param [in] fmt= (Y) Set to N to send back unformatted values
-  @param fref=(_webout) The fileref to which to write the JSON
+  @param [out] fref= (_webout) The fileref to which to write the JSON
+  @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
+    (eg `null`) or as STRING values (eg `".a"` or `".b"`)
+  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+    such as the column formats and types.  The metadata is contained inside an
+    object with the same name as the table but prefixed with a dollar sign - ie,
+    `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
 
   <h4> SAS Macros </h4>
   @li mp_jsonout.sas
@@ -39,7 +45,9 @@
 
 **/
 
-%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y);
+%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
+  ,showmeta=NO
+);
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
 
@@ -91,7 +99,7 @@
 
 %else %if &action=ARR or &action=OBJ %then %do;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=DATASTEP,dbg=%str(&_debug)
+    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta
   )
 %end;
 %else %if &action=CLOSE %then %do;
@@ -106,15 +114,12 @@
       set &tempds;
       if not (upcase(name) =:"DATA"); /* ignore temp datasets */
       i+1;
-      call symputx('wt'!!left(i),name,'l');
+      call symputx(cats('wt',i),name,'l');
       call symputx('wtcnt',i,'l');
     data _null_; file &fref mod encoding='utf-8' termstr=lf;
       put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
-      proc contents noprint data=&wt
-        out=_data_ (keep=name type length format:);
-      run;%let tempds=%scan(&syslast,2,.);
       data _null_; file &fref mod encoding='utf-8' termstr=lf;
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
@@ -124,8 +129,7 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES)
-      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
       data _null_; file &fref mod encoding='utf-8' termstr=lf;
         put "}";
     %end;
@@ -157,10 +161,11 @@
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
-    put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length autoexec $512;
     autoexec=quote(urlencode(trim(getoption('autoexec'))));
     put ',"AUTOEXEC" : ' autoexec;
+    length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
     memsize=quote(cats(memsize));
     put ',"MEMSIZE" : ' memsize;

tests/crossplatform/mcf_length.test.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Testing mcf_length.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mcf_length.sas
+  @li mp_assert.sas
+
+**/
+
+%mcf_length(wrap=YES, insert_cmplib=YES)
+
+data test;
+  call symputx('null',mcf_length(.));
+  call symputx('special',mcf_length(._));
+  call symputx('three',mcf_length(1));
+  call symputx('four',mcf_length(10000000));
+  call symputx('five',mcf_length(12345678));
+  call symputx('six',mcf_length(1234567890));
+  call symputx('seven',mcf_length(12345678901234));
+  call symputx('eight',mcf_length(12345678901234567));
+run;
+
+%mp_assert(
+  iftrue=(%str(&null)=%str(0)),
+  desc=Check if NULL returns 0
+)
+%mp_assert(
+  iftrue=(%str(&special)=%str(3)),
+  desc=Check if special missing ._ returns 3
+)
+%mp_assert(
+  iftrue=(%str(&three)=%str(3)),
+  desc=Check for length 3
+)
+%mp_assert(
+  iftrue=(%str(&four)=%str(4)),
+  desc=Check for length 4
+)
+%mp_assert(
+  iftrue=(%str(&five)=%str(5)),
+  desc=Check for length 5
+)
+%mp_assert(
+  iftrue=(%str(&six)=%str(6)),
+  desc=Check for length 6
+)
+%mp_assert(
+  iftrue=(%str(&seven)=%str(7)),
+  desc=Check for length 3
+)
+%mp_assert(
+  iftrue=(%str(&eight)=%str(8)),
+  desc=Check for length 8
+)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check syscc=0 before re-initialisation
+)
+
+/* test 2 - compile again test for warnings */
+%mcf_length(wrap=YES, insert_cmplib=YES)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check syscc=0 after re-initialisation
+)

tests/crossplatform/mp_assert.test.sas

@@ -0,0 +1,15 @@
+/**
+  @file
+  @brief Testing mp_assert macro
+  @details This is quite "meta".. it's just testing itself
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(1=1),
+  desc=Checking result was created,
+  outds=work.test_results
+)

tests/crossplatform/mp_assertscope.test.sas

@@ -0,0 +1,80 @@
+/**
+  @file
+  @brief Testing mp_assertscope macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getvalue.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+
+**/
+
+%macro dostuff(action);
+  %if &action=ADD %then %do;
+    %global NEWVAR1 NEWVAR2;
+  %end;
+  %else %if &action=DEL %then %do;
+    %symdel NEWVAR1 NEWVAR2;
+  %end;
+  %else %if &action=MOD %then %do;
+    %let NEWVAR1=Let us pray..;
+  %end;
+  %else %if &action=NOTHING %then %do;
+    %local a b c d e;
+  %end;
+%mend dostuff;
+
+
+/* check for adding variables */
+%mp_assertscope(SNAPSHOT)
+%dostuff(ADD)
+%mp_assertscope(COMPARE,outds=work.testing_the_tester1)
+%mp_assert(
+  iftrue=(
+    "%mf_getvalue(work.testing_the_tester1,test_comments)"
+      ="Mod:() Add:(NEWVAR1 NEWVAR2) Del:()"
+  ),
+  desc=Checking result when vars added,
+  outds=work.test_results
+)
+
+
+/* check for modifying variables */
+%mp_assertscope(SNAPSHOT)
+%dostuff(MOD)
+%mp_assertscope(COMPARE,outds=work.testing_the_tester2)
+%mp_assert(
+  iftrue=(
+    "%mf_getvalue(work.testing_the_tester2,test_comments)"
+      ="Mod:(NEWVAR1) Add:() Del:()"
+  ),
+  desc=Checking result when vars modified,
+  outds=work.test_results
+)
+
+/* check for deleting variables */
+%mp_assertscope(SNAPSHOT)
+%dostuff(DEL)
+%mp_assertscope(COMPARE,outds=work.testing_the_tester3)
+%mp_assert(
+  iftrue=(
+    "%mf_getvalue(work.testing_the_tester3,test_comments)"
+      ="Mod:() Add:() Del:(NEWVAR1 NEWVAR2)"
+  ),
+  desc=Checking result when vars deleted,
+  outds=work.test_results
+)
+
+/* check for doing nothing */
+%mp_assertscope(SNAPSHOT)
+%dostuff(NOTHING)
+%mp_assertscope(COMPARE,outds=work.testing_the_tester4)
+%mp_assert(
+  iftrue=(
+    "%mf_getvalue(work.testing_the_tester4,test_comments)"
+      ="GLOBAL Variables Unmodified"
+  ),
+  desc=Checking results when nothing created,
+  outds=work.test_results
+)

tests/crossplatform/mp_ds2squeeze.test.sas

@@ -0,0 +1,44 @@
+/**
+  @file
+  @brief Testing mp_ds2squeeze.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getvarlen.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mp_ds2squeeze.sas
+
+**/
+
+data big;
+  length my big $32000;
+  do i=1 to 1e4;
+    my=repeat('oh my',100);
+    big='dawg';
+    special=._;
+    missn=.;
+    missc='';
+    output;
+  end;
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mp_ds2squeeze(work.big,outds=work.smaller)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking syscc
+)
+%mp_assert(
+  iftrue=(%mf_getvarlen(work.smaller,missn)=3),
+  desc=Check missing numeric is 3
+)
+%mp_assert(
+  iftrue=(%mf_getvarlen(work.smaller,special)=3),
+  desc=Check missing special numeric is 3
+)
+%mp_assert(
+  iftrue=(%mf_getvarlen(work.smaller,missc)=1),
+  desc=Check missing char is 1
+)

tests/crossplatform/mp_getcols.test.sas

@@ -4,6 +4,7 @@
 
   <h4> SAS Macros </h4>
   @li mp_getcols.sas
+  @li mp_assertcols.sas
   @li mp_assertcolvals.sas
   @li mp_assertdsobs.sas
 
@@ -31,3 +32,9 @@ run;
   desc=All values have a match,
   test=ALLVALS
 )
+
+%mp_assertcols(work.info,
+  cols=name type length varnum format label ddtype fmtname,
+  test=ALL,
+  desc=check all columns exist
+)

tests/crossplatform/mp_getmaxvarlengths.test.sas

@@ -0,0 +1,80 @@
+/**
+  @file
+  @brief Testing mp_getmaxvarlengths macro
+
+  <h4> SAS Macros </h4>
+  @li mp_getmaxvarlengths.sas
+  @li mp_assert.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
+
+**/
+
+
+/* regular usage */
+%mp_assertscope(SNAPSHOT)
+%mp_getmaxvarlengths(sashelp.class,outds=work.myds)
+%mp_assertscope(COMPARE,desc=checking scope leakage on mp_getmaxvarlengths)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errs
+)
+%mp_assertdsobs(work.myds,
+  desc=Has 5 records,
+  test=EQUALS 5
+)
+data work.errs;
+  set work.myds;
+  if name='Name' and maxlen ne 7 then output;
+  if name='Sex' and maxlen ne 1 then output;
+  if name='Age' and maxlen ne 3 then output;
+  if name='Height' and maxlen ne 8 then output;
+  if name='Weight' and maxlen ne 3 then output;
+run;
+data _null_;
+  set work.errs;
+  putlog (_all_)(=);
+run;
+
+%mp_assertdsobs(work.errs,
+  desc=Err table has 0 records,
+  test=EQUALS 0
+)
+
+/* test2 */
+data work.test2;
+  length a 3 b 5;
+  a=1/3;
+  b=1/3;
+  c=1/3;
+  d=._;
+  e=.;
+  output;
+  output;
+run;
+%mp_getmaxvarlengths(work.test2,outds=work.myds2)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errs in second test (with nulls)
+)
+%mp_assertdsobs(work.myds2,
+  desc=Has 5 records,
+  test=EQUALS 5
+)
+data work.errs2;
+  set work.myds2;
+  if name='a' and maxlen ne 3 then output;
+  if name='b' and maxlen ne 5 then output;
+  if name='c' and maxlen ne 8 then output;
+  if name='d' and maxlen ne 3 then output;
+  if name='e' and maxlen ne 0 then output;
+run;
+data _null_;
+  set work.errs2;
+  putlog (_all_)(=);
+run;
+
+%mp_assertdsobs(work.errs2,
+  desc=Err table has 0 records,
+  test=EQUALS 0
+)

tests/crossplatform/mp_jsonout.test.1.sas

@@ -34,6 +34,11 @@ data work.test;
   call symputx('dtval',dtval);
 run;
 
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking for error condition,
+  outds=work.test_results
+)
 
 %mp_assert(
   iftrue=(&dtval=&compare),

tests/crossplatform/mp_jsonout.test.2.sas

@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief Testing mp_jsonout.sas macro with special missings
+
+  <h4> SAS Macros </h4>
+  @li mp_jsonout.sas
+  @li mp_assert.sas
+
+**/
+
+filename webref temp;
+
+data demo;
+  do x=._,.,.a,.b,.c,.d,.e,-99, 0, 1,2, 3.333333;
+    output;
+  end;
+run;
+%mp_jsonout(OPEN,jref=webref)
+%mp_jsonout(OBJ,demo,jref=webref,fmt=N,missing=STRING)
+%mp_jsonout(CLOSE,jref=webref)
+
+data _null_;
+  infile webref;
+  input;
+  putlog _infile_;
+run;
+
+libname web JSON fileref=webref;
+
+/* proc json turns to char - so switch back to numeric */
+data work.test(keep=x);
+  set web.demo(rename=(x=y));
+  if y ='_' then x=._;
+  else if anyalpha(y) then x=input(cats(".",y),best.);
+  else x=input(y,best.);
+  put (_all_)(=);
+run;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking for error condition with special missing export,
+  outds=work.test_results
+)
+
+proc compare base=work.demo compare=work.test;
+quit;
+
+%mp_assert(
+  iftrue=(&sysinfo=0),
+  desc=Returned json is identical to input table for all special missings,
+  outds=work.test_results
+)

tests/crossplatform/mp_lockanytable.test.sas

@@ -6,12 +6,13 @@
   @li mp_lockanytable.sas
   @li mp_assertcols.sas
   @li mp_assertcolvals.sas
+  @li mp_coretable.sas
 
 **/
 
 /* check create table */
 
-%mp_lockanytable(MAKETABLE, ctl_ds=work.controller)
+%mp_coretable(LOCKTABLE,libds=work.controller)
 
 %mp_assertcols(work.controller,
   cols=lock_status_cd lock_lib lock_ds lock_user_nm lock_ref lock_pid

tests/crossplatform/mp_validatecol.test.sas

@@ -112,6 +112,9 @@ datalines4;
 0
 above are good
 the rest are bad
+0.1
+1.1
+-0.001
 %abort
 1&somethingverybad.
 &

tests/testinit.sas

@@ -4,6 +4,7 @@
 
   <h4> SAS Macros </h4>
   @li mp_init.sas
+  @li mv_webout.sas
 
 **/
 

viya/mv_createwebservice.sas

@@ -54,9 +54,9 @@
     that location
   @param [in] adapter= the macro uses the sasjs adapter by default.  To use
     another adapter, add a (different) fileref here.
-  @param [in] contextname= Choose a specific context on which to run the Job.  Leave
+  @param [in] contextname= Choose a specific context on which to run the Job.
-    blank to use the default context.  From Viya 3.5 it is possible to configure
+    Leave blank to use the default context.  From Viya 3.5 it is possible to
-    a shared context - see
+    configure a shared context - see
 https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5pyhn1wg3ja0drdl6h.htm&docsetVersion=3.5&locale=en
   @param [in] mdebug=(0) set to 1 to enable DEBUG messages
 
@@ -237,9 +237,14 @@ data _null_;
   put "/* Created on %sysfunc(datetime(),datetime19.) by &sysuserid */";
 /* WEBOUT BEGIN */
   put ' ';
-  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0 ';
+  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
+  put '  ,engine=DATASTEP ';
+  put '  ,missing=NULL ';
+  put '  ,showmeta=NO ';
   put ')/*/STORE SOURCE*/; ';
-  put '%put output location=&jref; ';
+  put '%local tempds colinfo fmtds i numcols; ';
+  put '%let numcols=0; ';
+  put ' ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
   put '  data _null_;file &jref encoding=''utf-8'' ; ';
@@ -248,45 +253,25 @@ data _null_;
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
   put '  options validvarname=upcase; ';
-  put '  data _null_;file &jref mod encoding=''utf-8'' ; ';
+  put '  data _null_; file &jref encoding=''utf-8'' mod; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
   put ' ';
-  put '  %if &engine=PROCJSON %then %do; ';
+  put '  /* grab col defs */ ';
-  put '    data;run;%let tempds=&syslast; ';
-  put '    proc sql;drop table &tempds; ';
-  put '    data &tempds /view=&tempds;set &ds; ';
-  put '    %if &fmt=N %then format _numeric_ best32.;; ';
-  put '    proc json out=&jref pretty ';
-  put '        %if &action=ARR %then nokeys ; ';
-  put '        ;export &tempds / nosastags fmtnumeric; ';
-  put '    run; ';
-  put '    proc sql;drop view &tempds; ';
-  put '  %end; ';
-  put '  %else %if &engine=DATASTEP %then %do; ';
-  put '    %local cols i tempds; ';
-  put '    %let cols=0; ';
-  put '    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 %then %do; ';
-  put '      %put &sysmacroname:  &ds NOT FOUND!!!; ';
-  put '      %return; ';
-  put '    %end; ';
-  put '    %if &fmt=Y %then %do; ';
-  put '      %put converting every variable to a formatted variable; ';
-  put '      /* see mp_ds2fmtds.sas for source */ ';
   put '  proc contents noprint data=&ds ';
-  put '        out=_data_(keep=name type length format formatl formatd varnum); ';
+  put '    out=_data_(keep=name type length format formatl formatd varnum label); ';
   put '  run; ';
-  put '      proc sort; ';
+  put '  %let colinfo=%scan(&syslast,2,.); ';
+  put '  proc sort data=&colinfo; ';
   put '    by varnum; ';
   put '  run; ';
-  put '      %local fmtds; ';
+  put '  /* move meta to mac vars */ ';
-  put '      %let fmtds=%scan(&syslast,2,.); ';
-  put '      /* prepare formats and varnames */ ';
   put '  data _null_; ';
-  put '        if _n_=1 then call symputx(''nobs'',nobs,''l''); ';
+  put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
-  put '        set &fmtds end=last nobs=nobs; ';
+  put '    set &colinfo end=last nobs=nobs; ';
   put '    name=upcase(name); ';
   put '    /* fix formats */ ';
   put '    if type=2 or type=6 then do; ';
+  put '      typelong=''char''; ';
   put '      length fmt $49.; ';
   put '      if format='''' then fmt=cats(''$'',length,''.''); ';
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
@@ -294,6 +279,7 @@ data _null_;
   put '      newlen=max(formatl,length); ';
   put '    end; ';
   put '    else do; ';
+  put '      typelong=''num''; ';
   put '      if format='''' then fmt=''best.''; ';
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else if formatd=0 then fmt=cats(format,formatl,''.''); ';
@@ -307,59 +293,84 @@ data _null_;
   put '    call symputx(cats(''name'',_n_),name,''l''); ';
   put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
   put '    call symputx(cats(''len'',_n_),newlen,''l''); ';
+  put '    call symputx(cats(''length'',_n_),length,''l''); ';
   put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
+  put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
+  put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
   put '  run; ';
-  put '      data &fmtds; ';
+  put ' ';
+  put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put ' ';
+  put '  %if &engine=PROCJSON %then %do; ';
+  put '    %if &missing=STRING %then %do; ';
+  put '      %put &sysmacroname: Special Missings not supported in proc json.; ';
+  put '      %put &sysmacroname: Switching to DATASTEP engine; ';
+  put '      %goto datastep; ';
+  put '    %end; ';
+  put '    data &tempds /view=&tempds;set &ds; ';
+  put '    %if &fmt=N %then format _numeric_ best32.;; ';
+  put '    /* PRETTY is necessary to avoid line truncation in large files */ ';
+  put '    proc json out=&jref pretty ';
+  put '        %if &action=ARR %then nokeys ; ';
+  put '        ;export &tempds / nosastags fmtnumeric; ';
+  put '    run; ';
+  put '  %end; ';
+  put '  %else %if &engine=DATASTEP %then %do; ';
+  put '    %datastep: ';
+  put '    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 ';
+  put '    %then %do; ';
+  put '      %put &sysmacroname:  &ds NOT FOUND!!!; ';
+  put '      %return; ';
+  put '    %end; ';
+  put ' ';
+  put '    %if &fmt=Y %then %do; ';
+  put '      data _data_; ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
-  put '      %local i; ';
+  put '      %do i=1 %to &numcols; ';
-  put '      %do i=1 %to &nobs; ';
   put '        &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
-  put '      %do i=1 %to &nobs; ';
+  put '      %do i=1 %to &numcols; ';
   put '        length &&name&i $&&len&i; ';
   put '        &&name&i=left(put(&&newname&i,&&fmt&i)); ';
   put '        drop &&newname&i; ';
   put '      %end; ';
   put '        if _error_ then call symputx(''syscc'',1012); ';
   put '      run; ';
-  put '      %let ds=&fmtds; ';
+  put '      %let fmtds=&syslast; ';
-  put '    %end; /* &fmt=Y */ ';
+  put '    %end; ';
-  put '    data _null_;file &jref mod encoding=''utf-8'' ; ';
-  put '      put "["; call symputx(''cols'',0,''l''); ';
-  put '    proc sort ';
-  put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
-  put '      out=_data_; ';
-  put '      by varnum; ';
-  put ' ';
-  put '    data _null_; ';
-  put '      set _last_ end=last; ';
-  put '      call symputx(cats(''name'',_n_),name,''l''); ';
-  put '      call symputx(cats(''type'',_n_),type,''l''); ';
-  put '      call symputx(cats(''len'',_n_),length,''l''); ';
-  put '      if last then call symputx(''cols'',_n_,''l''); ';
-  put '    run; ';
   put ' ';
   put '    proc format; /* credit yabwon for special null removal */ ';
-  put '      value bart ._ - .z = null ';
+  put '    value bart (default=40) ';
+  put '    %if &missing=NULL %then %do; ';
+  put '      ._ - .z = null ';
+  put '    %end; ';
+  put '    %else %do; ';
+  put '      ._ = [quote()] ';
+  put '      . = null ';
+  put '      .a - .z = [quote()] ';
+  put '    %end; ';
   put '      other = [best.]; ';
   put ' ';
-  put '    data;run; %let tempds=&syslast; /* temp table for spesh char management */ ';
-  put '    proc sql; drop table &tempds; ';
   put '    data &tempds/view=&tempds; ';
   put '      attrib _all_ label=''''; ';
-  put '      %do i=1 %to &cols; ';
+  put '      %do i=1 %to &numcols; ';
-  put '        %if &&type&i=char %then %do; ';
+  put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
   put '          length &&name&i $32767; ';
   put '          format &&name&i $32767.; ';
   put '        %end; ';
   put '      %end; ';
+  put '      %if &fmt=Y %then %do; ';
+  put '        set &fmtds; ';
+  put '      %end; ';
+  put '      %else %do; ';
   put '        set &ds; ';
+  put '      %end; ';
   put '      format _numeric_ bart.; ';
-  put '    %do i=1 %to &cols; ';
+  put '    %do i=1 %to &numcols; ';
-  put '      %if &&type&i=char %then %do; ';
+  put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
   put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
   put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
   put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
@@ -369,21 +380,21 @@ data _null_;
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
+  put ' ';
   put '    /* write to temp loc to avoid _webout truncation ';
   put '      - https://support.sas.com/kb/49/325.html */ ';
   put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
   put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
+  put '      if _n_=1 then put "["; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
   put '      %if &action=ARR %then "[" ; %else "{" ; ';
-  put '      %do i=1 %to &cols; ';
+  put '      %do i=1 %to &numcols; ';
   put '        %if &i>1 %then  "," ; ';
   put '        %if &action=OBJ %then """&&name&i"":" ; ';
   put '        &&name&i ';
   put '      %end; ';
   put '      %if &action=ARR %then "]" ; %else "}" ; ; ';
-  put '    proc sql; ';
-  put '    drop view &tempds; ';
   put '    /* now write the long strings to _webout 1 byte at a time */ ';
   put '    data _null_; ';
   put '      length filein 8 fileid 8; ';
@@ -395,12 +406,33 @@ data _null_;
   put '        rc=fput(fileid, rec); ';
   put '        rc=fwrite(fileid); ';
   put '      end; ';
+  put '      /* close out the table */ ';
+  put '      rc=fput(fileid, "]"); ';
+  put '      rc=fwrite(fileid); ';
   put '      rc=fclose(filein); ';
   put '      rc=fclose(fileid); ';
   put '    run; ';
   put '    filename _sjs clear; ';
-  put '    data _null_; file &jref mod encoding=''utf-8'' ; ';
+  put '  %end; ';
-  put '      put "]"; ';
+  put ' ';
+  put '  proc sql; ';
+  put '  drop view &tempds; ';
+  put '  drop table &colinfo; ';
+  put ' ';
+  put '  %if &showmeta=YES %then %do; ';
+  put '    data _null_; file &jref encoding=''utf-8'' mod; ';
+  put '      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; ';
+  put '      do i=1 to &numcols; ';
+  put '        name=quote(trim(symget(cats(''name'',i)))); ';
+  put '        format=quote(trim(symget(cats(''fmt'',i)))); ';
+  put '        label=quote(trim(symget(cats(''label'',i)))); ';
+  put '        length=quote(trim(symget(cats(''length'',i)))); ';
+  put '        type=quote(trim(symget(cats(''typelong'',i)))); ';
+  put '        if i>1 then put "," @@; ';
+  put '        put name '':{"format":'' format '',"label":'' label ';
+  put '          '',"length":'' length '',"type":'' type ''}''; ';
+  put '      end; ';
+  put '      put ''}}''; ';
   put '    run; ';
   put '  %end; ';
   put '%end; ';
@@ -411,7 +443,9 @@ data _null_;
   put '  run; ';
   put '%end; ';
   put '%mend mp_jsonout; ';
-  put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y); ';
+  put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL ';
+  put '  ,showmeta=NO ';
+  put '); ';
   put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
   put '  sasjs_tables SYS_JES_JOB_URI; ';
   put '%if %index("&_debug",log) %then %let _debug=131; ';
@@ -533,12 +567,13 @@ data _null_;
   put ' ';
   put '  /* setup json */ ';
   put '  data _null_;file &fref; ';
-  put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
+  put '    put ''{"SYSDATE" : "'' "&SYSDATE" ''"''; ';
+  put '    put '',"SYSTIME" : "'' "&SYSTIME" ''"''; ';
   put '  run; ';
   put '%end; ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
   put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
-  put '      ,jref=&fref,engine=DATASTEP,dbg=%str(&_debug) ';
+  put '      ,jref=&fref,engine=DATASTEP,missing=&missing,showmeta=&showmeta ';
   put '    ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
@@ -553,14 +588,11 @@ data _null_;
   put '      set &tempds; ';
   put '      if not (upcase(name) =:"DATA"); /* ignore temp datasets */ ';
   put '      i+1; ';
-  put '      call symputx(''wt''!!left(i),name); ';
+  put '      call symputx(cats(''wt'',i),name,''l''); ';
-  put '      call symputx(''wtcnt'',i); ';
+  put '      call symputx(''wtcnt'',i,''l''); ';
   put '    data _null_; file &fref mod; put ",""WORK"":{"; ';
   put '    %do i=1 %to &wtcnt; ';
   put '      %let wt=&&wt&i; ';
-  put '      proc contents noprint data=&wt ';
-  put '        out=_data_ (keep=name type length format:); ';
-  put '      run;%let tempds=%scan(&syslast,2,.); ';
   put '      data _null_; file &fref mod; ';
   put '        dsid=open("WORK.&wt",''is''); ';
   put '        nlobs=attrn(dsid,''NLOBS''); ';
@@ -570,8 +602,7 @@ data _null_;
   put '        put " ""&wt"" : {"; ';
   put '        put ''"nlobs":'' nlobs; ';
   put '        put '',"nvars":'' nvars; ';
-  put '      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP) ';
+  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) ';
-  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP) ';
   put '      data _null_; file &fref mod;put "}"; ';
   put '    %end; ';
   put '    data _null_; file &fref mod;put "}";run; ';
@@ -595,7 +626,11 @@ data _null_;
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
   put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
-  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''" ''; ';
+  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
+  put '    length memsize $32; ';
+  put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
+  put '    memsize=quote(cats(memsize)); ';
+  put '    put '',"MEMSIZE" : '' memsize; ';
   put '    put "}"; ';
   put ' ';
   put '  %if %upcase(&fref) ne _WEBOUT and &stream=Y %then %do; ';
@@ -629,8 +664,10 @@ data _null_;
   put '%global __program _program;';
   put '%let _program=%sysfunc(coalescec(&__program,&_program));';
   put ' ';
-  put '%macro webout(action,ds,dslabel=,fmt=);';
+  put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO);';
-  put '  %mv_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt)';
+  put '  %mv_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt,missing=&missing';
+  put '    ,showmeta=&showmeta';
+  put '  )';
   put '%mend;';
 run;
 

viya/mv_deletefoldermember.sas

@@ -117,6 +117,7 @@ libname &libref1a JSON fileref=&fname1a;
 %let found=0;
 %put Getting object uri from &libref1a..items;
 data _null_;
+  length contenttype name $1000;
   set &libref1a..items;
   if contenttype="&contenttype" and upcase(name)="%upcase(&name)" then do;
     call symputx('uri',uri,'l');

viya/mv_deletejes.sas

@@ -114,6 +114,7 @@ libname &libref1a JSON fileref=&fname1a;
 %let found=0;
 %put Getting object uri from &libref1a..items;
 data _null_;
+  length contenttype name $1000;
   set &libref1a..items;
   if contenttype='jobDefinition' and upcase(name)="%upcase(&name)" then do;
     call symputx('uri',cats("&base_uri",uri),'l');

viya/mv_getaccesstoken.sas

@@ -1,32 +0,0 @@
-/**
-  @file mv_getaccesstoken.sas
-  @brief deprecated - replaced by mv_tokenrefresh.sas
-
-  @version VIYA V.03.04
-  @author Allan Bowe, source: https://github.com/sasjs/core
-
-  <h4> SAS Macros </h4>
-  @li mv_tokenrefresh.sas
-
-**/
-
-%macro mv_getaccesstoken(client_id=someclient
-    ,client_secret=somesecret
-    ,grant_type=authorization_code
-    ,code=
-    ,user=
-    ,pass=
-    ,access_token_var=ACCESS_TOKEN
-    ,refresh_token_var=REFRESH_TOKEN
-  );
-
-%mv_tokenrefresh(client_id=&client_id
-  ,client_secret=&client_secret
-  ,grant_type=&grant_type
-  ,user=&user
-  ,pass=&pass
-  ,access_token_var=&access_token_var
-  ,refresh_token_var=&refresh_token_var
-)
-
-%mend mv_getaccesstoken;

viya/mv_getapptoken.sas

@@ -1,23 +0,0 @@
-/**
-  @file
-  @brief deprecated - replaced by mv_registerclient.sas
-
-  @version VIYA V.03.04
-  @author Allan Bowe, source: https://github.com/sasjs/core
-
-  <h4> SAS Macros </h4>
-  @li mv_registerclient.sas
-
-**/
-
-%macro mv_getapptoken(client_id=someclient
-    ,client_secret=somesecret
-    ,grant_type=authorization_code
-  );
-
-%mv_registerclient(client_id=&client_id
-  ,client_secret=&client_secret
-  ,grant_type=&grant_type
-)
-
-%mend mv_getapptoken;

viya/mv_getrefreshtoken.sas

@@ -1,33 +0,0 @@
-/**
-  @file mv_getrefreshtoken.sas
-  @brief deprecated - replaced by mv_tokenauth.sas
-
-  @version VIYA V.03.04
-  @author Allan Bowe, source: https://github.com/sasjs/core
-
-  <h4> SAS Macros </h4>
-  @li mv_tokenauth.sas
-
-**/
-
-%macro mv_getrefreshtoken(client_id=someclient
-    ,client_secret=somesecret
-    ,grant_type=authorization_code
-    ,code=
-    ,user=
-    ,pass=
-    ,access_token_var=ACCESS_TOKEN
-    ,refresh_token_var=REFRESH_TOKEN
-  );
-
-%mv_tokenauth(client_id=&client_id
-  ,client_secret=&client_secret
-  ,grant_type=&grant_type
-  ,code=&code
-  ,user=&user
-  ,pass=&pass
-  ,access_token_var=&access_token_var
-  ,refresh_token_var=&refresh_token_var
-)
-
-%mend mv_getrefreshtoken;

viya/mv_jobflow.sas

@@ -272,7 +272,7 @@ data;run;%let jdswaitfor=&syslast;
       data _null_;
         infile &jfref lrecl=32767;
         input;
-        jparams='jparams'!!left(symget('jid'));
+        jparams=cats('jparams',symget('jid'));
         call symputx(jparams,substr(_infile_,3,length(_infile_)-4));
       run;
       %local jobuid&jid;

viya/mv_webout.sas

@@ -20,13 +20,19 @@
         %mv_webout(CLOSE)
 
 
-  @param action Either OPEN, ARR, OBJ or CLOSE
+  @param [in] action Either OPEN, ARR, OBJ or CLOSE
-  @param ds The dataset to send back to the frontend
+  @param [in] ds The dataset to send back to the frontend
-  @param _webout= fileref for returning the json
+  @param [in] _webout= fileref for returning the json
-  @param fref=(_mvwtemp) Temp fileref to which to write the output
+  @param [out] fref=(_mvwtemp) Temp fileref to which to write the output
-  @param dslabel= value to use instead of the real name for sending to JSON
+  @param [out] dslabel= value to use instead of table name for sending to JSON
-  @param fmt=(Y) change to N to strip formats from output
+  @param [in] fmt=(Y) change to N to strip formats from output
-  @param stream=(Y) Change to N if not streaming to _webout
+  @param [in] stream=(Y) Change to N if not streaming to _webout
+  @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
+    (eg `null`) or as STRING values (eg `".a"` or `".b"`)
+  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+    such as the column formats and types.  The metadata is contained inside an
+    object with the same name as the table but prefixed with a dollar sign - ie,
+    `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
 
   <h4> SAS Macros </h4>
   @li mp_jsonout.sas
@@ -36,7 +42,9 @@
   @author Allan Bowe, source: https://github.com/sasjs/core
 
 **/
-%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y);
+%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL
+  ,showmeta=NO
+);
 %global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name
   sasjs_tables SYS_JES_JOB_URI;
 %if %index("&_debug",log) %then %let _debug=131;
@@ -158,12 +166,13 @@
 
   /* setup json */
   data _null_;file &fref;
-    put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
+    put '{"SYSDATE" : "' "&SYSDATE" '"';
+    put ',"SYSTIME" : "' "&SYSTIME" '"';
   run;
 %end;
 %else %if &action=ARR or &action=OBJ %then %do;
     %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt
-      ,jref=&fref,engine=DATASTEP,dbg=%str(&_debug)
+      ,jref=&fref,engine=DATASTEP,missing=&missing,showmeta=&showmeta
     )
 %end;
 %else %if &action=CLOSE %then %do;
@@ -178,14 +187,11 @@
       set &tempds;
       if not (upcase(name) =:"DATA"); /* ignore temp datasets */
       i+1;
-      call symputx('wt'!!left(i),name);
+      call symputx(cats('wt',i),name,'l');
-      call symputx('wtcnt',i);
+      call symputx('wtcnt',i,'l');
     data _null_; file &fref mod; put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
-      proc contents noprint data=&wt
-        out=_data_ (keep=name type length format:);
-      run;%let tempds=%scan(&syslast,2,.);
       data _null_; file &fref mod;
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
@@ -195,8 +201,7 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES)
-      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
       data _null_; file &fref mod;put "}";
     %end;
     data _null_; file &fref mod;put "}";run;
@@ -220,7 +225,11 @@
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
-    put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
+    length memsize $32;
+    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
+    memsize=quote(cats(memsize));
+    put ',"MEMSIZE" : ' memsize;
     put "}";
 
   %if %upcase(&fref) ne _WEBOUT and &stream=Y %then %do;