Merge pull request #149 from sasjs/mcf_length

feat: new mcf_length.sas fcmp macro

README.md

@@ -188,6 +188,13 @@ When contributing to this library, it is therefore important to ensure that all
 
 - All macros should be compatible with SAS versions from support level B and above (so currently 9.2 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully (eg `%if %sysevalf(&sysver<9.3) %then %return`).
 
+## Breaking Changes
+
+We are currently on major release v3.  The following changes are planned when the next major (breaking) release becomes necessary:
+
+* Remove `dbg` parameter from mp_jsonout.sas (implement mdebug instead)
+* Remove `END_DTTM` and `START_DTTM` from mx_webout JSON
+
 ## Star Gazing
 
 If you find this library useful, please leave a [star](https://github.com/sasjs/core/stargazers) and help us grow our star graph!

base/mp_abort.sas

@@ -110,8 +110,8 @@
     %if %symexist(SYSPRINTTOLOG) %then %let logloc=&SYSPRINTTOLOG;
     %else %let logloc=%qsysfunc(getoption(LOG));
     proc printto log=log;run;
+    %let logline=0;
     %if %length(&logloc)>0 %then %do;
-      %let logline=0;
       data _null_;
         infile &logloc lrecl=5000;
         input; putlog _infile_;
@@ -160,7 +160,10 @@
       file _webout mod lrecl=32000 encoding='utf-8';
       length msg $32767 ;
       sasdatetime=datetime();
-      msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
+      msg=symget('msg');
+    %if &logline>0 %then %do;
+      msg=cats(msg,'\n\nLog Extract:\n',symget('logmsg'));
+    %end;
       /* escape the quotes */
       msg=tranwrd(msg,'"','\"');
       /* ditch the CRLFs as chrome complains */

base/mp_assertscope.sas

@@ -0,0 +1,119 @@
+/**
+  @file
+  @brief Used to capture scope leakage of macro variables
+  @details A common 'difficult to detect' bug in macros is where a nested
+    macro over-writes variables in a higher level macro.
+
+    This assertion takes a snapshot of the macro variables before and after
+    a macro invocation.  This makes it easy to detect whether any macro
+    variables were modified or changed.
+
+    Currently, the macro only checks for global scope variables.  In the future
+    it may be extended to work at multiple levels of nesting.
+
+    If you would like this feature, feel free to contribute / raise an issue /
+    engage the SASjs team directly.
+
+  Example usage:
+
+      %mp_assertscope(SNAPSHOT)
+
+      %let oops=I did it again;
+
+      %mp_assertscope(COMPARE,
+        desc=Checking macro variables against previous snapshot
+      )
+
+  @param [in] action (SNAPSHOT) The action to take.  Valid values:
+    @li SNAPSHOT - take a copy of the current macro variables
+    @li COMPARE - compare the current macro variables against previous values
+  @param [in] scope= (GLOBAL) The scope of the variables to be checked.  This
+    corresponds to the values in the SCOPE column in `sashelp.vmacro`.
+  @param [in] desc= (Testing variable scope) The user provided test description
+  @param [in,out] scopeds= (work.mp_assertscope) The dataset to contain the
+    scope snapshot
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|No out of scope variables created or modified|
+
+  <h4> Related Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertcols.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertscope(action,
+  desc=0,
+  scope=GLOBAL,
+  scopeds=work.mp_assertscope,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+%local ds test_result test_comments del add mod;
+
+/* get current variables */
+%if &action=SNAPSHOT %then %do;
+  proc sql;
+  create table &scopeds as
+    select name,offset,value
+    from dictionary.macros
+    where scope="&scope"
+    order by name,offset;
+%end;
+%else %if &action=COMPARE %then %do;
+
+  proc sql;
+  create table _data_ as
+    select name,offset,value
+    from dictionary.macros
+    where scope="&scope"
+    order by name,offset;
+
+  %let ds=&syslast;
+
+  proc compare base=&scopeds compare=&ds;
+  run;
+
+  %if &sysinfo=0 %then %do;
+    %let test_result=PASS;
+    %let test_comments=&scope Variables Unmodified;
+  %end;
+  %else %do;
+    proc sql noprint undo_policy=none;
+    select distinct name into: del separated by ' '  from &scopeds
+      where name not in (select name from &ds);
+    select distinct name into: add separated by ' '  from &ds
+      where name not in (select name from &scopeds);
+    select distinct a.name into: mod separated by ' '
+      from &scopeds a
+      inner join &ds b
+      on a.name=b.name
+        and a.offset=b.offset
+      where a.value ne b.value;
+    %let test_result=FAIL;
+    %let test_comments=%str(Mod:(&mod) Add:(&add) Del:(&del));
+  %end;
+
+
+  data ;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_comments=symget('test_comments');
+    test_result=symget('test_result');
+  run;
+
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+%end;
+
+%mend mp_assertscope;

base/mp_dirlist.sas

@@ -6,8 +6,7 @@
   Credit for the rename approach:
   https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003
 
-
-  usage:
+  Usage:
 
       %mp_dirlist(path=/some/location, outds=myTable, maxdepth=MAX)
 
@@ -23,12 +22,12 @@
   X CMD) do please raise an issue!
 
 
-  @param [in] path= for which to return contents
-  @param [in] fref= Provide a DISK engine fileref as an alternative to PATH
+  @param [in] path= (%sysfunc(pathname(work))) Path for which to return contents
+  @param [in] fref= (0) Provide a DISK engine fileref as an alternative to PATH
   @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
     subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
     recursion, set to MAX.
-  @param [out] outds= the output dataset to create
+  @param [out] outds= (work.mp_dirlist) The output dataset to create
   @param [out] getattrs= (NO)  If getattrs=YES then the doptname / foptname
     functions are used to scan all properties - any characters that are not
     valid in a SAS name (v7) are simply stripped, and the table is transposed
@@ -49,13 +48,15 @@
     - OS SPECIFIC variables, if <code>getattrs=</code> is used.
 
   <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_getvarlist.sas
+  @li mf_wordsinstr1butnotstr2.sas
   @li mp_dropmembers.sas
 
   <h4> Related Macros </h4>
   @li mp_dirlist.test.sas
 
   @version 9.2
-  @author Allan Bowe
 **/
 
 %macro mp_dirlist(path=%sysfunc(pathname(work))
@@ -193,9 +194,29 @@ data &out_ds;
   set &out_ds(where=(filepath ne ''));
 run;
 
-/* update main table */
-proc append base=&outds data=&out_ds;
-run;
+/**
+  * The above transpose can mean that some updates create additional columns.
+  * This necessitates the occasional use of datastep over proc append.
+  */
+%if %mf_existds(&outds) %then %do;
+  %local basevars appvars newvars;
+  %let basevars=%mf_getvarlist(&outds);
+  %let appvars=%mf_getvarlist(&out_ds);
+  %let newvars=%length(%mf_wordsinstr1butnotstr2(Str1=&appvars,Str2=&basevars));
+  %if &newvars>0 %then %do;
+    data &outds;
+      set &outds &out_ds;
+    run;
+  %end;
+  %else %do;
+    proc append base=&outds data=&out_ds force nowarn;
+    run;
+  %end;
+%end;
+%else %do;
+  proc append base=&outds data=&out_ds;
+  run;
+%end;
 
 /* recursive call */
 %if &maxdepth>&level or &maxdepth=MAX %then %do;

base/mp_ds2cards.sas

@@ -139,8 +139,9 @@ create table datalines1 as
 /**
   Due to long decimals cannot use best. format
   So - use bestd. format and then use character functions to strip trailing
-    zeros, if NOT an integer!!
-  resolved code = ifc(int(VARIABLE)=VARIABLE
+    zeros, if NOT an integer or missing!!  Cannot use int() as it upsets
+    note2err when there are missings.
+  resolved code = ifc( mod(coalesce(VARIABLE,0),1)=0
     ,put(VARIABLE,best32.)
     ,substrn(put(VARIABLE,bestd32.),1
     ,findc(put(VARIABLE,bestd32.),'0','TBK')));
@@ -151,7 +152,7 @@ data datalines_2;
   set datalines1 (where=(upcase(name) not in
     ('PROCESSED_DTTM','VALID_FROM_DTTM','VALID_TO_DTTM')));
   if type='num' then dataline=
-    cats('ifc(int(',name,')=',name,'
+    cats('ifc(mod(coalesce(',name,',0),1)=0
       ,put(',name,',best32.-l)
       ,substrn(put(',name,',bestd32.-l),1
       ,findc(put(',name,',bestd32.-l),"0","TBK")))');

base/mp_ds2csv.sas

@@ -41,7 +41,7 @@ data _null_;
   dsid=open("&ds.","i");
   num=attrn(dsid,"nvars");
   do i=1 to num;
-    header = trim(left(coalescec(varlabel(dsid,i),varname(dsid,i))));
+    header = cats(coalescec(varlabel(dsid,i),varname(dsid,i)));
     put header @;
   end;
   rc=close(dsid);

base/mp_filtercheck.sas

@@ -109,7 +109,7 @@ data &outds;
     output;
   end;
   if mod(SUBGROUP_ID,1) ne 0 then do;
-    REASON_CD='SUBGROUP_ID should be integer, not '!!left(subgroup_id);
+    REASON_CD='SUBGROUP_ID should be integer, not '!!cats(subgroup_id);
     putlog REASON_CD= SUBGROUP_ID=;
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');
@@ -127,7 +127,7 @@ data &outds;
   if OPERATOR_NM not in
   ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
   then do;
-    REASON_CD='Invalid OPERATOR_NM: '!!left(OPERATOR_NM);
+    REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');

base/mp_getddl.sas

@@ -158,7 +158,7 @@ run;
         lab=" label="!!cats("'",tranwrd(label,"'","''"),"'");
       if notnull='yes' then notnul=' not null';
       if type='char' then typ=cats('char(',length,')');
-      else if length ne 8 then typ='num length='!!left(length);
+      else if length ne 8 then typ='num length='!!cats(length);
       else typ='num';
       put name typ fmt notnul lab;
     run;

base/mp_init.sas

@@ -51,6 +51,7 @@
     noautocorrect           /* disallow misspelled procedure names            */
     compress=CHAR           /* default is none so ensure we have something!   */
     datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
+    dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
     %str(err)orcheck=STRICT /* catch errs in libname/filename statements      */
     fmterr                  /* ensure err when a format cannot be found       */
     mergenoby=%str(ERR)OR   /* throw err when a merge has no BY variables     */

base/mp_jsonout.sas

@@ -19,11 +19,12 @@
 
         %mp_jsonout(OPEN,jref=tmp)
         %mp_jsonout(OBJ,class,jref=tmp)
+        %mp_jsonout(OBJ,class,dslabel=class2,jref=tmp,showmeta=YES)
         %mp_jsonout(CLOSE,jref=tmp)
 
         data _null_;
         infile tmp;
-        input;list;
+        input;putlog _infile_;
         run;
 
   If you are building web apps with SAS then you are strongly encouraged to use
@@ -31,22 +32,24 @@
   [sasjs adapter](https://github.com/sasjs/adapter).
   For more information see https://sasjs.io
 
-  @param action Valid values:
+  @param [in] action Valid values:
     @li OPEN - opens the JSON
     @li OBJ - sends a table with each row as an object
     @li ARR - sends a table with each row in an array
     @li CLOSE - closes the JSON
-
-  @param ds the dataset to send.  Must be a work table.
-  @param jref= the fileref to which to send the JSON
-  @param dslabel= the name to give the table in the exported JSON
-  @param fmt= Whether to keep or strip formats from the table
-  @param engine= Which engine to use to send the JSON, valid options are:
+  @param [in] ds The dataset to send.  Must be a work table.
+  @param [out] jref= (_webout) The fileref to which to send the JSON
+  @param [out] dslabel= The name to give the table in the exported JSON
+  @param [in] fmt= (Y) Whether to keep (Y) or strip (N) formats from the table
+  @param [in] engine= (DATASTEP) Which engine to use to send the JSON. Options:
     @li PROCJSON (default)
     @li DATASTEP (more reliable when data has non standard characters)
-
-  @param dbg= DEPRECATED - was used to conditionally add PRETTY to
-    proc json but this can cause line truncation in large files.
+  @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
+    (eg `null`) or as STRING values (eg `".a"` or `".b"`)
+  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+    such as the column formats and types.  The metadata is contained inside an
+    object with the same name as the table but prefixed with a dollar sign - ie,
+    `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
 
   <h4> Related Macros <h4>
   @li mp_ds2fmtds.sas
@@ -57,9 +60,15 @@
 
 **/
 
-%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0
+%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y
+  ,engine=DATASTEP
+  ,dbg=0 /* DEPRECATED */
+  ,missing=NULL
+  ,showmeta=NO
 )/*/STORE SOURCE*/;
-%put output location=&jref;
+%local tempds colinfo fmtds i numcols;
+%let numcols=0;
+
 %if &action=OPEN %then %do;
   options nobomfile;
   data _null_;file &jref encoding='utf-8' ;
@@ -68,118 +77,124 @@
 %end;
 %else %if (&action=ARR or &action=OBJ) %then %do;
   options validvarname=upcase;
-  data _null_;file &jref mod encoding='utf-8' ;
+  data _null_; file &jref encoding='utf-8' mod;
     put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":";
 
+  /* grab col defs */
+  proc contents noprint data=&ds
+    out=_data_(keep=name type length format formatl formatd varnum label);
+  run;
+  %let colinfo=%scan(&syslast,2,.);
+  proc sort data=&colinfo;
+    by varnum;
+  run;
+  /* move meta to mac vars */
+  data _null_;
+    if _n_=1 then call symputx('numcols',nobs,'l');
+    set &colinfo end=last nobs=nobs;
+    name=upcase(name);
+    /* fix formats */
+    if type=2 or type=6 then do;
+      typelong='char';
+      length fmt $49.;
+      if format='' then fmt=cats('$',length,'.');
+      else if formatl=0 then fmt=cats(format,'.');
+      else fmt=cats(format,formatl,'.');
+      newlen=max(formatl,length);
+    end;
+    else do;
+      typelong='num';
+      if format='' then fmt='best.';
+      else if formatl=0 then fmt=cats(format,'.');
+      else if formatd=0 then fmt=cats(format,formatl,'.');
+      else fmt=cats(format,formatl,'.',formatd);
+      /* needs to be wide, for datetimes etc */
+      newlen=max(length,formatl,24);
+    end;
+    /* 32 char unique name */
+    newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
+
+    call symputx(cats('name',_n_),name,'l');
+    call symputx(cats('newname',_n_),newname,'l');
+    call symputx(cats('len',_n_),newlen,'l');
+    call symputx(cats('length',_n_),length,'l');
+    call symputx(cats('fmt',_n_),fmt,'l');
+    call symputx(cats('type',_n_),type,'l');
+    call symputx(cats('typelong',_n_),typelong,'l');
+    call symputx(cats('label',_n_),coalescec(label,name),'l');
+  run;
+
+  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+
   %if &engine=PROCJSON %then %do;
-    data;run;%let tempds=&syslast;
-    proc sql;drop table &tempds;
+    %if &missing=STRING %then %do;
+      %put &sysmacroname: Special Missings not supported in proc json.;
+      %put &sysmacroname: Switching to DATASTEP engine;
+      %goto datastep;
+    %end;
     data &tempds /view=&tempds;set &ds;
     %if &fmt=N %then format _numeric_ best32.;;
+    /* PRETTY is necessary to avoid line truncation in large files */
     proc json out=&jref pretty
         %if &action=ARR %then nokeys ;
         ;export &tempds / nosastags fmtnumeric;
     run;
-    proc sql;drop view &tempds;
   %end;
   %else %if &engine=DATASTEP %then %do;
-    %local cols i tempds;
-    %let cols=0;
-    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 %then %do;
+    %datastep:
+    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1
+    %then %do;
       %put &sysmacroname:  &ds NOT FOUND!!!;
       %return;
     %end;
-    %if &fmt=Y %then %do;
-      %put converting every variable to a formatted variable;
-      /* see mp_ds2fmtds.sas for source */
-      proc contents noprint data=&ds
-        out=_data_(keep=name type length format formatl formatd varnum);
-      run;
-      proc sort;
-        by varnum;
-      run;
-      %local fmtds;
-      %let fmtds=%scan(&syslast,2,.);
-      /* prepare formats and varnames */
-      data _null_;
-        if _n_=1 then call symputx('nobs',nobs,'l');
-        set &fmtds end=last nobs=nobs;
-        name=upcase(name);
-        /* fix formats */
-        if type=2 or type=6 then do;
-          length fmt $49.;
-          if format='' then fmt=cats('$',length,'.');
-          else if formatl=0 then fmt=cats(format,'.');
-          else fmt=cats(format,formatl,'.');
-          newlen=max(formatl,length);
-        end;
-        else do;
-          if format='' then fmt='best.';
-          else if formatl=0 then fmt=cats(format,'.');
-          else if formatd=0 then fmt=cats(format,formatl,'.');
-          else fmt=cats(format,formatl,'.',formatd);
-          /* needs to be wide, for datetimes etc */
-          newlen=max(length,formatl,24);
-        end;
-        /* 32 char unique name */
-        newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
 
-        call symputx(cats('name',_n_),name,'l');
-        call symputx(cats('newname',_n_),newname,'l');
-        call symputx(cats('len',_n_),newlen,'l');
-        call symputx(cats('fmt',_n_),fmt,'l');
-        call symputx(cats('type',_n_),type,'l');
-      run;
-      data &fmtds;
+    %if &fmt=Y %then %do;
+      data _data_;
         /* rename on entry */
         set &ds(rename=(
-      %local i;
-      %do i=1 %to &nobs;
+      %do i=1 %to &numcols;
         &&name&i=&&newname&i
       %end;
         ));
-      %do i=1 %to &nobs;
+      %do i=1 %to &numcols;
         length &&name&i $&&len&i;
         &&name&i=left(put(&&newname&i,&&fmt&i));
         drop &&newname&i;
       %end;
         if _error_ then call symputx('syscc',1012);
       run;
-      %let ds=&fmtds;
-    %end; /* &fmt=Y */
-    data _null_;file &jref mod encoding='utf-8' ;
-      put "["; call symputx('cols',0,'l');
-    proc sort
-      data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
-      out=_data_;
-      by varnum;
-
-    data _null_;
-      set _last_ end=last;
-      call symputx(cats('name',_n_),name,'l');
-      call symputx(cats('type',_n_),type,'l');
-      call symputx(cats('len',_n_),length,'l');
-      if last then call symputx('cols',_n_,'l');
-    run;
+      %let fmtds=&syslast;
+    %end;
 
     proc format; /* credit yabwon for special null removal */
-      value bart ._ - .z = null
+    value bart (default=40)
+    %if &missing=NULL %then %do;
+      ._ - .z = null
+    %end;
+    %else %do;
+      ._ = [quote()]
+      . = null
+      .a - .z = [quote()]
+    %end;
       other = [best.];
 
-    data;run; %let tempds=&syslast; /* temp table for spesh char management */
-    proc sql; drop table &tempds;
     data &tempds/view=&tempds;
       attrib _all_ label='';
-      %do i=1 %to &cols;
-        %if &&type&i=char %then %do;
+      %do i=1 %to &numcols;
+        %if &&typelong&i=char or &fmt=Y %then %do;
           length &&name&i $32767;
           format &&name&i $32767.;
         %end;
       %end;
-      set &ds;
+      %if &fmt=Y %then %do;
+        set &fmtds;
+      %end;
+      %else %do;
+        set &ds;
+      %end;
       format _numeric_ bart.;
-    %do i=1 %to &cols;
-      %if &&type&i=char %then %do;
+    %do i=1 %to &numcols;
+      %if &&typelong&i=char or &fmt=Y %then %do;
         &&name&i='"'!!trim(prxchange('s/"/\"/',-1,
                     prxchange('s/'!!'0A'x!!'/\n/',-1,
                     prxchange('s/'!!'0D'x!!'/\r/',-1,
@@ -189,44 +204,65 @@
       %end;
     %end;
     run;
+
     /* write to temp loc to avoid _webout truncation
       - https://support.sas.com/kb/49/325.html */
     filename _sjs temp lrecl=131068 encoding='utf-8';
     data _null_; file _sjs lrecl=131068 encoding='utf-8' mod ;
+      if _n_=1 then put "[";
       set &tempds;
       if _n_>1 then put "," @; put
       %if &action=ARR %then "[" ; %else "{" ;
-      %do i=1 %to &cols;
+      %do i=1 %to &numcols;
         %if &i>1 %then  "," ;
         %if &action=OBJ %then """&&name&i"":" ;
         &&name&i
       %end;
       %if &action=ARR %then "]" ; %else "}" ; ;
-    proc sql;
-    drop view &tempds;
     /* now write the long strings to _webout 1 byte at a time */
     data _null_;
       length filein 8 fileid 8;
-      filein = fopen("_sjs",'I',1,'B');
-      fileid = fopen("&jref",'A',1,'B');
-      rec = '20'x;
+      filein=fopen("_sjs",'I',1,'B');
+      fileid=fopen("&jref",'A',1,'B');
+      rec='20'x;
       do while(fread(filein)=0);
-        rc = fget(filein,rec,1);
-        rc = fput(fileid, rec);
-        rc =fwrite(fileid);
+        rc=fget(filein,rec,1);
+        rc=fput(fileid, rec);
+        rc=fwrite(fileid);
       end;
-      rc = fclose(filein);
-      rc = fclose(fileid);
+      /* close out the table */
+      rc=fput(fileid, "]");
+      rc=fwrite(fileid);
+      rc=fclose(filein);
+      rc=fclose(fileid);
     run;
     filename _sjs clear;
-    data _null_; file &jref mod encoding='utf-8' ;
-      put "]";
+  %end;
+
+  proc sql;
+  drop view &tempds;
+  drop table &colinfo;
+
+  %if &showmeta=YES %then %do;
+    data _null_; file &jref encoding='utf-8' mod;
+      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{";
+      do i=1 to &numcols;
+        name=quote(trim(symget(cats('name',i))));
+        format=quote(trim(symget(cats('fmt',i))));
+        label=quote(trim(symget(cats('label',i))));
+        length=quote(trim(symget(cats('length',i))));
+        type=quote(trim(symget(cats('typelong',i))));
+        if i>1 then put "," @@;
+        put name ':{"format":' format ',"label":' label
+          ',"length":' length ',"type":' type '}';
+      end;
+      put '}}';
     run;
   %end;
 %end;
 
 %else %if &action=CLOSE %then %do;
-  data _null_;file &jref encoding='utf-8' mod ;
+  data _null_; file &jref encoding='utf-8' mod ;
     put "}";
   run;
 %end;

base/mp_lockanytable.sas

@@ -15,7 +15,7 @@
     length is 200 characters.
   @param [out] ctl_ds= (0) The control table which controls the actual locking.
     Should already be assigned and available.  The definition is available by
-    running mp_coretable.sas as follows:  `mp_coretable(LOCKTABLE)`.
+    running mp_coretable.sas as follows:  `%mp_coretable(LOCKTABLE)`.
 
   @param [in] loops= (25) Number of times to check for a lock.
   @param [in] loop_secs= (1) Seconds to wait between each lock attempt

base/mp_testservice.sas

@@ -76,7 +76,7 @@
     else do;
       x+1;
       call symputx(name,quote(cats(value)),'l');
-      call symputx('pval'!!left(x),name,'l');
+      call symputx(cats('pval',x),name,'l');
       call symputx('pcnt',x,'l');
     end;
   run;

base/mp_validatecol.sas

@@ -48,9 +48,11 @@
 %let tempcol=%mf_getuniquename();
 
 %if &rule=ISINT %then %do;
-  &tempcol=input(&incol,?? best32.);
   &outcol=0;
-  if not missing(&tempcol) then if mod(&incol,1)=0 then &outcol=1;
+  if not missing(&incol) then do;
+    &tempcol=input(&incol,?? best32.);
+    if not missing(&tempcol) then if mod(&tempcol,1)=0 then &outcol=1;
+  end;
   drop &tempcol;
 %end;
 %else %if &rule=ISNUM %then %do;

fcmp/mcf_length.sas

@@ -0,0 +1,77 @@
+/**
+  @file
+  @brief Returns the length of a numeric value
+  @details
+  Returns the length, in bytes, of a numeric value.  If the value is
+  missing, then 0 is returned.
+
+  The function itself takes the following (positional) parameters:
+
+  | PARAMETER | DESCRIPTION |
+  |---|---|
+  | var | variable (or value) to be tested|
+
+  Usage:
+
+      %mcf_length(wrap=YES, insert_cmplib=YES)
+
+      data _null_;
+        ina=1;
+        inb=10000000;
+        inc=12345678;
+        ind=.;
+        outa=mcf_length(ina);
+        outb=mcf_length(inb);
+        outc=mcf_length(inc);
+        outd=mcf_length(ind);
+        put (out:)(=);
+      run;
+
+  Returns:
+
+  > outa=3 outb=4 outc=5 outd=0
+
+  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
+  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
+    CMPLIB reference.
+  @param [out] lib= (work) The output library in which to create the catalog.
+  @param [out] cat= (sasjs) The output catalog in which to create the package.
+  @param [out] pkg= (utils) The output package in which to create the function.
+    Uses a 3 part format:  libref.catalog.package
+
+  <h4> Related Macros </h4>
+  @li mcf_length.test.sas
+
+**/
+
+%macro mcf_length(wrap=NO
+  ,insert_cmplib=NO
+  ,lib=WORK
+  ,cat=SASJS
+  ,pkg=UTILS
+)/*/STORE SOURCE*/;
+
+%if &wrap=YES  %then %do;
+  proc fcmp outlib=&lib..&cat..&pkg;
+%end;
+
+function mcf_length(var);
+  if missing(var) then len=0;
+  else if trunc(var,3)=var then len=3;
+  else if trunc(var,4)=var then len=4;
+  else if trunc(var,5)=var then len=5;
+  else if trunc(var,6)=var then len=6;
+  else if trunc(var,7)=var then len=7;
+  else len=8;
+  return(len);
+endsub;
+
+%if &wrap=YES %then %do;
+  quit;
+%end;
+
+%if &insert_cmplib=YES %then %do;
+  options insert=(CMPLIB=(&lib..&cat));
+%end;
+
+%mend mcf_length;

fcmp/mcf_stpsrv_header.sas

@@ -69,7 +69,7 @@
 %if %mf_existfunction(stpsrv_header)=1 %then %return;
 
 %if &wrap=YES  %then %do;
-  proc fcmp outcat=&lib..&cat..&pkg;
+  proc fcmp outlib=&lib..&cat..&pkg;
 %end;
 
 function stpsrv_header(name $, value $);

fcmp/mcf_string2file.sas

@@ -49,7 +49,7 @@
 )/*/STORE SOURCE*/;
 
 %if &wrap=YES  %then %do;
-  proc fcmp outcat=&lib..&cat..&pkg;
+  proc fcmp outlib=&lib..&cat..&pkg;
 %end;
 
 function mcf_string2file(filepath $, string $, mode $);

meta/mm_createstp.sas

@@ -39,14 +39,6 @@
         ,Server=SASApp
         ,stptype=2)
 
-  <h4> SAS Macros </h4>
-  @li mf_nobs.sas
-  @li mf_verifymacvars.sas
-  @li mm_getdirectories.sas
-  @li mm_updatestpsourcecode.sas
-  @li mp_dropmembers.sas
-  @li mm_getservercontexts.sas
-
   @param stpname= Stored Process name.  Avoid spaces - testing has shown that
     the check to avoid creating multiple STPs in the same folder with the same
     name does not work when the name contains spaces.
@@ -77,6 +69,17 @@
     - fileuri
     - texturi
 
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mf_verifymacvars.sas
+  @li mm_getdirectories.sas
+  @li mm_updatestpsourcecode.sas
+  @li mp_dropmembers.sas
+  @li mm_getservercontexts.sas
+
+  <h4> Related Macros </h4>
+  @li mm_createwebservice.sas
+
   @version 9.2
   @author Allan Bowe
 

meta/mm_createwebservice.sas

@@ -12,6 +12,7 @@ Usage:
     %* parmcards lets us write to a text file from open code ;
     filename ft15f001 temp;
     parmcards4;
+        %webout(FETCH)
         %* do some sas, any inputs are now already WORK tables;
         data example1 example2;
           set sashelp.class;
@@ -24,11 +25,8 @@ Usage:
     ;;;;
     %mm_createwebservice(path=/Public/app/common,name=appInit,code=ft15f001)
 
-  <h4> SAS Macros </h4>
-  @li mm_createstp.sas
-  @li mf_getuser.sas
-  @li mm_createfolder.sas
-  @li mm_deletestp.sas
+  For more examples of using these web services with the SASjs Adapter, see:
+  https://github.com/sasjs/adapter#readme
 
   @param path= The full path (in SAS Metadata) where the service will be created
   @param name= Stored Process name.  Avoid spaces - testing has shown that
@@ -37,16 +35,22 @@ Usage:
   @param desc= The description of the service (optional)
   @param precode= Space separated list of filerefs, pointing to the code that
     needs to be attached to the beginning of the service (optional)
-  @param code=(ft15f001) Space seperated fileref(s) of the actual code to be
+  @param code= (ft15f001) Space seperated fileref(s) of the actual code to be
     added
-  @param server=(SASApp) The server which will run the STP.  Server name or uri
+  @param server= (SASApp) The server which will run the STP.  Server name or uri
     is fine.
-  @param mDebug=(0) set to 1 to show debug messages in the log
-  @param replace=(YES) select NO to avoid replacing an existing service in that
+  @param mDebug= (0) set to 1 to show debug messages in the log
+  @param replace= (YES) select NO to avoid replacing an existing service in that
     location
-  @param adapter=(sasjs) the macro uses the sasjs adapter by default.  To use
+  @param adapter= (sasjs) the macro uses the sasjs adapter by default.  To use
     another adapter, add a (different) fileref here.
 
+  <h4> SAS Macros </h4>
+  @li mm_createstp.sas
+  @li mf_getuser.sas
+  @li mm_createfolder.sas
+  @li mm_deletestp.sas
+
   @version 9.2
   @author Allan Bowe
 
@@ -89,9 +93,15 @@ data _null_;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
   put ' ';
-  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0 ';
+  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
+  put '  ,engine=DATASTEP ';
+  put '  ,dbg=0 /* DEPRECATED */ ';
+  put '  ,missing=NULL ';
+  put '  ,showmeta=NO ';
   put ')/*/STORE SOURCE*/; ';
-  put '%put output location=&jref; ';
+  put '%local tempds colinfo fmtds i numcols; ';
+  put '%let numcols=0; ';
+  put ' ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
   put '  data _null_;file &jref encoding=''utf-8'' ; ';
@@ -100,118 +110,124 @@ data _null_;
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
   put '  options validvarname=upcase; ';
-  put '  data _null_;file &jref mod encoding=''utf-8'' ; ';
+  put '  data _null_; file &jref encoding=''utf-8'' mod; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
   put ' ';
+  put '  /* grab col defs */ ';
+  put '  proc contents noprint data=&ds ';
+  put '    out=_data_(keep=name type length format formatl formatd varnum label); ';
+  put '  run; ';
+  put '  %let colinfo=%scan(&syslast,2,.); ';
+  put '  proc sort data=&colinfo; ';
+  put '    by varnum; ';
+  put '  run; ';
+  put '  /* move meta to mac vars */ ';
+  put '  data _null_; ';
+  put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
+  put '    set &colinfo end=last nobs=nobs; ';
+  put '    name=upcase(name); ';
+  put '    /* fix formats */ ';
+  put '    if type=2 or type=6 then do; ';
+  put '      typelong=''char''; ';
+  put '      length fmt $49.; ';
+  put '      if format='''' then fmt=cats(''$'',length,''.''); ';
+  put '      else if formatl=0 then fmt=cats(format,''.''); ';
+  put '      else fmt=cats(format,formatl,''.''); ';
+  put '      newlen=max(formatl,length); ';
+  put '    end; ';
+  put '    else do; ';
+  put '      typelong=''num''; ';
+  put '      if format='''' then fmt=''best.''; ';
+  put '      else if formatl=0 then fmt=cats(format,''.''); ';
+  put '      else if formatd=0 then fmt=cats(format,formatl,''.''); ';
+  put '      else fmt=cats(format,formatl,''.'',formatd); ';
+  put '      /* needs to be wide, for datetimes etc */ ';
+  put '      newlen=max(length,formatl,24); ';
+  put '    end; ';
+  put '    /* 32 char unique name */ ';
+  put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put ' ';
+  put '    call symputx(cats(''name'',_n_),name,''l''); ';
+  put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '    call symputx(cats(''len'',_n_),newlen,''l''); ';
+  put '    call symputx(cats(''length'',_n_),length,''l''); ';
+  put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
+  put '    call symputx(cats(''type'',_n_),type,''l''); ';
+  put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
+  put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
+  put '  run; ';
+  put ' ';
+  put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
-  put '    data;run;%let tempds=&syslast; ';
-  put '    proc sql;drop table &tempds; ';
+  put '    %if &missing=STRING %then %do; ';
+  put '      %put &sysmacroname: Special Missings not supported in proc json.; ';
+  put '      %put &sysmacroname: Switching to DATASTEP engine; ';
+  put '      %goto datastep; ';
+  put '    %end; ';
   put '    data &tempds /view=&tempds;set &ds; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
+  put '    /* PRETTY is necessary to avoid line truncation in large files */ ';
   put '    proc json out=&jref pretty ';
   put '        %if &action=ARR %then nokeys ; ';
   put '        ;export &tempds / nosastags fmtnumeric; ';
   put '    run; ';
-  put '    proc sql;drop view &tempds; ';
   put '  %end; ';
   put '  %else %if &engine=DATASTEP %then %do; ';
-  put '    %local cols i tempds; ';
-  put '    %let cols=0; ';
-  put '    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 %then %do; ';
+  put '    %datastep: ';
+  put '    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 ';
+  put '    %then %do; ';
   put '      %put &sysmacroname:  &ds NOT FOUND!!!; ';
   put '      %return; ';
   put '    %end; ';
-  put '    %if &fmt=Y %then %do; ';
-  put '      %put converting every variable to a formatted variable; ';
-  put '      /* see mp_ds2fmtds.sas for source */ ';
-  put '      proc contents noprint data=&ds ';
-  put '        out=_data_(keep=name type length format formatl formatd varnum); ';
-  put '      run; ';
-  put '      proc sort; ';
-  put '        by varnum; ';
-  put '      run; ';
-  put '      %local fmtds; ';
-  put '      %let fmtds=%scan(&syslast,2,.); ';
-  put '      /* prepare formats and varnames */ ';
-  put '      data _null_; ';
-  put '        if _n_=1 then call symputx(''nobs'',nobs,''l''); ';
-  put '        set &fmtds end=last nobs=nobs; ';
-  put '        name=upcase(name); ';
-  put '        /* fix formats */ ';
-  put '        if type=2 or type=6 then do; ';
-  put '          length fmt $49.; ';
-  put '          if format='''' then fmt=cats(''$'',length,''.''); ';
-  put '          else if formatl=0 then fmt=cats(format,''.''); ';
-  put '          else fmt=cats(format,formatl,''.''); ';
-  put '          newlen=max(formatl,length); ';
-  put '        end; ';
-  put '        else do; ';
-  put '          if format='''' then fmt=''best.''; ';
-  put '          else if formatl=0 then fmt=cats(format,''.''); ';
-  put '          else if formatd=0 then fmt=cats(format,formatl,''.''); ';
-  put '          else fmt=cats(format,formatl,''.'',formatd); ';
-  put '          /* needs to be wide, for datetimes etc */ ';
-  put '          newlen=max(length,formatl,24); ';
-  put '        end; ';
-  put '        /* 32 char unique name */ ';
-  put '        newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
   put ' ';
-  put '        call symputx(cats(''name'',_n_),name,''l''); ';
-  put '        call symputx(cats(''newname'',_n_),newname,''l''); ';
-  put '        call symputx(cats(''len'',_n_),newlen,''l''); ';
-  put '        call symputx(cats(''fmt'',_n_),fmt,''l''); ';
-  put '        call symputx(cats(''type'',_n_),type,''l''); ';
-  put '      run; ';
-  put '      data &fmtds; ';
+  put '    %if &fmt=Y %then %do; ';
+  put '      data _data_; ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
-  put '      %local i; ';
-  put '      %do i=1 %to &nobs; ';
+  put '      %do i=1 %to &numcols; ';
   put '        &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
-  put '      %do i=1 %to &nobs; ';
+  put '      %do i=1 %to &numcols; ';
   put '        length &&name&i $&&len&i; ';
   put '        &&name&i=left(put(&&newname&i,&&fmt&i)); ';
   put '        drop &&newname&i; ';
   put '      %end; ';
   put '        if _error_ then call symputx(''syscc'',1012); ';
   put '      run; ';
-  put '      %let ds=&fmtds; ';
-  put '    %end; /* &fmt=Y */ ';
-  put '    data _null_;file &jref mod encoding=''utf-8'' ; ';
-  put '      put "["; call symputx(''cols'',0,''l''); ';
-  put '    proc sort ';
-  put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
-  put '      out=_data_; ';
-  put '      by varnum; ';
-  put ' ';
-  put '    data _null_; ';
-  put '      set _last_ end=last; ';
-  put '      call symputx(cats(''name'',_n_),name,''l''); ';
-  put '      call symputx(cats(''type'',_n_),type,''l''); ';
-  put '      call symputx(cats(''len'',_n_),length,''l''); ';
-  put '      if last then call symputx(''cols'',_n_,''l''); ';
-  put '    run; ';
+  put '      %let fmtds=&syslast; ';
+  put '    %end; ';
   put ' ';
   put '    proc format; /* credit yabwon for special null removal */ ';
-  put '      value bart ._ - .z = null ';
+  put '    value bart (default=40) ';
+  put '    %if &missing=NULL %then %do; ';
+  put '      ._ - .z = null ';
+  put '    %end; ';
+  put '    %else %do; ';
+  put '      ._ = [quote()] ';
+  put '      . = null ';
+  put '      .a - .z = [quote()] ';
+  put '    %end; ';
   put '      other = [best.]; ';
   put ' ';
-  put '    data;run; %let tempds=&syslast; /* temp table for spesh char management */ ';
-  put '    proc sql; drop table &tempds; ';
   put '    data &tempds/view=&tempds; ';
   put '      attrib _all_ label=''''; ';
-  put '      %do i=1 %to &cols; ';
-  put '        %if &&type&i=char %then %do; ';
+  put '      %do i=1 %to &numcols; ';
+  put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
   put '          length &&name&i $32767; ';
   put '          format &&name&i $32767.; ';
   put '        %end; ';
   put '      %end; ';
-  put '      set &ds; ';
+  put '      %if &fmt=Y %then %do; ';
+  put '        set &fmtds; ';
+  put '      %end; ';
+  put '      %else %do; ';
+  put '        set &ds; ';
+  put '      %end; ';
   put '      format _numeric_ bart.; ';
-  put '    %do i=1 %to &cols; ';
-  put '      %if &&type&i=char %then %do; ';
+  put '    %do i=1 %to &numcols; ';
+  put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
   put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
   put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
   put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
@@ -221,49 +237,72 @@ data _null_;
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
+  put ' ';
   put '    /* write to temp loc to avoid _webout truncation ';
   put '      - https://support.sas.com/kb/49/325.html */ ';
   put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
   put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
+  put '      if _n_=1 then put "["; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
   put '      %if &action=ARR %then "[" ; %else "{" ; ';
-  put '      %do i=1 %to &cols; ';
+  put '      %do i=1 %to &numcols; ';
   put '        %if &i>1 %then  "," ; ';
   put '        %if &action=OBJ %then """&&name&i"":" ; ';
   put '        &&name&i ';
   put '      %end; ';
   put '      %if &action=ARR %then "]" ; %else "}" ; ; ';
-  put '    proc sql; ';
-  put '    drop view &tempds; ';
   put '    /* now write the long strings to _webout 1 byte at a time */ ';
   put '    data _null_; ';
   put '      length filein 8 fileid 8; ';
-  put '      filein = fopen("_sjs",''I'',1,''B''); ';
-  put '      fileid = fopen("&jref",''A'',1,''B''); ';
-  put '      rec = ''20''x; ';
+  put '      filein=fopen("_sjs",''I'',1,''B''); ';
+  put '      fileid=fopen("&jref",''A'',1,''B''); ';
+  put '      rec=''20''x; ';
   put '      do while(fread(filein)=0); ';
-  put '        rc = fget(filein,rec,1); ';
-  put '        rc = fput(fileid, rec); ';
-  put '        rc =fwrite(fileid); ';
+  put '        rc=fget(filein,rec,1); ';
+  put '        rc=fput(fileid, rec); ';
+  put '        rc=fwrite(fileid); ';
   put '      end; ';
-  put '      rc = fclose(filein); ';
-  put '      rc = fclose(fileid); ';
+  put '      /* close out the table */ ';
+  put '      rc=fput(fileid, "]"); ';
+  put '      rc=fwrite(fileid); ';
+  put '      rc=fclose(filein); ';
+  put '      rc=fclose(fileid); ';
   put '    run; ';
   put '    filename _sjs clear; ';
-  put '    data _null_; file &jref mod encoding=''utf-8'' ; ';
-  put '      put "]"; ';
+  put '  %end; ';
+  put ' ';
+  put '  proc sql; ';
+  put '  drop view &tempds; ';
+  put '  drop table &colinfo; ';
+  put ' ';
+  put '  %if &showmeta=YES %then %do; ';
+  put '    data _null_; file &jref encoding=''utf-8'' mod; ';
+  put '      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; ';
+  put '      do i=1 to &numcols; ';
+  put '        name=quote(trim(symget(cats(''name'',i)))); ';
+  put '        format=quote(trim(symget(cats(''fmt'',i)))); ';
+  put '        label=quote(trim(symget(cats(''label'',i)))); ';
+  put '        length=quote(trim(symget(cats(''length'',i)))); ';
+  put '        type=quote(trim(symget(cats(''typelong'',i)))); ';
+  put '        if i>1 then put "," @@; ';
+  put '        put name '':{"format":'' format '',"label":'' label ';
+  put '          '',"length":'' length '',"type":'' type ''}''; ';
+  put '      end; ';
+  put '      put ''}}''; ';
   put '    run; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8'' mod ; ';
+  put '  data _null_; file &jref encoding=''utf-8'' mod ; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
   put '%mend mp_jsonout; ';
-  put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y); ';
+  put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
+  put '  ,showmeta=NO ';
+  put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
   put '%local i tempds jsonengine; ';
@@ -321,14 +360,15 @@ data _null_;
   put '  %if %str(&_debug) ge 131 %then %do; ';
   put '    put ''>>weboutBEGIN<<''; ';
   put '  %end; ';
-  put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
+  put '    put ''{"SYSDATE" : "'' "&SYSDATE" ''"''; ';
+  put '    put '',"SYSTIME" : "'' "&SYSTIME" ''"''; ';
   put '  run; ';
   put ' ';
   put '%end; ';
   put ' ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '    ,engine=&jsonengine,dbg=%str(&_debug) ';
+  put '    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta ';
   put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
@@ -343,15 +383,12 @@ data _null_;
   put '      set &tempds; ';
   put '      if not (upcase(name) =:"DATA"); /* ignore temp datasets */ ';
   put '      i+1; ';
-  put '      call symputx(''wt''!!left(i),name,''l''); ';
+  put '      call symputx(cats(''wt'',i),name,''l''); ';
   put '      call symputx(''wtcnt'',i,''l''); ';
   put '    data _null_; file &fref mod encoding=''utf-8''; ';
   put '      put ",""WORK"":{"; ';
   put '    %do i=1 %to &wtcnt; ';
   put '      %let wt=&&wt&i; ';
-  put '      proc contents noprint data=&wt ';
-  put '        out=_data_ (keep=name type length format:); ';
-  put '      run;%let tempds=%scan(&syslast,2,.); ';
   put '      data _null_; file &fref mod encoding=''utf-8''; ';
   put '        dsid=open("WORK.&wt",''is''); ';
   put '        nlobs=attrn(dsid,''NLOBS''); ';
@@ -361,8 +398,7 @@ data _null_;
   put '        put " ""&wt"" : {"; ';
   put '        put ''"nlobs":'' nlobs; ';
   put '        put '',"nvars":'' nvars; ';
-  put '      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=&jsonengine) ';
-  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=&jsonengine) ';
+  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) ';
   put '      data _null_; file &fref mod encoding=''utf-8''; ';
   put '        put "}"; ';
   put '    %end; ';
@@ -391,6 +427,10 @@ data _null_;
   put '    put '',"SYSVLONG" : '' sysvlong; ';
   put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''" ''; ';
+  put '    length memsize $32; ';
+  put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
+  put '    memsize=quote(cats(memsize)); ';
+  put '    put '',"MEMSIZE" : '' memsize; ';
   put '    put "}" @; ';
   put '  %if %str(&_debug) ge 131 %then %do; ';
   put '    put ''>>weboutEND<<''; ';
@@ -418,8 +458,10 @@ data _null_;
   put ' ';
   put '%mend mf_getuser; ';
 /* WEBOUT END */
-  put '%macro webout(action,ds,dslabel=,fmt=);';
-  put '  %mm_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt)';
+  put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO);';
+  put '  %mm_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt,missing=&missing';
+  put '    ,showmeta=&showmeta';
+  put '  )';
   put '%mend;';
 run;
 

meta/mm_getservercontexts.sas

@@ -35,7 +35,7 @@ data _null_;
   set repos;
   where repositorytype in('CUSTOM','FOUNDATION');
   keep id name ;
-  call symputx('repo'!!left(_n_),name,'l');
+  call symputx(cats('repo',_n_),name,'l');
   call symputx('repocnt',_n_,'l');
 run;
 

meta/mm_webout.sas

@@ -23,17 +23,25 @@
         %mm_webout(CLOSE)
 
 
-  @param action Either FETCH, OPEN, ARR, OBJ or CLOSE
-  @param ds The dataset to send back to the frontend
-  @param dslabel= value to use instead of the real name for sending to JSON
-  @param fmt=(Y) Set to N to send back unformatted values
-  @param fref=(_webout) The fileref to which to write the JSON
+  @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE
+  @param [in] ds The dataset to send back to the frontend
+  @param [out] dslabel= Value to use instead of table name for sending to JSON
+  @param [in] fmt=(Y) Set to N to send back unformatted values
+  @param [out] fref= (_webout) The fileref to which to write the JSON
+  @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
+    (eg `null`) or as STRING values (eg `".a"` or `".b"`)
+  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+    such as the column formats and types.  The metadata is contained inside an
+    object with the same name as the table but prefixed with a dollar sign - ie,
+    `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
 
   @version 9.3
   @author Allan Bowe
 
 **/
-%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y);
+%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
+  ,showmeta=NO
+);
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
 %local i tempds jsonengine;
@@ -91,14 +99,15 @@
   %if %str(&_debug) ge 131 %then %do;
     put '>>weboutBEGIN<<';
   %end;
-    put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
+    put '{"SYSDATE" : "' "&SYSDATE" '"';
+    put ',"SYSTIME" : "' "&SYSTIME" '"';
   run;
 
 %end;
 
 %else %if &action=ARR or &action=OBJ %then %do;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=&jsonengine,dbg=%str(&_debug)
+    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta
   )
 %end;
 %else %if &action=CLOSE %then %do;
@@ -113,15 +122,12 @@
       set &tempds;
       if not (upcase(name) =:"DATA"); /* ignore temp datasets */
       i+1;
-      call symputx('wt'!!left(i),name,'l');
+      call symputx(cats('wt',i),name,'l');
       call symputx('wtcnt',i,'l');
     data _null_; file &fref mod encoding='utf-8';
       put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
-      proc contents noprint data=&wt
-        out=_data_ (keep=name type length format:);
-      run;%let tempds=%scan(&syslast,2,.);
       data _null_; file &fref mod encoding='utf-8';
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
@@ -131,8 +137,7 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=&jsonengine)
-      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=&jsonengine)
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES)
       data _null_; file &fref mod encoding='utf-8';
         put "}";
     %end;
@@ -161,6 +166,10 @@
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
     put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+    length memsize $32;
+    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
+    memsize=quote(cats(memsize));
+    put ',"MEMSIZE" : ' memsize;
     put "}" @;
   %if %str(&_debug) ge 131 %then %do;
     put '>>weboutEND<<';

package-lock.json

@@ -10,73 +10,25 @@
         "ts-loader": "^9.2.6"
       },
       "devDependencies": {
-        "@sasjs/cli": "^2.39.0"
+        "@sasjs/cli": "3.6.0"
       }
     },
     "node_modules/@sasjs/adapter": {
-      "version": "2.12.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.12.0.tgz",
-      "integrity": "sha512-zzIuohhR8KUDl3DfIFOW38gv3LADPnOBCLOvLoKu4hH5R/UJDkjZ/Gdgc8B35vI7aOprYOLK/T5D/Z44OaTkqw==",
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-3.4.1.tgz",
+      "integrity": "sha512-FbsvYDaoJAuH8FMidXhX3Kh4Eb8qIcxy5iiCxHgcSRWM89W29W21WfCqCw0F28yFr+V5vZkvphwXMKFdOGGxlw==",
       "dev": true,
       "hasInstallScript": true,
       "dependencies": {
-        "@sasjs/utils": "^2.32.0",
-        "axios": "^0.21.4",
-        "axios-cookiejar-support": "^1.0.1",
-        "form-data": "^4.0.0",
-        "https": "^1.0.0",
-        "tough-cookie": "^4.0.0"
-      }
-    },
-    "node_modules/@sasjs/cli": {
-      "version": "2.39.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.39.0.tgz",
-      "integrity": "sha512-n2LcU4n0QCEbUpXqZnBz/Ey5Td0nMJmgJpZRymMGfYEM0Y0x/CeXemd+kXHPjUvgQ+FX+SQzcvUQTEY/YlT4hA==",
-      "dev": true,
-      "hasInstallScript": true,
-      "dependencies": {
-        "@sasjs/adapter": "2.12.0",
-        "@sasjs/core": "2.45.2",
-        "@sasjs/lint": "1.11.2",
         "@sasjs/utils": "2.32.0",
-        "chalk": "4.1.2",
-        "csv-stringify": "5.6.5",
-        "dotenv": "10.0.0",
-        "esm": "3.2.25",
-        "find": "0.3.0",
-        "get-installed-path": "4.0.8",
-        "js-base64": "3.7.2",
-        "jsdom": "17.0.0",
-        "jwt-decode": "3.1.2",
-        "lodash.groupby": "4.6.0",
-        "lodash.uniqby": "4.7.0",
-        "node-graphviz": "0.1.0",
-        "ora": "5.4.1",
-        "rimraf": "3.0.2",
-        "shelljs": "0.8.4",
-        "xml": "1.0.1",
-        "yargs": "17.2.1"
-      },
-      "bin": {
-        "sasjs": "build/index.js"
+        "axios": "0.25.0",
+        "axios-cookiejar-support": "1.0.1",
+        "form-data": "4.0.0",
+        "https": "1.0.0",
+        "tough-cookie": "4.0.0"
       }
     },
-    "node_modules/@sasjs/core": {
-      "version": "2.45.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.45.2.tgz",
-      "integrity": "sha512-tg+oZCD8GFMXsg+vDL66LMnyU+t151Hrqd7yl+pMXH2qwkA14N/j6QdkTBZOchskqOA/3PnpOlAZN/xxMW2gdg==",
-      "dev": true
-    },
-    "node_modules/@sasjs/lint": {
-      "version": "1.11.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.2.tgz",
-      "integrity": "sha512-zEonhvha9kwrD+hxhG0hEhtfqpXwffH4vRDIr6eDiXkC7S8M3yImpjyFBvX/THJO5+8iuY8TYkOXKl7+nK/wAg==",
-      "dev": true,
-      "dependencies": {
-        "@sasjs/utils": "^2.19.0"
-      }
-    },
-    "node_modules/@sasjs/utils": {
+    "node_modules/@sasjs/adapter/node_modules/@sasjs/utils": {
       "version": "2.32.0",
       "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.32.0.tgz",
       "integrity": "sha512-xnvdEuI4PhTtulcdDEIMK7IxVj9bOMU1JTnxRuSEKWcsclY9P9Fw3cnMOOEgXCDffrOPn3f54DP7Wb1GXd+f8g==",
@@ -96,6 +48,101 @@
         "valid-url": "^1.0.9"
       }
     },
+    "node_modules/@sasjs/cli": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-3.6.0.tgz",
+      "integrity": "sha512-px5aFXNoTlML7g4TvP92q/uqWnHGbxnwEzy+4D+sJbXgpzLlCyyuUU7rbd/9sXT+/Pi7vjfHfBCv0JhoGYr6gw==",
+      "dev": true,
+      "hasInstallScript": true,
+      "dependencies": {
+        "@sasjs/adapter": "3.4.1",
+        "@sasjs/core": "3.10.0",
+        "@sasjs/lint": "1.11.2",
+        "@sasjs/utils": "2.35.0",
+        "chalk": "4.1.2",
+        "csv-stringify": "5.6.5",
+        "dotenv": "10.0.0",
+        "esm": "3.2.25",
+        "find": "0.3.0",
+        "get-installed-path": "4.0.8",
+        "js-base64": "3.7.2",
+        "jsdom": "17.0.0",
+        "jwt-decode": "3.1.2",
+        "lodash.groupby": "4.6.0",
+        "lodash.uniqby": "4.7.0",
+        "node-graphviz": "0.1.0",
+        "ora": "5.4.1",
+        "rimraf": "3.0.2",
+        "shelljs": "0.8.5",
+        "xml": "1.0.1",
+        "yargs": "17.2.1"
+      },
+      "bin": {
+        "sasjs": "build/index.js"
+      }
+    },
+    "node_modules/@sasjs/cli/node_modules/@sasjs/utils": {
+      "version": "2.35.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.35.0.tgz",
+      "integrity": "sha512-q9ZKV+TXqwiaj+0z5U7/00eBpp2QpjKfC9BKx7A6rQjBl10WtoWd5C9Em+RQULWVEdRbVS2XcnNsWelbKq/Zsw==",
+      "dev": true,
+      "hasInstallScript": true,
+      "dependencies": {
+        "@types/fs-extra": "^9.0.13",
+        "@types/prompts": "^2.0.13",
+        "chalk": "^4.1.1",
+        "cli-table": "^0.3.6",
+        "consola": "^2.15.0",
+        "csv-stringify": "^5.6.5",
+        "find": "0.3.0",
+        "fs-extra": "^10.0.0",
+        "jwt-decode": "^3.1.2",
+        "prompts": "^2.4.1",
+        "rimraf": "^3.0.2",
+        "valid-url": "^1.0.9"
+      }
+    },
+    "node_modules/@sasjs/core": {
+      "version": "3.10.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-3.10.0.tgz",
+      "integrity": "sha512-lgLxDYpIvwSrXFaUaTFCR0KXHQEc5QIOL4DU87TvBHEUUAWNQHzuVQWkavLtW5hbvLGnPXnyvspzoSzmBojXzg==",
+      "dev": true,
+      "dependencies": {
+        "ts-loader": "^9.2.6"
+      }
+    },
+    "node_modules/@sasjs/lint": {
+      "version": "1.11.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.2.tgz",
+      "integrity": "sha512-zEonhvha9kwrD+hxhG0hEhtfqpXwffH4vRDIr6eDiXkC7S8M3yImpjyFBvX/THJO5+8iuY8TYkOXKl7+nK/wAg==",
+      "dev": true,
+      "dependencies": {
+        "@sasjs/utils": "^2.19.0"
+      }
+    },
+    "node_modules/@sasjs/utils": {
+      "version": "2.34.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.34.1.tgz",
+      "integrity": "sha512-hd1qieH3d7+xH96n5DpRGTEazeAhYyBBKCdnKhOXMgF2TZVoHFdRs5REfT88CKza6DHBGRVGnIVm5ORGP4cVLg==",
+      "dev": true,
+      "hasInstallScript": true,
+      "dependencies": {
+        "@types/fs-extra": "^9.0.11",
+        "@types/prompts": "^2.0.13",
+        "chalk": "^4.1.1",
+        "cli-table": "^0.3.6",
+        "consola": "^2.15.0",
+        "csv-stringify": "^5.6.5",
+        "find": "0.3.0",
+        "fs-extra": "^10.0.0",
+        "jwt-decode": "^3.1.2",
+        "lodash.groupby": "4.6.0",
+        "lodash.uniqby": "4.7.0",
+        "prompts": "^2.4.1",
+        "rimraf": "^3.0.2",
+        "valid-url": "^1.0.9"
+      }
+    },
     "node_modules/@tootallnate/once": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz",
@@ -132,9 +179,9 @@
       "peer": true
     },
     "node_modules/@types/fs-extra": {
-      "version": "9.0.12",
-      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.12.tgz",
-      "integrity": "sha512-I+bsBr67CurCGnSenZZ7v94gd3tc3+Aj2taxMT4yu4ABLuOgOjeFxX3dokG24ztSRg5tnT00sL8BszO7gSMoIw==",
+      "version": "9.0.13",
+      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.13.tgz",
+      "integrity": "sha512-nEnwB++1u5lVDM2UI4c1+5R+FYaKfaAzS4OococimjVm3nQw3TuzH5UNsocrcTBbhnerblyHj4A49qXbIiZdpA==",
       "dev": true,
       "dependencies": {
         "@types/node": "*"
@@ -449,12 +496,12 @@
       "dev": true
     },
     "node_modules/axios": {
-      "version": "0.21.4",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
-      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
+      "version": "0.25.0",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.25.0.tgz",
+      "integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==",
       "dev": true,
       "dependencies": {
-        "follow-redirects": "^1.14.0"
+        "follow-redirects": "^1.14.7"
       }
     },
     "node_modules/axios-cookiejar-support": {
@@ -1051,9 +1098,9 @@
       }
     },
     "node_modules/follow-redirects": {
-      "version": "1.14.5",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.5.tgz",
-      "integrity": "sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA==",
+      "version": "1.14.7",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz",
+      "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==",
       "dev": true,
       "funding": [
         {
@@ -1335,9 +1382,9 @@
       }
     },
     "node_modules/is-core-module": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.4.0.tgz",
-      "integrity": "sha512-6A2fkfq1rfeQZjxrZJGerpLCTHRNEBiSgnu0+obeJpEPZRUooHgsizvzv0ZjJwOz3iWIHdJtVWJ/tmPr3D21/A==",
+      "version": "2.8.1",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.1.tgz",
+      "integrity": "sha512-SdNCUs284hr40hFTFP6l0IfZ/RSrMXF3qgoRHd3/79unUTvrFO/JoXwkGm+5J/Oe3E/b5GsnG330uUNgRpu1PA==",
       "dev": true,
       "dependencies": {
         "has": "^1.0.3"
@@ -1887,13 +1934,17 @@
       }
     },
     "node_modules/resolve": {
-      "version": "1.20.0",
-      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz",
-      "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==",
+      "version": "1.21.0",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.21.0.tgz",
+      "integrity": "sha512-3wCbTpk5WJlyE4mSOtDLhqQmGFi0/TD9VPwmiolnk8U0wRgMEktqCXd3vy5buTO3tljvalNvKrjHEfrd2WpEKA==",
       "dev": true,
       "dependencies": {
-        "is-core-module": "^2.2.0",
-        "path-parse": "^1.0.6"
+        "is-core-module": "^2.8.0",
+        "path-parse": "^1.0.7",
+        "supports-preserve-symlinks-flag": "^1.0.0"
+      },
+      "bin": {
+        "resolve": "bin/resolve"
       },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
@@ -2019,9 +2070,9 @@
       }
     },
     "node_modules/shelljs": {
-      "version": "0.8.4",
-      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.4.tgz",
-      "integrity": "sha512-7gk3UZ9kOfPLIAbslLzyWeGiEqx9e3rxwZM0KE6EL8GlGwjym9Mrlx5/p33bWTu9YG6vcS4MBxYZDHYr5lr8BQ==",
+      "version": "0.8.5",
+      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz",
+      "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==",
       "dev": true,
       "dependencies": {
         "glob": "^7.0.0",
@@ -2111,6 +2162,18 @@
         "node": ">=8"
       }
     },
+    "node_modules/supports-preserve-symlinks-flag": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz",
+      "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/symbol-tree": {
       "version": "3.2.4",
       "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
@@ -2581,29 +2644,50 @@
   },
   "dependencies": {
     "@sasjs/adapter": {
-      "version": "2.12.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.12.0.tgz",
-      "integrity": "sha512-zzIuohhR8KUDl3DfIFOW38gv3LADPnOBCLOvLoKu4hH5R/UJDkjZ/Gdgc8B35vI7aOprYOLK/T5D/Z44OaTkqw==",
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-3.4.1.tgz",
+      "integrity": "sha512-FbsvYDaoJAuH8FMidXhX3Kh4Eb8qIcxy5iiCxHgcSRWM89W29W21WfCqCw0F28yFr+V5vZkvphwXMKFdOGGxlw==",
       "dev": true,
       "requires": {
-        "@sasjs/utils": "^2.32.0",
-        "axios": "^0.21.4",
-        "axios-cookiejar-support": "^1.0.1",
-        "form-data": "^4.0.0",
-        "https": "^1.0.0",
-        "tough-cookie": "^4.0.0"
+        "@sasjs/utils": "2.32.0",
+        "axios": "0.25.0",
+        "axios-cookiejar-support": "1.0.1",
+        "form-data": "4.0.0",
+        "https": "1.0.0",
+        "tough-cookie": "4.0.0"
+      },
+      "dependencies": {
+        "@sasjs/utils": {
+          "version": "2.32.0",
+          "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.32.0.tgz",
+          "integrity": "sha512-xnvdEuI4PhTtulcdDEIMK7IxVj9bOMU1JTnxRuSEKWcsclY9P9Fw3cnMOOEgXCDffrOPn3f54DP7Wb1GXd+f8g==",
+          "dev": true,
+          "requires": {
+            "@types/fs-extra": "^9.0.11",
+            "@types/prompts": "^2.0.13",
+            "chalk": "^4.1.1",
+            "cli-table": "^0.3.6",
+            "consola": "^2.15.0",
+            "csv-stringify": "^5.6.5",
+            "fs-extra": "^10.0.0",
+            "jwt-decode": "^3.1.2",
+            "prompts": "^2.4.1",
+            "rimraf": "^3.0.2",
+            "valid-url": "^1.0.9"
+          }
+        }
       }
     },
     "@sasjs/cli": {
-      "version": "2.39.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.39.0.tgz",
-      "integrity": "sha512-n2LcU4n0QCEbUpXqZnBz/Ey5Td0nMJmgJpZRymMGfYEM0Y0x/CeXemd+kXHPjUvgQ+FX+SQzcvUQTEY/YlT4hA==",
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-3.6.0.tgz",
+      "integrity": "sha512-px5aFXNoTlML7g4TvP92q/uqWnHGbxnwEzy+4D+sJbXgpzLlCyyuUU7rbd/9sXT+/Pi7vjfHfBCv0JhoGYr6gw==",
       "dev": true,
       "requires": {
-        "@sasjs/adapter": "2.12.0",
-        "@sasjs/core": "2.45.2",
+        "@sasjs/adapter": "3.4.1",
+        "@sasjs/core": "3.10.0",
         "@sasjs/lint": "1.11.2",
-        "@sasjs/utils": "2.32.0",
+        "@sasjs/utils": "2.35.0",
         "chalk": "4.1.2",
         "csv-stringify": "5.6.5",
         "dotenv": "10.0.0",
@@ -2618,16 +2702,41 @@
         "node-graphviz": "0.1.0",
         "ora": "5.4.1",
         "rimraf": "3.0.2",
-        "shelljs": "0.8.4",
+        "shelljs": "0.8.5",
         "xml": "1.0.1",
         "yargs": "17.2.1"
+      },
+      "dependencies": {
+        "@sasjs/utils": {
+          "version": "2.35.0",
+          "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.35.0.tgz",
+          "integrity": "sha512-q9ZKV+TXqwiaj+0z5U7/00eBpp2QpjKfC9BKx7A6rQjBl10WtoWd5C9Em+RQULWVEdRbVS2XcnNsWelbKq/Zsw==",
+          "dev": true,
+          "requires": {
+            "@types/fs-extra": "^9.0.13",
+            "@types/prompts": "^2.0.13",
+            "chalk": "^4.1.1",
+            "cli-table": "^0.3.6",
+            "consola": "^2.15.0",
+            "csv-stringify": "^5.6.5",
+            "find": "0.3.0",
+            "fs-extra": "^10.0.0",
+            "jwt-decode": "^3.1.2",
+            "prompts": "^2.4.1",
+            "rimraf": "^3.0.2",
+            "valid-url": "^1.0.9"
+          }
+        }
       }
     },
     "@sasjs/core": {
-      "version": "2.45.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.45.2.tgz",
-      "integrity": "sha512-tg+oZCD8GFMXsg+vDL66LMnyU+t151Hrqd7yl+pMXH2qwkA14N/j6QdkTBZOchskqOA/3PnpOlAZN/xxMW2gdg==",
-      "dev": true
+      "version": "3.10.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-3.10.0.tgz",
+      "integrity": "sha512-lgLxDYpIvwSrXFaUaTFCR0KXHQEc5QIOL4DU87TvBHEUUAWNQHzuVQWkavLtW5hbvLGnPXnyvspzoSzmBojXzg==",
+      "dev": true,
+      "requires": {
+        "ts-loader": "^9.2.6"
+      }
     },
     "@sasjs/lint": {
       "version": "1.11.2",
@@ -2639,9 +2748,9 @@
       }
     },
     "@sasjs/utils": {
-      "version": "2.32.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.32.0.tgz",
-      "integrity": "sha512-xnvdEuI4PhTtulcdDEIMK7IxVj9bOMU1JTnxRuSEKWcsclY9P9Fw3cnMOOEgXCDffrOPn3f54DP7Wb1GXd+f8g==",
+      "version": "2.34.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.34.1.tgz",
+      "integrity": "sha512-hd1qieH3d7+xH96n5DpRGTEazeAhYyBBKCdnKhOXMgF2TZVoHFdRs5REfT88CKza6DHBGRVGnIVm5ORGP4cVLg==",
       "dev": true,
       "requires": {
         "@types/fs-extra": "^9.0.11",
@@ -2650,8 +2759,11 @@
         "cli-table": "^0.3.6",
         "consola": "^2.15.0",
         "csv-stringify": "^5.6.5",
+        "find": "0.3.0",
         "fs-extra": "^10.0.0",
         "jwt-decode": "^3.1.2",
+        "lodash.groupby": "4.6.0",
+        "lodash.uniqby": "4.7.0",
         "prompts": "^2.4.1",
         "rimraf": "^3.0.2",
         "valid-url": "^1.0.9"
@@ -2690,9 +2802,9 @@
       "peer": true
     },
     "@types/fs-extra": {
-      "version": "9.0.12",
-      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.12.tgz",
-      "integrity": "sha512-I+bsBr67CurCGnSenZZ7v94gd3tc3+Aj2taxMT4yu4ABLuOgOjeFxX3dokG24ztSRg5tnT00sL8BszO7gSMoIw==",
+      "version": "9.0.13",
+      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.13.tgz",
+      "integrity": "sha512-nEnwB++1u5lVDM2UI4c1+5R+FYaKfaAzS4OococimjVm3nQw3TuzH5UNsocrcTBbhnerblyHj4A49qXbIiZdpA==",
       "dev": true,
       "requires": {
         "@types/node": "*"
@@ -2974,12 +3086,12 @@
       "dev": true
     },
     "axios": {
-      "version": "0.21.4",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
-      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
+      "version": "0.25.0",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.25.0.tgz",
+      "integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==",
       "dev": true,
       "requires": {
-        "follow-redirects": "^1.14.0"
+        "follow-redirects": "^1.14.7"
       }
     },
     "axios-cookiejar-support": {
@@ -3421,9 +3533,9 @@
       }
     },
     "follow-redirects": {
-      "version": "1.14.5",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.5.tgz",
-      "integrity": "sha512-wtphSXy7d4/OR+MvIFbCVBDzZ5520qV8XfPklSN5QtxuMUJZ+b0Wnst1e1lCDocfzuCkHqj8k0FpZqO+UIaKNA==",
+      "version": "1.14.7",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz",
+      "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==",
       "dev": true
     },
     "form-data": {
@@ -3627,9 +3739,9 @@
       "dev": true
     },
     "is-core-module": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.4.0.tgz",
-      "integrity": "sha512-6A2fkfq1rfeQZjxrZJGerpLCTHRNEBiSgnu0+obeJpEPZRUooHgsizvzv0ZjJwOz3iWIHdJtVWJ/tmPr3D21/A==",
+      "version": "2.8.1",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.1.tgz",
+      "integrity": "sha512-SdNCUs284hr40hFTFP6l0IfZ/RSrMXF3qgoRHd3/79unUTvrFO/JoXwkGm+5J/Oe3E/b5GsnG330uUNgRpu1PA==",
       "dev": true,
       "requires": {
         "has": "^1.0.3"
@@ -4051,13 +4163,14 @@
       "dev": true
     },
     "resolve": {
-      "version": "1.20.0",
-      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz",
-      "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==",
+      "version": "1.21.0",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.21.0.tgz",
+      "integrity": "sha512-3wCbTpk5WJlyE4mSOtDLhqQmGFi0/TD9VPwmiolnk8U0wRgMEktqCXd3vy5buTO3tljvalNvKrjHEfrd2WpEKA==",
       "dev": true,
       "requires": {
-        "is-core-module": "^2.2.0",
-        "path-parse": "^1.0.6"
+        "is-core-module": "^2.8.0",
+        "path-parse": "^1.0.7",
+        "supports-preserve-symlinks-flag": "^1.0.0"
       }
     },
     "resolve-dir": {
@@ -4138,9 +4251,9 @@
       }
     },
     "shelljs": {
-      "version": "0.8.4",
-      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.4.tgz",
-      "integrity": "sha512-7gk3UZ9kOfPLIAbslLzyWeGiEqx9e3rxwZM0KE6EL8GlGwjym9Mrlx5/p33bWTu9YG6vcS4MBxYZDHYr5lr8BQ==",
+      "version": "0.8.5",
+      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz",
+      "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==",
       "dev": true,
       "requires": {
         "glob": "^7.0.0",
@@ -4212,6 +4325,12 @@
         "has-flag": "^4.0.0"
       }
     },
+    "supports-preserve-symlinks-flag": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz",
+      "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==",
+      "dev": true
+    },
     "symbol-tree": {
       "version": "3.2.4",
       "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",

package.json

@@ -33,9 +33,6 @@
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "^2.39.0"
-  },
-  "dependencies": {
-    "ts-loader": "^9.2.6"
+    "@sasjs/cli": "3.6.0"
   }
-}
+}

server/ms_webout.sas

@@ -20,11 +20,17 @@
         %ms_webout(CLOSE)
 
 
-  @param action Either FETCH, OPEN, ARR, OBJ or CLOSE
-  @param ds The dataset to send back to the frontend
-  @param dslabel= value to use instead of the real name for sending to JSON
-  @param fmt=(Y) Set to N to send back unformatted values
-  @param fref=(_webout) The fileref to which to write the JSON
+  @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE
+  @param [in] ds The dataset to send back to the frontend
+  @param [out] dslabel= value to use instead of table name for sending to JSON
+  @param [in] fmt= (Y) Set to N to send back unformatted values
+  @param [out] fref= (_webout) The fileref to which to write the JSON
+  @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
+    (eg `null`) or as STRING values (eg `".a"` or `".b"`)
+  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+    such as the column formats and types.  The metadata is contained inside an
+    object with the same name as the table but prefixed with a dollar sign - ie,
+    `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
 
   <h4> SAS Macros </h4>
   @li mp_jsonout.sas
@@ -39,7 +45,9 @@
 
 **/
 
-%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y);
+%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
+  ,showmeta=NO
+);
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
 
@@ -91,7 +99,7 @@
 
 %else %if &action=ARR or &action=OBJ %then %do;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=DATASTEP,dbg=%str(&_debug)
+    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta
   )
 %end;
 %else %if &action=CLOSE %then %do;
@@ -106,15 +114,12 @@
       set &tempds;
       if not (upcase(name) =:"DATA"); /* ignore temp datasets */
       i+1;
-      call symputx('wt'!!left(i),name,'l');
+      call symputx(cats('wt',i),name,'l');
       call symputx('wtcnt',i,'l');
     data _null_; file &fref mod encoding='utf-8' termstr=lf;
       put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
-      proc contents noprint data=&wt
-        out=_data_ (keep=name type length format:);
-      run;%let tempds=%scan(&syslast,2,.);
       data _null_; file &fref mod encoding='utf-8' termstr=lf;
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
@@ -124,8 +129,7 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
-      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES)
       data _null_; file &fref mod encoding='utf-8' termstr=lf;
         put "}";
     %end;
@@ -161,6 +165,7 @@
     length autoexec $512;
     autoexec=quote(urlencode(trim(getoption('autoexec'))));
     put ',"AUTOEXEC" : ' autoexec;
+    length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
     memsize=quote(cats(memsize));
     put ',"MEMSIZE" : ' memsize;

tests/crossplatform/mcf_length.test.sas

@@ -0,0 +1,50 @@
+/**
+  @file
+  @brief Testing mcf_length.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mcf_length.sas
+  @li mp_assert.sas
+
+**/
+
+%mcf_length(wrap=YES, insert_cmplib=YES)
+
+data test;
+  call symputx('null',mcf_length(.));
+  call symputx('three',mcf_length(1));
+  call symputx('four',mcf_length(10000000));
+  call symputx('five',mcf_length(12345678));
+  call symputx('six',mcf_length(1234567890));
+  call symputx('seven',mcf_length(12345678901234));
+  call symputx('eight',mcf_length(12345678901234567));
+run;
+
+%mp_assert(
+  iftrue=(%str(&null)=%str(0)),
+  desc=Check if NULL returns 0
+)
+%mp_assert(
+  iftrue=(%str(&three)=%str(3)),
+  desc=Check for length 3
+)
+%mp_assert(
+  iftrue=(%str(&four)=%str(4)),
+  desc=Check for length 4
+)
+%mp_assert(
+  iftrue=(%str(&five)=%str(5)),
+  desc=Check for length 5
+)
+%mp_assert(
+  iftrue=(%str(&six)=%str(6)),
+  desc=Check for length 6
+)
+%mp_assert(
+  iftrue=(%str(&seven)=%str(7)),
+  desc=Check for length 3
+)
+%mp_assert(
+  iftrue=(%str(&eight)=%str(8)),
+  desc=Check for length 8
+)

tests/crossplatform/mp_assert.test.sas

@@ -0,0 +1,15 @@
+/**
+  @file
+  @brief Testing mp_assert macro
+  @details This is quite "meta".. it's just testing itself
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(1=1),
+  desc=Checking result was created,
+  outds=work.test_results
+)

tests/crossplatform/mp_assertscope.test.sas

@@ -0,0 +1,80 @@
+/**
+  @file
+  @brief Testing mp_assertscope macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getvalue.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+
+**/
+
+%macro dostuff(action);
+  %if &action=ADD %then %do;
+    %global NEWVAR1 NEWVAR2;
+  %end;
+  %else %if &action=DEL %then %do;
+    %symdel NEWVAR1 NEWVAR2;
+  %end;
+  %else %if &action=MOD %then %do;
+    %let NEWVAR1=Let us pray..;
+  %end;
+  %else %if &action=NOTHING %then %do;
+    %local a b c d e;
+  %end;
+%mend dostuff;
+
+
+/* check for adding variables */
+%mp_assertscope(SNAPSHOT)
+%dostuff(ADD)
+%mp_assertscope(COMPARE,outds=work.testing_the_tester1)
+%mp_assert(
+  iftrue=(
+    "%mf_getvalue(work.testing_the_tester1,test_comments)"
+      ="Mod:() Add:(NEWVAR1 NEWVAR2) Del:()"
+  ),
+  desc=Checking result when vars added,
+  outds=work.test_results
+)
+
+
+/* check for modifying variables */
+%mp_assertscope(SNAPSHOT)
+%dostuff(MOD)
+%mp_assertscope(COMPARE,outds=work.testing_the_tester2)
+%mp_assert(
+  iftrue=(
+    "%mf_getvalue(work.testing_the_tester2,test_comments)"
+      ="Mod:(NEWVAR1) Add:() Del:()"
+  ),
+  desc=Checking result when vars modified,
+  outds=work.test_results
+)
+
+/* check for deleting variables */
+%mp_assertscope(SNAPSHOT)
+%dostuff(DEL)
+%mp_assertscope(COMPARE,outds=work.testing_the_tester3)
+%mp_assert(
+  iftrue=(
+    "%mf_getvalue(work.testing_the_tester3,test_comments)"
+      ="Mod:() Add:() Del:(NEWVAR1 NEWVAR2)"
+  ),
+  desc=Checking result when vars deleted,
+  outds=work.test_results
+)
+
+/* check for doing nothing */
+%mp_assertscope(SNAPSHOT)
+%dostuff(NOTHING)
+%mp_assertscope(COMPARE,outds=work.testing_the_tester4)
+%mp_assert(
+  iftrue=(
+    "%mf_getvalue(work.testing_the_tester4,test_comments)"
+      ="GLOBAL Variables Unmodified"
+  ),
+  desc=Checking results when nothing created,
+  outds=work.test_results
+)

tests/crossplatform/mp_jsonout.test.1.sas

@@ -34,6 +34,11 @@ data work.test;
   call symputx('dtval',dtval);
 run;
 
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking for error condition,
+  outds=work.test_results
+)
 
 %mp_assert(
   iftrue=(&dtval=&compare),

tests/crossplatform/mp_jsonout.test.2.sas

@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief Testing mp_jsonout.sas macro with special missings
+
+  <h4> SAS Macros </h4>
+  @li mp_jsonout.sas
+  @li mp_assert.sas
+
+**/
+
+filename webref temp;
+
+data demo;
+  do x=._,.,.a,.b,.c,.d,.e,-99, 0, 1,2, 3.333333;
+    output;
+  end;
+run;
+%mp_jsonout(OPEN,jref=webref)
+%mp_jsonout(OBJ,demo,jref=webref,fmt=N,missing=STRING)
+%mp_jsonout(CLOSE,jref=webref)
+
+data _null_;
+  infile webref;
+  input;
+  putlog _infile_;
+run;
+
+libname web JSON fileref=webref;
+
+/* proc json turns to char - so switch back to numeric */
+data work.test(keep=x);
+  set web.demo(rename=(x=y));
+  if y ='_' then x=._;
+  else if anyalpha(y) then x=input(cats(".",y),best.);
+  else x=input(y,best.);
+  put (_all_)(=);
+run;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking for error condition with special missing export,
+  outds=work.test_results
+)
+
+proc compare base=work.demo compare=work.test;
+quit;
+
+%mp_assert(
+  iftrue=(&sysinfo=0),
+  desc=Returned json is identical to input table for all special missings,
+  outds=work.test_results
+)

tests/crossplatform/mp_lockanytable.test.sas

@@ -6,12 +6,13 @@
   @li mp_lockanytable.sas
   @li mp_assertcols.sas
   @li mp_assertcolvals.sas
+  @li mp_coretable.sas
 
 **/
 
 /* check create table */
 
-%mp_lockanytable(MAKETABLE, ctl_ds=work.controller)
+%mp_coretable(LOCKTABLE,libds=work.controller)
 
 %mp_assertcols(work.controller,
   cols=lock_status_cd lock_lib lock_ds lock_user_nm lock_ref lock_pid

tests/crossplatform/mp_validatecol.test.sas

@@ -112,6 +112,9 @@ datalines4;
 0
 above are good
 the rest are bad
+0.1
+1.1
+-0.001
 %abort
 1&somethingverybad.
 &

tests/testinit.sas

@@ -4,6 +4,7 @@
 
   <h4> SAS Macros </h4>
   @li mp_init.sas
+  @li mv_webout.sas
 
 **/
 

viya/mv_createwebservice.sas

@@ -54,9 +54,9 @@
     that location
   @param [in] adapter= the macro uses the sasjs adapter by default.  To use
     another adapter, add a (different) fileref here.
-  @param [in] contextname= Choose a specific context on which to run the Job.  Leave
-    blank to use the default context.  From Viya 3.5 it is possible to configure
-    a shared context - see
+  @param [in] contextname= Choose a specific context on which to run the Job.
+    Leave blank to use the default context.  From Viya 3.5 it is possible to
+    configure a shared context - see
 https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5pyhn1wg3ja0drdl6h.htm&docsetVersion=3.5&locale=en
   @param [in] mdebug=(0) set to 1 to enable DEBUG messages
 
@@ -237,9 +237,15 @@ data _null_;
   put "/* Created on %sysfunc(datetime(),datetime19.) by &sysuserid */";
 /* WEBOUT BEGIN */
   put ' ';
-  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0 ';
+  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
+  put '  ,engine=DATASTEP ';
+  put '  ,dbg=0 /* DEPRECATED */ ';
+  put '  ,missing=NULL ';
+  put '  ,showmeta=NO ';
   put ')/*/STORE SOURCE*/; ';
-  put '%put output location=&jref; ';
+  put '%local tempds colinfo fmtds i numcols; ';
+  put '%let numcols=0; ';
+  put ' ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
   put '  data _null_;file &jref encoding=''utf-8'' ; ';
@@ -248,118 +254,124 @@ data _null_;
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
   put '  options validvarname=upcase; ';
-  put '  data _null_;file &jref mod encoding=''utf-8'' ; ';
+  put '  data _null_; file &jref encoding=''utf-8'' mod; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
   put ' ';
+  put '  /* grab col defs */ ';
+  put '  proc contents noprint data=&ds ';
+  put '    out=_data_(keep=name type length format formatl formatd varnum label); ';
+  put '  run; ';
+  put '  %let colinfo=%scan(&syslast,2,.); ';
+  put '  proc sort data=&colinfo; ';
+  put '    by varnum; ';
+  put '  run; ';
+  put '  /* move meta to mac vars */ ';
+  put '  data _null_; ';
+  put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
+  put '    set &colinfo end=last nobs=nobs; ';
+  put '    name=upcase(name); ';
+  put '    /* fix formats */ ';
+  put '    if type=2 or type=6 then do; ';
+  put '      typelong=''char''; ';
+  put '      length fmt $49.; ';
+  put '      if format='''' then fmt=cats(''$'',length,''.''); ';
+  put '      else if formatl=0 then fmt=cats(format,''.''); ';
+  put '      else fmt=cats(format,formatl,''.''); ';
+  put '      newlen=max(formatl,length); ';
+  put '    end; ';
+  put '    else do; ';
+  put '      typelong=''num''; ';
+  put '      if format='''' then fmt=''best.''; ';
+  put '      else if formatl=0 then fmt=cats(format,''.''); ';
+  put '      else if formatd=0 then fmt=cats(format,formatl,''.''); ';
+  put '      else fmt=cats(format,formatl,''.'',formatd); ';
+  put '      /* needs to be wide, for datetimes etc */ ';
+  put '      newlen=max(length,formatl,24); ';
+  put '    end; ';
+  put '    /* 32 char unique name */ ';
+  put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put ' ';
+  put '    call symputx(cats(''name'',_n_),name,''l''); ';
+  put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '    call symputx(cats(''len'',_n_),newlen,''l''); ';
+  put '    call symputx(cats(''length'',_n_),length,''l''); ';
+  put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
+  put '    call symputx(cats(''type'',_n_),type,''l''); ';
+  put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
+  put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
+  put '  run; ';
+  put ' ';
+  put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
-  put '    data;run;%let tempds=&syslast; ';
-  put '    proc sql;drop table &tempds; ';
+  put '    %if &missing=STRING %then %do; ';
+  put '      %put &sysmacroname: Special Missings not supported in proc json.; ';
+  put '      %put &sysmacroname: Switching to DATASTEP engine; ';
+  put '      %goto datastep; ';
+  put '    %end; ';
   put '    data &tempds /view=&tempds;set &ds; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
+  put '    /* PRETTY is necessary to avoid line truncation in large files */ ';
   put '    proc json out=&jref pretty ';
   put '        %if &action=ARR %then nokeys ; ';
   put '        ;export &tempds / nosastags fmtnumeric; ';
   put '    run; ';
-  put '    proc sql;drop view &tempds; ';
   put '  %end; ';
   put '  %else %if &engine=DATASTEP %then %do; ';
-  put '    %local cols i tempds; ';
-  put '    %let cols=0; ';
-  put '    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 %then %do; ';
+  put '    %datastep: ';
+  put '    %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 ';
+  put '    %then %do; ';
   put '      %put &sysmacroname:  &ds NOT FOUND!!!; ';
   put '      %return; ';
   put '    %end; ';
-  put '    %if &fmt=Y %then %do; ';
-  put '      %put converting every variable to a formatted variable; ';
-  put '      /* see mp_ds2fmtds.sas for source */ ';
-  put '      proc contents noprint data=&ds ';
-  put '        out=_data_(keep=name type length format formatl formatd varnum); ';
-  put '      run; ';
-  put '      proc sort; ';
-  put '        by varnum; ';
-  put '      run; ';
-  put '      %local fmtds; ';
-  put '      %let fmtds=%scan(&syslast,2,.); ';
-  put '      /* prepare formats and varnames */ ';
-  put '      data _null_; ';
-  put '        if _n_=1 then call symputx(''nobs'',nobs,''l''); ';
-  put '        set &fmtds end=last nobs=nobs; ';
-  put '        name=upcase(name); ';
-  put '        /* fix formats */ ';
-  put '        if type=2 or type=6 then do; ';
-  put '          length fmt $49.; ';
-  put '          if format='''' then fmt=cats(''$'',length,''.''); ';
-  put '          else if formatl=0 then fmt=cats(format,''.''); ';
-  put '          else fmt=cats(format,formatl,''.''); ';
-  put '          newlen=max(formatl,length); ';
-  put '        end; ';
-  put '        else do; ';
-  put '          if format='''' then fmt=''best.''; ';
-  put '          else if formatl=0 then fmt=cats(format,''.''); ';
-  put '          else if formatd=0 then fmt=cats(format,formatl,''.''); ';
-  put '          else fmt=cats(format,formatl,''.'',formatd); ';
-  put '          /* needs to be wide, for datetimes etc */ ';
-  put '          newlen=max(length,formatl,24); ';
-  put '        end; ';
-  put '        /* 32 char unique name */ ';
-  put '        newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
   put ' ';
-  put '        call symputx(cats(''name'',_n_),name,''l''); ';
-  put '        call symputx(cats(''newname'',_n_),newname,''l''); ';
-  put '        call symputx(cats(''len'',_n_),newlen,''l''); ';
-  put '        call symputx(cats(''fmt'',_n_),fmt,''l''); ';
-  put '        call symputx(cats(''type'',_n_),type,''l''); ';
-  put '      run; ';
-  put '      data &fmtds; ';
+  put '    %if &fmt=Y %then %do; ';
+  put '      data _data_; ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
-  put '      %local i; ';
-  put '      %do i=1 %to &nobs; ';
+  put '      %do i=1 %to &numcols; ';
   put '        &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
-  put '      %do i=1 %to &nobs; ';
+  put '      %do i=1 %to &numcols; ';
   put '        length &&name&i $&&len&i; ';
   put '        &&name&i=left(put(&&newname&i,&&fmt&i)); ';
   put '        drop &&newname&i; ';
   put '      %end; ';
   put '        if _error_ then call symputx(''syscc'',1012); ';
   put '      run; ';
-  put '      %let ds=&fmtds; ';
-  put '    %end; /* &fmt=Y */ ';
-  put '    data _null_;file &jref mod encoding=''utf-8'' ; ';
-  put '      put "["; call symputx(''cols'',0,''l''); ';
-  put '    proc sort ';
-  put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
-  put '      out=_data_; ';
-  put '      by varnum; ';
-  put ' ';
-  put '    data _null_; ';
-  put '      set _last_ end=last; ';
-  put '      call symputx(cats(''name'',_n_),name,''l''); ';
-  put '      call symputx(cats(''type'',_n_),type,''l''); ';
-  put '      call symputx(cats(''len'',_n_),length,''l''); ';
-  put '      if last then call symputx(''cols'',_n_,''l''); ';
-  put '    run; ';
+  put '      %let fmtds=&syslast; ';
+  put '    %end; ';
   put ' ';
   put '    proc format; /* credit yabwon for special null removal */ ';
-  put '      value bart ._ - .z = null ';
+  put '    value bart (default=40) ';
+  put '    %if &missing=NULL %then %do; ';
+  put '      ._ - .z = null ';
+  put '    %end; ';
+  put '    %else %do; ';
+  put '      ._ = [quote()] ';
+  put '      . = null ';
+  put '      .a - .z = [quote()] ';
+  put '    %end; ';
   put '      other = [best.]; ';
   put ' ';
-  put '    data;run; %let tempds=&syslast; /* temp table for spesh char management */ ';
-  put '    proc sql; drop table &tempds; ';
   put '    data &tempds/view=&tempds; ';
   put '      attrib _all_ label=''''; ';
-  put '      %do i=1 %to &cols; ';
-  put '        %if &&type&i=char %then %do; ';
+  put '      %do i=1 %to &numcols; ';
+  put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
   put '          length &&name&i $32767; ';
   put '          format &&name&i $32767.; ';
   put '        %end; ';
   put '      %end; ';
-  put '      set &ds; ';
+  put '      %if &fmt=Y %then %do; ';
+  put '        set &fmtds; ';
+  put '      %end; ';
+  put '      %else %do; ';
+  put '        set &ds; ';
+  put '      %end; ';
   put '      format _numeric_ bart.; ';
-  put '    %do i=1 %to &cols; ';
-  put '      %if &&type&i=char %then %do; ';
+  put '    %do i=1 %to &numcols; ';
+  put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
   put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
   put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
   put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
@@ -369,49 +381,72 @@ data _null_;
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
+  put ' ';
   put '    /* write to temp loc to avoid _webout truncation ';
   put '      - https://support.sas.com/kb/49/325.html */ ';
   put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
   put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
+  put '      if _n_=1 then put "["; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
   put '      %if &action=ARR %then "[" ; %else "{" ; ';
-  put '      %do i=1 %to &cols; ';
+  put '      %do i=1 %to &numcols; ';
   put '        %if &i>1 %then  "," ; ';
   put '        %if &action=OBJ %then """&&name&i"":" ; ';
   put '        &&name&i ';
   put '      %end; ';
   put '      %if &action=ARR %then "]" ; %else "}" ; ; ';
-  put '    proc sql; ';
-  put '    drop view &tempds; ';
   put '    /* now write the long strings to _webout 1 byte at a time */ ';
   put '    data _null_; ';
   put '      length filein 8 fileid 8; ';
-  put '      filein = fopen("_sjs",''I'',1,''B''); ';
-  put '      fileid = fopen("&jref",''A'',1,''B''); ';
-  put '      rec = ''20''x; ';
+  put '      filein=fopen("_sjs",''I'',1,''B''); ';
+  put '      fileid=fopen("&jref",''A'',1,''B''); ';
+  put '      rec=''20''x; ';
   put '      do while(fread(filein)=0); ';
-  put '        rc = fget(filein,rec,1); ';
-  put '        rc = fput(fileid, rec); ';
-  put '        rc =fwrite(fileid); ';
+  put '        rc=fget(filein,rec,1); ';
+  put '        rc=fput(fileid, rec); ';
+  put '        rc=fwrite(fileid); ';
   put '      end; ';
-  put '      rc = fclose(filein); ';
-  put '      rc = fclose(fileid); ';
+  put '      /* close out the table */ ';
+  put '      rc=fput(fileid, "]"); ';
+  put '      rc=fwrite(fileid); ';
+  put '      rc=fclose(filein); ';
+  put '      rc=fclose(fileid); ';
   put '    run; ';
   put '    filename _sjs clear; ';
-  put '    data _null_; file &jref mod encoding=''utf-8'' ; ';
-  put '      put "]"; ';
+  put '  %end; ';
+  put ' ';
+  put '  proc sql; ';
+  put '  drop view &tempds; ';
+  put '  drop table &colinfo; ';
+  put ' ';
+  put '  %if &showmeta=YES %then %do; ';
+  put '    data _null_; file &jref encoding=''utf-8'' mod; ';
+  put '      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; ';
+  put '      do i=1 to &numcols; ';
+  put '        name=quote(trim(symget(cats(''name'',i)))); ';
+  put '        format=quote(trim(symget(cats(''fmt'',i)))); ';
+  put '        label=quote(trim(symget(cats(''label'',i)))); ';
+  put '        length=quote(trim(symget(cats(''length'',i)))); ';
+  put '        type=quote(trim(symget(cats(''typelong'',i)))); ';
+  put '        if i>1 then put "," @@; ';
+  put '        put name '':{"format":'' format '',"label":'' label ';
+  put '          '',"length":'' length '',"type":'' type ''}''; ';
+  put '      end; ';
+  put '      put ''}}''; ';
   put '    run; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8'' mod ; ';
+  put '  data _null_; file &jref encoding=''utf-8'' mod ; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
   put '%mend mp_jsonout; ';
-  put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y); ';
+  put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL ';
+  put '  ,showmeta=NO ';
+  put '); ';
   put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
   put '  sasjs_tables SYS_JES_JOB_URI; ';
   put '%if %index("&_debug",log) %then %let _debug=131; ';
@@ -533,12 +568,13 @@ data _null_;
   put ' ';
   put '  /* setup json */ ';
   put '  data _null_;file &fref; ';
-  put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
+  put '    put ''{"SYSDATE" : "'' "&SYSDATE" ''"''; ';
+  put '    put '',"SYSTIME" : "'' "&SYSTIME" ''"''; ';
   put '  run; ';
   put '%end; ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
   put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
-  put '      ,jref=&fref,engine=DATASTEP,dbg=%str(&_debug) ';
+  put '      ,jref=&fref,engine=DATASTEP,missing=&missing,showmeta=&showmeta ';
   put '    ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
@@ -553,14 +589,11 @@ data _null_;
   put '      set &tempds; ';
   put '      if not (upcase(name) =:"DATA"); /* ignore temp datasets */ ';
   put '      i+1; ';
-  put '      call symputx(''wt''!!left(i),name); ';
-  put '      call symputx(''wtcnt'',i); ';
+  put '      call symputx(cats(''wt'',i),name,''l''); ';
+  put '      call symputx(''wtcnt'',i,''l''); ';
   put '    data _null_; file &fref mod; put ",""WORK"":{"; ';
   put '    %do i=1 %to &wtcnt; ';
   put '      %let wt=&&wt&i; ';
-  put '      proc contents noprint data=&wt ';
-  put '        out=_data_ (keep=name type length format:); ';
-  put '      run;%let tempds=%scan(&syslast,2,.); ';
   put '      data _null_; file &fref mod; ';
   put '        dsid=open("WORK.&wt",''is''); ';
   put '        nlobs=attrn(dsid,''NLOBS''); ';
@@ -570,8 +603,7 @@ data _null_;
   put '        put " ""&wt"" : {"; ';
   put '        put ''"nlobs":'' nlobs; ';
   put '        put '',"nvars":'' nvars; ';
-  put '      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP) ';
-  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP) ';
+  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) ';
   put '      data _null_; file &fref mod;put "}"; ';
   put '    %end; ';
   put '    data _null_; file &fref mod;put "}";run; ';
@@ -596,6 +628,10 @@ data _null_;
   put '    put '',"SYSVLONG" : '' sysvlong; ';
   put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''" ''; ';
+  put '    length memsize $32; ';
+  put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
+  put '    memsize=quote(cats(memsize)); ';
+  put '    put '',"MEMSIZE" : '' memsize; ';
   put '    put "}"; ';
   put ' ';
   put '  %if %upcase(&fref) ne _WEBOUT and &stream=Y %then %do; ';
@@ -629,8 +665,10 @@ data _null_;
   put '%global __program _program;';
   put '%let _program=%sysfunc(coalescec(&__program,&_program));';
   put ' ';
-  put '%macro webout(action,ds,dslabel=,fmt=);';
-  put '  %mv_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt)';
+  put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO);';
+  put '  %mv_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt,missing=&missing';
+  put '    ,showmeta=&showmeta';
+  put '  )';
   put '%mend;';
 run;
 

viya/mv_jobflow.sas

@@ -272,7 +272,7 @@ data;run;%let jdswaitfor=&syslast;
       data _null_;
         infile &jfref lrecl=32767;
         input;
-        jparams='jparams'!!left(symget('jid'));
+        jparams=cats('jparams',symget('jid'));
         call symputx(jparams,substr(_infile_,3,length(_infile_)-4));
       run;
       %local jobuid&jid;

viya/mv_webout.sas

@@ -20,13 +20,19 @@
         %mv_webout(CLOSE)
 
 
-  @param action Either OPEN, ARR, OBJ or CLOSE
-  @param ds The dataset to send back to the frontend
-  @param _webout= fileref for returning the json
-  @param fref=(_mvwtemp) Temp fileref to which to write the output
-  @param dslabel= value to use instead of the real name for sending to JSON
-  @param fmt=(Y) change to N to strip formats from output
-  @param stream=(Y) Change to N if not streaming to _webout
+  @param [in] action Either OPEN, ARR, OBJ or CLOSE
+  @param [in] ds The dataset to send back to the frontend
+  @param [in] _webout= fileref for returning the json
+  @param [out] fref=(_mvwtemp) Temp fileref to which to write the output
+  @param [out] dslabel= value to use instead of table name for sending to JSON
+  @param [in] fmt=(Y) change to N to strip formats from output
+  @param [in] stream=(Y) Change to N if not streaming to _webout
+  @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
+    (eg `null`) or as STRING values (eg `".a"` or `".b"`)
+  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+    such as the column formats and types.  The metadata is contained inside an
+    object with the same name as the table but prefixed with a dollar sign - ie,
+    `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
 
   <h4> SAS Macros </h4>
   @li mp_jsonout.sas
@@ -36,7 +42,9 @@
   @author Allan Bowe, source: https://github.com/sasjs/core
 
 **/
-%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y);
+%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL
+  ,showmeta=NO
+);
 %global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name
   sasjs_tables SYS_JES_JOB_URI;
 %if %index("&_debug",log) %then %let _debug=131;
@@ -158,12 +166,13 @@
 
   /* setup json */
   data _null_;file &fref;
-    put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
+    put '{"SYSDATE" : "' "&SYSDATE" '"';
+    put ',"SYSTIME" : "' "&SYSTIME" '"';
   run;
 %end;
 %else %if &action=ARR or &action=OBJ %then %do;
     %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt
-      ,jref=&fref,engine=DATASTEP,dbg=%str(&_debug)
+      ,jref=&fref,engine=DATASTEP,missing=&missing,showmeta=&showmeta
     )
 %end;
 %else %if &action=CLOSE %then %do;
@@ -178,14 +187,11 @@
       set &tempds;
       if not (upcase(name) =:"DATA"); /* ignore temp datasets */
       i+1;
-      call symputx('wt'!!left(i),name);
-      call symputx('wtcnt',i);
+      call symputx(cats('wt',i),name,'l');
+      call symputx('wtcnt',i,'l');
     data _null_; file &fref mod; put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
-      proc contents noprint data=&wt
-        out=_data_ (keep=name type length format:);
-      run;%let tempds=%scan(&syslast,2,.);
       data _null_; file &fref mod;
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
@@ -195,8 +201,7 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
-      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES)
       data _null_; file &fref mod;put "}";
     %end;
     data _null_; file &fref mod;put "}";run;
@@ -221,6 +226,10 @@
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
     put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+    length memsize $32;
+    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
+    memsize=quote(cats(memsize));
+    put ',"MEMSIZE" : ' memsize;
     put "}";
 
   %if %upcase(&fref) ne _WEBOUT and &stream=Y %then %do;