Merge branch 'main' of github.com:sasjs/core

fix: Non-stp propagation of syscc

README.md

@@ -116,19 +116,19 @@ The **Macro Core** documentation is created using [doxygen](http://www.doxygen.n
 All macros must be commented in the doxygen format, to enable the [online documentation](https://core.sasjs.io).
 
 ### Dependencies
-SAS code can contain one of two types of dependency - SAS Macros, and SAS Programs.  When compiling projects using the [SASjs CLI](https://cli.sasjs.io) the doxygen header is scanned for `  @li` items under the following headers:
+SAS code can contain one of two types of dependency - SAS Macros, and SAS Includes.  When compiling projects using the [SASjs CLI](https://cli.sasjs.io) the doxygen header is scanned for `  @li` items under the following headers:
 
 ```sas
   <h4> SAS Macros </h4>
   @li mf_nobs.sas
   @li mm_assignlib.sas
 
-  <h4> SAS Programs </h4>
+  <h4> SAS Includes </h4>
   @li somefile.ddl SOMEFREF
   @li someprogram.sas FREFTWO
 ```
 
-The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Programs) when creating SAS Jobs and Services.
+The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Includes) when creating SAS Jobs and Services.
 
 When contributing to this library, it is therefore important to ensure that all dependencies are listed in the header in this format.
 

all.sas

@@ -18,24 +18,12 @@
 options noquotelenmax;
 /**
   @file
-  @brief abort gracefully according to context
-  @details Do not use directly!  See bottom of explanation for details.
+  @brief to be deprecated
+  @details We will deprecate this macro in 2022
 
-  Configures an abort mechanism according to site specific policies or the
-    particulars of an environment.  For instance, can stream custom
-    results back to the client in an STP Web App context, or completely stop
-    in the case of a batch run.
+  As you can see, it's not a macro function.
 
-  For the sharp eyed readers - this is no longer a macro function!! It became
-  a macro procedure during a project and now it's kinda stuck that way until
-  that project is updated (if it's ever updated).  In the meantime we created
-  `mp_abort` which is just a wrapper for this one, and so we recomend you use
-  that for forwards compatibility reasons.
-
-  @param mac= to contain the name of the calling macro
-  @param type= deprecated.  Not used.
-  @param msg= message to be returned
-  @param iftrue= supply a condition under which the macro should be executed.
+  Use mp_abort.sas instead.
 
   @version 9.2
   @author Allan Bowe
@@ -801,8 +789,8 @@ options noquotelenmax;
 
 
 %macro mf_getuniquename(prefix=MC);
-  &prefix.%substr(%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32-%length(&prefix))
-%mend;/**
+&prefix.%substr(%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32-%length(&prefix))
+%mend mf_getuniquename;/**
   @file
   @brief Returns a userid according to session context
   @details In a workspace session, a user is generally represented by <code>
@@ -3254,6 +3242,103 @@ run;
 
 
 %mend;/**
+  @file
+  @brief Converts every value in a dataset to it's formatted value
+  @details Converts every value to it's formatted value.  All variables will
+  become character, and will be in the same order.
+
+  Usage:
+
+      %mp_ds2fmtds(sashelp.cars,work.cars)
+
+  @param [in] libds The library.dataset to be converted
+  @param [out] outds The dataset to create.
+
+  <h4> Related Macros <h4>
+  @li mp_jsonout.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_ds2fmtds(libds, outds
+)/*/STORE SOURCE*/;
+
+/* validations */
+%if not %sysfunc(exist(&libds)) %then %do;
+  %put %str(WARN)ING:  &libds does not exist;
+  %return;
+%end;
+%if %index(&libds,.)=0 %then %let libds=WORK.&libds;
+
+/* grab metadata */
+proc contents noprint data=&libds
+  out=_data_(keep=name type length format formatl formatd varnum);
+run;
+proc sort;
+  by varnum;
+run;
+
+/* prepare formats and varnames */
+data _null_;
+  set &syslast end=last;
+  name=upcase(name);
+  /* fix formats */
+  if type=2 or type=6 then do;
+    length fmt $49.;
+    if format='' then fmt=cats('$',length,'.');
+    else if formatl=0 then fmt=cats(format,'.');
+    else fmt=cats(format,formatl,'.');
+    newlen=max(formatl,length);
+  end;
+  else do;
+    if format='' then fmt='best.';
+    else if formatl=0 then fmt=cats(format,'.');
+    else if formatd=0 then fmt=cats(format,formatl,'.');
+    else fmt=cats(format,formatl,'.',formatd);
+    /* needs to be wide, for datetimes etc */
+    newlen=max(length,formatl,24);
+  end;
+  /* 32 char unique name */
+  newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
+
+  call symputx(cats('name',_n_),name,'l');
+  call symputx(cats('newname',_n_),newname,'l');
+  call symputx(cats('len',_n_),newlen,'l');
+  call symputx(cats('fmt',_n_),fmt,'l');
+  call symputx(cats('type',_n_),type,'l');
+  if last then call symputx('nobs',_n_,'l');
+run;
+
+/* clean up */
+proc sql;
+drop table &syslast;
+
+%if &nobs=0 %then %do;
+  %put Dataset &libds has no columns!
+  data &outds;
+    set &libds;
+  run;
+  %return;
+%end;
+
+data &outds;
+  /* rename on entry */
+  set &libds(rename=(
+%local i;
+%do i=1 %to &nobs;
+  &&name&i=&&newname&i
+%end;
+  ));
+%do i=1 %to &nobs;
+  length &&name&i $&&len&i;
+  &&name&i=left(put(&&newname&i,&&fmt&i));
+  drop &&newname&i;
+%end;
+  if _error_ then call symputx('syscc',1012);
+run;
+
+%mend mp_ds2fmtds;/**
   @file
   @brief Checks an input filter table for validity
   @details Performs checks on the input table to ensure it arrives in the
@@ -3300,7 +3385,6 @@ run;
   @li mf_getuniquefileref.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
-  @li mf_nobs.sas
   @li mp_filtergenerate.sas
   @li mp_filtervalidate.sas
 
@@ -3413,18 +3497,23 @@ data &outds;
 
 run;
 
+%local nobs;
+%let nobs=0;
 data _null_;
-  set &outds;
-  call symputx('REASON_CD',reason_cd,'l');
-  stop;
+  set &outds end=last;
+  putlog (_all_)(=);
+  if last then do;
+    call symputx('REASON_CD',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
+  end;
 run;
 
-%mp_abort(iftrue=(&abort=YES and %mf_nobs(&outds)>0),
+%mp_abort(iftrue=(&abort=YES and &nobs>0),
   mac=&sysmacroname,
-  msg=%str(Filter issues in &inds, reason: &reason_cd, details in &outds)
+  msg=%str(&nobs filter issues in &inds, reason: &reason_cd, details in &outds)
 )
 
-%if %mf_nobs(&outds)>0 %then %do;
+%if &nobs>0 %then %do;
   %let syscc=1008;
   %return;
 %end;
@@ -4919,6 +5008,8 @@ create table &outds (rename=(
   @param dbg= DEPRECATED - was used to conditionally add PRETTY to
     proc json but this can cause line truncation in large files.
 
+  <h4> Related Macros <h4>
+  @li mp_ds2fmtds.sas
 
   @version 9.2
   @author Allan Bowe
@@ -4926,10 +5017,11 @@ create table &outds (rename=(
 
 **/
 
-%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=PROCJSON,dbg=0
+%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0
 )/*/STORE SOURCE*/;
 %put output location=&jref;
 %if &action=OPEN %then %do;
+  OPTIONS NOBOMFILE;
   data _null_;file &jref encoding='utf-8';
     put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
   run;
@@ -4957,6 +5049,64 @@ create table &outds (rename=(
       %put &sysmacroname:  &ds NOT FOUND!!!;
       %return;
     %end;
+    %if &fmt=Y %then %do;
+      %put converting every variable to a formatted variable;
+      /* see mp_ds2fmtds.sas for source */
+      proc contents noprint data=&ds
+        out=_data_(keep=name type length format formatl formatd varnum);
+      run;
+      proc sort;
+        by varnum;
+      run;
+      %local fmtds;
+      %let fmtds=%scan(&syslast,2,.);
+      /* prepare formats and varnames */
+      data _null_;
+        set &fmtds end=last;
+        name=upcase(name);
+        /* fix formats */
+        if type=2 or type=6 then do;
+          length fmt $49.;
+          if format='' then fmt=cats('$',length,'.');
+          else if formatl=0 then fmt=cats(format,'.');
+          else fmt=cats(format,formatl,'.');
+          newlen=max(formatl,length);
+        end;
+        else do;
+          if format='' then fmt='best.';
+          else if formatl=0 then fmt=cats(format,'.');
+          else if formatd=0 then fmt=cats(format,formatl,'.');
+          else fmt=cats(format,formatl,'.',formatd);
+          /* needs to be wide, for datetimes etc */
+          newlen=max(length,formatl,24);
+        end;
+        /* 32 char unique name */
+        newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
+
+        call symputx(cats('name',_n_),name,'l');
+        call symputx(cats('newname',_n_),newname,'l');
+        call symputx(cats('len',_n_),newlen,'l');
+        call symputx(cats('fmt',_n_),fmt,'l');
+        call symputx(cats('type',_n_),type,'l');
+        if last then call symputx('nobs',_n_,'l');
+      run;
+      data &fmtds;
+        /* rename on entry */
+        set &ds(rename=(
+      %local i;
+      %do i=1 %to &nobs;
+        &&name&i=&&newname&i
+      %end;
+        ));
+      %do i=1 %to &nobs;
+        length &&name&i $&&len&i;
+        &&name&i=left(put(&&newname&i,&&fmt&i));
+        drop &&newname&i;
+      %end;
+        if _error_ then call symputx('syscc',1012);
+      run;
+      %let ds=&fmtds;
+    %end; /* &fmt=Y */
     data _null_;file &jref mod ;
       put "["; call symputx('cols',0,'l');
     proc sort
@@ -5040,7 +5190,7 @@ create table &outds (rename=(
     put "}";
   run;
 %end;
-%mend;
+%mend mp_jsonout;
 /**
   @file
   @brief Convert all library members to CARDS files
@@ -8697,10 +8847,11 @@ data _null_;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
   put ' ';
-  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=PROCJSON,dbg=0 ';
+  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0 ';
   put ')/*/STORE SOURCE*/; ';
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
+  put '  OPTIONS NOBOMFILE; ';
   put '  data _null_;file &jref encoding=''utf-8''; ';
   put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
   put '  run; ';
@@ -8728,6 +8879,64 @@ data _null_;
   put '      %put &sysmacroname:  &ds NOT FOUND!!!; ';
   put '      %return; ';
   put '    %end; ';
+  put '    %if &fmt=Y %then %do; ';
+  put '      %put converting every variable to a formatted variable; ';
+  put '      /* see mp_ds2fmtds.sas for source */ ';
+  put '      proc contents noprint data=&ds ';
+  put '        out=_data_(keep=name type length format formatl formatd varnum); ';
+  put '      run; ';
+  put '      proc sort; ';
+  put '        by varnum; ';
+  put '      run; ';
+  put '      %local fmtds; ';
+  put '      %let fmtds=%scan(&syslast,2,.); ';
+  put '      /* prepare formats and varnames */ ';
+  put '      data _null_; ';
+  put '        set &fmtds end=last; ';
+  put '        name=upcase(name); ';
+  put '        /* fix formats */ ';
+  put '        if type=2 or type=6 then do; ';
+  put '          length fmt $49.; ';
+  put '          if format='''' then fmt=cats(''$'',length,''.''); ';
+  put '          else if formatl=0 then fmt=cats(format,''.''); ';
+  put '          else fmt=cats(format,formatl,''.''); ';
+  put '          newlen=max(formatl,length); ';
+  put '        end; ';
+  put '        else do; ';
+  put '          if format='''' then fmt=''best.''; ';
+  put '          else if formatl=0 then fmt=cats(format,''.''); ';
+  put '          else if formatd=0 then fmt=cats(format,formatl,''.''); ';
+  put '          else fmt=cats(format,formatl,''.'',formatd); ';
+  put '          /* needs to be wide, for datetimes etc */ ';
+  put '          newlen=max(length,formatl,24); ';
+  put '        end; ';
+  put '        /* 32 char unique name */ ';
+  put '        newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put ' ';
+  put '        call symputx(cats(''name'',_n_),name,''l''); ';
+  put '        call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '        call symputx(cats(''len'',_n_),newlen,''l''); ';
+  put '        call symputx(cats(''fmt'',_n_),fmt,''l''); ';
+  put '        call symputx(cats(''type'',_n_),type,''l''); ';
+  put '        if last then call symputx(''nobs'',_n_,''l''); ';
+  put '      run; ';
+  put '      data &fmtds; ';
+  put '        /* rename on entry */ ';
+  put '        set &ds(rename=( ';
+  put '      %local i; ';
+  put '      %do i=1 %to &nobs; ';
+  put '        &&name&i=&&newname&i ';
+  put '      %end; ';
+  put '        )); ';
+  put '      %do i=1 %to &nobs; ';
+  put '        length &&name&i $&&len&i; ';
+  put '        &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '        drop &&newname&i; ';
+  put '      %end; ';
+  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '      run; ';
+  put '      %let ds=&fmtds; ';
+  put '    %end; /* &fmt=Y */ ';
   put '    data _null_;file &jref mod ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
@@ -8811,7 +9020,7 @@ data _null_;
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
-  put '%mend; ';
+  put '%mend mp_jsonout; ';
   put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
@@ -13253,10 +13462,11 @@ data _null_;
   put "/* Created on %sysfunc(datetime(),datetime19.) by &sysuserid */";
 /* WEBOUT BEGIN */
   put ' ';
-  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=PROCJSON,dbg=0 ';
+  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0 ';
   put ')/*/STORE SOURCE*/; ';
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
+  put '  OPTIONS NOBOMFILE; ';
   put '  data _null_;file &jref encoding=''utf-8''; ';
   put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
   put '  run; ';
@@ -13284,6 +13494,64 @@ data _null_;
   put '      %put &sysmacroname:  &ds NOT FOUND!!!; ';
   put '      %return; ';
   put '    %end; ';
+  put '    %if &fmt=Y %then %do; ';
+  put '      %put converting every variable to a formatted variable; ';
+  put '      /* see mp_ds2fmtds.sas for source */ ';
+  put '      proc contents noprint data=&ds ';
+  put '        out=_data_(keep=name type length format formatl formatd varnum); ';
+  put '      run; ';
+  put '      proc sort; ';
+  put '        by varnum; ';
+  put '      run; ';
+  put '      %local fmtds; ';
+  put '      %let fmtds=%scan(&syslast,2,.); ';
+  put '      /* prepare formats and varnames */ ';
+  put '      data _null_; ';
+  put '        set &fmtds end=last; ';
+  put '        name=upcase(name); ';
+  put '        /* fix formats */ ';
+  put '        if type=2 or type=6 then do; ';
+  put '          length fmt $49.; ';
+  put '          if format='''' then fmt=cats(''$'',length,''.''); ';
+  put '          else if formatl=0 then fmt=cats(format,''.''); ';
+  put '          else fmt=cats(format,formatl,''.''); ';
+  put '          newlen=max(formatl,length); ';
+  put '        end; ';
+  put '        else do; ';
+  put '          if format='''' then fmt=''best.''; ';
+  put '          else if formatl=0 then fmt=cats(format,''.''); ';
+  put '          else if formatd=0 then fmt=cats(format,formatl,''.''); ';
+  put '          else fmt=cats(format,formatl,''.'',formatd); ';
+  put '          /* needs to be wide, for datetimes etc */ ';
+  put '          newlen=max(length,formatl,24); ';
+  put '        end; ';
+  put '        /* 32 char unique name */ ';
+  put '        newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put ' ';
+  put '        call symputx(cats(''name'',_n_),name,''l''); ';
+  put '        call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '        call symputx(cats(''len'',_n_),newlen,''l''); ';
+  put '        call symputx(cats(''fmt'',_n_),fmt,''l''); ';
+  put '        call symputx(cats(''type'',_n_),type,''l''); ';
+  put '        if last then call symputx(''nobs'',_n_,''l''); ';
+  put '      run; ';
+  put '      data &fmtds; ';
+  put '        /* rename on entry */ ';
+  put '        set &ds(rename=( ';
+  put '      %local i; ';
+  put '      %do i=1 %to &nobs; ';
+  put '        &&name&i=&&newname&i ';
+  put '      %end; ';
+  put '        )); ';
+  put '      %do i=1 %to &nobs; ';
+  put '        length &&name&i $&&len&i; ';
+  put '        &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '        drop &&newname&i; ';
+  put '      %end; ';
+  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '      run; ';
+  put '      %let ds=&fmtds; ';
+  put '    %end; /* &fmt=Y */ ';
   put '    data _null_;file &jref mod ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
@@ -13367,7 +13635,7 @@ data _null_;
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
-  put '%mend; ';
+  put '%mend mp_jsonout; ';
   put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y); ';
   put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
   put '  sasjs_tables SYS_JES_JOB_URI; ';

base/mf_abort.sas

@@ -1,23 +1,11 @@
 /**
   @file
-  @brief abort gracefully according to context
-  @details Do not use directly!  See bottom of explanation for details.
+  @brief to be deprecated
+  @details We will deprecate this macro in 2022
 
-  Configures an abort mechanism according to site specific policies or the
-    particulars of an environment.  For instance, can stream custom
-    results back to the client in an STP Web App context, or completely stop
-    in the case of a batch run.
+  As you can see, it's not a macro function.
 
-  For the sharp eyed readers - this is no longer a macro function!! It became
-  a macro procedure during a project and now it's kinda stuck that way until
-  that project is updated (if it's ever updated).  In the meantime we created
-  `mp_abort` which is just a wrapper for this one, and so we recomend you use
-  that for forwards compatibility reasons.
-
-  @param mac= to contain the name of the calling macro
-  @param type= deprecated.  Not used.
-  @param msg= message to be returned
-  @param iftrue= supply a condition under which the macro should be executed.
+  Use mp_abort.sas instead.
 
   @version 9.2
   @author Allan Bowe

base/mf_getuniquename.sas

@@ -18,5 +18,5 @@
 
 
 %macro mf_getuniquename(prefix=MC);
-  &prefix.%substr(%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32-%length(&prefix))
-%mend;
+&prefix.%substr(%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32-%length(&prefix))
+%mend mf_getuniquename;

base/mp_abort.sas

@@ -128,12 +128,13 @@
       if debug ge '"131"' then put '>>weboutEND<<';
     run;
 
-    %let syscc=0;
     %if %symexist(_metaport) %then %do;
       data _null_;
-        if symexist('sysprocessmode')
-          then if symget("sysprocessmode")="SAS Stored Process Server"
-            then rc=stpsrvset('program error', 0);
+        if symexist('sysprocessmode') then
+          if symget("sysprocessmode")="SAS Stored Process Server" then do;
+            rc=stpsrvset('program error', 0);
+            call symputx("syscc",0,"g");
+          end;
       run;
     %end;
     /**

base/mp_ds2fmtds.sas

@@ -0,0 +1,98 @@
+/**
+  @file
+  @brief Converts every value in a dataset to it's formatted value
+  @details Converts every value to it's formatted value.  All variables will
+  become character, and will be in the same order.
+
+  Usage:
+
+      %mp_ds2fmtds(sashelp.cars,work.cars)
+
+  @param [in] libds The library.dataset to be converted
+  @param [out] outds The dataset to create.
+
+  <h4> Related Macros <h4>
+  @li mp_jsonout.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_ds2fmtds(libds, outds
+)/*/STORE SOURCE*/;
+
+/* validations */
+%if not %sysfunc(exist(&libds)) %then %do;
+  %put %str(WARN)ING:  &libds does not exist;
+  %return;
+%end;
+%if %index(&libds,.)=0 %then %let libds=WORK.&libds;
+
+/* grab metadata */
+proc contents noprint data=&libds
+  out=_data_(keep=name type length format formatl formatd varnum);
+run;
+proc sort;
+  by varnum;
+run;
+
+/* prepare formats and varnames */
+data _null_;
+  set &syslast end=last;
+  name=upcase(name);
+  /* fix formats */
+  if type=2 or type=6 then do;
+    length fmt $49.;
+    if format='' then fmt=cats('$',length,'.');
+    else if formatl=0 then fmt=cats(format,'.');
+    else fmt=cats(format,formatl,'.');
+    newlen=max(formatl,length);
+  end;
+  else do;
+    if format='' then fmt='best.';
+    else if formatl=0 then fmt=cats(format,'.');
+    else if formatd=0 then fmt=cats(format,formatl,'.');
+    else fmt=cats(format,formatl,'.',formatd);
+    /* needs to be wide, for datetimes etc */
+    newlen=max(length,formatl,24);
+  end;
+  /* 32 char unique name */
+  newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
+
+  call symputx(cats('name',_n_),name,'l');
+  call symputx(cats('newname',_n_),newname,'l');
+  call symputx(cats('len',_n_),newlen,'l');
+  call symputx(cats('fmt',_n_),fmt,'l');
+  call symputx(cats('type',_n_),type,'l');
+  if last then call symputx('nobs',_n_,'l');
+run;
+
+/* clean up */
+proc sql;
+drop table &syslast;
+
+%if &nobs=0 %then %do;
+  %put Dataset &libds has no columns!
+  data &outds;
+    set &libds;
+  run;
+  %return;
+%end;
+
+data &outds;
+  /* rename on entry */
+  set &libds(rename=(
+%local i;
+%do i=1 %to &nobs;
+  &&name&i=&&newname&i
+%end;
+  ));
+%do i=1 %to &nobs;
+  length &&name&i $&&len&i;
+  &&name&i=left(put(&&newname&i,&&fmt&i));
+  drop &&newname&i;
+%end;
+  if _error_ then call symputx('syscc',1012);
+run;
+
+%mend mp_ds2fmtds;

base/mp_filtercheck.sas

@@ -45,7 +45,6 @@
   @li mf_getuniquefileref.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
-  @li mf_nobs.sas
   @li mp_filtergenerate.sas
   @li mp_filtervalidate.sas
 
@@ -158,18 +157,23 @@ data &outds;
 
 run;
 
+%local nobs;
+%let nobs=0;
 data _null_;
-  set &outds;
-  call symputx('REASON_CD',reason_cd,'l');
-  stop;
+  set &outds end=last;
+  putlog (_all_)(=);
+  if last then do;
+    call symputx('REASON_CD',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
+  end;
 run;
 
-%mp_abort(iftrue=(&abort=YES and %mf_nobs(&outds)>0),
+%mp_abort(iftrue=(&abort=YES and &nobs>0),
   mac=&sysmacroname,
-  msg=%str(Filter issues in &inds, reason: &reason_cd, details in &outds)
+  msg=%str(&nobs filter issues in &inds, reason: &reason_cd, details in &outds)
 )
 
-%if %mf_nobs(&outds)>0 %then %do;
+%if &nobs>0 %then %do;
   %let syscc=1008;
   %return;
 %end;

base/mp_jsonout.sas

@@ -48,6 +48,8 @@
   @param dbg= DEPRECATED - was used to conditionally add PRETTY to
     proc json but this can cause line truncation in large files.
 
+  <h4> Related Macros <h4>
+  @li mp_ds2fmtds.sas
 
   @version 9.2
   @author Allan Bowe
@@ -55,10 +57,11 @@
 
 **/
 
-%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=PROCJSON,dbg=0
+%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0
 )/*/STORE SOURCE*/;
 %put output location=&jref;
 %if &action=OPEN %then %do;
+  OPTIONS NOBOMFILE;
   data _null_;file &jref encoding='utf-8';
     put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
   run;
@@ -86,6 +89,64 @@
       %put &sysmacroname:  &ds NOT FOUND!!!;
       %return;
     %end;
+    %if &fmt=Y %then %do;
+      %put converting every variable to a formatted variable;
+      /* see mp_ds2fmtds.sas for source */
+      proc contents noprint data=&ds
+        out=_data_(keep=name type length format formatl formatd varnum);
+      run;
+      proc sort;
+        by varnum;
+      run;
+      %local fmtds;
+      %let fmtds=%scan(&syslast,2,.);
+      /* prepare formats and varnames */
+      data _null_;
+        set &fmtds end=last;
+        name=upcase(name);
+        /* fix formats */
+        if type=2 or type=6 then do;
+          length fmt $49.;
+          if format='' then fmt=cats('$',length,'.');
+          else if formatl=0 then fmt=cats(format,'.');
+          else fmt=cats(format,formatl,'.');
+          newlen=max(formatl,length);
+        end;
+        else do;
+          if format='' then fmt='best.';
+          else if formatl=0 then fmt=cats(format,'.');
+          else if formatd=0 then fmt=cats(format,formatl,'.');
+          else fmt=cats(format,formatl,'.',formatd);
+          /* needs to be wide, for datetimes etc */
+          newlen=max(length,formatl,24);
+        end;
+        /* 32 char unique name */
+        newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
+
+        call symputx(cats('name',_n_),name,'l');
+        call symputx(cats('newname',_n_),newname,'l');
+        call symputx(cats('len',_n_),newlen,'l');
+        call symputx(cats('fmt',_n_),fmt,'l');
+        call symputx(cats('type',_n_),type,'l');
+        if last then call symputx('nobs',_n_,'l');
+      run;
+      data &fmtds;
+        /* rename on entry */
+        set &ds(rename=(
+      %local i;
+      %do i=1 %to &nobs;
+        &&name&i=&&newname&i
+      %end;
+        ));
+      %do i=1 %to &nobs;
+        length &&name&i $&&len&i;
+        &&name&i=left(put(&&newname&i,&&fmt&i));
+        drop &&newname&i;
+      %end;
+        if _error_ then call symputx('syscc',1012);
+      run;
+      %let ds=&fmtds;
+    %end; /* &fmt=Y */
     data _null_;file &jref mod ;
       put "["; call symputx('cols',0,'l');
     proc sort
@@ -169,4 +230,4 @@
     put "}";
   run;
 %end;
-%mend;
+%mend mp_jsonout;

meta/mm_createwebservice.sas

@@ -86,10 +86,11 @@ data _null_;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
   put ' ';
-  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=PROCJSON,dbg=0 ';
+  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0 ';
   put ')/*/STORE SOURCE*/; ';
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
+  put '  OPTIONS NOBOMFILE; ';
   put '  data _null_;file &jref encoding=''utf-8''; ';
   put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
   put '  run; ';
@@ -117,6 +118,64 @@ data _null_;
   put '      %put &sysmacroname:  &ds NOT FOUND!!!; ';
   put '      %return; ';
   put '    %end; ';
+  put '    %if &fmt=Y %then %do; ';
+  put '      %put converting every variable to a formatted variable; ';
+  put '      /* see mp_ds2fmtds.sas for source */ ';
+  put '      proc contents noprint data=&ds ';
+  put '        out=_data_(keep=name type length format formatl formatd varnum); ';
+  put '      run; ';
+  put '      proc sort; ';
+  put '        by varnum; ';
+  put '      run; ';
+  put '      %local fmtds; ';
+  put '      %let fmtds=%scan(&syslast,2,.); ';
+  put '      /* prepare formats and varnames */ ';
+  put '      data _null_; ';
+  put '        set &fmtds end=last; ';
+  put '        name=upcase(name); ';
+  put '        /* fix formats */ ';
+  put '        if type=2 or type=6 then do; ';
+  put '          length fmt $49.; ';
+  put '          if format='''' then fmt=cats(''$'',length,''.''); ';
+  put '          else if formatl=0 then fmt=cats(format,''.''); ';
+  put '          else fmt=cats(format,formatl,''.''); ';
+  put '          newlen=max(formatl,length); ';
+  put '        end; ';
+  put '        else do; ';
+  put '          if format='''' then fmt=''best.''; ';
+  put '          else if formatl=0 then fmt=cats(format,''.''); ';
+  put '          else if formatd=0 then fmt=cats(format,formatl,''.''); ';
+  put '          else fmt=cats(format,formatl,''.'',formatd); ';
+  put '          /* needs to be wide, for datetimes etc */ ';
+  put '          newlen=max(length,formatl,24); ';
+  put '        end; ';
+  put '        /* 32 char unique name */ ';
+  put '        newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put ' ';
+  put '        call symputx(cats(''name'',_n_),name,''l''); ';
+  put '        call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '        call symputx(cats(''len'',_n_),newlen,''l''); ';
+  put '        call symputx(cats(''fmt'',_n_),fmt,''l''); ';
+  put '        call symputx(cats(''type'',_n_),type,''l''); ';
+  put '        if last then call symputx(''nobs'',_n_,''l''); ';
+  put '      run; ';
+  put '      data &fmtds; ';
+  put '        /* rename on entry */ ';
+  put '        set &ds(rename=( ';
+  put '      %local i; ';
+  put '      %do i=1 %to &nobs; ';
+  put '        &&name&i=&&newname&i ';
+  put '      %end; ';
+  put '        )); ';
+  put '      %do i=1 %to &nobs; ';
+  put '        length &&name&i $&&len&i; ';
+  put '        &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '        drop &&newname&i; ';
+  put '      %end; ';
+  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '      run; ';
+  put '      %let ds=&fmtds; ';
+  put '    %end; /* &fmt=Y */ ';
   put '    data _null_;file &jref mod ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
@@ -200,7 +259,7 @@ data _null_;
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
-  put '%mend; ';
+  put '%mend mp_jsonout; ';
   put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';

tests/base/mp_ds2fmtds.test.sas

@@ -0,0 +1,28 @@
+/**
+  @file
+  @brief Testing mp_ds2fmtds.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2fmtds.sas
+  @li mp_assert.sas
+
+**/
+
+proc sql;
+create table test as select * from dictionary.tables where libname='SASHELP';
+
+filename inc temp;
+data _null_;
+  set work.test;
+  file inc;
+  line=cats('%mp_ds2fmtds(sashelp.',memname,',',memname,')');
+  put line;
+run;
+
+%inc inc;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking tables were created successfully,
+  outds=work.test_results
+)

tests/base/mp_jsonout.test.sas

@@ -0,0 +1,42 @@
+/**
+  @file
+  @brief Testing mp_jsonout.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_jsonout.sas
+  @li mp_assert.sas
+
+**/
+
+filename webref temp;
+
+data demo;
+  dtval=date();
+  format dtval date9.;
+  compare=put(date(),date9.);
+  call symputx('compare',compare);
+run;
+
+%mp_jsonout(OPEN,jref=webref)
+%mp_jsonout(OBJ,demo,jref=webref,fmt=Y)
+%mp_jsonout(CLOSE,jref=webref)
+
+data _null_;
+  infile webref;
+  input;
+  putlog _infile_;
+run;
+
+libname web JSON fileref=webref;
+%let dtval=0;
+data work.test;
+  set web.demo;
+  call symputx('dtval',dtval);
+run;
+
+
+%mp_assert(
+  iftrue=(&dtval=&compare),
+  desc=Checking tables were created successfully,
+  outds=work.test_results
+)

viya/mv_createwebservice.sas

@@ -237,10 +237,11 @@ data _null_;
   put "/* Created on %sysfunc(datetime(),datetime19.) by &sysuserid */";
 /* WEBOUT BEGIN */
   put ' ';
-  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=PROCJSON,dbg=0 ';
+  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0 ';
   put ')/*/STORE SOURCE*/; ';
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
+  put '  OPTIONS NOBOMFILE; ';
   put '  data _null_;file &jref encoding=''utf-8''; ';
   put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
   put '  run; ';
@@ -268,6 +269,64 @@ data _null_;
   put '      %put &sysmacroname:  &ds NOT FOUND!!!; ';
   put '      %return; ';
   put '    %end; ';
+  put '    %if &fmt=Y %then %do; ';
+  put '      %put converting every variable to a formatted variable; ';
+  put '      /* see mp_ds2fmtds.sas for source */ ';
+  put '      proc contents noprint data=&ds ';
+  put '        out=_data_(keep=name type length format formatl formatd varnum); ';
+  put '      run; ';
+  put '      proc sort; ';
+  put '        by varnum; ';
+  put '      run; ';
+  put '      %local fmtds; ';
+  put '      %let fmtds=%scan(&syslast,2,.); ';
+  put '      /* prepare formats and varnames */ ';
+  put '      data _null_; ';
+  put '        set &fmtds end=last; ';
+  put '        name=upcase(name); ';
+  put '        /* fix formats */ ';
+  put '        if type=2 or type=6 then do; ';
+  put '          length fmt $49.; ';
+  put '          if format='''' then fmt=cats(''$'',length,''.''); ';
+  put '          else if formatl=0 then fmt=cats(format,''.''); ';
+  put '          else fmt=cats(format,formatl,''.''); ';
+  put '          newlen=max(formatl,length); ';
+  put '        end; ';
+  put '        else do; ';
+  put '          if format='''' then fmt=''best.''; ';
+  put '          else if formatl=0 then fmt=cats(format,''.''); ';
+  put '          else if formatd=0 then fmt=cats(format,formatl,''.''); ';
+  put '          else fmt=cats(format,formatl,''.'',formatd); ';
+  put '          /* needs to be wide, for datetimes etc */ ';
+  put '          newlen=max(length,formatl,24); ';
+  put '        end; ';
+  put '        /* 32 char unique name */ ';
+  put '        newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put ' ';
+  put '        call symputx(cats(''name'',_n_),name,''l''); ';
+  put '        call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '        call symputx(cats(''len'',_n_),newlen,''l''); ';
+  put '        call symputx(cats(''fmt'',_n_),fmt,''l''); ';
+  put '        call symputx(cats(''type'',_n_),type,''l''); ';
+  put '        if last then call symputx(''nobs'',_n_,''l''); ';
+  put '      run; ';
+  put '      data &fmtds; ';
+  put '        /* rename on entry */ ';
+  put '        set &ds(rename=( ';
+  put '      %local i; ';
+  put '      %do i=1 %to &nobs; ';
+  put '        &&name&i=&&newname&i ';
+  put '      %end; ';
+  put '        )); ';
+  put '      %do i=1 %to &nobs; ';
+  put '        length &&name&i $&&len&i; ';
+  put '        &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '        drop &&newname&i; ';
+  put '      %end; ';
+  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '      run; ';
+  put '      %let ds=&fmtds; ';
+  put '    %end; /* &fmt=Y */ ';
   put '    data _null_;file &jref mod ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
@@ -351,7 +410,7 @@ data _null_;
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
-  put '%mend; ';
+  put '%mend mp_jsonout; ';
   put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y); ';
   put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
   put '  sasjs_tables SYS_JES_JOB_URI; ';