Merge pull request #296 from sasjs/allanbowe/improve-efficiency-of-295

fix: performance optimisations. closes #295

all.sas

@@ -3775,7 +3775,7 @@ run;
 
 %mend mp_chop;
 /**
-  @file mp_cleancsv.sas
+  @file
   @brief Fixes embedded cr / lf / crlf in CSV
   @details CSVs will sometimes contain lf or crlf within quotes (eg when
   saved by excel).  When the termstr is ALSO lf or crlf that can be tricky
@@ -3783,14 +3783,16 @@ run;
   This macro converts any csv to follow the convention of a windows excel file,
   applying CRLF line endings and converting embedded cr and crlf to lf.
 
-  usage:
+  Usage:
 
       fileref mycsv "/path/your/csv";
       %mp_cleancsv(in=mycsv,out=/path/new.csv)
 
-  @param in= provide path or fileref to input csv
+  @param in= (NOTPROVIDED) Provide path or fileref to input csv. If a period is
-  @param out= output path or fileref to output csv
+    found, it is assumed to be a file.
-  @param qchar= quote char - hex code 22 is the double quote.
+  @param out= (NOTPROVIDED) Output path or fileref to output csv.  If a period
+    is found, it is assumed to be a file.
+  @param qchar= ('22'x) Quote char - hex code 22 is the double quote.
 
   @version 9.2
   @author Allan Bowe
@@ -3832,9 +3834,14 @@ run;
     else do;
       /* outside a quote, change cr and lf to crlf */
       if inchar='0D'x then do;
+        crblank:
         put '0D0A'x;
         input inchar $char1.;
-        if inchar ne '0A'x then do;
+        if inchar='0D'x then do;
+          /* multiple CR indicates CR formatted file with blank lines */
+          goto crblank;
+        end;
+        else if inchar ne '0A'x then do;
           put inchar $char1.;
           if inchar=qchar then isq = mod(isq+1,2);
         end;
@@ -5333,7 +5340,7 @@ run;
   <h4> SAS Macros </h4>
   @li mf_existds.sas
 
-  <h4> Related Macros <h4>
+  <h4> Related Macros </h4>
   @li mp_jsonout.sas
 
   @version 9.2
@@ -8702,7 +8709,7 @@ options
   @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
     that should be converted to JSON
 
-  <h4> Related Macros <h4>
+  <h4> Related Macros </h4>
   @li mp_ds2fmtds.sas
 
   @version 9.2
@@ -8710,14 +8717,13 @@ options
   @source https://github.com/sasjs/core
 
 **/
-
 %macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y
   ,engine=DATASTEP
   ,missing=NULL
   ,showmeta=N
   ,maxobs=MAX
 )/*/STORE SOURCE*/;
-%local tempds colinfo fmtds i numcols stmt_obs;
+%local tempds colinfo fmtds i numcols stmt_obs tempvar lastobs optval;
 %let numcols=0;
 %if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;);
 
@@ -8776,10 +8782,14 @@ options
     end;
     /* 32 char unique name */
     newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
+    maxlenv='maxlen'!!substr(cats(put(md5(name),$hex32.)),1,26);
 
     call symputx(cats('name',_n_),name,'l');
     call symputx(cats('newname',_n_),newname,'l');
+    call symputx(cats('maxlenv',_n_),maxlenv,'l');
     call symputx(cats('length',_n_),length,'l');
+    /* overwritten when fmt=Y */
+    call symputx(cats('fmtlen',_n_),min(32767,ceil((length+3)*1.2)),'l');
     call symputx(cats('fmt',_n_),fmt,'l');
     call symputx(cats('type',_n_),type,'l');
     call symputx(cats('typelong',_n_),typelong,'l');
@@ -8787,6 +8797,9 @@ options
   run;
 
   %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+  proc sql;
+  select count(*) into: lastobs from &ds;
+  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs));
 
   %if &engine=PROCJSON %then %do;
     %if &missing=STRING %then %do;
@@ -8823,24 +8836,49 @@ options
     %end;
 
     %if &fmt=Y %then %do;
-      data _data_;
+      %let tempvar=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      /* need to find max length of formatted values */
+      data _data_(compress=char);
         /* rename on entry */
         set &ds(rename=(
       %do i=1 %to &numcols;
         &&name&i=&&newname&i
       %end;
         ));
+      &stmt_obs;
+      /* formatted values can be up to length 32767 */
+      length
+      %do i=1 %to &numcols;
+        &&name&i
+      %end;
+        $32767;
+      retain &tempvar
+      %do i=1 %to &numcols;
+        &&maxlenv&i
+      %end;
+        0;
+      drop &tempvar
+      %do i=1 %to &numcols;
+        &&newname&i &&maxlenv&i
+      %end;
+        ;
       %do i=1 %to &numcols;
-        /* formatted values can be up to length 32767 */
-        length &&name&i $32767;
         %if &&typelong&i=num %then %do;
-          &&name&i=left(put(&&newname&i,&&fmt&i));
+          &&name&i=cats(put(&&newname&i,&&fmt&i));
         %end;
         %else %do;
           &&name&i=put(&&newname&i,&&fmt&i);
         %end;
-        drop &&newname&i;
+        /* grab max length of each value as we move down the data step */
+        &tempvar=length(&&name&i);
+        if &tempvar>&&maxlenv&i then &&maxlenv&i=&tempvar;
       %end;
+        if _n_=&lastobs then do;
+          /* add a 20% buffer in case of special chars that need to be escaped */
+      %do i=1 %to &numcols;
+          call symputx("fmtlen&i",min(32767,ceil((&&maxlenv&i+3)*1.2)),'l');
+      %end;
+        end;
         if _error_ then call symputx('syscc',1012);
       run;
       %let fmtds=&syslast;
@@ -8858,12 +8896,15 @@ options
     %end;
       other = [best.];
 
+    /* configure varlenchk - as we are explicitly shortening the variables */
+    %let optval=%sysfunc(getoption(varlenchk));
+    options varlenchk=NOWARN;
     data &tempds;
       attrib _all_ label='';
       %do i=1 %to &numcols;
         %if &&typelong&i=char or &fmt=Y %then %do;
-          length &&name&i $32767;
+          length &&name&i $&&fmtlen&i...;
-          format &&name&i $32767.;
+          format &&name&i $&&fmtlen&i...;
         %end;
       %end;
       %if &fmt=Y %then %do;
@@ -8896,6 +8937,7 @@ options
       %end;
     %end;
     run;
+    options varlenchk=&optval;
 
     filename _sjs3 temp lrecl=131068 ;
     data _null_;
@@ -9932,14 +9974,15 @@ put(md5(
 %mend mp_md5;
 /**
   @file
-  @brief Logs the time the macro was executed in a control dataset.
+  @brief Logs a message in a dataset every time it is invoked
-  @details If the dataset does not exist, it is created.  Usage:
+  @details If the dataset does not exist, it is created.
+  Usage:
 
-    %mp_perflog(started)
+      %mp_perflog(started)
-    %mp_perflog()
+      %mp_perflog()
-    %mp_perflog(startanew,libds=work.newdataset)
+      %mp_perflog(startanew,libds=work.newdataset)
-    %mp_perflog(finished,libds=work.newdataset)
+      %mp_perflog(finished,libds=work.newdataset)
-    %mp_perflog(finished)
+      %mp_perflog(finished)
 
 
   @param label Provide label to go into the control dataset
@@ -15126,14 +15169,13 @@ data _null_;
   file sasjs lrecl=3000 ;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
-  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
   put '  ,showmeta=N ';
   put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols stmt_obs; ';
+  put '%local tempds colinfo fmtds i numcols stmt_obs tempvar lastobs optval; ';
   put '%let numcols=0; ';
   put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
@@ -15192,10 +15234,14 @@ data _null_;
   put '    end; ';
   put '    /* 32 char unique name */ ';
   put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put '    maxlenv=''maxlen''!!substr(cats(put(md5(name),$hex32.)),1,26); ';
   put ' ';
   put '    call symputx(cats(''name'',_n_),name,''l''); ';
   put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '    call symputx(cats(''maxlenv'',_n_),maxlenv,''l''); ';
   put '    call symputx(cats(''length'',_n_),length,''l''); ';
+  put '    /* overwritten when fmt=Y */ ';
+  put '    call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.2)),''l''); ';
   put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
@@ -15203,6 +15249,9 @@ data _null_;
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '  proc sql; ';
+  put '  select count(*) into: lastobs from &ds; ';
+  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -15239,24 +15288,49 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      data _data_; ';
+  put '      %let tempvar=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      /* need to find max length of formatted values */ ';
+  put '      data _data_(compress=char); ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
   put '        &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
+  put '      &stmt_obs; ';
+  put '      /* formatted values can be up to length 32767 */ ';
+  put '      length ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&name&i ';
+  put '      %end; ';
+  put '        $32767; ';
+  put '      retain &tempvar ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&maxlenv&i ';
+  put '      %end; ';
+  put '        0; ';
+  put '      drop &tempvar ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&newname&i &&maxlenv&i ';
+  put '      %end; ';
+  put '        ; ';
   put '      %do i=1 %to &numcols; ';
-  put '        /* formatted values can be up to length 32767 */ ';
-  put '        length &&name&i $32767; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
-  put '        drop &&newname&i; ';
+  put '        /* grab max length of each value as we move down the data step */ ';
+  put '        &tempvar=length(&&name&i); ';
+  put '        if &tempvar>&&maxlenv&i then &&maxlenv&i=&tempvar; ';
   put '      %end; ';
+  put '        if _n_=&lastobs then do; ';
+  put '          /* add a 20% buffer in case of special chars that need to be escaped */ ';
+  put '      %do i=1 %to &numcols; ';
+  put '          call symputx("fmtlen&i",min(32767,ceil((&&maxlenv&i+3)*1.2)),''l''); ';
+  put '      %end; ';
+  put '        end; ';
   put '        if _error_ then call symputx(''syscc'',1012); ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
@@ -15274,12 +15348,15 @@ data _null_;
   put '    %end; ';
   put '      other = [best.]; ';
   put ' ';
+  put '    /* configure varlenchk - as we are explicitly shortening the variables */ ';
+  put '    %let optval=%sysfunc(getoption(varlenchk)); ';
+  put '    options varlenchk=NOWARN; ';
   put '    data &tempds; ';
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $32767; ';
+  put '          length &&name&i $&&fmtlen&i...; ';
-  put '          format &&name&i $32767.; ';
+  put '          format &&name&i $&&fmtlen&i...; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -15312,6 +15389,7 @@ data _null_;
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
+  put '    options varlenchk=&optval; ';
   put ' ';
   put '    filename _sjs3 temp lrecl=131068 ; ';
   put '    data _null_; ';
@@ -15402,7 +15480,7 @@ data _null_;
   put ' ';
   put '%mend mf_getuser; ';
   put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
-  put '  ,showmeta=N ';
+  put '  ,showmeta=N,maxobs=MAX ';
   put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
@@ -15469,7 +15547,7 @@ data _null_;
   put ' ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta ';
+  put '    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
   put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
@@ -15510,23 +15588,30 @@ data _null_;
   put '  %end; ';
   put '  /* close off json */ ';
   put '  data _null_;file _sjsref mod encoding=''utf-8''; ';
-  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
+  put '    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; ';
-  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
-  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""_DEBUG"" : ""&_debug"" "; ';
   put '    _METAUSER=quote(trim(symget(''_METAUSER''))); ';
   put '    put ",""_METAUSER"": " _METAUSER; ';
   put '    _METAPERSON=quote(trim(symget(''_METAPERSON''))); ';
   put '    put '',"_METAPERSON": '' _METAPERSON; ';
+  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
+  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
+  put '    put '',"AUTOEXEC" : '' autoexec; ';
+  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
   put '    put ",""SYSENCODING"" : ""&sysencoding"" "; ';
   put '    syserrortext=cats(''"'',tranwrd(symget(''syserrortext''),''"'',''\"''),''"''); ';
   put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
+  put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
+  put '    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; ';
+  put '    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); ';
+  put '    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
+  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
   put '    syswarningtext=cats(''"'',tranwrd(symget(''syswarningtext''),''"'',''\"''),''"''); ';
@@ -18954,7 +19039,8 @@ run;
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
-
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to output JSON
 
   <h4> SAS Macros </h4>
   @li mp_jsonout.sas
@@ -18964,7 +19050,7 @@ run;
 
 **/
 %macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
-  ,showmeta=N
+  ,showmeta=N,maxobs=MAX
 );
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
@@ -19031,7 +19117,7 @@ run;
 
 %else %if &action=ARR or &action=OBJ %then %do;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta
+    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta,maxobs=&maxobs
   )
 %end;
 %else %if &action=CLOSE %then %do;
@@ -19072,23 +19158,30 @@ run;
   %end;
   /* close off json */
   data _null_;file _sjsref mod encoding='utf-8';
-    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
+    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512;
-    put ",""SYSUSERID"" : ""&sysuserid"" ";
-    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""_DEBUG"" : ""&_debug"" ";
     _METAUSER=quote(trim(symget('_METAUSER')));
     put ",""_METAUSER"": " _METAUSER;
     _METAPERSON=quote(trim(symget('_METAPERSON')));
     put ',"_METAPERSON": ' _METAPERSON;
+    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ',"_PROGRAM" : ' _PROGRAM ;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
+    put ',"AUTOEXEC" : ' autoexec;
+    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""SYSCC"" : ""&syscc"" ";
     put ",""SYSENCODING"" : ""&sysencoding"" ";
     syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
     put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
+    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
+    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
+    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
     syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');
@@ -19976,14 +20069,13 @@ data _null_;
   file &sasjsref termstr=crlf lrecl=512;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
-  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
   put '  ,showmeta=N ';
   put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols stmt_obs; ';
+  put '%local tempds colinfo fmtds i numcols stmt_obs tempvar lastobs optval; ';
   put '%let numcols=0; ';
   put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
@@ -20042,10 +20134,14 @@ data _null_;
   put '    end; ';
   put '    /* 32 char unique name */ ';
   put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put '    maxlenv=''maxlen''!!substr(cats(put(md5(name),$hex32.)),1,26); ';
   put ' ';
   put '    call symputx(cats(''name'',_n_),name,''l''); ';
   put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '    call symputx(cats(''maxlenv'',_n_),maxlenv,''l''); ';
   put '    call symputx(cats(''length'',_n_),length,''l''); ';
+  put '    /* overwritten when fmt=Y */ ';
+  put '    call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.2)),''l''); ';
   put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
@@ -20053,6 +20149,9 @@ data _null_;
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '  proc sql; ';
+  put '  select count(*) into: lastobs from &ds; ';
+  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -20089,24 +20188,49 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      data _data_; ';
+  put '      %let tempvar=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      /* need to find max length of formatted values */ ';
+  put '      data _data_(compress=char); ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
   put '        &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
+  put '      &stmt_obs; ';
+  put '      /* formatted values can be up to length 32767 */ ';
+  put '      length ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&name&i ';
+  put '      %end; ';
+  put '        $32767; ';
+  put '      retain &tempvar ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&maxlenv&i ';
+  put '      %end; ';
+  put '        0; ';
+  put '      drop &tempvar ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&newname&i &&maxlenv&i ';
+  put '      %end; ';
+  put '        ; ';
   put '      %do i=1 %to &numcols; ';
-  put '        /* formatted values can be up to length 32767 */ ';
-  put '        length &&name&i $32767; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
-  put '        drop &&newname&i; ';
+  put '        /* grab max length of each value as we move down the data step */ ';
+  put '        &tempvar=length(&&name&i); ';
+  put '        if &tempvar>&&maxlenv&i then &&maxlenv&i=&tempvar; ';
   put '      %end; ';
+  put '        if _n_=&lastobs then do; ';
+  put '          /* add a 20% buffer in case of special chars that need to be escaped */ ';
+  put '      %do i=1 %to &numcols; ';
+  put '          call symputx("fmtlen&i",min(32767,ceil((&&maxlenv&i+3)*1.2)),''l''); ';
+  put '      %end; ';
+  put '        end; ';
   put '        if _error_ then call symputx(''syscc'',1012); ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
@@ -20124,12 +20248,15 @@ data _null_;
   put '    %end; ';
   put '      other = [best.]; ';
   put ' ';
+  put '    /* configure varlenchk - as we are explicitly shortening the variables */ ';
+  put '    %let optval=%sysfunc(getoption(varlenchk)); ';
+  put '    options varlenchk=NOWARN; ';
   put '    data &tempds; ';
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $32767; ';
+  put '          length &&name&i $&&fmtlen&i...; ';
-  put '          format &&name&i $32767.; ';
+  put '          format &&name&i $&&fmtlen&i...; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -20162,6 +20289,7 @@ data _null_;
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
+  put '    options varlenchk=&optval; ';
   put ' ';
   put '    filename _sjs3 temp lrecl=131068 ; ';
   put '    data _null_; ';
@@ -20253,7 +20381,7 @@ data _null_;
   put '%mend mf_getuser; ';
   put ' ';
   put '%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
-  put '  ,showmeta=N ';
+  put '  ,showmeta=N,maxobs=MAX ';
   put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
@@ -20312,7 +20440,7 @@ data _null_;
   put '    %let missing=NULL; ';
   put '  %end; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta ';
+  put '    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
   put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
@@ -20352,11 +20480,13 @@ data _null_;
   put '  %end; ';
   put '  /* close off json */ ';
   put '  data _null_;file &fref mod encoding=''utf-8'' termstr=lf lrecl=32767; ';
-  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
+  put '    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; ';
-  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
-  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""_DEBUG"" : ""&_debug"" "; ';
+  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
+  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
+  put '    put '',"AUTOEXEC" : '' autoexec; ';
+  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
   put '    put ",""SYSENCODING"" : ""&sysencoding"" "; ';
   put '    syserrortext=cats(''"'',tranwrd(symget(''syserrortext''),''"'',''\"''),''"''); ';
@@ -20366,21 +20496,18 @@ data _null_;
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
   put '    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; ';
-  put '    length SYSPROCESSNAME $512; ';
   put '    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); ';
   put '    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" "; ';
+  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
   put '    syswarningtext=cats(''"'',tranwrd(symget(''syswarningtext''),''"'',''\"''),''"''); ';
   put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
-  put '    length autoexec $512; ';
-  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
-  put '    put '',"AUTOEXEC" : '' autoexec; ';
   put '    length memsize $32; ';
   put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
   put '    memsize=quote(cats(memsize)); ';
@@ -21228,6 +21355,8 @@ run;
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to output JSON
 
   <h4> SAS Macros </h4>
   @li mf_getuser.sas
@@ -21244,7 +21373,7 @@ run;
 **/
 
 %macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
-  ,showmeta=N
+  ,showmeta=N,maxobs=MAX
 );
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
@@ -21303,7 +21432,7 @@ run;
     %let missing=NULL;
   %end;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta
+    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs
   )
 %end;
 %else %if &action=CLOSE %then %do;
@@ -21343,11 +21472,13 @@ run;
   %end;
   /* close off json */
   data _null_;file &fref mod encoding='utf-8' termstr=lf lrecl=32767;
-    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
+    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512;
-    put ",""SYSUSERID"" : ""&sysuserid"" ";
-    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""_DEBUG"" : ""&_debug"" ";
+    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ',"_PROGRAM" : ' _PROGRAM ;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
+    put ',"AUTOEXEC" : ' autoexec;
+    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""SYSCC"" : ""&syscc"" ";
     put ",""SYSENCODING"" : ""&sysencoding"" ";
     syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
@@ -21357,21 +21488,18 @@ run;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
     put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
-    length SYSPROCESSNAME $512;
     SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
     put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" ";
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
     syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');
     put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
-    length autoexec $512;
-    autoexec=quote(urlencode(trim(getoption('autoexec'))));
-    put ',"AUTOEXEC" : ' autoexec;
     length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
     memsize=quote(cats(memsize));
@@ -22354,14 +22482,13 @@ data _null_;
   file &adapter;
   put "/* Created on %sysfunc(datetime(),datetime19.) by &sysuserid */";
 /* WEBOUT BEGIN */
-  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
   put '  ,showmeta=N ';
   put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols stmt_obs; ';
+  put '%local tempds colinfo fmtds i numcols stmt_obs tempvar lastobs optval; ';
   put '%let numcols=0; ';
   put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
@@ -22420,10 +22547,14 @@ data _null_;
   put '    end; ';
   put '    /* 32 char unique name */ ';
   put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put '    maxlenv=''maxlen''!!substr(cats(put(md5(name),$hex32.)),1,26); ';
   put ' ';
   put '    call symputx(cats(''name'',_n_),name,''l''); ';
   put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '    call symputx(cats(''maxlenv'',_n_),maxlenv,''l''); ';
   put '    call symputx(cats(''length'',_n_),length,''l''); ';
+  put '    /* overwritten when fmt=Y */ ';
+  put '    call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.2)),''l''); ';
   put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
@@ -22431,6 +22562,9 @@ data _null_;
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '  proc sql; ';
+  put '  select count(*) into: lastobs from &ds; ';
+  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -22467,24 +22601,49 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      data _data_; ';
+  put '      %let tempvar=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      /* need to find max length of formatted values */ ';
+  put '      data _data_(compress=char); ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
   put '        &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
+  put '      &stmt_obs; ';
+  put '      /* formatted values can be up to length 32767 */ ';
+  put '      length ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&name&i ';
+  put '      %end; ';
+  put '        $32767; ';
+  put '      retain &tempvar ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&maxlenv&i ';
+  put '      %end; ';
+  put '        0; ';
+  put '      drop &tempvar ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&newname&i &&maxlenv&i ';
+  put '      %end; ';
+  put '        ; ';
   put '      %do i=1 %to &numcols; ';
-  put '        /* formatted values can be up to length 32767 */ ';
-  put '        length &&name&i $32767; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
-  put '        drop &&newname&i; ';
+  put '        /* grab max length of each value as we move down the data step */ ';
+  put '        &tempvar=length(&&name&i); ';
+  put '        if &tempvar>&&maxlenv&i then &&maxlenv&i=&tempvar; ';
   put '      %end; ';
+  put '        if _n_=&lastobs then do; ';
+  put '          /* add a 20% buffer in case of special chars that need to be escaped */ ';
+  put '      %do i=1 %to &numcols; ';
+  put '          call symputx("fmtlen&i",min(32767,ceil((&&maxlenv&i+3)*1.2)),''l''); ';
+  put '      %end; ';
+  put '        end; ';
   put '        if _error_ then call symputx(''syscc'',1012); ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
@@ -22502,12 +22661,15 @@ data _null_;
   put '    %end; ';
   put '      other = [best.]; ';
   put ' ';
+  put '    /* configure varlenchk - as we are explicitly shortening the variables */ ';
+  put '    %let optval=%sysfunc(getoption(varlenchk)); ';
+  put '    options varlenchk=NOWARN; ';
   put '    data &tempds; ';
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $32767; ';
+  put '          length &&name&i $&&fmtlen&i...; ';
-  put '          format &&name&i $32767.; ';
+  put '          format &&name&i $&&fmtlen&i...; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -22540,6 +22702,7 @@ data _null_;
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
+  put '    options varlenchk=&optval; ';
   put ' ';
   put '    filename _sjs3 temp lrecl=131068 ; ';
   put '    data _null_; ';
@@ -22630,7 +22793,7 @@ data _null_;
   put ' ';
   put '%mend mf_getuser; ';
   put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL ';
-  put '  ,showmeta=N ';
+  put '  ,showmeta=N,maxobs=MAX ';
   put '); ';
   put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
   put '  sasjs_tables SYS_JES_JOB_URI; ';
@@ -22732,8 +22895,8 @@ data _null_;
   put '  run; ';
   put '%end; ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
-  put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
+  put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '      ,jref=&fref,engine=DATASTEP,missing=&missing,showmeta=&showmeta ';
+  put '      ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
   put '    ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
@@ -22769,20 +22932,28 @@ data _null_;
   put ' ';
   put '  /* close off json */ ';
   put '  data _null_;file &fref mod; ';
+  put '    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; ';
+  put '    put ",""_DEBUG"" : ""&_debug"" "; ';
   put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
-  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
+  put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
+  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
+  put '    put '',"AUTOEXEC" : '' autoexec; ';
   put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    SYS_JES_JOB_URI=quote(trim(resolve(symget(''SYS_JES_JOB_URI'')))); ';
   put '    put '',"SYS_JES_JOB_URI" : '' SYS_JES_JOB_URI ; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
-  put '    put ",""_DEBUG"" : ""&_debug"" "; ';
-  put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
   put '    syserrortext=cats(''"'',tranwrd(symget(''syserrortext''),''"'',''\"''),''"''); ';
   put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
+  put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
+  put '    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; ';
+  put '    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); ';
+  put '    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; ';
+  put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
+  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
   put '    syswarningtext=cats(''"'',tranwrd(symget(''syswarningtext''),''"'',''\"''),''"''); ';
@@ -26408,6 +26579,8 @@ filename &fref1 clear;
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to output JSON
 
   <h4> SAS Macros </h4>
   @li mp_jsonout.sas
@@ -26418,7 +26591,7 @@ filename &fref1 clear;
 
 **/
 %macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL
-  ,showmeta=N
+  ,showmeta=N,maxobs=MAX
 );
 %global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name
   sasjs_tables SYS_JES_JOB_URI;
@@ -26520,8 +26693,8 @@ filename &fref1 clear;
   run;
 %end;
 %else %if &action=ARR or &action=OBJ %then %do;
-    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt
+    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-      ,jref=&fref,engine=DATASTEP,missing=&missing,showmeta=&showmeta
+      ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs
     )
 %end;
 %else %if &action=CLOSE %then %do;
@@ -26557,20 +26730,28 @@ filename &fref1 clear;
 
   /* close off json */
   data _null_;file &fref mod;
+    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512;
+    put ",""_DEBUG"" : ""&_debug"" ";
     _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
-    put ",""SYSUSERID"" : ""&sysuserid"" ";
+    put ',"_PROGRAM" : ' _PROGRAM ;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
+    put ',"AUTOEXEC" : ' autoexec;
     put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     SYS_JES_JOB_URI=quote(trim(resolve(symget('SYS_JES_JOB_URI'))));
     put ',"SYS_JES_JOB_URI" : ' SYS_JES_JOB_URI ;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
-    put ",""_DEBUG"" : ""&_debug"" ";
-    put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
     syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
     put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
+    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
+    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
+    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
+    put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
     syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');

base/mp_cleancsv.sas

@@ -1,5 +1,5 @@
 /**
-  @file mp_cleancsv.sas
+  @file
   @brief Fixes embedded cr / lf / crlf in CSV
   @details CSVs will sometimes contain lf or crlf within quotes (eg when
   saved by excel).  When the termstr is ALSO lf or crlf that can be tricky
@@ -7,14 +7,16 @@
   This macro converts any csv to follow the convention of a windows excel file,
   applying CRLF line endings and converting embedded cr and crlf to lf.
 
-  usage:
+  Usage:
 
       fileref mycsv "/path/your/csv";
       %mp_cleancsv(in=mycsv,out=/path/new.csv)
 
-  @param in= provide path or fileref to input csv
+  @param in= (NOTPROVIDED) Provide path or fileref to input csv. If a period is
-  @param out= output path or fileref to output csv
+    found, it is assumed to be a file.
-  @param qchar= quote char - hex code 22 is the double quote.
+  @param out= (NOTPROVIDED) Output path or fileref to output csv.  If a period
+    is found, it is assumed to be a file.
+  @param qchar= ('22'x) Quote char - hex code 22 is the double quote.
 
   @version 9.2
   @author Allan Bowe
@@ -56,9 +58,14 @@
     else do;
       /* outside a quote, change cr and lf to crlf */
       if inchar='0D'x then do;
+        crblank:
         put '0D0A'x;
         input inchar $char1.;
-        if inchar ne '0A'x then do;
+        if inchar='0D'x then do;
+          /* multiple CR indicates CR formatted file with blank lines */
+          goto crblank;
+        end;
+        else if inchar ne '0A'x then do;
           put inchar $char1.;
           if inchar=qchar then isq = mod(isq+1,2);
         end;

base/mp_ds2fmtds.sas

@@ -17,7 +17,7 @@
   <h4> SAS Macros </h4>
   @li mf_existds.sas
 
-  <h4> Related Macros <h4>
+  <h4> Related Macros </h4>
   @li mp_jsonout.sas
 
   @version 9.2

base/mp_jsonout.sas

@@ -62,7 +62,7 @@
   @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
     that should be converted to JSON
 
-  <h4> Related Macros <h4>
+  <h4> Related Macros </h4>
   @li mp_ds2fmtds.sas
 
   @version 9.2
@@ -70,14 +70,13 @@
   @source https://github.com/sasjs/core
 
 **/
-
 %macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y
   ,engine=DATASTEP
   ,missing=NULL
   ,showmeta=N
   ,maxobs=MAX
 )/*/STORE SOURCE*/;
-%local tempds colinfo fmtds i numcols stmt_obs;
+%local tempds colinfo fmtds i numcols stmt_obs tempvar lastobs optval;
 %let numcols=0;
 %if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;);
 
@@ -136,10 +135,14 @@
     end;
     /* 32 char unique name */
     newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
+    maxlenv='maxlen'!!substr(cats(put(md5(name),$hex32.)),1,26);
 
     call symputx(cats('name',_n_),name,'l');
     call symputx(cats('newname',_n_),newname,'l');
+    call symputx(cats('maxlenv',_n_),maxlenv,'l');
     call symputx(cats('length',_n_),length,'l');
+    /* overwritten when fmt=Y */
+    call symputx(cats('fmtlen',_n_),min(32767,ceil((length+3)*1.2)),'l');
     call symputx(cats('fmt',_n_),fmt,'l');
     call symputx(cats('type',_n_),type,'l');
     call symputx(cats('typelong',_n_),typelong,'l');
@@ -147,6 +150,9 @@
   run;
 
   %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+  proc sql;
+  select count(*) into: lastobs from &ds;
+  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs));
 
   %if &engine=PROCJSON %then %do;
     %if &missing=STRING %then %do;
@@ -183,24 +189,49 @@
     %end;
 
     %if &fmt=Y %then %do;
-      data _data_;
+      %let tempvar=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      /* need to find max length of formatted values */
+      data _data_(compress=char);
         /* rename on entry */
         set &ds(rename=(
       %do i=1 %to &numcols;
         &&name&i=&&newname&i
       %end;
         ));
+      &stmt_obs;
+      /* formatted values can be up to length 32767 */
+      length
+      %do i=1 %to &numcols;
+        &&name&i
+      %end;
+        $32767;
+      retain &tempvar
+      %do i=1 %to &numcols;
+        &&maxlenv&i
+      %end;
+        0;
+      drop &tempvar
+      %do i=1 %to &numcols;
+        &&newname&i &&maxlenv&i
+      %end;
+        ;
       %do i=1 %to &numcols;
-        /* formatted values can be up to length 32767 */
-        length &&name&i $32767;
         %if &&typelong&i=num %then %do;
-          &&name&i=left(put(&&newname&i,&&fmt&i));
+          &&name&i=cats(put(&&newname&i,&&fmt&i));
         %end;
         %else %do;
           &&name&i=put(&&newname&i,&&fmt&i);
         %end;
-        drop &&newname&i;
+        /* grab max length of each value as we move down the data step */
+        &tempvar=length(&&name&i);
+        if &tempvar>&&maxlenv&i then &&maxlenv&i=&tempvar;
       %end;
+        if _n_=&lastobs then do;
+          /* add a 20% buffer in case of special chars that need to be escaped */
+      %do i=1 %to &numcols;
+          call symputx("fmtlen&i",min(32767,ceil((&&maxlenv&i+3)*1.2)),'l');
+      %end;
+        end;
         if _error_ then call symputx('syscc',1012);
       run;
       %let fmtds=&syslast;
@@ -218,12 +249,15 @@
     %end;
       other = [best.];
 
+    /* configure varlenchk - as we are explicitly shortening the variables */
+    %let optval=%sysfunc(getoption(varlenchk));
+    options varlenchk=NOWARN;
     data &tempds;
       attrib _all_ label='';
       %do i=1 %to &numcols;
         %if &&typelong&i=char or &fmt=Y %then %do;
-          length &&name&i $32767;
+          length &&name&i $&&fmtlen&i...;
-          format &&name&i $32767.;
+          format &&name&i $&&fmtlen&i...;
         %end;
       %end;
       %if &fmt=Y %then %do;
@@ -256,6 +290,7 @@
       %end;
     %end;
     run;
+    options varlenchk=&optval;
 
     filename _sjs3 temp lrecl=131068 ;
     data _null_;

base/mp_perflog.sas

@@ -1,13 +1,14 @@
 /**
   @file
-  @brief Logs the time the macro was executed in a control dataset.
+  @brief Logs a message in a dataset every time it is invoked
-  @details If the dataset does not exist, it is created.  Usage:
+  @details If the dataset does not exist, it is created.
+  Usage:
 
-    %mp_perflog(started)
+      %mp_perflog(started)
-    %mp_perflog()
+      %mp_perflog()
-    %mp_perflog(startanew,libds=work.newdataset)
+      %mp_perflog(startanew,libds=work.newdataset)
-    %mp_perflog(finished,libds=work.newdataset)
+      %mp_perflog(finished,libds=work.newdataset)
-    %mp_perflog(finished)
+      %mp_perflog(finished)
 
 
   @param label Provide label to go into the control dataset

meta/mm_createwebservice.sas

@@ -93,14 +93,13 @@ data _null_;
   file sasjs lrecl=3000 ;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
-  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
   put '  ,showmeta=N ';
   put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols stmt_obs; ';
+  put '%local tempds colinfo fmtds i numcols stmt_obs tempvar lastobs optval; ';
   put '%let numcols=0; ';
   put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
@@ -159,10 +158,14 @@ data _null_;
   put '    end; ';
   put '    /* 32 char unique name */ ';
   put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put '    maxlenv=''maxlen''!!substr(cats(put(md5(name),$hex32.)),1,26); ';
   put ' ';
   put '    call symputx(cats(''name'',_n_),name,''l''); ';
   put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '    call symputx(cats(''maxlenv'',_n_),maxlenv,''l''); ';
   put '    call symputx(cats(''length'',_n_),length,''l''); ';
+  put '    /* overwritten when fmt=Y */ ';
+  put '    call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.2)),''l''); ';
   put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
@@ -170,6 +173,9 @@ data _null_;
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '  proc sql; ';
+  put '  select count(*) into: lastobs from &ds; ';
+  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -206,24 +212,49 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      data _data_; ';
+  put '      %let tempvar=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      /* need to find max length of formatted values */ ';
+  put '      data _data_(compress=char); ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
   put '        &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
+  put '      &stmt_obs; ';
+  put '      /* formatted values can be up to length 32767 */ ';
+  put '      length ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&name&i ';
+  put '      %end; ';
+  put '        $32767; ';
+  put '      retain &tempvar ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&maxlenv&i ';
+  put '      %end; ';
+  put '        0; ';
+  put '      drop &tempvar ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&newname&i &&maxlenv&i ';
+  put '      %end; ';
+  put '        ; ';
   put '      %do i=1 %to &numcols; ';
-  put '        /* formatted values can be up to length 32767 */ ';
-  put '        length &&name&i $32767; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
-  put '        drop &&newname&i; ';
+  put '        /* grab max length of each value as we move down the data step */ ';
+  put '        &tempvar=length(&&name&i); ';
+  put '        if &tempvar>&&maxlenv&i then &&maxlenv&i=&tempvar; ';
   put '      %end; ';
+  put '        if _n_=&lastobs then do; ';
+  put '          /* add a 20% buffer in case of special chars that need to be escaped */ ';
+  put '      %do i=1 %to &numcols; ';
+  put '          call symputx("fmtlen&i",min(32767,ceil((&&maxlenv&i+3)*1.2)),''l''); ';
+  put '      %end; ';
+  put '        end; ';
   put '        if _error_ then call symputx(''syscc'',1012); ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
@@ -241,12 +272,15 @@ data _null_;
   put '    %end; ';
   put '      other = [best.]; ';
   put ' ';
+  put '    /* configure varlenchk - as we are explicitly shortening the variables */ ';
+  put '    %let optval=%sysfunc(getoption(varlenchk)); ';
+  put '    options varlenchk=NOWARN; ';
   put '    data &tempds; ';
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $32767; ';
+  put '          length &&name&i $&&fmtlen&i...; ';
-  put '          format &&name&i $32767.; ';
+  put '          format &&name&i $&&fmtlen&i...; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -279,6 +313,7 @@ data _null_;
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
+  put '    options varlenchk=&optval; ';
   put ' ';
   put '    filename _sjs3 temp lrecl=131068 ; ';
   put '    data _null_; ';
@@ -369,7 +404,7 @@ data _null_;
   put ' ';
   put '%mend mf_getuser; ';
   put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
-  put '  ,showmeta=N ';
+  put '  ,showmeta=N,maxobs=MAX ';
   put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
@@ -436,7 +471,7 @@ data _null_;
   put ' ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta ';
+  put '    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
   put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
@@ -477,23 +512,30 @@ data _null_;
   put '  %end; ';
   put '  /* close off json */ ';
   put '  data _null_;file _sjsref mod encoding=''utf-8''; ';
-  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
+  put '    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; ';
-  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
-  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""_DEBUG"" : ""&_debug"" "; ';
   put '    _METAUSER=quote(trim(symget(''_METAUSER''))); ';
   put '    put ",""_METAUSER"": " _METAUSER; ';
   put '    _METAPERSON=quote(trim(symget(''_METAPERSON''))); ';
   put '    put '',"_METAPERSON": '' _METAPERSON; ';
+  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
+  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
+  put '    put '',"AUTOEXEC" : '' autoexec; ';
+  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
   put '    put ",""SYSENCODING"" : ""&sysencoding"" "; ';
   put '    syserrortext=cats(''"'',tranwrd(symget(''syserrortext''),''"'',''\"''),''"''); ';
   put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
+  put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
+  put '    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; ';
+  put '    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); ';
+  put '    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
+  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
   put '    syswarningtext=cats(''"'',tranwrd(symget(''syswarningtext''),''"'',''\"''),''"''); ';

meta/mm_webout.sas

@@ -34,7 +34,8 @@
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
-
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to output JSON
 
   <h4> SAS Macros </h4>
   @li mp_jsonout.sas
@@ -44,7 +45,7 @@
 
 **/
 %macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
-  ,showmeta=N
+  ,showmeta=N,maxobs=MAX
 );
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
@@ -111,7 +112,7 @@
 
 %else %if &action=ARR or &action=OBJ %then %do;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta
+    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta,maxobs=&maxobs
   )
 %end;
 %else %if &action=CLOSE %then %do;
@@ -152,23 +153,30 @@
   %end;
   /* close off json */
   data _null_;file _sjsref mod encoding='utf-8';
-    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
+    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512;
-    put ",""SYSUSERID"" : ""&sysuserid"" ";
-    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""_DEBUG"" : ""&_debug"" ";
     _METAUSER=quote(trim(symget('_METAUSER')));
     put ",""_METAUSER"": " _METAUSER;
     _METAPERSON=quote(trim(symget('_METAPERSON')));
     put ',"_METAPERSON": ' _METAPERSON;
+    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ',"_PROGRAM" : ' _PROGRAM ;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
+    put ',"AUTOEXEC" : ' autoexec;
+    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""SYSCC"" : ""&syscc"" ";
     put ",""SYSENCODING"" : ""&sysencoding"" ";
     syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
     put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
+    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
+    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
+    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
     syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');

server/ms_createwebservice.sas

@@ -94,14 +94,13 @@ data _null_;
   file &sasjsref termstr=crlf lrecl=512;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
-  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
   put '  ,showmeta=N ';
   put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols stmt_obs; ';
+  put '%local tempds colinfo fmtds i numcols stmt_obs tempvar lastobs optval; ';
   put '%let numcols=0; ';
   put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
@@ -160,10 +159,14 @@ data _null_;
   put '    end; ';
   put '    /* 32 char unique name */ ';
   put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put '    maxlenv=''maxlen''!!substr(cats(put(md5(name),$hex32.)),1,26); ';
   put ' ';
   put '    call symputx(cats(''name'',_n_),name,''l''); ';
   put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '    call symputx(cats(''maxlenv'',_n_),maxlenv,''l''); ';
   put '    call symputx(cats(''length'',_n_),length,''l''); ';
+  put '    /* overwritten when fmt=Y */ ';
+  put '    call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.2)),''l''); ';
   put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
@@ -171,6 +174,9 @@ data _null_;
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '  proc sql; ';
+  put '  select count(*) into: lastobs from &ds; ';
+  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -207,24 +213,49 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      data _data_; ';
+  put '      %let tempvar=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      /* need to find max length of formatted values */ ';
+  put '      data _data_(compress=char); ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
   put '        &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
+  put '      &stmt_obs; ';
+  put '      /* formatted values can be up to length 32767 */ ';
+  put '      length ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&name&i ';
+  put '      %end; ';
+  put '        $32767; ';
+  put '      retain &tempvar ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&maxlenv&i ';
+  put '      %end; ';
+  put '        0; ';
+  put '      drop &tempvar ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&newname&i &&maxlenv&i ';
+  put '      %end; ';
+  put '        ; ';
   put '      %do i=1 %to &numcols; ';
-  put '        /* formatted values can be up to length 32767 */ ';
-  put '        length &&name&i $32767; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
-  put '        drop &&newname&i; ';
+  put '        /* grab max length of each value as we move down the data step */ ';
+  put '        &tempvar=length(&&name&i); ';
+  put '        if &tempvar>&&maxlenv&i then &&maxlenv&i=&tempvar; ';
   put '      %end; ';
+  put '        if _n_=&lastobs then do; ';
+  put '          /* add a 20% buffer in case of special chars that need to be escaped */ ';
+  put '      %do i=1 %to &numcols; ';
+  put '          call symputx("fmtlen&i",min(32767,ceil((&&maxlenv&i+3)*1.2)),''l''); ';
+  put '      %end; ';
+  put '        end; ';
   put '        if _error_ then call symputx(''syscc'',1012); ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
@@ -242,12 +273,15 @@ data _null_;
   put '    %end; ';
   put '      other = [best.]; ';
   put ' ';
+  put '    /* configure varlenchk - as we are explicitly shortening the variables */ ';
+  put '    %let optval=%sysfunc(getoption(varlenchk)); ';
+  put '    options varlenchk=NOWARN; ';
   put '    data &tempds; ';
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $32767; ';
+  put '          length &&name&i $&&fmtlen&i...; ';
-  put '          format &&name&i $32767.; ';
+  put '          format &&name&i $&&fmtlen&i...; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -280,6 +314,7 @@ data _null_;
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
+  put '    options varlenchk=&optval; ';
   put ' ';
   put '    filename _sjs3 temp lrecl=131068 ; ';
   put '    data _null_; ';
@@ -371,7 +406,7 @@ data _null_;
   put '%mend mf_getuser; ';
   put ' ';
   put '%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
-  put '  ,showmeta=N ';
+  put '  ,showmeta=N,maxobs=MAX ';
   put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
@@ -430,7 +465,7 @@ data _null_;
   put '    %let missing=NULL; ';
   put '  %end; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta ';
+  put '    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
   put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
@@ -470,11 +505,13 @@ data _null_;
   put '  %end; ';
   put '  /* close off json */ ';
   put '  data _null_;file &fref mod encoding=''utf-8'' termstr=lf lrecl=32767; ';
-  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
+  put '    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; ';
-  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
-  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""_DEBUG"" : ""&_debug"" "; ';
+  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
+  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
+  put '    put '',"AUTOEXEC" : '' autoexec; ';
+  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
   put '    put ",""SYSENCODING"" : ""&sysencoding"" "; ';
   put '    syserrortext=cats(''"'',tranwrd(symget(''syserrortext''),''"'',''\"''),''"''); ';
@@ -484,21 +521,18 @@ data _null_;
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
   put '    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; ';
-  put '    length SYSPROCESSNAME $512; ';
   put '    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); ';
   put '    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" "; ';
+  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
   put '    syswarningtext=cats(''"'',tranwrd(symget(''syswarningtext''),''"'',''\"''),''"''); ';
   put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
-  put '    length autoexec $512; ';
-  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
-  put '    put '',"AUTOEXEC" : '' autoexec; ';
   put '    length memsize $32; ';
   put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
   put '    memsize=quote(cats(memsize)); ';

server/ms_webout.sas

@@ -31,6 +31,8 @@
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to output JSON
 
   <h4> SAS Macros </h4>
   @li mf_getuser.sas
@@ -47,7 +49,7 @@
 **/
 
 %macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
-  ,showmeta=N
+  ,showmeta=N,maxobs=MAX
 );
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
@@ -106,7 +108,7 @@
     %let missing=NULL;
   %end;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta
+    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs
   )
 %end;
 %else %if &action=CLOSE %then %do;
@@ -146,11 +148,13 @@
   %end;
   /* close off json */
   data _null_;file &fref mod encoding='utf-8' termstr=lf lrecl=32767;
-    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
+    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512;
-    put ",""SYSUSERID"" : ""&sysuserid"" ";
-    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""_DEBUG"" : ""&_debug"" ";
+    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ',"_PROGRAM" : ' _PROGRAM ;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
+    put ',"AUTOEXEC" : ' autoexec;
+    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""SYSCC"" : ""&syscc"" ";
     put ",""SYSENCODING"" : ""&sysencoding"" ";
     syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
@@ -160,21 +164,18 @@
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
     put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
-    length SYSPROCESSNAME $512;
     SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
     put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" ";
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
     syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');
     put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
-    length autoexec $512;
-    autoexec=quote(urlencode(trim(getoption('autoexec'))));
-    put ',"AUTOEXEC" : ' autoexec;
     length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
     memsize=quote(cats(memsize));

tests/base/mp_cleancsv.test.sas

@@ -0,0 +1,43 @@
+/**
+  @file
+  @brief Testing mp_cleancsv.sas macro
+  @details Credit for test 1 goes to
+  [Tom](https://communities.sas.com/t5/user/viewprofilepage/user-id/159) from
+  SAS Communities:
+https://communities.sas.com/t5/SAS-Programming/Removing-embedded-carriage-returns/m-p/824790#M325761
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_cleancsv.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+/* test 1 - cope with empty rows on CR formatted file */
+
+filename crlf "%sysfunc(pathname(work))/crlf";
+filename cr "%sysfunc(pathname(work))/cr";
+data _null_;
+  file cr termstr=cr ;
+  put 'line 1'///'line 4'/'line 5';
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mp_cleancsv(in=cr,out=crlf)
+%mp_assertscope(COMPARE)
+
+/* 5 rows as all converted to OD0A */
+data test1;
+  infile "%sysfunc(pathname(work))/crlf" lrecl=100 termstr=crlf;
+  input;
+  list;
+run;
+
+%put test1=%mf_nobs(test1);
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.test1)=5),
+  desc=Checking blank rows on CR formatted file,
+  outds=work.test_results
+)

viya/mv_createwebservice.sas

@@ -236,14 +236,13 @@ data _null_;
   file &adapter;
   put "/* Created on %sysfunc(datetime(),datetime19.) by &sysuserid */";
 /* WEBOUT BEGIN */
-  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
   put '  ,showmeta=N ';
   put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols stmt_obs; ';
+  put '%local tempds colinfo fmtds i numcols stmt_obs tempvar lastobs optval; ';
   put '%let numcols=0; ';
   put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
@@ -302,10 +301,14 @@ data _null_;
   put '    end; ';
   put '    /* 32 char unique name */ ';
   put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put '    maxlenv=''maxlen''!!substr(cats(put(md5(name),$hex32.)),1,26); ';
   put ' ';
   put '    call symputx(cats(''name'',_n_),name,''l''); ';
   put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '    call symputx(cats(''maxlenv'',_n_),maxlenv,''l''); ';
   put '    call symputx(cats(''length'',_n_),length,''l''); ';
+  put '    /* overwritten when fmt=Y */ ';
+  put '    call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.2)),''l''); ';
   put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
@@ -313,6 +316,9 @@ data _null_;
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '  proc sql; ';
+  put '  select count(*) into: lastobs from &ds; ';
+  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -349,24 +355,49 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      data _data_; ';
+  put '      %let tempvar=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      /* need to find max length of formatted values */ ';
+  put '      data _data_(compress=char); ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
   put '        &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
+  put '      &stmt_obs; ';
+  put '      /* formatted values can be up to length 32767 */ ';
+  put '      length ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&name&i ';
+  put '      %end; ';
+  put '        $32767; ';
+  put '      retain &tempvar ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&maxlenv&i ';
+  put '      %end; ';
+  put '        0; ';
+  put '      drop &tempvar ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&newname&i &&maxlenv&i ';
+  put '      %end; ';
+  put '        ; ';
   put '      %do i=1 %to &numcols; ';
-  put '        /* formatted values can be up to length 32767 */ ';
-  put '        length &&name&i $32767; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
-  put '        drop &&newname&i; ';
+  put '        /* grab max length of each value as we move down the data step */ ';
+  put '        &tempvar=length(&&name&i); ';
+  put '        if &tempvar>&&maxlenv&i then &&maxlenv&i=&tempvar; ';
   put '      %end; ';
+  put '        if _n_=&lastobs then do; ';
+  put '          /* add a 20% buffer in case of special chars that need to be escaped */ ';
+  put '      %do i=1 %to &numcols; ';
+  put '          call symputx("fmtlen&i",min(32767,ceil((&&maxlenv&i+3)*1.2)),''l''); ';
+  put '      %end; ';
+  put '        end; ';
   put '        if _error_ then call symputx(''syscc'',1012); ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
@@ -384,12 +415,15 @@ data _null_;
   put '    %end; ';
   put '      other = [best.]; ';
   put ' ';
+  put '    /* configure varlenchk - as we are explicitly shortening the variables */ ';
+  put '    %let optval=%sysfunc(getoption(varlenchk)); ';
+  put '    options varlenchk=NOWARN; ';
   put '    data &tempds; ';
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $32767; ';
+  put '          length &&name&i $&&fmtlen&i...; ';
-  put '          format &&name&i $32767.; ';
+  put '          format &&name&i $&&fmtlen&i...; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -422,6 +456,7 @@ data _null_;
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
+  put '    options varlenchk=&optval; ';
   put ' ';
   put '    filename _sjs3 temp lrecl=131068 ; ';
   put '    data _null_; ';
@@ -512,7 +547,7 @@ data _null_;
   put ' ';
   put '%mend mf_getuser; ';
   put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL ';
-  put '  ,showmeta=N ';
+  put '  ,showmeta=N,maxobs=MAX ';
   put '); ';
   put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
   put '  sasjs_tables SYS_JES_JOB_URI; ';
@@ -614,8 +649,8 @@ data _null_;
   put '  run; ';
   put '%end; ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
-  put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
+  put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '      ,jref=&fref,engine=DATASTEP,missing=&missing,showmeta=&showmeta ';
+  put '      ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
   put '    ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
@@ -651,20 +686,28 @@ data _null_;
   put ' ';
   put '  /* close off json */ ';
   put '  data _null_;file &fref mod; ';
+  put '    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; ';
+  put '    put ",""_DEBUG"" : ""&_debug"" "; ';
   put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
-  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
+  put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
+  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
+  put '    put '',"AUTOEXEC" : '' autoexec; ';
   put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    SYS_JES_JOB_URI=quote(trim(resolve(symget(''SYS_JES_JOB_URI'')))); ';
   put '    put '',"SYS_JES_JOB_URI" : '' SYS_JES_JOB_URI ; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
-  put '    put ",""_DEBUG"" : ""&_debug"" "; ';
-  put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
   put '    syserrortext=cats(''"'',tranwrd(symget(''syserrortext''),''"'',''\"''),''"''); ';
   put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
+  put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
+  put '    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; ';
+  put '    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); ';
+  put '    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; ';
+  put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
+  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
   put '    syswarningtext=cats(''"'',tranwrd(symget(''syswarningtext''),''"'',''\"''),''"''); ';

viya/mv_webout.sas

@@ -33,6 +33,8 @@
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to output JSON
 
   <h4> SAS Macros </h4>
   @li mp_jsonout.sas
@@ -43,7 +45,7 @@
 
 **/
 %macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL
-  ,showmeta=N
+  ,showmeta=N,maxobs=MAX
 );
 %global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name
   sasjs_tables SYS_JES_JOB_URI;
@@ -145,8 +147,8 @@
   run;
 %end;
 %else %if &action=ARR or &action=OBJ %then %do;
-    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt
+    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-      ,jref=&fref,engine=DATASTEP,missing=&missing,showmeta=&showmeta
+      ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs
     )
 %end;
 %else %if &action=CLOSE %then %do;
@@ -182,20 +184,28 @@
 
   /* close off json */
   data _null_;file &fref mod;
+    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512;
+    put ",""_DEBUG"" : ""&_debug"" ";
     _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
-    put ",""SYSUSERID"" : ""&sysuserid"" ";
+    put ',"_PROGRAM" : ' _PROGRAM ;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
+    put ',"AUTOEXEC" : ' autoexec;
     put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     SYS_JES_JOB_URI=quote(trim(resolve(symget('SYS_JES_JOB_URI'))));
     put ',"SYS_JES_JOB_URI" : ' SYS_JES_JOB_URI ;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
-    put ",""_DEBUG"" : ""&_debug"" ";
-    put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
     syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
     put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
+    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
+    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
+    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
+    put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
     syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');