Merge pull request #297 from sasjs/allanbowe/improve-efficiency-of-295

Further improvements to mp_jsonout

all.sas

@@ -5340,7 +5340,7 @@ run;
   <h4> SAS Macros </h4>
   @li mf_existds.sas
 
-  <h4> Related Macros <h4>
+  <h4> Related Macros </h4>
   @li mp_jsonout.sas
 
   @version 9.2
@@ -8709,7 +8709,7 @@ options
   @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
     that should be converted to JSON
 
-  <h4> Related Macros <h4>
+  <h4> Related Macros </h4>
   @li mp_ds2fmtds.sas
 
   @version 9.2
@@ -8717,14 +8717,14 @@ options
   @source https://github.com/sasjs/core
 
 **/
-
 %macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y
   ,engine=DATASTEP
   ,missing=NULL
   ,showmeta=N
   ,maxobs=MAX
 )/*/STORE SOURCE*/;
-%local tempds colinfo fmtds i numcols stmt_obs;
+%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval
+  tmpds1 tmpds2 tmpds3 tmpds4;
 %let numcols=0;
 %if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;);
 
@@ -8762,7 +8762,7 @@ options
     by varnum;
   run;
   /* move meta to mac vars */
-  data _null_;
+  data &colinfo;
     if _n_=1 then call symputx('numcols',nobs,'l');
     set &colinfo end=last nobs=nobs;
     name=upcase(name);
@@ -8791,9 +8791,15 @@ options
     call symputx(cats('type',_n_),type,'l');
     call symputx(cats('typelong',_n_),typelong,'l');
     call symputx(cats('label',_n_),coalescec(label,name),'l');
+    /* overwritten when fmt=Y and a custom format exists in catalog */
+    if typelong='num' then call symputx(cats('fmtlen',_n_),200,'l');
+    else call symputx(cats('fmtlen',_n_),min(32767,ceil((length+3)*1.5)),'l');
   run;
 
   %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+  proc sql;
+  select count(*) into: lastobs from &ds;
+  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs));
 
   %if &engine=PROCJSON %then %do;
     %if &missing=STRING %then %do;
@@ -8830,27 +8836,99 @@ options
     %end;
 
     %if &fmt=Y %then %do;
-      data _data_;
+      /**
+        * Extract format definitions
+        * First, by getting library locations from dictionary.formats
+        * Then, by exporting the width using proc format
+        * Cannot use maxw from sashelp.vformat as not always populated
+        * Cannot use fmtinfo() as not supported in all flavours
+        */
+      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      proc sql noprint;
+      create table &tmpds1 as
+          select cats(libname,'.',memname) as fmtcat,
+          fmtname
+        from dictionary.formats
+        where fmttype='F' and libname is not null
+          and fmtname in (select format from &colinfo where format is not null)
+        order by 1;
+      create table &tmpds2(
+          FMTNAME char(32),
+          MAX num length=3
+      );
+      %local catlist cat fmtlist i;
+      select distinct fmtcat into: catlist separated by ' ' from &tmpds1;
+      %do i=1 %to %sysfunc(countw(&catlist,%str( )));
+        %let cat=%scan(&catlist,&i,%str( ));
+        proc sql;
+        select distinct fmtname into: fmtlist separated by ' '
+          from &tmpds1 where fmtcat="&cat";
+        proc format lib=&cat cntlout=&tmpds3(keep=fmtname max);
+          select &fmtlist;
+        run;
+        proc sql;
+        insert into &tmpds2 select distinct fmtname,max from &tmpds3;
+      %end;
+
+      proc sql;
+      create table &tmpds4 as
+        select a.*, b.max as maxw
+        from &colinfo a
+        left join &tmpds2 b
+        on cats(a.format)=cats(upcase(b.fmtname))
+        order by a.varnum;
+      data _null_;
+        set &tmpds4;
+        if not missing(maxw);
+        call symputx(
+          cats('fmtlen',_n_),
+          /* vars need extra padding due to JSON escaping of special chars */
+          min(32767,ceil((max(length,maxw)+3)*1.5))
+          ,'l'
+        );
+      run;
+
+      /* configure varlenchk - as we are explicitly shortening the variables */
+      %let optval=%sysfunc(getoption(varlenchk));
+      options varlenchk=NOWARN;
+      data _data_(compress=char);
+        /* shorten the new vars */
+        length
+      %do i=1 %to &numcols;
+          &&name&i $&&fmtlen&i
+      %end;
+          ;
         /* rename on entry */
         set &ds(rename=(
       %do i=1 %to &numcols;
-        &&name&i=&&newname&i
+          &&name&i=&&newname&i
       %end;
         ));
+      &stmt_obs;
+
+      drop
+      %do i=1 %to &numcols;
+        &&newname&i
+      %end;
+        ;
       %do i=1 %to &numcols;
-        /* formatted values can be up to length 32767 */
-        length &&name&i $32767;
         %if &&typelong&i=num %then %do;
-          &&name&i=left(put(&&newname&i,&&fmt&i));
+          &&name&i=cats(put(&&newname&i,&&fmt&i));
         %end;
         %else %do;
           &&name&i=put(&&newname&i,&&fmt&i);
         %end;
-        drop &&newname&i;
       %end;
-        if _error_ then call symputx('syscc',1012);
+        if _error_ then do;
+          call symputx('syscc',1012);
+          stop;
+        end;
       run;
       %let fmtds=&syslast;
+      options varlenchk=&optval;
     %end;
 
     proc format; /* credit yabwon for special null removal */
@@ -8869,8 +8947,8 @@ options
       attrib _all_ label='';
       %do i=1 %to &numcols;
         %if &&typelong&i=char or &fmt=Y %then %do;
-          length &&name&i $32767;
-          format &&name&i $32767.;
+          length &&name&i $&&fmtlen&i...;
+          format &&name&i $&&fmtlen&i...;
         %end;
       %end;
       %if &fmt=Y %then %do;
@@ -9939,14 +10017,15 @@ put(md5(
 %mend mp_md5;
 /**
   @file
-  @brief Logs the time the macro was executed in a control dataset.
-  @details If the dataset does not exist, it is created.  Usage:
+  @brief Logs a message in a dataset every time it is invoked
+  @details If the dataset does not exist, it is created.
+  Usage:
 
-    %mp_perflog(started)
-    %mp_perflog()
-    %mp_perflog(startanew,libds=work.newdataset)
-    %mp_perflog(finished,libds=work.newdataset)
-    %mp_perflog(finished)
+      %mp_perflog(started)
+      %mp_perflog()
+      %mp_perflog(startanew,libds=work.newdataset)
+      %mp_perflog(finished,libds=work.newdataset)
+      %mp_perflog(finished)
 
 
   @param label Provide label to go into the control dataset
@@ -15133,14 +15212,14 @@ data _null_;
   file sasjs lrecl=3000 ;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
-  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
   put '  ,showmeta=N ';
   put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols stmt_obs; ';
+  put '%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval ';
+  put '  tmpds1 tmpds2 tmpds3 tmpds4; ';
   put '%let numcols=0; ';
   put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
@@ -15178,7 +15257,7 @@ data _null_;
   put '    by varnum; ';
   put '  run; ';
   put '  /* move meta to mac vars */ ';
-  put '  data _null_; ';
+  put '  data &colinfo; ';
   put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
   put '    set &colinfo end=last nobs=nobs; ';
   put '    name=upcase(name); ';
@@ -15207,9 +15286,15 @@ data _null_;
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
+  put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
+  put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
+  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '  proc sql; ';
+  put '  select count(*) into: lastobs from &ds; ';
+  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -15246,27 +15331,99 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      data _data_; ';
+  put '      /** ';
+  put '        * Extract format definitions ';
+  put '        * First, by getting library locations from dictionary.formats ';
+  put '        * Then, by exporting the width using proc format ';
+  put '        * Cannot use maxw from sashelp.vformat as not always populated ';
+  put '        * Cannot use fmtinfo() as not supported in all flavours ';
+  put '        */ ';
+  put '      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      proc sql noprint; ';
+  put '      create table &tmpds1 as ';
+  put '          select cats(libname,''.'',memname) as fmtcat, ';
+  put '          fmtname ';
+  put '        from dictionary.formats ';
+  put '        where fmttype=''F'' and libname is not null ';
+  put '          and fmtname in (select format from &colinfo where format is not null) ';
+  put '        order by 1; ';
+  put '      create table &tmpds2( ';
+  put '          FMTNAME char(32), ';
+  put '          MAX num length=3 ';
+  put '      ); ';
+  put '      %local catlist cat fmtlist i; ';
+  put '      select distinct fmtcat into: catlist separated by '' '' from &tmpds1; ';
+  put '      %do i=1 %to %sysfunc(countw(&catlist,%str( ))); ';
+  put '        %let cat=%scan(&catlist,&i,%str( )); ';
+  put '        proc sql; ';
+  put '        select distinct fmtname into: fmtlist separated by '' '' ';
+  put '          from &tmpds1 where fmtcat="&cat"; ';
+  put '        proc format lib=&cat cntlout=&tmpds3(keep=fmtname max); ';
+  put '          select &fmtlist; ';
+  put '        run; ';
+  put '        proc sql; ';
+  put '        insert into &tmpds2 select distinct fmtname,max from &tmpds3; ';
+  put '      %end; ';
+  put ' ';
+  put '      proc sql; ';
+  put '      create table &tmpds4 as ';
+  put '        select a.*, b.max as maxw ';
+  put '        from &colinfo a ';
+  put '        left join &tmpds2 b ';
+  put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
+  put '        order by a.varnum; ';
+  put '      data _null_; ';
+  put '        set &tmpds4; ';
+  put '        if not missing(maxw); ';
+  put '        call symputx( ';
+  put '          cats(''fmtlen'',_n_), ';
+  put '          /* vars need extra padding due to JSON escaping of special chars */ ';
+  put '          min(32767,ceil((max(length,maxw)+3)*1.5)) ';
+  put '          ,''l'' ';
+  put '        ); ';
+  put '      run; ';
+  put ' ';
+  put '      /* configure varlenchk - as we are explicitly shortening the variables */ ';
+  put '      %let optval=%sysfunc(getoption(varlenchk)); ';
+  put '      options varlenchk=NOWARN; ';
+  put '      data _data_(compress=char); ';
+  put '        /* shorten the new vars */ ';
+  put '        length ';
+  put '      %do i=1 %to &numcols; ';
+  put '          &&name&i $&&fmtlen&i ';
+  put '      %end; ';
+  put '          ; ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
-  put '        &&name&i=&&newname&i ';
+  put '          &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
+  put '      &stmt_obs; ';
+  put ' ';
+  put '      drop ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&newname&i ';
+  put '      %end; ';
+  put '        ; ';
   put '      %do i=1 %to &numcols; ';
-  put '        /* formatted values can be up to length 32767 */ ';
-  put '        length &&name&i $32767; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
-  put '        drop &&newname&i; ';
   put '      %end; ';
-  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '        if _error_ then do; ';
+  put '          call symputx(''syscc'',1012); ';
+  put '          stop; ';
+  put '        end; ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
+  put '      options varlenchk=&optval; ';
   put '    %end; ';
   put ' ';
   put '    proc format; /* credit yabwon for special null removal */ ';
@@ -15285,8 +15442,8 @@ data _null_;
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $32767; ';
-  put '          format &&name&i $32767.; ';
+  put '          length &&name&i $&&fmtlen&i...; ';
+  put '          format &&name&i $&&fmtlen&i...; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -15408,8 +15565,8 @@ data _null_;
   put '  %quote(&user) ';
   put ' ';
   put '%mend mf_getuser; ';
-  put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
-  put '  ,showmeta=N ';
+  put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL ';
+  put '  ,showmeta=N,maxobs=MAX ';
   put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
@@ -15476,7 +15633,7 @@ data _null_;
   put ' ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta ';
+  put '    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
   put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
@@ -18960,7 +19117,7 @@ run;
   @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE
   @param [in] ds The dataset to send back to the frontend
   @param [out] dslabel= Value to use instead of table name for sending to JSON
-  @param [in] fmt=(Y) Set to N to send back unformatted values
+  @param [in] fmt= (N) Setting Y converts all vars to their formatted values
   @param [out] fref= (_webout) The fileref to which to write the JSON
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
@@ -18968,17 +19125,22 @@ run;
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
-
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to output JSON
 
   <h4> SAS Macros </h4>
   @li mp_jsonout.sas
 
+  <h4> Related Macros </h4>
+  @li ms_webout.sas
+  @li mv_webout.sas
+
   @version 9.3
   @author Allan Bowe
 
 **/
-%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
-  ,showmeta=N
+%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL
+  ,showmeta=N,maxobs=MAX
 );
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
@@ -19045,7 +19207,7 @@ run;
 
 %else %if &action=ARR or &action=OBJ %then %do;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta
+    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta,maxobs=&maxobs
   )
 %end;
 %else %if &action=CLOSE %then %do;
@@ -19997,14 +20159,14 @@ data _null_;
   file &sasjsref termstr=crlf lrecl=512;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
-  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
   put '  ,showmeta=N ';
   put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols stmt_obs; ';
+  put '%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval ';
+  put '  tmpds1 tmpds2 tmpds3 tmpds4; ';
   put '%let numcols=0; ';
   put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
@@ -20042,7 +20204,7 @@ data _null_;
   put '    by varnum; ';
   put '  run; ';
   put '  /* move meta to mac vars */ ';
-  put '  data _null_; ';
+  put '  data &colinfo; ';
   put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
   put '    set &colinfo end=last nobs=nobs; ';
   put '    name=upcase(name); ';
@@ -20071,9 +20233,15 @@ data _null_;
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
+  put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
+  put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
+  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '  proc sql; ';
+  put '  select count(*) into: lastobs from &ds; ';
+  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -20110,27 +20278,99 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      data _data_; ';
+  put '      /** ';
+  put '        * Extract format definitions ';
+  put '        * First, by getting library locations from dictionary.formats ';
+  put '        * Then, by exporting the width using proc format ';
+  put '        * Cannot use maxw from sashelp.vformat as not always populated ';
+  put '        * Cannot use fmtinfo() as not supported in all flavours ';
+  put '        */ ';
+  put '      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      proc sql noprint; ';
+  put '      create table &tmpds1 as ';
+  put '          select cats(libname,''.'',memname) as fmtcat, ';
+  put '          fmtname ';
+  put '        from dictionary.formats ';
+  put '        where fmttype=''F'' and libname is not null ';
+  put '          and fmtname in (select format from &colinfo where format is not null) ';
+  put '        order by 1; ';
+  put '      create table &tmpds2( ';
+  put '          FMTNAME char(32), ';
+  put '          MAX num length=3 ';
+  put '      ); ';
+  put '      %local catlist cat fmtlist i; ';
+  put '      select distinct fmtcat into: catlist separated by '' '' from &tmpds1; ';
+  put '      %do i=1 %to %sysfunc(countw(&catlist,%str( ))); ';
+  put '        %let cat=%scan(&catlist,&i,%str( )); ';
+  put '        proc sql; ';
+  put '        select distinct fmtname into: fmtlist separated by '' '' ';
+  put '          from &tmpds1 where fmtcat="&cat"; ';
+  put '        proc format lib=&cat cntlout=&tmpds3(keep=fmtname max); ';
+  put '          select &fmtlist; ';
+  put '        run; ';
+  put '        proc sql; ';
+  put '        insert into &tmpds2 select distinct fmtname,max from &tmpds3; ';
+  put '      %end; ';
+  put ' ';
+  put '      proc sql; ';
+  put '      create table &tmpds4 as ';
+  put '        select a.*, b.max as maxw ';
+  put '        from &colinfo a ';
+  put '        left join &tmpds2 b ';
+  put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
+  put '        order by a.varnum; ';
+  put '      data _null_; ';
+  put '        set &tmpds4; ';
+  put '        if not missing(maxw); ';
+  put '        call symputx( ';
+  put '          cats(''fmtlen'',_n_), ';
+  put '          /* vars need extra padding due to JSON escaping of special chars */ ';
+  put '          min(32767,ceil((max(length,maxw)+3)*1.5)) ';
+  put '          ,''l'' ';
+  put '        ); ';
+  put '      run; ';
+  put ' ';
+  put '      /* configure varlenchk - as we are explicitly shortening the variables */ ';
+  put '      %let optval=%sysfunc(getoption(varlenchk)); ';
+  put '      options varlenchk=NOWARN; ';
+  put '      data _data_(compress=char); ';
+  put '        /* shorten the new vars */ ';
+  put '        length ';
+  put '      %do i=1 %to &numcols; ';
+  put '          &&name&i $&&fmtlen&i ';
+  put '      %end; ';
+  put '          ; ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
-  put '        &&name&i=&&newname&i ';
+  put '          &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
+  put '      &stmt_obs; ';
+  put ' ';
+  put '      drop ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&newname&i ';
+  put '      %end; ';
+  put '        ; ';
   put '      %do i=1 %to &numcols; ';
-  put '        /* formatted values can be up to length 32767 */ ';
-  put '        length &&name&i $32767; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
-  put '        drop &&newname&i; ';
   put '      %end; ';
-  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '        if _error_ then do; ';
+  put '          call symputx(''syscc'',1012); ';
+  put '          stop; ';
+  put '        end; ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
+  put '      options varlenchk=&optval; ';
   put '    %end; ';
   put ' ';
   put '    proc format; /* credit yabwon for special null removal */ ';
@@ -20149,8 +20389,8 @@ data _null_;
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $32767; ';
-  put '          format &&name&i $32767.; ';
+  put '          length &&name&i $&&fmtlen&i...; ';
+  put '          format &&name&i $&&fmtlen&i...; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -20273,8 +20513,8 @@ data _null_;
   put ' ';
   put '%mend mf_getuser; ';
   put ' ';
-  put '%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
-  put '  ,showmeta=N ';
+  put '%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL ';
+  put '  ,showmeta=N,maxobs=MAX ';
   put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
@@ -20333,7 +20573,7 @@ data _null_;
   put '    %let missing=NULL; ';
   put '  %end; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta ';
+  put '    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
   put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
@@ -21240,7 +21480,7 @@ run;
   @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE
   @param [in] ds The dataset to send back to the frontend
   @param [out] dslabel= value to use instead of table name for sending to JSON
-  @param [in] fmt= (Y) Set to N to send back unformatted values
+  @param [in] fmt= (N) Setting Y converts all vars to their formatted values
   @param [out] fref= (_webout) The fileref to which to write the JSON
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
@@ -21248,6 +21488,8 @@ run;
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to output JSON
 
   <h4> SAS Macros </h4>
   @li mf_getuser.sas
@@ -21263,8 +21505,8 @@ run;
 
 **/
 
-%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
-  ,showmeta=N
+%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL
+  ,showmeta=N,maxobs=MAX
 );
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
@@ -21323,7 +21565,7 @@ run;
     %let missing=NULL;
   %end;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta
+    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs
   )
 %end;
 %else %if &action=CLOSE %then %do;
@@ -22373,14 +22615,14 @@ data _null_;
   file &adapter;
   put "/* Created on %sysfunc(datetime(),datetime19.) by &sysuserid */";
 /* WEBOUT BEGIN */
-  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
   put '  ,showmeta=N ';
   put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols stmt_obs; ';
+  put '%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval ';
+  put '  tmpds1 tmpds2 tmpds3 tmpds4; ';
   put '%let numcols=0; ';
   put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
@@ -22418,7 +22660,7 @@ data _null_;
   put '    by varnum; ';
   put '  run; ';
   put '  /* move meta to mac vars */ ';
-  put '  data _null_; ';
+  put '  data &colinfo; ';
   put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
   put '    set &colinfo end=last nobs=nobs; ';
   put '    name=upcase(name); ';
@@ -22447,9 +22689,15 @@ data _null_;
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
+  put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
+  put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
+  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '  proc sql; ';
+  put '  select count(*) into: lastobs from &ds; ';
+  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -22486,27 +22734,99 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      data _data_; ';
+  put '      /** ';
+  put '        * Extract format definitions ';
+  put '        * First, by getting library locations from dictionary.formats ';
+  put '        * Then, by exporting the width using proc format ';
+  put '        * Cannot use maxw from sashelp.vformat as not always populated ';
+  put '        * Cannot use fmtinfo() as not supported in all flavours ';
+  put '        */ ';
+  put '      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      proc sql noprint; ';
+  put '      create table &tmpds1 as ';
+  put '          select cats(libname,''.'',memname) as fmtcat, ';
+  put '          fmtname ';
+  put '        from dictionary.formats ';
+  put '        where fmttype=''F'' and libname is not null ';
+  put '          and fmtname in (select format from &colinfo where format is not null) ';
+  put '        order by 1; ';
+  put '      create table &tmpds2( ';
+  put '          FMTNAME char(32), ';
+  put '          MAX num length=3 ';
+  put '      ); ';
+  put '      %local catlist cat fmtlist i; ';
+  put '      select distinct fmtcat into: catlist separated by '' '' from &tmpds1; ';
+  put '      %do i=1 %to %sysfunc(countw(&catlist,%str( ))); ';
+  put '        %let cat=%scan(&catlist,&i,%str( )); ';
+  put '        proc sql; ';
+  put '        select distinct fmtname into: fmtlist separated by '' '' ';
+  put '          from &tmpds1 where fmtcat="&cat"; ';
+  put '        proc format lib=&cat cntlout=&tmpds3(keep=fmtname max); ';
+  put '          select &fmtlist; ';
+  put '        run; ';
+  put '        proc sql; ';
+  put '        insert into &tmpds2 select distinct fmtname,max from &tmpds3; ';
+  put '      %end; ';
+  put ' ';
+  put '      proc sql; ';
+  put '      create table &tmpds4 as ';
+  put '        select a.*, b.max as maxw ';
+  put '        from &colinfo a ';
+  put '        left join &tmpds2 b ';
+  put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
+  put '        order by a.varnum; ';
+  put '      data _null_; ';
+  put '        set &tmpds4; ';
+  put '        if not missing(maxw); ';
+  put '        call symputx( ';
+  put '          cats(''fmtlen'',_n_), ';
+  put '          /* vars need extra padding due to JSON escaping of special chars */ ';
+  put '          min(32767,ceil((max(length,maxw)+3)*1.5)) ';
+  put '          ,''l'' ';
+  put '        ); ';
+  put '      run; ';
+  put ' ';
+  put '      /* configure varlenchk - as we are explicitly shortening the variables */ ';
+  put '      %let optval=%sysfunc(getoption(varlenchk)); ';
+  put '      options varlenchk=NOWARN; ';
+  put '      data _data_(compress=char); ';
+  put '        /* shorten the new vars */ ';
+  put '        length ';
+  put '      %do i=1 %to &numcols; ';
+  put '          &&name&i $&&fmtlen&i ';
+  put '      %end; ';
+  put '          ; ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
-  put '        &&name&i=&&newname&i ';
+  put '          &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
+  put '      &stmt_obs; ';
+  put ' ';
+  put '      drop ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&newname&i ';
+  put '      %end; ';
+  put '        ; ';
   put '      %do i=1 %to &numcols; ';
-  put '        /* formatted values can be up to length 32767 */ ';
-  put '        length &&name&i $32767; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
-  put '        drop &&newname&i; ';
   put '      %end; ';
-  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '        if _error_ then do; ';
+  put '          call symputx(''syscc'',1012); ';
+  put '          stop; ';
+  put '        end; ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
+  put '      options varlenchk=&optval; ';
   put '    %end; ';
   put ' ';
   put '    proc format; /* credit yabwon for special null removal */ ';
@@ -22525,8 +22845,8 @@ data _null_;
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $32767; ';
-  put '          format &&name&i $32767.; ';
+  put '          length &&name&i $&&fmtlen&i...; ';
+  put '          format &&name&i $&&fmtlen&i...; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -22648,8 +22968,8 @@ data _null_;
   put '  %quote(&user) ';
   put ' ';
   put '%mend mf_getuser; ';
-  put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL ';
-  put '  ,showmeta=N ';
+  put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=N,stream=Y,missing=NULL ';
+  put '  ,showmeta=N,maxobs=MAX ';
   put '); ';
   put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
   put '  sasjs_tables SYS_JES_JOB_URI; ';
@@ -22751,8 +23071,8 @@ data _null_;
   put '  run; ';
   put '%end; ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
-  put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
-  put '      ,jref=&fref,engine=DATASTEP,missing=&missing,showmeta=&showmeta ';
+  put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
+  put '      ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
   put '    ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
@@ -26427,7 +26747,7 @@ filename &fref1 clear;
   @param [in] _webout= fileref for returning the json
   @param [out] fref=(_mvwtemp) Temp fileref to which to write the output
   @param [out] dslabel= value to use instead of table name for sending to JSON
-  @param [in] fmt=(Y) change to N to strip formats from output
+  @param [in] fmt= (N) Setting Y converts all vars to their formatted values
   @param [in] stream=(Y) Change to N if not streaming to _webout
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
@@ -26435,17 +26755,23 @@ filename &fref1 clear;
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to output JSON
 
   <h4> SAS Macros </h4>
   @li mp_jsonout.sas
   @li mf_getuser.sas
 
+  <h4> Related Macros </h4>
+  @li ms_webout.sas
+  @li mm_webout.sas
+
   @version Viya 3.3
   @author Allan Bowe, source: https://github.com/sasjs/core
 
 **/
-%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL
-  ,showmeta=N
+%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=N,stream=Y,missing=NULL
+  ,showmeta=N,maxobs=MAX
 );
 %global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name
   sasjs_tables SYS_JES_JOB_URI;
@@ -26547,8 +26873,8 @@ filename &fref1 clear;
   run;
 %end;
 %else %if &action=ARR or &action=OBJ %then %do;
-    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt
-      ,jref=&fref,engine=DATASTEP,missing=&missing,showmeta=&showmeta
+    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
+      ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs
     )
 %end;
 %else %if &action=CLOSE %then %do;

base/mp_ds2fmtds.sas

@@ -17,7 +17,7 @@
   <h4> SAS Macros </h4>
   @li mf_existds.sas
 
-  <h4> Related Macros <h4>
+  <h4> Related Macros </h4>
   @li mp_jsonout.sas
 
   @version 9.2

base/mp_jsonout.sas

@@ -62,7 +62,7 @@
   @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
     that should be converted to JSON
 
-  <h4> Related Macros <h4>
+  <h4> Related Macros </h4>
   @li mp_ds2fmtds.sas
 
   @version 9.2
@@ -70,14 +70,14 @@
   @source https://github.com/sasjs/core
 
 **/
-
 %macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y
   ,engine=DATASTEP
   ,missing=NULL
   ,showmeta=N
   ,maxobs=MAX
 )/*/STORE SOURCE*/;
-%local tempds colinfo fmtds i numcols stmt_obs;
+%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval
+  tmpds1 tmpds2 tmpds3 tmpds4;
 %let numcols=0;
 %if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;);
 
@@ -115,7 +115,7 @@
     by varnum;
   run;
   /* move meta to mac vars */
-  data _null_;
+  data &colinfo;
     if _n_=1 then call symputx('numcols',nobs,'l');
     set &colinfo end=last nobs=nobs;
     name=upcase(name);
@@ -144,9 +144,15 @@
     call symputx(cats('type',_n_),type,'l');
     call symputx(cats('typelong',_n_),typelong,'l');
     call symputx(cats('label',_n_),coalescec(label,name),'l');
+    /* overwritten when fmt=Y and a custom format exists in catalog */
+    if typelong='num' then call symputx(cats('fmtlen',_n_),200,'l');
+    else call symputx(cats('fmtlen',_n_),min(32767,ceil((length+3)*1.5)),'l');
   run;
 
   %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+  proc sql;
+  select count(*) into: lastobs from &ds;
+  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs));
 
   %if &engine=PROCJSON %then %do;
     %if &missing=STRING %then %do;
@@ -183,27 +189,99 @@
     %end;
 
     %if &fmt=Y %then %do;
-      data _data_;
+      /**
+        * Extract format definitions
+        * First, by getting library locations from dictionary.formats
+        * Then, by exporting the width using proc format
+        * Cannot use maxw from sashelp.vformat as not always populated
+        * Cannot use fmtinfo() as not supported in all flavours
+        */
+      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      proc sql noprint;
+      create table &tmpds1 as
+          select cats(libname,'.',memname) as fmtcat,
+          fmtname
+        from dictionary.formats
+        where fmttype='F' and libname is not null
+          and fmtname in (select format from &colinfo where format is not null)
+        order by 1;
+      create table &tmpds2(
+          FMTNAME char(32),
+          MAX num length=3
+      );
+      %local catlist cat fmtlist i;
+      select distinct fmtcat into: catlist separated by ' ' from &tmpds1;
+      %do i=1 %to %sysfunc(countw(&catlist,%str( )));
+        %let cat=%scan(&catlist,&i,%str( ));
+        proc sql;
+        select distinct fmtname into: fmtlist separated by ' '
+          from &tmpds1 where fmtcat="&cat";
+        proc format lib=&cat cntlout=&tmpds3(keep=fmtname max);
+          select &fmtlist;
+        run;
+        proc sql;
+        insert into &tmpds2 select distinct fmtname,max from &tmpds3;
+      %end;
+
+      proc sql;
+      create table &tmpds4 as
+        select a.*, b.max as maxw
+        from &colinfo a
+        left join &tmpds2 b
+        on cats(a.format)=cats(upcase(b.fmtname))
+        order by a.varnum;
+      data _null_;
+        set &tmpds4;
+        if not missing(maxw);
+        call symputx(
+          cats('fmtlen',_n_),
+          /* vars need extra padding due to JSON escaping of special chars */
+          min(32767,ceil((max(length,maxw)+3)*1.5))
+          ,'l'
+        );
+      run;
+
+      /* configure varlenchk - as we are explicitly shortening the variables */
+      %let optval=%sysfunc(getoption(varlenchk));
+      options varlenchk=NOWARN;
+      data _data_(compress=char);
+        /* shorten the new vars */
+        length
+      %do i=1 %to &numcols;
+          &&name&i $&&fmtlen&i
+      %end;
+          ;
         /* rename on entry */
         set &ds(rename=(
       %do i=1 %to &numcols;
-        &&name&i=&&newname&i
+          &&name&i=&&newname&i
       %end;
         ));
+      &stmt_obs;
+
+      drop
+      %do i=1 %to &numcols;
+        &&newname&i
+      %end;
+        ;
       %do i=1 %to &numcols;
-        /* formatted values can be up to length 32767 */
-        length &&name&i $32767;
         %if &&typelong&i=num %then %do;
-          &&name&i=left(put(&&newname&i,&&fmt&i));
+          &&name&i=cats(put(&&newname&i,&&fmt&i));
         %end;
         %else %do;
           &&name&i=put(&&newname&i,&&fmt&i);
         %end;
-        drop &&newname&i;
       %end;
-        if _error_ then call symputx('syscc',1012);
+        if _error_ then do;
+          call symputx('syscc',1012);
+          stop;
+        end;
       run;
       %let fmtds=&syslast;
+      options varlenchk=&optval;
     %end;
 
     proc format; /* credit yabwon for special null removal */
@@ -222,8 +300,8 @@
       attrib _all_ label='';
       %do i=1 %to &numcols;
         %if &&typelong&i=char or &fmt=Y %then %do;
-          length &&name&i $32767;
-          format &&name&i $32767.;
+          length &&name&i $&&fmtlen&i...;
+          format &&name&i $&&fmtlen&i...;
         %end;
       %end;
       %if &fmt=Y %then %do;

base/mp_perflog.sas

@@ -1,13 +1,14 @@
 /**
   @file
-  @brief Logs the time the macro was executed in a control dataset.
-  @details If the dataset does not exist, it is created.  Usage:
+  @brief Logs a message in a dataset every time it is invoked
+  @details If the dataset does not exist, it is created.
+  Usage:
 
-    %mp_perflog(started)
-    %mp_perflog()
-    %mp_perflog(startanew,libds=work.newdataset)
-    %mp_perflog(finished,libds=work.newdataset)
-    %mp_perflog(finished)
+      %mp_perflog(started)
+      %mp_perflog()
+      %mp_perflog(startanew,libds=work.newdataset)
+      %mp_perflog(finished,libds=work.newdataset)
+      %mp_perflog(finished)
 
 
   @param label Provide label to go into the control dataset

meta/mm_createwebservice.sas

@@ -93,14 +93,14 @@ data _null_;
   file sasjs lrecl=3000 ;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
-  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
   put '  ,showmeta=N ';
   put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols stmt_obs; ';
+  put '%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval ';
+  put '  tmpds1 tmpds2 tmpds3 tmpds4; ';
   put '%let numcols=0; ';
   put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
@@ -138,7 +138,7 @@ data _null_;
   put '    by varnum; ';
   put '  run; ';
   put '  /* move meta to mac vars */ ';
-  put '  data _null_; ';
+  put '  data &colinfo; ';
   put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
   put '    set &colinfo end=last nobs=nobs; ';
   put '    name=upcase(name); ';
@@ -167,9 +167,15 @@ data _null_;
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
+  put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
+  put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
+  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '  proc sql; ';
+  put '  select count(*) into: lastobs from &ds; ';
+  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -206,27 +212,99 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      data _data_; ';
+  put '      /** ';
+  put '        * Extract format definitions ';
+  put '        * First, by getting library locations from dictionary.formats ';
+  put '        * Then, by exporting the width using proc format ';
+  put '        * Cannot use maxw from sashelp.vformat as not always populated ';
+  put '        * Cannot use fmtinfo() as not supported in all flavours ';
+  put '        */ ';
+  put '      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      proc sql noprint; ';
+  put '      create table &tmpds1 as ';
+  put '          select cats(libname,''.'',memname) as fmtcat, ';
+  put '          fmtname ';
+  put '        from dictionary.formats ';
+  put '        where fmttype=''F'' and libname is not null ';
+  put '          and fmtname in (select format from &colinfo where format is not null) ';
+  put '        order by 1; ';
+  put '      create table &tmpds2( ';
+  put '          FMTNAME char(32), ';
+  put '          MAX num length=3 ';
+  put '      ); ';
+  put '      %local catlist cat fmtlist i; ';
+  put '      select distinct fmtcat into: catlist separated by '' '' from &tmpds1; ';
+  put '      %do i=1 %to %sysfunc(countw(&catlist,%str( ))); ';
+  put '        %let cat=%scan(&catlist,&i,%str( )); ';
+  put '        proc sql; ';
+  put '        select distinct fmtname into: fmtlist separated by '' '' ';
+  put '          from &tmpds1 where fmtcat="&cat"; ';
+  put '        proc format lib=&cat cntlout=&tmpds3(keep=fmtname max); ';
+  put '          select &fmtlist; ';
+  put '        run; ';
+  put '        proc sql; ';
+  put '        insert into &tmpds2 select distinct fmtname,max from &tmpds3; ';
+  put '      %end; ';
+  put ' ';
+  put '      proc sql; ';
+  put '      create table &tmpds4 as ';
+  put '        select a.*, b.max as maxw ';
+  put '        from &colinfo a ';
+  put '        left join &tmpds2 b ';
+  put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
+  put '        order by a.varnum; ';
+  put '      data _null_; ';
+  put '        set &tmpds4; ';
+  put '        if not missing(maxw); ';
+  put '        call symputx( ';
+  put '          cats(''fmtlen'',_n_), ';
+  put '          /* vars need extra padding due to JSON escaping of special chars */ ';
+  put '          min(32767,ceil((max(length,maxw)+3)*1.5)) ';
+  put '          ,''l'' ';
+  put '        ); ';
+  put '      run; ';
+  put ' ';
+  put '      /* configure varlenchk - as we are explicitly shortening the variables */ ';
+  put '      %let optval=%sysfunc(getoption(varlenchk)); ';
+  put '      options varlenchk=NOWARN; ';
+  put '      data _data_(compress=char); ';
+  put '        /* shorten the new vars */ ';
+  put '        length ';
+  put '      %do i=1 %to &numcols; ';
+  put '          &&name&i $&&fmtlen&i ';
+  put '      %end; ';
+  put '          ; ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
-  put '        &&name&i=&&newname&i ';
+  put '          &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
+  put '      &stmt_obs; ';
+  put ' ';
+  put '      drop ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&newname&i ';
+  put '      %end; ';
+  put '        ; ';
   put '      %do i=1 %to &numcols; ';
-  put '        /* formatted values can be up to length 32767 */ ';
-  put '        length &&name&i $32767; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
-  put '        drop &&newname&i; ';
   put '      %end; ';
-  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '        if _error_ then do; ';
+  put '          call symputx(''syscc'',1012); ';
+  put '          stop; ';
+  put '        end; ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
+  put '      options varlenchk=&optval; ';
   put '    %end; ';
   put ' ';
   put '    proc format; /* credit yabwon for special null removal */ ';
@@ -245,8 +323,8 @@ data _null_;
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $32767; ';
-  put '          format &&name&i $32767.; ';
+  put '          length &&name&i $&&fmtlen&i...; ';
+  put '          format &&name&i $&&fmtlen&i...; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -368,8 +446,8 @@ data _null_;
   put '  %quote(&user) ';
   put ' ';
   put '%mend mf_getuser; ';
-  put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
-  put '  ,showmeta=N ';
+  put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL ';
+  put '  ,showmeta=N,maxobs=MAX ';
   put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
@@ -436,7 +514,7 @@ data _null_;
   put ' ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta ';
+  put '    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
   put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';

meta/mm_webout.sas

@@ -26,7 +26,7 @@
   @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE
   @param [in] ds The dataset to send back to the frontend
   @param [out] dslabel= Value to use instead of table name for sending to JSON
-  @param [in] fmt=(Y) Set to N to send back unformatted values
+  @param [in] fmt= (N) Setting Y converts all vars to their formatted values
   @param [out] fref= (_webout) The fileref to which to write the JSON
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
@@ -34,17 +34,22 @@
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
-
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to output JSON
 
   <h4> SAS Macros </h4>
   @li mp_jsonout.sas
 
+  <h4> Related Macros </h4>
+  @li ms_webout.sas
+  @li mv_webout.sas
+
   @version 9.3
   @author Allan Bowe
 
 **/
-%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
-  ,showmeta=N
+%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL
+  ,showmeta=N,maxobs=MAX
 );
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
@@ -111,7 +116,7 @@
 
 %else %if &action=ARR or &action=OBJ %then %do;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta
+    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta,maxobs=&maxobs
   )
 %end;
 %else %if &action=CLOSE %then %do;

server/ms_createwebservice.sas

@@ -94,14 +94,14 @@ data _null_;
   file &sasjsref termstr=crlf lrecl=512;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
-  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
   put '  ,showmeta=N ';
   put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols stmt_obs; ';
+  put '%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval ';
+  put '  tmpds1 tmpds2 tmpds3 tmpds4; ';
   put '%let numcols=0; ';
   put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
@@ -139,7 +139,7 @@ data _null_;
   put '    by varnum; ';
   put '  run; ';
   put '  /* move meta to mac vars */ ';
-  put '  data _null_; ';
+  put '  data &colinfo; ';
   put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
   put '    set &colinfo end=last nobs=nobs; ';
   put '    name=upcase(name); ';
@@ -168,9 +168,15 @@ data _null_;
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
+  put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
+  put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
+  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '  proc sql; ';
+  put '  select count(*) into: lastobs from &ds; ';
+  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -207,27 +213,99 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      data _data_; ';
+  put '      /** ';
+  put '        * Extract format definitions ';
+  put '        * First, by getting library locations from dictionary.formats ';
+  put '        * Then, by exporting the width using proc format ';
+  put '        * Cannot use maxw from sashelp.vformat as not always populated ';
+  put '        * Cannot use fmtinfo() as not supported in all flavours ';
+  put '        */ ';
+  put '      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      proc sql noprint; ';
+  put '      create table &tmpds1 as ';
+  put '          select cats(libname,''.'',memname) as fmtcat, ';
+  put '          fmtname ';
+  put '        from dictionary.formats ';
+  put '        where fmttype=''F'' and libname is not null ';
+  put '          and fmtname in (select format from &colinfo where format is not null) ';
+  put '        order by 1; ';
+  put '      create table &tmpds2( ';
+  put '          FMTNAME char(32), ';
+  put '          MAX num length=3 ';
+  put '      ); ';
+  put '      %local catlist cat fmtlist i; ';
+  put '      select distinct fmtcat into: catlist separated by '' '' from &tmpds1; ';
+  put '      %do i=1 %to %sysfunc(countw(&catlist,%str( ))); ';
+  put '        %let cat=%scan(&catlist,&i,%str( )); ';
+  put '        proc sql; ';
+  put '        select distinct fmtname into: fmtlist separated by '' '' ';
+  put '          from &tmpds1 where fmtcat="&cat"; ';
+  put '        proc format lib=&cat cntlout=&tmpds3(keep=fmtname max); ';
+  put '          select &fmtlist; ';
+  put '        run; ';
+  put '        proc sql; ';
+  put '        insert into &tmpds2 select distinct fmtname,max from &tmpds3; ';
+  put '      %end; ';
+  put ' ';
+  put '      proc sql; ';
+  put '      create table &tmpds4 as ';
+  put '        select a.*, b.max as maxw ';
+  put '        from &colinfo a ';
+  put '        left join &tmpds2 b ';
+  put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
+  put '        order by a.varnum; ';
+  put '      data _null_; ';
+  put '        set &tmpds4; ';
+  put '        if not missing(maxw); ';
+  put '        call symputx( ';
+  put '          cats(''fmtlen'',_n_), ';
+  put '          /* vars need extra padding due to JSON escaping of special chars */ ';
+  put '          min(32767,ceil((max(length,maxw)+3)*1.5)) ';
+  put '          ,''l'' ';
+  put '        ); ';
+  put '      run; ';
+  put ' ';
+  put '      /* configure varlenchk - as we are explicitly shortening the variables */ ';
+  put '      %let optval=%sysfunc(getoption(varlenchk)); ';
+  put '      options varlenchk=NOWARN; ';
+  put '      data _data_(compress=char); ';
+  put '        /* shorten the new vars */ ';
+  put '        length ';
+  put '      %do i=1 %to &numcols; ';
+  put '          &&name&i $&&fmtlen&i ';
+  put '      %end; ';
+  put '          ; ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
-  put '        &&name&i=&&newname&i ';
+  put '          &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
+  put '      &stmt_obs; ';
+  put ' ';
+  put '      drop ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&newname&i ';
+  put '      %end; ';
+  put '        ; ';
   put '      %do i=1 %to &numcols; ';
-  put '        /* formatted values can be up to length 32767 */ ';
-  put '        length &&name&i $32767; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
-  put '        drop &&newname&i; ';
   put '      %end; ';
-  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '        if _error_ then do; ';
+  put '          call symputx(''syscc'',1012); ';
+  put '          stop; ';
+  put '        end; ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
+  put '      options varlenchk=&optval; ';
   put '    %end; ';
   put ' ';
   put '    proc format; /* credit yabwon for special null removal */ ';
@@ -246,8 +324,8 @@ data _null_;
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $32767; ';
-  put '          format &&name&i $32767.; ';
+  put '          length &&name&i $&&fmtlen&i...; ';
+  put '          format &&name&i $&&fmtlen&i...; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -370,8 +448,8 @@ data _null_;
   put ' ';
   put '%mend mf_getuser; ';
   put ' ';
-  put '%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
-  put '  ,showmeta=N ';
+  put '%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL ';
+  put '  ,showmeta=N,maxobs=MAX ';
   put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
@@ -430,7 +508,7 @@ data _null_;
   put '    %let missing=NULL; ';
   put '  %end; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta ';
+  put '    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
   put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';

server/ms_webout.sas

@@ -23,7 +23,7 @@
   @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE
   @param [in] ds The dataset to send back to the frontend
   @param [out] dslabel= value to use instead of table name for sending to JSON
-  @param [in] fmt= (Y) Set to N to send back unformatted values
+  @param [in] fmt= (N) Setting Y converts all vars to their formatted values
   @param [out] fref= (_webout) The fileref to which to write the JSON
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
@@ -31,6 +31,8 @@
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to output JSON
 
   <h4> SAS Macros </h4>
   @li mf_getuser.sas
@@ -46,8 +48,8 @@
 
 **/
 
-%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
-  ,showmeta=N
+%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL
+  ,showmeta=N,maxobs=MAX
 );
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
@@ -106,7 +108,7 @@
     %let missing=NULL;
   %end;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta
+    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs
   )
 %end;
 %else %if &action=CLOSE %then %do;

viya/mv_createwebservice.sas

@@ -236,14 +236,14 @@ data _null_;
   file &adapter;
   put "/* Created on %sysfunc(datetime(),datetime19.) by &sysuserid */";
 /* WEBOUT BEGIN */
-  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
   put '  ,showmeta=N ';
   put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols stmt_obs; ';
+  put '%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval ';
+  put '  tmpds1 tmpds2 tmpds3 tmpds4; ';
   put '%let numcols=0; ';
   put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
@@ -281,7 +281,7 @@ data _null_;
   put '    by varnum; ';
   put '  run; ';
   put '  /* move meta to mac vars */ ';
-  put '  data _null_; ';
+  put '  data &colinfo; ';
   put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
   put '    set &colinfo end=last nobs=nobs; ';
   put '    name=upcase(name); ';
@@ -310,9 +310,15 @@ data _null_;
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
+  put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
+  put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
+  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '  proc sql; ';
+  put '  select count(*) into: lastobs from &ds; ';
+  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -349,27 +355,99 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      data _data_; ';
+  put '      /** ';
+  put '        * Extract format definitions ';
+  put '        * First, by getting library locations from dictionary.formats ';
+  put '        * Then, by exporting the width using proc format ';
+  put '        * Cannot use maxw from sashelp.vformat as not always populated ';
+  put '        * Cannot use fmtinfo() as not supported in all flavours ';
+  put '        */ ';
+  put '      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      proc sql noprint; ';
+  put '      create table &tmpds1 as ';
+  put '          select cats(libname,''.'',memname) as fmtcat, ';
+  put '          fmtname ';
+  put '        from dictionary.formats ';
+  put '        where fmttype=''F'' and libname is not null ';
+  put '          and fmtname in (select format from &colinfo where format is not null) ';
+  put '        order by 1; ';
+  put '      create table &tmpds2( ';
+  put '          FMTNAME char(32), ';
+  put '          MAX num length=3 ';
+  put '      ); ';
+  put '      %local catlist cat fmtlist i; ';
+  put '      select distinct fmtcat into: catlist separated by '' '' from &tmpds1; ';
+  put '      %do i=1 %to %sysfunc(countw(&catlist,%str( ))); ';
+  put '        %let cat=%scan(&catlist,&i,%str( )); ';
+  put '        proc sql; ';
+  put '        select distinct fmtname into: fmtlist separated by '' '' ';
+  put '          from &tmpds1 where fmtcat="&cat"; ';
+  put '        proc format lib=&cat cntlout=&tmpds3(keep=fmtname max); ';
+  put '          select &fmtlist; ';
+  put '        run; ';
+  put '        proc sql; ';
+  put '        insert into &tmpds2 select distinct fmtname,max from &tmpds3; ';
+  put '      %end; ';
+  put ' ';
+  put '      proc sql; ';
+  put '      create table &tmpds4 as ';
+  put '        select a.*, b.max as maxw ';
+  put '        from &colinfo a ';
+  put '        left join &tmpds2 b ';
+  put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
+  put '        order by a.varnum; ';
+  put '      data _null_; ';
+  put '        set &tmpds4; ';
+  put '        if not missing(maxw); ';
+  put '        call symputx( ';
+  put '          cats(''fmtlen'',_n_), ';
+  put '          /* vars need extra padding due to JSON escaping of special chars */ ';
+  put '          min(32767,ceil((max(length,maxw)+3)*1.5)) ';
+  put '          ,''l'' ';
+  put '        ); ';
+  put '      run; ';
+  put ' ';
+  put '      /* configure varlenchk - as we are explicitly shortening the variables */ ';
+  put '      %let optval=%sysfunc(getoption(varlenchk)); ';
+  put '      options varlenchk=NOWARN; ';
+  put '      data _data_(compress=char); ';
+  put '        /* shorten the new vars */ ';
+  put '        length ';
+  put '      %do i=1 %to &numcols; ';
+  put '          &&name&i $&&fmtlen&i ';
+  put '      %end; ';
+  put '          ; ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
-  put '        &&name&i=&&newname&i ';
+  put '          &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
+  put '      &stmt_obs; ';
+  put ' ';
+  put '      drop ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&newname&i ';
+  put '      %end; ';
+  put '        ; ';
   put '      %do i=1 %to &numcols; ';
-  put '        /* formatted values can be up to length 32767 */ ';
-  put '        length &&name&i $32767; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
-  put '        drop &&newname&i; ';
   put '      %end; ';
-  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '        if _error_ then do; ';
+  put '          call symputx(''syscc'',1012); ';
+  put '          stop; ';
+  put '        end; ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
+  put '      options varlenchk=&optval; ';
   put '    %end; ';
   put ' ';
   put '    proc format; /* credit yabwon for special null removal */ ';
@@ -388,8 +466,8 @@ data _null_;
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $32767; ';
-  put '          format &&name&i $32767.; ';
+  put '          length &&name&i $&&fmtlen&i...; ';
+  put '          format &&name&i $&&fmtlen&i...; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -511,8 +589,8 @@ data _null_;
   put '  %quote(&user) ';
   put ' ';
   put '%mend mf_getuser; ';
-  put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL ';
-  put '  ,showmeta=N ';
+  put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=N,stream=Y,missing=NULL ';
+  put '  ,showmeta=N,maxobs=MAX ';
   put '); ';
   put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
   put '  sasjs_tables SYS_JES_JOB_URI; ';
@@ -614,8 +692,8 @@ data _null_;
   put '  run; ';
   put '%end; ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
-  put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
-  put '      ,jref=&fref,engine=DATASTEP,missing=&missing,showmeta=&showmeta ';
+  put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
+  put '      ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
   put '    ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';

viya/mv_webout.sas

@@ -25,7 +25,7 @@
   @param [in] _webout= fileref for returning the json
   @param [out] fref=(_mvwtemp) Temp fileref to which to write the output
   @param [out] dslabel= value to use instead of table name for sending to JSON
-  @param [in] fmt=(Y) change to N to strip formats from output
+  @param [in] fmt= (N) Setting Y converts all vars to their formatted values
   @param [in] stream=(Y) Change to N if not streaming to _webout
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
@@ -33,17 +33,23 @@
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to output JSON
 
   <h4> SAS Macros </h4>
   @li mp_jsonout.sas
   @li mf_getuser.sas
 
+  <h4> Related Macros </h4>
+  @li ms_webout.sas
+  @li mm_webout.sas
+
   @version Viya 3.3
   @author Allan Bowe, source: https://github.com/sasjs/core
 
 **/
-%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL
-  ,showmeta=N
+%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=N,stream=Y,missing=NULL
+  ,showmeta=N,maxobs=MAX
 );
 %global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name
   sasjs_tables SYS_JES_JOB_URI;
@@ -145,8 +151,8 @@
   run;
 %end;
 %else %if &action=ARR or &action=OBJ %then %do;
-    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt
-      ,jref=&fref,engine=DATASTEP,missing=&missing,showmeta=&showmeta
+    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
+      ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs
     )
 %end;
 %else %if &action=CLOSE %then %do;