sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2025-12-10 22:14:35 +00:00
Code Issues Projects Releases Wiki Activity

fix: improved JSON performance for wide tables with a lot of formatted values. 50% improvement!

Browse Source
This commit is contained in:
Allan Bowe
2022-08-14 23:43:19 +00:00
parent 594a895ddd
commit 793319fe38
4 changed files with 92 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
base/mp_jsonout.sas
View File

@@ -76,7 +76,8 @@
,showmeta=N
,maxobs=MAX
)/*/STORE SOURCE*/;
%local tempds colinfo fmtds i numcols stmt_obs tempvar lastobs optval;
%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval
tmpds1 tmpds2 tmpds3 tmpds4;
%let numcols=0;
%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;);
@@ -114,7 +115,7 @@
by varnum;
run;
/* move meta to mac vars */
data _null_;
data &colinfo;
if _n_=1 then call symputx('numcols',nobs,'l');
set &colinfo end=last nobs=nobs;
name=upcase(name);
@@ -135,18 +136,17 @@
end;
/* 32 char unique name */
newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
maxlenv='maxlen'!!substr(cats(put(md5(name),$hex32.)),1,26);
call symputx(cats('name',_n_),name,'l');
call symputx(cats('newname',_n_),newname,'l');
call symputx(cats('maxlenv',_n_),maxlenv,'l');
call symputx(cats('length',_n_),length,'l');
/* overwritten when fmt=Y */
call symputx(cats('fmtlen',_n_),min(32767,ceil((length+3)*1.2)),'l');
call symputx(cats('fmt',_n_),fmt,'l');
call symputx(cats('type',_n_),type,'l');
call symputx(cats('typelong',_n_),typelong,'l');
call symputx(cats('label',_n_),coalescec(label,name),'l');
/* overwritten when fmt=Y and a custom format exists in catalog */
if typelong='num' then call symputx(cats('fmtlen',_n_),200,'l');
else call symputx(cats('fmtlen',_n_),min(32767,ceil((length+3)*1.5)),'l');
run;
%let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
@@ -189,30 +189,82 @@
%end;
%if &fmt=Y %then %do;
%let tempvar=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
/* need to find max length of formatted values */
/**
* Extract format definitions
* First, by getting library locations from dictionary.formats
* Then, by exporting the width using proc format
* Cannot use maxw from sashelp.vformat as not always populated
* Cannot use fmtinfo() as not supported in all flavours
*/
%let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
%let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
%let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
%let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
proc sql noprint;
create table &tmpds1 as
select cats(libname,'.',memname) as fmtcat,
fmtname
from dictionary.formats
where fmttype='F' and libname is not null
and fmtname in (select format from &colinfo where format is not null)
order by 1;
create table &tmpds2(
FMTNAME char(32),
MAX num length=3
);
%local catlist cat fmtlist i;
select distinct fmtcat into: catlist separated by ' ' from &tmpds1;
%do i=1 %to %sysfunc(countw(&catlist,%str( )));
%let cat=%scan(&catlist,&i,%str( ));
proc sql;
select distinct fmtname into: fmtlist separated by ' '
from &tmpds1 where fmtcat="&cat";
proc format lib=&cat cntlout=&tmpds3(keep=fmtname max);
select &fmtlist;
run;
proc sql;
insert into &tmpds2 select distinct fmtname,max from &tmpds3;
%end;
proc sql;
create table &tmpds4 as
select a.*, b.max as maxw
from &colinfo a
left join &tmpds2 b
on cats(a.format)=cats(upcase(b.fmtname))
order by a.varnum;
data _null_;
set &tmpds4;
if not missing(maxw);
call symputx(
cats('fmtlen',_n_),
/* vars need extra padding due to JSON escaping of special chars */
min(32767,ceil((max(length,maxw)+3)*1.5))
,'l'
);
run;
/* configure varlenchk - as we are explicitly shortening the variables */
%let optval=%sysfunc(getoption(varlenchk));
options varlenchk=NOWARN;
data _data_(compress=char);
/* shorten the new vars */
length
%do i=1 %to &numcols;
&&name&i $&&fmtlen&i
%end;
;
/* rename on entry */
set &ds(rename=(
%do i=1 %to &numcols;
&&name&i=&&newname&i
&&name&i=&&newname&i
%end;
));
&stmt_obs;
/* formatted values can be up to length 32767 */
length
drop
%do i=1 %to &numcols;
&&name&i
%end;
$32767;
retain &tempvar
%do i=1 %to &numcols;
&&maxlenv&i
%end;
0;
drop &tempvar
%do i=1 %to &numcols;
&&newname&i &&maxlenv&i
&&newname&i
%end;
;
%do i=1 %to &numcols;
@@ -222,19 +274,14 @@
%else %do;
&&name&i=put(&&newname&i,&&fmt&i);
%end;
/* grab max length of each value as we move down the data step */
&tempvar=length(&&name&i);
if &tempvar>&&maxlenv&i then &&maxlenv&i=&tempvar;
%end;
if _n_=&lastobs then do;
/* add a 20% buffer in case of special chars that need to be escaped */
%do i=1 %to &numcols;
call symputx("fmtlen&i",min(32767,ceil((&&maxlenv&i+3)*1.2)),'l');
%end;
if _error_ then do;
call symputx('syscc',1012);
stop;
end;
if _error_ then call symputx('syscc',1012);
run;
%let fmtds=&syslast;
options varlenchk=&optval;
%end;
proc format; /* credit yabwon for special null removal */
@@ -249,9 +296,6 @@
%end;
other = [best.];
/* configure varlenchk - as we are explicitly shortening the variables */
%let optval=%sysfunc(getoption(varlenchk));
options varlenchk=NOWARN;
data &tempds;
attrib _all_ label='';
%do i=1 %to &numcols;
@@ -290,7 +334,6 @@
%end;
%end;
run;
options varlenchk=&optval;
filename _sjs3 temp lrecl=131068 ;
data _null_;

8
meta/mm_webout.sas
View File

@@ -26,7 +26,7 @@
@param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE
@param [in] ds The dataset to send back to the frontend
@param [out] dslabel= Value to use instead of table name for sending to JSON
@param [in] fmt=(Y) Set to N to send back unformatted values
@param [in] fmt= (N) Setting Y converts all vars to their formatted values
@param [out] fref= (_webout) The fileref to which to write the JSON
@param [in] missing= (NULL) Special numeric missing values can be sent as NULL
(eg `null`) or as STRING values (eg `".a"` or `".b"`)
@@ -40,11 +40,15 @@
<h4> SAS Macros </h4>
@li mp_jsonout.sas
<h4> Related Macros </h4>
@li ms_webout.sas
@li mv_webout.sas
@version 9.3
@author Allan Bowe
**/
%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL
,showmeta=N,maxobs=MAX
);
%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug

4
server/ms_webout.sas
View File

@@ -23,7 +23,7 @@
@param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE
@param [in] ds The dataset to send back to the frontend
@param [out] dslabel= value to use instead of table name for sending to JSON
@param [in] fmt= (Y) Set to N to send back unformatted values
@param [in] fmt= (N) Setting Y converts all vars to their formatted values
@param [out] fref= (_webout) The fileref to which to write the JSON
@param [in] missing= (NULL) Special numeric missing values can be sent as NULL
(eg `null`) or as STRING values (eg `".a"` or `".b"`)
@@ -48,7 +48,7 @@
**/
%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL
,showmeta=N,maxobs=MAX
);
%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug

8
viya/mv_webout.sas
View File

@@ -25,7 +25,7 @@
@param [in] _webout= fileref for returning the json
@param [out] fref=(_mvwtemp) Temp fileref to which to write the output
@param [out] dslabel= value to use instead of table name for sending to JSON
@param [in] fmt=(Y) change to N to strip formats from output
@param [in] fmt= (N) Setting Y converts all vars to their formatted values
@param [in] stream=(Y) Change to N if not streaming to _webout
@param [in] missing= (NULL) Special numeric missing values can be sent as NULL
(eg `null`) or as STRING values (eg `".a"` or `".b"`)
@@ -40,11 +40,15 @@
@li mp_jsonout.sas
@li mf_getuser.sas
<h4> Related Macros </h4>
@li ms_webout.sas
@li mm_webout.sas
@version Viya 3.3
@author Allan Bowe, source: https://github.com/sasjs/core
**/
%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL
%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=N,stream=Y,missing=NULL
,showmeta=N,maxobs=MAX
);
%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name