sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2026-01-03 15:40:05 +00:00
Code Issues Projects Releases Wiki Activity

feat: adding SHOWMETA option to mp_jsonout

Browse Source 
Includes updates to associated wrapper macros, and a 30% performance improvement (for small tables).  Addresses https://github.com/sasjs/adapter/issues/607
This commit is contained in:
munja
2022-01-06 23:36:54 +01:00
parent ebd567af48
commit cb07305a87
8 changed files with 778 additions and 666 deletions
Download Patch File Download Diff File Expand all files Collapse all files

211
viya/mv_createwebservice.sas
View File

@@ -237,11 +237,16 @@ data _null_;
put "/* Created on %sysfunc(datetime(),datetime19.) by &sysuserid */";
/* WEBOUT BEGIN */
put ' ';
put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP ';
put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
put ' ,engine=DATASTEP ';
put ' ,dbg=0 /* DEPRECATED */ ';
put ' ,missing=NULL ';
put ' ,showmeta=NO ';
put ')/*/STORE SOURCE*/; ';
put '%put &sysmacroname: output location=&jref; ';
put '%local tempds colinfo fmtds i numcols; ';
put '%let numcols=0; ';
put '%let fmtds=_null_; ';
put ' ';
put '%if &action=OPEN %then %do; ';
put ' options nobomfile; ';
put ' data _null_;file &jref encoding=''utf-8'' ; ';
@@ -250,17 +255,60 @@ data _null_;
put '%end; ';
put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
put ' options validvarname=upcase; ';
put ' data _null_;file &jref mod encoding=''utf-8'' ; ';
put ' data _null_; file &jref encoding=''utf-8'' mod; ';
put ' put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
put ' ';
put ' /* grab col defs */ ';
put ' proc contents noprint data=&ds ';
put ' out=_data_(keep=name type length format formatl formatd varnum label); ';
put ' run; ';
put ' %let colinfo=%scan(&syslast,2,.); ';
put ' proc sort data=&colinfo; ';
put ' by varnum; ';
put ' run; ';
put ' /* move meta to mac vars */ ';
put ' data _null_; ';
put ' if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
put ' set &colinfo end=last nobs=nobs; ';
put ' name=upcase(name); ';
put ' /* fix formats */ ';
put ' if type=2 or type=6 then do; ';
put ' typelong=''char''; ';
put ' length fmt $49.; ';
put ' if format='''' then fmt=cats(''$'',length,''.''); ';
put ' else if formatl=0 then fmt=cats(format,''.''); ';
put ' else fmt=cats(format,formatl,''.''); ';
put ' newlen=max(formatl,length); ';
put ' end; ';
put ' else do; ';
put ' typelong=''num''; ';
put ' if format='''' then fmt=''best.''; ';
put ' else if formatl=0 then fmt=cats(format,''.''); ';
put ' else if formatd=0 then fmt=cats(format,formatl,''.''); ';
put ' else fmt=cats(format,formatl,''.'',formatd); ';
put ' /* needs to be wide, for datetimes etc */ ';
put ' newlen=max(length,formatl,24); ';
put ' end; ';
put ' /* 32 char unique name */ ';
put ' newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
put ' ';
put ' call symputx(cats(''name'',_n_),name,''l''); ';
put ' call symputx(cats(''newname'',_n_),newname,''l''); ';
put ' call symputx(cats(''len'',_n_),newlen,''l''); ';
put ' call symputx(cats(''fmt'',_n_),fmt,''l''); ';
put ' call symputx(cats(''type'',_n_),type,''l''); ';
put ' call symputx(cats(''typelong'',_n_),typelong,''l''); ';
put ' call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
put ' run; ';
put ' ';
put ' %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
put ' ';
put ' %if &engine=PROCJSON %then %do; ';
put ' %if &missing=STRING %then %do; ';
put ' %put &sysmacroname: Special Missings are not supported in proc json.; ';
put ' %put &sysmacroname: Special Missings not supported in proc json.; ';
put ' %put &sysmacroname: Switching to DATASTEP engine; ';
put ' %goto datastep; ';
put ' %end; ';
put ' data;run;%let tempds=&syslast; ';
put ' proc sql;drop table &tempds; ';
put ' data &tempds /view=&tempds;set &ds; ';
put ' %if &fmt=N %then format _numeric_ best32.;; ';
put ' /* PRETTY is necessary to avoid line truncation in large files */ ';
@@ -268,91 +316,35 @@ data _null_;
put ' %if &action=ARR %then nokeys ; ';
put ' ;export &tempds / nosastags fmtnumeric; ';
put ' run; ';
put ' proc sql;drop view &tempds; ';
put ' %end; ';
put ' %else %if &engine=DATASTEP %then %do; ';
put ' %datastep: ';
put ' %local cols i tempds; ';
put ' %let cols=0; ';
put ' %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 %then %do; ';
put ' %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 ';
put ' %then %do; ';
put ' %put &sysmacroname: &ds NOT FOUND!!!; ';
put ' %return; ';
put ' %end; ';
put ' %if &fmt=Y %then %do; ';
put ' %put converting every variable to a formatted variable; ';
put ' /* see mp_ds2fmtds.sas for source */ ';
put ' proc contents noprint data=&ds ';
put ' out=_data_(keep=name type length format formatl formatd varnum); ';
put ' run; ';
put ' proc sort; ';
put ' by varnum; ';
put ' run; ';
put ' %local fmtds; ';
put ' %let fmtds=%scan(&syslast,2,.); ';
put ' /* prepare formats and varnames */ ';
put ' data _null_; ';
put ' if _n_=1 then call symputx(''nobs'',nobs,''l''); ';
put ' set &fmtds end=last nobs=nobs; ';
put ' name=upcase(name); ';
put ' /* fix formats */ ';
put ' if type=2 or type=6 then do; ';
put ' length fmt $49.; ';
put ' if format='''' then fmt=cats(''$'',length,''.''); ';
put ' else if formatl=0 then fmt=cats(format,''.''); ';
put ' else fmt=cats(format,formatl,''.''); ';
put ' newlen=max(formatl,length); ';
put ' end; ';
put ' else do; ';
put ' if format='''' then fmt=''best.''; ';
put ' else if formatl=0 then fmt=cats(format,''.''); ';
put ' else if formatd=0 then fmt=cats(format,formatl,''.''); ';
put ' else fmt=cats(format,formatl,''.'',formatd); ';
put ' /* needs to be wide, for datetimes etc */ ';
put ' newlen=max(length,formatl,24); ';
put ' end; ';
put ' /* 32 char unique name */ ';
put ' newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
put ' ';
put ' call symputx(cats(''name'',_n_),name,''l''); ';
put ' call symputx(cats(''newname'',_n_),newname,''l''); ';
put ' call symputx(cats(''len'',_n_),newlen,''l''); ';
put ' call symputx(cats(''fmt'',_n_),fmt,''l''); ';
put ' call symputx(cats(''type'',_n_),type,''l''); ';
put ' run; ';
put ' data &fmtds; ';
put ' %if &fmt=Y %then %do; ';
put ' data _data_; ';
put ' /* rename on entry */ ';
put ' set &ds(rename=( ';
put ' %local i; ';
put ' %do i=1 %to &nobs; ';
put ' %do i=1 %to &numcols; ';
put ' &&name&i=&&newname&i ';
put ' %end; ';
put ' )); ';
put ' %do i=1 %to &nobs; ';
put ' %do i=1 %to &numcols; ';
put ' length &&name&i $&&len&i; ';
put ' &&name&i=left(put(&&newname&i,&&fmt&i)); ';
put ' drop &&newname&i; ';
put ' %end; ';
put ' if _error_ then call symputx(''syscc'',1012); ';
put ' run; ';
put ' %let ds=&fmtds; ';
put ' %end; /* &fmt=Y */ ';
put ' data _null_;file &jref mod encoding=''utf-8'' ; ';
put ' put "["; call symputx(''cols'',0,''l''); ';
put ' proc sort ';
put ' data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
put ' out=_data_; ';
put ' by varnum; ';
put ' ';
put ' data _null_; ';
put ' set _last_ end=last; ';
put ' call symputx(cats(''name'',_n_),name,''l''); ';
put ' call symputx(cats(''type'',_n_),type,''l''); ';
put ' call symputx(cats(''len'',_n_),length,''l''); ';
put ' if last then call symputx(''cols'',_n_,''l''); ';
put ' run; ';
put ' %let fmtds=&syslast; ';
put ' %end; ';
put ' ';
put ' proc format; /* credit yabwon for special null removal */ ';
put ' value bart ';
put ' value bart (default=40) ';
put ' %if &missing=NULL %then %do; ';
put ' ._ - .z = null ';
put ' %end; ';
@@ -363,20 +355,18 @@ data _null_;
put ' %end; ';
put ' other = [best.]; ';
put ' ';
put ' data;run; %let tempds=&syslast; /* temp table for spesh char management */ ';
put ' proc sql; drop table &tempds; ';
put ' data &tempds/view=&tempds; ';
put ' attrib _all_ label=''''; ';
put ' %do i=1 %to &cols; ';
put ' %if &&type&i=char %then %do; ';
put ' %do i=1 %to &numcols; ';
put ' %if &&typelong&i=char %then %do; ';
put ' length &&name&i $32767; ';
put ' format &&name&i $32767.; ';
put ' %end; ';
put ' %end; ';
put ' set &ds; ';
put ' set &fmtds &ds; ';
put ' format _numeric_ bart.; ';
put ' %do i=1 %to &cols; ';
put ' %if &&type&i=char %then %do; ';
put ' %do i=1 %to &numcols; ';
put ' %if &&typelong&i=char %then %do; ';
put ' &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
put ' prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
put ' prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
@@ -386,49 +376,70 @@ data _null_;
put ' %end; ';
put ' %end; ';
put ' run; ';
put ' ';
put ' /* write to temp loc to avoid _webout truncation ';
put ' - https://support.sas.com/kb/49/325.html */ ';
put ' filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
put ' data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
put ' if _n_=1 then put "["; ';
put ' set &tempds; ';
put ' if _n_>1 then put "," @; put ';
put ' %if &action=ARR %then "[" ; %else "{" ; ';
put ' %do i=1 %to &cols; ';
put ' %do i=1 %to &numcols; ';
put ' %if &i>1 %then "," ; ';
put ' %if &action=OBJ %then """&&name&i"":" ; ';
put ' &&name&i ';
put ' %end; ';
put ' %if &action=ARR %then "]" ; %else "}" ; ; ';
put ' proc sql; ';
put ' drop view &tempds; ';
put ' /* now write the long strings to _webout 1 byte at a time */ ';
put ' data _null_; ';
put ' length filein 8 fileid 8; ';
put ' filein = fopen("_sjs",''I'',1,''B''); ';
put ' fileid = fopen("&jref",''A'',1,''B''); ';
put ' rec = ''20''x; ';
put ' filein=fopen("_sjs",''I'',1,''B''); ';
put ' fileid=fopen("&jref",''A'',1,''B''); ';
put ' rec=''20''x; ';
put ' do while(fread(filein)=0); ';
put ' rc = fget(filein,rec,1); ';
put ' rc = fput(fileid, rec); ';
put ' rc =fwrite(fileid); ';
put ' rc=fget(filein,rec,1); ';
put ' rc=fput(fileid, rec); ';
put ' rc=fwrite(fileid); ';
put ' end; ';
put ' rc = fclose(filein); ';
put ' rc = fclose(fileid); ';
put ' /* close out the table */ ';
put ' rc=fput(fileid, "]"); ';
put ' rc=fwrite(fileid); ';
put ' rc=fclose(filein); ';
put ' rc=fclose(fileid); ';
put ' run; ';
put ' filename _sjs clear; ';
put ' data _null_; file &jref mod encoding=''utf-8'' ; ';
put ' put "]"; ';
put ' %end; ';
put ' ';
put ' proc sql; ';
put ' drop view &tempds; ';
put ' drop table &colinfo; ';
put ' ';
put ' %if &showmeta=YES %then %do; ';
put ' data _null_; file &jref encoding=''utf-8'' mod; ';
put ' put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; ';
put ' do i=1 to &numcols; ';
put ' name=quote(trim(symget(cats(''name'',i)))); ';
put ' format=quote(trim(symget(cats(''fmt'',i)))); ';
put ' label=quote(trim(symget(cats(''label'',i)))); ';
put ' type=quote(trim(symget(cats(''typelong'',i)))); ';
put ' if i>1 then put "," @@; ';
put ' put name '':{"format":'' format '',"label":'' label '',"type":'' type ''}''; ';
put ' end; ';
put ' put ''}}''; ';
put ' run; ';
put ' %end; ';
put '%end; ';
put ' ';
put '%else %if &action=CLOSE %then %do; ';
put ' data _null_;file &jref encoding=''utf-8'' mod ; ';
put ' data _null_; file &jref encoding=''utf-8'' mod ; ';
put ' put "}"; ';
put ' run; ';
put '%end; ';
put '%mend mp_jsonout; ';
put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL); ';
put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL ';
put ' ,showmeta=NO ';
put '); ';
put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
put ' sasjs_tables SYS_JES_JOB_URI; ';
put '%if %index("&_debug",log) %then %let _debug=131; ';
@@ -550,12 +561,13 @@ data _null_;
put ' ';
put ' /* setup json */ ';
put ' data _null_;file &fref; ';
put ' put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
put ' put ''{"SYSDATE" : "'' "&SYSDATE" ''"''; ';
put ' put '',"SYSTIME" : "'' "&SYSTIME" ''"''; ';
put ' run; ';
put '%end; ';
put '%else %if &action=ARR or &action=OBJ %then %do; ';
put ' %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
put ' ,jref=&fref,engine=DATASTEP,missing=&missing ';
put ' ,jref=&fref,engine=DATASTEP,missing=&missing,showmeta=&showmeta ';
put ' ) ';
put '%end; ';
put '%else %if &action=CLOSE %then %do; ';
@@ -575,9 +587,6 @@ data _null_;
put ' data _null_; file &fref mod; put ",""WORK"":{"; ';
put ' %do i=1 %to &wtcnt; ';
put ' %let wt=&&wt&i; ';
put ' proc contents noprint data=&wt ';
put ' out=_data_ (keep=name type length format:); ';
put ' run;%let tempds=%scan(&syslast,2,.); ';
put ' data _null_; file &fref mod; ';
put ' dsid=open("WORK.&wt",''is''); ';
put ' nlobs=attrn(dsid,''NLOBS''); ';
@@ -587,8 +596,7 @@ data _null_;
put ' put " ""&wt"" : {"; ';
put ' put ''"nlobs":'' nlobs; ';
put ' put '',"nvars":'' nvars; ';
put ' %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP) ';
put ' %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP) ';
put ' %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) ';
put ' data _null_; file &fref mod;put "}"; ';
put ' %end; ';
put ' data _null_; file &fref mod;put "}";run; ';
@@ -613,6 +621,9 @@ data _null_;
put ' put '',"SYSVLONG" : '' sysvlong; ';
put ' put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
put ' put '',"END_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''" ''; ';
put ' memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
put ' memsize=quote(cats(memsize)); ';
put ' put '',"MEMSIZE" : '' memsize; ';
put ' put "}"; ';
put ' ';
put ' %if %upcase(&fref) ne _WEBOUT and &stream=Y %then %do; ';