From 2372ff5f4f3e63adbc31a01de9f41424fa977370 Mon Sep 17 00:00:00 2001 From: munja Date: Sun, 26 Jun 2022 22:09:54 +0100 Subject: [PATCH] fix: writing utf-8 to _webout on windows in a latin9 session causes problems with subsequent (regular) put statements. The workaround is to write to a different file and stream it back to _webout. --- all.sas | 343 +++++++++++++++++++++++++-------- base/mp_jsonout.sas | 75 +++++-- meta/mm_createwebservice.sas | 88 +++++++-- meta/mm_webout.sas | 52 +++-- server/ms_createwebservice.sas | 64 ++++-- viya/mv_createwebservice.sas | 64 ++++-- 6 files changed, 516 insertions(+), 170 deletions(-) diff --git a/all.sas b/all.sas index f3bb790..ef9703a 100644 --- a/all.sas +++ b/all.sas @@ -8575,7 +8575,12 @@ options /** @file mp_jsonout.sas @brief Writes JSON in SASjs format to a fileref - @details PROC JSON is faster but will produce errs like the ones below if + @details This macro can be used to OPEN a JSON stream and send one or more + tables as arrays of rows, where each row can be an object or a nested array. + + There are two engines available - DATASTEP or PROCJSON. + + PROC JSON is fast but will produce errs like the ones below if special chars are encountered. > (ERR)OR: Some code points did not transcode. @@ -8586,6 +8591,10 @@ options If this happens, try running with ENGINE=DATASTEP. + The DATASTEP engine is used to handle special SAS missing numerics, and + can also convert entire datasets to formatted values. Output JSON is always + in UTF-8. + Usage: filename tmp temp; @@ -8649,9 +8658,23 @@ options run; %end; %else %if (&action=ARR or &action=OBJ) %then %do; + /* force variable names to always be uppercase in the JSON */ options validvarname=upcase; - data _null_; file &jref encoding='utf-8' mod; + /* To avoid issues with _webout on EBI - such as encoding diffs and truncation + (https://support.sas.com/kb/49/325.html) we use temporary files */ + filename _sjs1 temp lrecl=200 ; + data _null_; file _sjs1 encoding='utf-8'; put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; + run; + /* now write to _webout 1 char at a time */ + data _null_; + infile _sjs1 lrecl=1 recfm=n; + file &jref mod lrecl=1 recfm=n; + input sourcechar $char1. @@; + format sourcechar hex2.; + put sourcechar char1. @@; + run; + filename _sjs1 clear; /* grab col defs */ proc contents noprint data=&ds @@ -8708,10 +8731,20 @@ options data &tempds;set &ds; %if &fmt=N %then format _numeric_ best32.;; /* PRETTY is necessary to avoid line truncation in large files */ - proc json out=&jref pretty + filename _sjs2 temp lrecl=131068 encoding='utf-8'; + proc json out=_sjs2 pretty %if &action=ARR %then nokeys ; ;export &tempds / nosastags fmtnumeric; run; + /* send back to webout */ + data _null_; + infile _sjs2 lrecl=1 recfm=n; + file &jref mod lrecl=1 recfm=n; + input sourcechar $char1. @@; + format sourcechar hex2.; + put sourcechar char1. @@; + run; + filename _sjs2 clear; %end; %else %if &engine=DATASTEP %then %do; %datastep: @@ -8794,10 +8827,9 @@ options %end; run; - /* write to temp loc to avoid _webout truncation - - https://support.sas.com/kb/49/325.html */ - filename _sjs temp lrecl=131068 encoding='utf-8'; - data _null_; file _sjs lrecl=131068 encoding='utf-8' mod ; + filename _sjs3 temp lrecl=131068 ; + data _null_; + file _sjs3 encoding='utf-8'; if _n_=1 then put "["; set &tempds; if _n_>1 then put "," @; put @@ -8808,27 +8840,29 @@ options "&&name&i"n /* name literal for reserved variable names */ %end; %if &action=ARR %then "]" ; %else "}" ; ; - /* now write the long strings to _webout 1 char at a time */ + + /* close out the table */ data _null_; - infile _sjs lrecl=1 recfm=n; + file _sjs3 mod encoding='utf-8'; + put ']'; + run; + data _null_; + infile _sjs3 lrecl=1 recfm=n; file &jref mod lrecl=1 recfm=n; input sourcechar $char1. @@; format sourcechar hex2.; put sourcechar char1. @@; run; - /* close out the table */ - data _null_; - file &jref mod; - put ']'; - run; - filename _sjs clear; + filename _sjs3 clear; %end; proc sql; drop table &colinfo, &tempds; %if &showmeta=YES %then %do; - data _null_; file &jref encoding='utf-8' mod; + filename _sjs4 temp lrecl=131068 encoding='utf-8'; + data _null_; + file _sjs4; put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; do i=1 to &numcols; name=quote(trim(symget(cats('name',i)))); @@ -8842,6 +8876,15 @@ options end; put '}}'; run; + /* send back to webout */ + data _null_; + infile _sjs4 lrecl=1 recfm=n; + file &jref mod lrecl=1 recfm=n; + input sourcechar $char1. @@; + format sourcechar hex2.; + put sourcechar char1. @@; + run; + filename _sjs4 clear; %end; %end; @@ -15026,9 +15069,23 @@ data _null_; put ' run; '; put '%end; '; put '%else %if (&action=ARR or &action=OBJ) %then %do; '; + put ' /* force variable names to always be uppercase in the JSON */ '; put ' options validvarname=upcase; '; - put ' data _null_; file &jref encoding=''utf-8'' mod; '; + put ' /* To avoid issues with _webout on EBI - such as encoding diffs and truncation '; + put ' (https://support.sas.com/kb/49/325.html) we use temporary files */ '; + put ' filename _sjs1 temp lrecl=200 ; '; + put ' data _null_; file _sjs1 encoding=''utf-8''; '; put ' put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; '; + put ' run; '; + put ' /* now write to _webout 1 char at a time */ '; + put ' data _null_; '; + put ' infile _sjs1 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs1 clear; '; put ' '; put ' /* grab col defs */ '; put ' proc contents noprint data=&ds '; @@ -15085,10 +15142,20 @@ data _null_; put ' data &tempds;set &ds; '; put ' %if &fmt=N %then format _numeric_ best32.;; '; put ' /* PRETTY is necessary to avoid line truncation in large files */ '; - put ' proc json out=&jref pretty '; + put ' filename _sjs2 temp lrecl=131068 encoding=''utf-8''; '; + put ' proc json out=_sjs2 pretty '; put ' %if &action=ARR %then nokeys ; '; put ' ;export &tempds / nosastags fmtnumeric; '; put ' run; '; + put ' /* send back to webout */ '; + put ' data _null_; '; + put ' infile _sjs2 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs2 clear; '; put ' %end; '; put ' %else %if &engine=DATASTEP %then %do; '; put ' %datastep: '; @@ -15171,10 +15238,9 @@ data _null_; put ' %end; '; put ' run; '; put ' '; - put ' /* write to temp loc to avoid _webout truncation '; - put ' - https://support.sas.com/kb/49/325.html */ '; - put ' filename _sjs temp lrecl=131068 encoding=''utf-8''; '; - put ' data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; '; + put ' filename _sjs3 temp lrecl=131068 ; '; + put ' data _null_; '; + put ' file _sjs3 encoding=''utf-8''; '; put ' if _n_=1 then put "["; '; put ' set &tempds; '; put ' if _n_>1 then put "," @; put '; @@ -15185,27 +15251,29 @@ data _null_; put ' "&&name&i"n /* name literal for reserved variable names */ '; put ' %end; '; put ' %if &action=ARR %then "]" ; %else "}" ; ; '; - put ' /* now write the long strings to _webout 1 char at a time */ '; + put ' '; + put ' /* close out the table */ '; put ' data _null_; '; - put ' infile _sjs lrecl=1 recfm=n; '; + put ' file _sjs3 mod encoding=''utf-8''; '; + put ' put '']''; '; + put ' run; '; + put ' data _null_; '; + put ' infile _sjs3 lrecl=1 recfm=n; '; put ' file &jref mod lrecl=1 recfm=n; '; put ' input sourcechar $char1. @@; '; put ' format sourcechar hex2.; '; put ' put sourcechar char1. @@; '; put ' run; '; - put ' /* close out the table */ '; - put ' data _null_; '; - put ' file &jref mod; '; - put ' put '']''; '; - put ' run; '; - put ' filename _sjs clear; '; + put ' filename _sjs3 clear; '; put ' %end; '; put ' '; put ' proc sql; '; put ' drop table &colinfo, &tempds; '; put ' '; put ' %if &showmeta=YES %then %do; '; - put ' data _null_; file &jref encoding=''utf-8'' mod; '; + put ' filename _sjs4 temp lrecl=131068 encoding=''utf-8''; '; + put ' data _null_; '; + put ' file _sjs4; '; put ' put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; '; put ' do i=1 to &numcols; '; put ' name=quote(trim(symget(cats(''name'',i)))); '; @@ -15219,6 +15287,15 @@ data _null_; put ' end; '; put ' put ''}}''; '; put ' run; '; + put ' /* send back to webout */ '; + put ' data _null_; '; + put ' infile _sjs4 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs4 clear; '; put ' %end; '; put '%end; '; put ' '; @@ -15320,6 +15397,8 @@ data _null_; put ' ) '; put '%end; '; put '%else %if &action=CLOSE %then %do; '; + put ' /* To avoid issues with _webout on EBI we use a temporary file */ '; + put ' filename _sjsref temp lrecl=131068; '; put ' %if %str(&_debug) ge 131 %then %do; '; put ' /* if debug mode, send back first 10 records of each work table also */ '; put ' options obs=10; '; @@ -15333,11 +15412,11 @@ data _null_; put ' i+1; '; put ' call symputx(cats(''wt'',i),name,''l''); '; put ' call symputx(''wtcnt'',i,''l''); '; - put ' data _null_; file &fref mod encoding=''utf-8''; '; + put ' data _null_; file _sjsref mod encoding=''utf-8''; '; put ' put ",""WORK"":{"; '; put ' %do i=1 %to &wtcnt; '; put ' %let wt=&&wt&i; '; - put ' data _null_; file &fref mod encoding=''utf-8''; '; + put ' data _null_; file _sjsref mod encoding=''utf-8''; '; put ' dsid=open("WORK.&wt",''is''); '; put ' nlobs=attrn(dsid,''NLOBS''); '; put ' nvars=attrn(dsid,''NVARS''); '; @@ -15346,16 +15425,16 @@ data _null_; put ' put " ""&wt"" : {"; '; put ' put ''"nlobs":'' nlobs; '; put ' put '',"nvars":'' nvars; '; - put ' %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) '; - put ' data _null_; file &fref mod encoding=''utf-8''; '; + put ' %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=YES) '; + put ' data _null_; file _sjsref mod encoding=''utf-8''; '; put ' put "}"; '; put ' %end; '; - put ' data _null_; file &fref mod encoding=''utf-8''; '; + put ' data _null_; file _sjsref mod encoding=''utf-8''; '; put ' put "}"; '; put ' run; '; put ' %end; '; put ' /* close off json */ '; - put ' data _null_;file &fref mod encoding=''utf-8''; '; + put ' data _null_;file _sjsref mod encoding=''utf-8''; '; put ' _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); '; put ' put ",""SYSUSERID"" : ""&sysuserid"" "; '; put ' put ",""MF_GETUSER"" : ""%mf_getuser()"" "; '; @@ -15387,6 +15466,16 @@ data _null_; put ' put ''>>weboutEND<<''; '; put ' %end; '; put ' run; '; + put ' /* now write to _webout 1 char at a time */ '; + put ' data _null_; '; + put ' infile _sjsref lrecl=1 recfm=n; '; + put ' file &fref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjsref clear; '; + put ' '; put '%end; '; put ' '; put '%mend mm_webout; '; @@ -18757,23 +18846,23 @@ run; @details This macro should be added to the start of each Stored Process, **immediately** followed by a call to: - %mm_webout(FETCH) + %mm_webout(FETCH) - This will read all the input data and create same-named SAS datasets in the - WORK library. You can then insert your code, and send data back using the - following syntax: + This will read all the input data and create same-named SAS datasets in the + WORK library. You can then insert your code, and send data back using the + following syntax: - data some datasets; * make some data ; - retain some columns; - run; + data some datasets; * make some data ; + retain some columns; + run; - %mm_webout(OPEN) - %mm_webout(ARR,some) * Array format, fast, suitable for large tables ; - %mm_webout(OBJ,datasets) * Object format, easier to work with ; + %mm_webout(OPEN) + %mm_webout(ARR,some) * Array format, fast, suitable for large tables ; + %mm_webout(OBJ,datasets) * Object format, easier to work with ; - Finally, wrap everything up send some helpful system variables too + Finally, wrap everything up send some helpful system variables too - %mm_webout(CLOSE) + %mm_webout(CLOSE) @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE @@ -18788,6 +18877,10 @@ run; object with the same name as the table but prefixed with a dollar sign - ie, `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}` + +

SAS Macros

+ @li mp_jsonout.sas + @version 9.3 @author Allan Bowe @@ -18864,6 +18957,8 @@ run; ) %end; %else %if &action=CLOSE %then %do; + /* To avoid issues with _webout on EBI we use a temporary file */ + filename _sjsref temp lrecl=131068; %if %str(&_debug) ge 131 %then %do; /* if debug mode, send back first 10 records of each work table also */ options obs=10; @@ -18877,11 +18972,11 @@ run; i+1; call symputx(cats('wt',i),name,'l'); call symputx('wtcnt',i,'l'); - data _null_; file &fref mod encoding='utf-8'; + data _null_; file _sjsref mod encoding='utf-8'; put ",""WORK"":{"; %do i=1 %to &wtcnt; %let wt=&&wt&i; - data _null_; file &fref mod encoding='utf-8'; + data _null_; file _sjsref mod encoding='utf-8'; dsid=open("WORK.&wt",'is'); nlobs=attrn(dsid,'NLOBS'); nvars=attrn(dsid,'NVARS'); @@ -18890,16 +18985,16 @@ run; put " ""&wt"" : {"; put '"nlobs":' nlobs; put ',"nvars":' nvars; - %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) - data _null_; file &fref mod encoding='utf-8'; + %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=YES) + data _null_; file _sjsref mod encoding='utf-8'; put "}"; %end; - data _null_; file &fref mod encoding='utf-8'; + data _null_; file _sjsref mod encoding='utf-8'; put "}"; run; %end; /* close off json */ - data _null_;file &fref mod encoding='utf-8'; + data _null_;file _sjsref mod encoding='utf-8'; _PROGRAM=quote(trim(resolve(symget('_PROGRAM')))); put ",""SYSUSERID"" : ""&sysuserid"" "; put ",""MF_GETUSER"" : ""%mf_getuser()"" "; @@ -18931,6 +19026,16 @@ run; put '>>weboutEND<<'; %end; run; + /* now write to _webout 1 char at a time */ + data _null_; + infile _sjsref lrecl=1 recfm=n; + file &fref mod lrecl=1 recfm=n; + input sourcechar $char1. @@; + format sourcechar hex2.; + put sourcechar char1. @@; + run; + filename _sjsref clear; + %end; %mend mm_webout; @@ -19759,9 +19864,23 @@ data _null_; put ' run; '; put '%end; '; put '%else %if (&action=ARR or &action=OBJ) %then %do; '; + put ' /* force variable names to always be uppercase in the JSON */ '; put ' options validvarname=upcase; '; - put ' data _null_; file &jref encoding=''utf-8'' mod; '; + put ' /* To avoid issues with _webout on EBI - such as encoding diffs and truncation '; + put ' (https://support.sas.com/kb/49/325.html) we use temporary files */ '; + put ' filename _sjs1 temp lrecl=200 ; '; + put ' data _null_; file _sjs1 encoding=''utf-8''; '; put ' put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; '; + put ' run; '; + put ' /* now write to _webout 1 char at a time */ '; + put ' data _null_; '; + put ' infile _sjs1 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs1 clear; '; put ' '; put ' /* grab col defs */ '; put ' proc contents noprint data=&ds '; @@ -19818,10 +19937,20 @@ data _null_; put ' data &tempds;set &ds; '; put ' %if &fmt=N %then format _numeric_ best32.;; '; put ' /* PRETTY is necessary to avoid line truncation in large files */ '; - put ' proc json out=&jref pretty '; + put ' filename _sjs2 temp lrecl=131068 encoding=''utf-8''; '; + put ' proc json out=_sjs2 pretty '; put ' %if &action=ARR %then nokeys ; '; put ' ;export &tempds / nosastags fmtnumeric; '; put ' run; '; + put ' /* send back to webout */ '; + put ' data _null_; '; + put ' infile _sjs2 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs2 clear; '; put ' %end; '; put ' %else %if &engine=DATASTEP %then %do; '; put ' %datastep: '; @@ -19904,10 +20033,9 @@ data _null_; put ' %end; '; put ' run; '; put ' '; - put ' /* write to temp loc to avoid _webout truncation '; - put ' - https://support.sas.com/kb/49/325.html */ '; - put ' filename _sjs temp lrecl=131068 encoding=''utf-8''; '; - put ' data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; '; + put ' filename _sjs3 temp lrecl=131068 ; '; + put ' data _null_; '; + put ' file _sjs3 encoding=''utf-8''; '; put ' if _n_=1 then put "["; '; put ' set &tempds; '; put ' if _n_>1 then put "," @; put '; @@ -19918,27 +20046,29 @@ data _null_; put ' "&&name&i"n /* name literal for reserved variable names */ '; put ' %end; '; put ' %if &action=ARR %then "]" ; %else "}" ; ; '; - put ' /* now write the long strings to _webout 1 char at a time */ '; + put ' '; + put ' /* close out the table */ '; put ' data _null_; '; - put ' infile _sjs lrecl=1 recfm=n; '; + put ' file _sjs3 mod encoding=''utf-8''; '; + put ' put '']''; '; + put ' run; '; + put ' data _null_; '; + put ' infile _sjs3 lrecl=1 recfm=n; '; put ' file &jref mod lrecl=1 recfm=n; '; put ' input sourcechar $char1. @@; '; put ' format sourcechar hex2.; '; put ' put sourcechar char1. @@; '; put ' run; '; - put ' /* close out the table */ '; - put ' data _null_; '; - put ' file &jref mod; '; - put ' put '']''; '; - put ' run; '; - put ' filename _sjs clear; '; + put ' filename _sjs3 clear; '; put ' %end; '; put ' '; put ' proc sql; '; put ' drop table &colinfo, &tempds; '; put ' '; put ' %if &showmeta=YES %then %do; '; - put ' data _null_; file &jref encoding=''utf-8'' mod; '; + put ' filename _sjs4 temp lrecl=131068 encoding=''utf-8''; '; + put ' data _null_; '; + put ' file _sjs4; '; put ' put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; '; put ' do i=1 to &numcols; '; put ' name=quote(trim(symget(cats(''name'',i)))); '; @@ -19952,6 +20082,15 @@ data _null_; put ' end; '; put ' put ''}}''; '; put ' run; '; + put ' /* send back to webout */ '; + put ' data _null_; '; + put ' infile _sjs4 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs4 clear; '; put ' %end; '; put '%end; '; put ' '; @@ -22100,9 +22239,23 @@ data _null_; put ' run; '; put '%end; '; put '%else %if (&action=ARR or &action=OBJ) %then %do; '; + put ' /* force variable names to always be uppercase in the JSON */ '; put ' options validvarname=upcase; '; - put ' data _null_; file &jref encoding=''utf-8'' mod; '; + put ' /* To avoid issues with _webout on EBI - such as encoding diffs and truncation '; + put ' (https://support.sas.com/kb/49/325.html) we use temporary files */ '; + put ' filename _sjs1 temp lrecl=200 ; '; + put ' data _null_; file _sjs1 encoding=''utf-8''; '; put ' put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; '; + put ' run; '; + put ' /* now write to _webout 1 char at a time */ '; + put ' data _null_; '; + put ' infile _sjs1 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs1 clear; '; put ' '; put ' /* grab col defs */ '; put ' proc contents noprint data=&ds '; @@ -22159,10 +22312,20 @@ data _null_; put ' data &tempds;set &ds; '; put ' %if &fmt=N %then format _numeric_ best32.;; '; put ' /* PRETTY is necessary to avoid line truncation in large files */ '; - put ' proc json out=&jref pretty '; + put ' filename _sjs2 temp lrecl=131068 encoding=''utf-8''; '; + put ' proc json out=_sjs2 pretty '; put ' %if &action=ARR %then nokeys ; '; put ' ;export &tempds / nosastags fmtnumeric; '; put ' run; '; + put ' /* send back to webout */ '; + put ' data _null_; '; + put ' infile _sjs2 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs2 clear; '; put ' %end; '; put ' %else %if &engine=DATASTEP %then %do; '; put ' %datastep: '; @@ -22245,10 +22408,9 @@ data _null_; put ' %end; '; put ' run; '; put ' '; - put ' /* write to temp loc to avoid _webout truncation '; - put ' - https://support.sas.com/kb/49/325.html */ '; - put ' filename _sjs temp lrecl=131068 encoding=''utf-8''; '; - put ' data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; '; + put ' filename _sjs3 temp lrecl=131068 ; '; + put ' data _null_; '; + put ' file _sjs3 encoding=''utf-8''; '; put ' if _n_=1 then put "["; '; put ' set &tempds; '; put ' if _n_>1 then put "," @; put '; @@ -22259,27 +22421,29 @@ data _null_; put ' "&&name&i"n /* name literal for reserved variable names */ '; put ' %end; '; put ' %if &action=ARR %then "]" ; %else "}" ; ; '; - put ' /* now write the long strings to _webout 1 char at a time */ '; + put ' '; + put ' /* close out the table */ '; put ' data _null_; '; - put ' infile _sjs lrecl=1 recfm=n; '; + put ' file _sjs3 mod encoding=''utf-8''; '; + put ' put '']''; '; + put ' run; '; + put ' data _null_; '; + put ' infile _sjs3 lrecl=1 recfm=n; '; put ' file &jref mod lrecl=1 recfm=n; '; put ' input sourcechar $char1. @@; '; put ' format sourcechar hex2.; '; put ' put sourcechar char1. @@; '; put ' run; '; - put ' /* close out the table */ '; - put ' data _null_; '; - put ' file &jref mod; '; - put ' put '']''; '; - put ' run; '; - put ' filename _sjs clear; '; + put ' filename _sjs3 clear; '; put ' %end; '; put ' '; put ' proc sql; '; put ' drop table &colinfo, &tempds; '; put ' '; put ' %if &showmeta=YES %then %do; '; - put ' data _null_; file &jref encoding=''utf-8'' mod; '; + put ' filename _sjs4 temp lrecl=131068 encoding=''utf-8''; '; + put ' data _null_; '; + put ' file _sjs4; '; put ' put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; '; put ' do i=1 to &numcols; '; put ' name=quote(trim(symget(cats(''name'',i)))); '; @@ -22293,6 +22457,15 @@ data _null_; put ' end; '; put ' put ''}}''; '; put ' run; '; + put ' /* send back to webout */ '; + put ' data _null_; '; + put ' infile _sjs4 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs4 clear; '; put ' %end; '; put '%end; '; put ' '; diff --git a/base/mp_jsonout.sas b/base/mp_jsonout.sas index 924f333..79760c7 100644 --- a/base/mp_jsonout.sas +++ b/base/mp_jsonout.sas @@ -1,7 +1,12 @@ /** @file mp_jsonout.sas @brief Writes JSON in SASjs format to a fileref - @details PROC JSON is faster but will produce errs like the ones below if + @details This macro can be used to OPEN a JSON stream and send one or more + tables as arrays of rows, where each row can be an object or a nested array. + + There are two engines available - DATASTEP or PROCJSON. + + PROC JSON is fast but will produce errs like the ones below if special chars are encountered. > (ERR)OR: Some code points did not transcode. @@ -12,6 +17,10 @@ If this happens, try running with ENGINE=DATASTEP. + The DATASTEP engine is used to handle special SAS missing numerics, and + can also convert entire datasets to formatted values. Output JSON is always + in UTF-8. + Usage: filename tmp temp; @@ -75,9 +84,23 @@ run; %end; %else %if (&action=ARR or &action=OBJ) %then %do; + /* force variable names to always be uppercase in the JSON */ options validvarname=upcase; - data _null_; file &jref encoding='utf-8' mod; + /* To avoid issues with _webout on EBI - such as encoding diffs and truncation + (https://support.sas.com/kb/49/325.html) we use temporary files */ + filename _sjs1 temp lrecl=200 ; + data _null_; file _sjs1 encoding='utf-8'; put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; + run; + /* now write to _webout 1 char at a time */ + data _null_; + infile _sjs1 lrecl=1 recfm=n; + file &jref mod lrecl=1 recfm=n; + input sourcechar $char1. @@; + format sourcechar hex2.; + put sourcechar char1. @@; + run; + filename _sjs1 clear; /* grab col defs */ proc contents noprint data=&ds @@ -134,10 +157,20 @@ data &tempds;set &ds; %if &fmt=N %then format _numeric_ best32.;; /* PRETTY is necessary to avoid line truncation in large files */ - proc json out=&jref pretty + filename _sjs2 temp lrecl=131068 encoding='utf-8'; + proc json out=_sjs2 pretty %if &action=ARR %then nokeys ; ;export &tempds / nosastags fmtnumeric; run; + /* send back to webout */ + data _null_; + infile _sjs2 lrecl=1 recfm=n; + file &jref mod lrecl=1 recfm=n; + input sourcechar $char1. @@; + format sourcechar hex2.; + put sourcechar char1. @@; + run; + filename _sjs2 clear; %end; %else %if &engine=DATASTEP %then %do; %datastep: @@ -220,10 +253,9 @@ %end; run; - /* write to temp loc to avoid _webout truncation - - https://support.sas.com/kb/49/325.html */ - filename _sjs temp lrecl=131068 encoding='utf-8'; - data _null_; file _sjs lrecl=131068 encoding='utf-8' mod ; + filename _sjs3 temp lrecl=131068 ; + data _null_; + file _sjs3 encoding='utf-8'; if _n_=1 then put "["; set &tempds; if _n_>1 then put "," @; put @@ -234,27 +266,29 @@ "&&name&i"n /* name literal for reserved variable names */ %end; %if &action=ARR %then "]" ; %else "}" ; ; - /* now write the long strings to _webout 1 char at a time */ + + /* close out the table */ data _null_; - infile _sjs lrecl=1 recfm=n; + file _sjs3 mod encoding='utf-8'; + put ']'; + run; + data _null_; + infile _sjs3 lrecl=1 recfm=n; file &jref mod lrecl=1 recfm=n; input sourcechar $char1. @@; format sourcechar hex2.; put sourcechar char1. @@; run; - /* close out the table */ - data _null_; - file &jref mod; - put ']'; - run; - filename _sjs clear; + filename _sjs3 clear; %end; proc sql; drop table &colinfo, &tempds; %if &showmeta=YES %then %do; - data _null_; file &jref encoding='utf-8' mod; + filename _sjs4 temp lrecl=131068 encoding='utf-8'; + data _null_; + file _sjs4; put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; do i=1 to &numcols; name=quote(trim(symget(cats('name',i)))); @@ -268,6 +302,15 @@ end; put '}}'; run; + /* send back to webout */ + data _null_; + infile _sjs4 lrecl=1 recfm=n; + file &jref mod lrecl=1 recfm=n; + input sourcechar $char1. @@; + format sourcechar hex2.; + put sourcechar char1. @@; + run; + filename _sjs4 clear; %end; %end; diff --git a/meta/mm_createwebservice.sas b/meta/mm_createwebservice.sas index 9285e5c..4d33822 100644 --- a/meta/mm_createwebservice.sas +++ b/meta/mm_createwebservice.sas @@ -109,9 +109,23 @@ data _null_; put ' run; '; put '%end; '; put '%else %if (&action=ARR or &action=OBJ) %then %do; '; + put ' /* force variable names to always be uppercase in the JSON */ '; put ' options validvarname=upcase; '; - put ' data _null_; file &jref encoding=''utf-8'' mod; '; + put ' /* To avoid issues with _webout on EBI - such as encoding diffs and truncation '; + put ' (https://support.sas.com/kb/49/325.html) we use temporary files */ '; + put ' filename _sjs1 temp lrecl=200 ; '; + put ' data _null_; file _sjs1 encoding=''utf-8''; '; put ' put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; '; + put ' run; '; + put ' /* now write to _webout 1 char at a time */ '; + put ' data _null_; '; + put ' infile _sjs1 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs1 clear; '; put ' '; put ' /* grab col defs */ '; put ' proc contents noprint data=&ds '; @@ -168,10 +182,20 @@ data _null_; put ' data &tempds;set &ds; '; put ' %if &fmt=N %then format _numeric_ best32.;; '; put ' /* PRETTY is necessary to avoid line truncation in large files */ '; - put ' proc json out=&jref pretty '; + put ' filename _sjs2 temp lrecl=131068 encoding=''utf-8''; '; + put ' proc json out=_sjs2 pretty '; put ' %if &action=ARR %then nokeys ; '; put ' ;export &tempds / nosastags fmtnumeric; '; put ' run; '; + put ' /* send back to webout */ '; + put ' data _null_; '; + put ' infile _sjs2 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs2 clear; '; put ' %end; '; put ' %else %if &engine=DATASTEP %then %do; '; put ' %datastep: '; @@ -254,10 +278,9 @@ data _null_; put ' %end; '; put ' run; '; put ' '; - put ' /* write to temp loc to avoid _webout truncation '; - put ' - https://support.sas.com/kb/49/325.html */ '; - put ' filename _sjs temp lrecl=131068 encoding=''utf-8''; '; - put ' data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; '; + put ' filename _sjs3 temp lrecl=131068 ; '; + put ' data _null_; '; + put ' file _sjs3 encoding=''utf-8''; '; put ' if _n_=1 then put "["; '; put ' set &tempds; '; put ' if _n_>1 then put "," @; put '; @@ -268,27 +291,29 @@ data _null_; put ' "&&name&i"n /* name literal for reserved variable names */ '; put ' %end; '; put ' %if &action=ARR %then "]" ; %else "}" ; ; '; - put ' /* now write the long strings to _webout 1 char at a time */ '; + put ' '; + put ' /* close out the table */ '; put ' data _null_; '; - put ' infile _sjs lrecl=1 recfm=n; '; + put ' file _sjs3 mod encoding=''utf-8''; '; + put ' put '']''; '; + put ' run; '; + put ' data _null_; '; + put ' infile _sjs3 lrecl=1 recfm=n; '; put ' file &jref mod lrecl=1 recfm=n; '; put ' input sourcechar $char1. @@; '; put ' format sourcechar hex2.; '; put ' put sourcechar char1. @@; '; put ' run; '; - put ' /* close out the table */ '; - put ' data _null_; '; - put ' file &jref mod; '; - put ' put '']''; '; - put ' run; '; - put ' filename _sjs clear; '; + put ' filename _sjs3 clear; '; put ' %end; '; put ' '; put ' proc sql; '; put ' drop table &colinfo, &tempds; '; put ' '; put ' %if &showmeta=YES %then %do; '; - put ' data _null_; file &jref encoding=''utf-8'' mod; '; + put ' filename _sjs4 temp lrecl=131068 encoding=''utf-8''; '; + put ' data _null_; '; + put ' file _sjs4; '; put ' put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; '; put ' do i=1 to &numcols; '; put ' name=quote(trim(symget(cats(''name'',i)))); '; @@ -302,6 +327,15 @@ data _null_; put ' end; '; put ' put ''}}''; '; put ' run; '; + put ' /* send back to webout */ '; + put ' data _null_; '; + put ' infile _sjs4 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs4 clear; '; put ' %end; '; put '%end; '; put ' '; @@ -403,6 +437,8 @@ data _null_; put ' ) '; put '%end; '; put '%else %if &action=CLOSE %then %do; '; + put ' /* To avoid issues with _webout on EBI we use a temporary file */ '; + put ' filename _sjsref temp lrecl=131068; '; put ' %if %str(&_debug) ge 131 %then %do; '; put ' /* if debug mode, send back first 10 records of each work table also */ '; put ' options obs=10; '; @@ -416,11 +452,11 @@ data _null_; put ' i+1; '; put ' call symputx(cats(''wt'',i),name,''l''); '; put ' call symputx(''wtcnt'',i,''l''); '; - put ' data _null_; file &fref mod encoding=''utf-8''; '; + put ' data _null_; file _sjsref mod encoding=''utf-8''; '; put ' put ",""WORK"":{"; '; put ' %do i=1 %to &wtcnt; '; put ' %let wt=&&wt&i; '; - put ' data _null_; file &fref mod encoding=''utf-8''; '; + put ' data _null_; file _sjsref mod encoding=''utf-8''; '; put ' dsid=open("WORK.&wt",''is''); '; put ' nlobs=attrn(dsid,''NLOBS''); '; put ' nvars=attrn(dsid,''NVARS''); '; @@ -429,16 +465,16 @@ data _null_; put ' put " ""&wt"" : {"; '; put ' put ''"nlobs":'' nlobs; '; put ' put '',"nvars":'' nvars; '; - put ' %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) '; - put ' data _null_; file &fref mod encoding=''utf-8''; '; + put ' %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=YES) '; + put ' data _null_; file _sjsref mod encoding=''utf-8''; '; put ' put "}"; '; put ' %end; '; - put ' data _null_; file &fref mod encoding=''utf-8''; '; + put ' data _null_; file _sjsref mod encoding=''utf-8''; '; put ' put "}"; '; put ' run; '; put ' %end; '; put ' /* close off json */ '; - put ' data _null_;file &fref mod encoding=''utf-8''; '; + put ' data _null_;file _sjsref mod encoding=''utf-8''; '; put ' _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); '; put ' put ",""SYSUSERID"" : ""&sysuserid"" "; '; put ' put ",""MF_GETUSER"" : ""%mf_getuser()"" "; '; @@ -470,6 +506,16 @@ data _null_; put ' put ''>>weboutEND<<''; '; put ' %end; '; put ' run; '; + put ' /* now write to _webout 1 char at a time */ '; + put ' data _null_; '; + put ' infile _sjsref lrecl=1 recfm=n; '; + put ' file &fref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjsref clear; '; + put ' '; put '%end; '; put ' '; put '%mend mm_webout; '; diff --git a/meta/mm_webout.sas b/meta/mm_webout.sas index a0cc878..f1817d8 100644 --- a/meta/mm_webout.sas +++ b/meta/mm_webout.sas @@ -4,23 +4,23 @@ @details This macro should be added to the start of each Stored Process, **immediately** followed by a call to: - %mm_webout(FETCH) + %mm_webout(FETCH) - This will read all the input data and create same-named SAS datasets in the - WORK library. You can then insert your code, and send data back using the - following syntax: + This will read all the input data and create same-named SAS datasets in the + WORK library. You can then insert your code, and send data back using the + following syntax: - data some datasets; * make some data ; - retain some columns; - run; + data some datasets; * make some data ; + retain some columns; + run; - %mm_webout(OPEN) - %mm_webout(ARR,some) * Array format, fast, suitable for large tables ; - %mm_webout(OBJ,datasets) * Object format, easier to work with ; + %mm_webout(OPEN) + %mm_webout(ARR,some) * Array format, fast, suitable for large tables ; + %mm_webout(OBJ,datasets) * Object format, easier to work with ; - Finally, wrap everything up send some helpful system variables too + Finally, wrap everything up send some helpful system variables too - %mm_webout(CLOSE) + %mm_webout(CLOSE) @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE @@ -35,6 +35,10 @@ object with the same name as the table but prefixed with a dollar sign - ie, `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}` + +

SAS Macros

+ @li mp_jsonout.sas + @version 9.3 @author Allan Bowe @@ -111,6 +115,8 @@ ) %end; %else %if &action=CLOSE %then %do; + /* To avoid issues with _webout on EBI we use a temporary file */ + filename _sjsref temp lrecl=131068; %if %str(&_debug) ge 131 %then %do; /* if debug mode, send back first 10 records of each work table also */ options obs=10; @@ -124,11 +130,11 @@ i+1; call symputx(cats('wt',i),name,'l'); call symputx('wtcnt',i,'l'); - data _null_; file &fref mod encoding='utf-8'; + data _null_; file _sjsref mod encoding='utf-8'; put ",""WORK"":{"; %do i=1 %to &wtcnt; %let wt=&&wt&i; - data _null_; file &fref mod encoding='utf-8'; + data _null_; file _sjsref mod encoding='utf-8'; dsid=open("WORK.&wt",'is'); nlobs=attrn(dsid,'NLOBS'); nvars=attrn(dsid,'NVARS'); @@ -137,16 +143,16 @@ put " ""&wt"" : {"; put '"nlobs":' nlobs; put ',"nvars":' nvars; - %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) - data _null_; file &fref mod encoding='utf-8'; + %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=YES) + data _null_; file _sjsref mod encoding='utf-8'; put "}"; %end; - data _null_; file &fref mod encoding='utf-8'; + data _null_; file _sjsref mod encoding='utf-8'; put "}"; run; %end; /* close off json */ - data _null_;file &fref mod encoding='utf-8'; + data _null_;file _sjsref mod encoding='utf-8'; _PROGRAM=quote(trim(resolve(symget('_PROGRAM')))); put ",""SYSUSERID"" : ""&sysuserid"" "; put ",""MF_GETUSER"" : ""%mf_getuser()"" "; @@ -178,6 +184,16 @@ put '>>weboutEND<<'; %end; run; + /* now write to _webout 1 char at a time */ + data _null_; + infile _sjsref lrecl=1 recfm=n; + file &fref mod lrecl=1 recfm=n; + input sourcechar $char1. @@; + format sourcechar hex2.; + put sourcechar char1. @@; + run; + filename _sjsref clear; + %end; %mend mm_webout; diff --git a/server/ms_createwebservice.sas b/server/ms_createwebservice.sas index e3d392e..a5f3e9d 100644 --- a/server/ms_createwebservice.sas +++ b/server/ms_createwebservice.sas @@ -110,9 +110,23 @@ data _null_; put ' run; '; put '%end; '; put '%else %if (&action=ARR or &action=OBJ) %then %do; '; + put ' /* force variable names to always be uppercase in the JSON */ '; put ' options validvarname=upcase; '; - put ' data _null_; file &jref encoding=''utf-8'' mod; '; + put ' /* To avoid issues with _webout on EBI - such as encoding diffs and truncation '; + put ' (https://support.sas.com/kb/49/325.html) we use temporary files */ '; + put ' filename _sjs1 temp lrecl=200 ; '; + put ' data _null_; file _sjs1 encoding=''utf-8''; '; put ' put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; '; + put ' run; '; + put ' /* now write to _webout 1 char at a time */ '; + put ' data _null_; '; + put ' infile _sjs1 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs1 clear; '; put ' '; put ' /* grab col defs */ '; put ' proc contents noprint data=&ds '; @@ -169,10 +183,20 @@ data _null_; put ' data &tempds;set &ds; '; put ' %if &fmt=N %then format _numeric_ best32.;; '; put ' /* PRETTY is necessary to avoid line truncation in large files */ '; - put ' proc json out=&jref pretty '; + put ' filename _sjs2 temp lrecl=131068 encoding=''utf-8''; '; + put ' proc json out=_sjs2 pretty '; put ' %if &action=ARR %then nokeys ; '; put ' ;export &tempds / nosastags fmtnumeric; '; put ' run; '; + put ' /* send back to webout */ '; + put ' data _null_; '; + put ' infile _sjs2 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs2 clear; '; put ' %end; '; put ' %else %if &engine=DATASTEP %then %do; '; put ' %datastep: '; @@ -255,10 +279,9 @@ data _null_; put ' %end; '; put ' run; '; put ' '; - put ' /* write to temp loc to avoid _webout truncation '; - put ' - https://support.sas.com/kb/49/325.html */ '; - put ' filename _sjs temp lrecl=131068 encoding=''utf-8''; '; - put ' data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; '; + put ' filename _sjs3 temp lrecl=131068 ; '; + put ' data _null_; '; + put ' file _sjs3 encoding=''utf-8''; '; put ' if _n_=1 then put "["; '; put ' set &tempds; '; put ' if _n_>1 then put "," @; put '; @@ -269,27 +292,29 @@ data _null_; put ' "&&name&i"n /* name literal for reserved variable names */ '; put ' %end; '; put ' %if &action=ARR %then "]" ; %else "}" ; ; '; - put ' /* now write the long strings to _webout 1 char at a time */ '; + put ' '; + put ' /* close out the table */ '; put ' data _null_; '; - put ' infile _sjs lrecl=1 recfm=n; '; + put ' file _sjs3 mod encoding=''utf-8''; '; + put ' put '']''; '; + put ' run; '; + put ' data _null_; '; + put ' infile _sjs3 lrecl=1 recfm=n; '; put ' file &jref mod lrecl=1 recfm=n; '; put ' input sourcechar $char1. @@; '; put ' format sourcechar hex2.; '; put ' put sourcechar char1. @@; '; put ' run; '; - put ' /* close out the table */ '; - put ' data _null_; '; - put ' file &jref mod; '; - put ' put '']''; '; - put ' run; '; - put ' filename _sjs clear; '; + put ' filename _sjs3 clear; '; put ' %end; '; put ' '; put ' proc sql; '; put ' drop table &colinfo, &tempds; '; put ' '; put ' %if &showmeta=YES %then %do; '; - put ' data _null_; file &jref encoding=''utf-8'' mod; '; + put ' filename _sjs4 temp lrecl=131068 encoding=''utf-8''; '; + put ' data _null_; '; + put ' file _sjs4; '; put ' put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; '; put ' do i=1 to &numcols; '; put ' name=quote(trim(symget(cats(''name'',i)))); '; @@ -303,6 +328,15 @@ data _null_; put ' end; '; put ' put ''}}''; '; put ' run; '; + put ' /* send back to webout */ '; + put ' data _null_; '; + put ' infile _sjs4 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs4 clear; '; put ' %end; '; put '%end; '; put ' '; diff --git a/viya/mv_createwebservice.sas b/viya/mv_createwebservice.sas index 5b3f18a..6d032b3 100644 --- a/viya/mv_createwebservice.sas +++ b/viya/mv_createwebservice.sas @@ -252,9 +252,23 @@ data _null_; put ' run; '; put '%end; '; put '%else %if (&action=ARR or &action=OBJ) %then %do; '; + put ' /* force variable names to always be uppercase in the JSON */ '; put ' options validvarname=upcase; '; - put ' data _null_; file &jref encoding=''utf-8'' mod; '; + put ' /* To avoid issues with _webout on EBI - such as encoding diffs and truncation '; + put ' (https://support.sas.com/kb/49/325.html) we use temporary files */ '; + put ' filename _sjs1 temp lrecl=200 ; '; + put ' data _null_; file _sjs1 encoding=''utf-8''; '; put ' put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; '; + put ' run; '; + put ' /* now write to _webout 1 char at a time */ '; + put ' data _null_; '; + put ' infile _sjs1 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs1 clear; '; put ' '; put ' /* grab col defs */ '; put ' proc contents noprint data=&ds '; @@ -311,10 +325,20 @@ data _null_; put ' data &tempds;set &ds; '; put ' %if &fmt=N %then format _numeric_ best32.;; '; put ' /* PRETTY is necessary to avoid line truncation in large files */ '; - put ' proc json out=&jref pretty '; + put ' filename _sjs2 temp lrecl=131068 encoding=''utf-8''; '; + put ' proc json out=_sjs2 pretty '; put ' %if &action=ARR %then nokeys ; '; put ' ;export &tempds / nosastags fmtnumeric; '; put ' run; '; + put ' /* send back to webout */ '; + put ' data _null_; '; + put ' infile _sjs2 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs2 clear; '; put ' %end; '; put ' %else %if &engine=DATASTEP %then %do; '; put ' %datastep: '; @@ -397,10 +421,9 @@ data _null_; put ' %end; '; put ' run; '; put ' '; - put ' /* write to temp loc to avoid _webout truncation '; - put ' - https://support.sas.com/kb/49/325.html */ '; - put ' filename _sjs temp lrecl=131068 encoding=''utf-8''; '; - put ' data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; '; + put ' filename _sjs3 temp lrecl=131068 ; '; + put ' data _null_; '; + put ' file _sjs3 encoding=''utf-8''; '; put ' if _n_=1 then put "["; '; put ' set &tempds; '; put ' if _n_>1 then put "," @; put '; @@ -411,27 +434,29 @@ data _null_; put ' "&&name&i"n /* name literal for reserved variable names */ '; put ' %end; '; put ' %if &action=ARR %then "]" ; %else "}" ; ; '; - put ' /* now write the long strings to _webout 1 char at a time */ '; + put ' '; + put ' /* close out the table */ '; put ' data _null_; '; - put ' infile _sjs lrecl=1 recfm=n; '; + put ' file _sjs3 mod encoding=''utf-8''; '; + put ' put '']''; '; + put ' run; '; + put ' data _null_; '; + put ' infile _sjs3 lrecl=1 recfm=n; '; put ' file &jref mod lrecl=1 recfm=n; '; put ' input sourcechar $char1. @@; '; put ' format sourcechar hex2.; '; put ' put sourcechar char1. @@; '; put ' run; '; - put ' /* close out the table */ '; - put ' data _null_; '; - put ' file &jref mod; '; - put ' put '']''; '; - put ' run; '; - put ' filename _sjs clear; '; + put ' filename _sjs3 clear; '; put ' %end; '; put ' '; put ' proc sql; '; put ' drop table &colinfo, &tempds; '; put ' '; put ' %if &showmeta=YES %then %do; '; - put ' data _null_; file &jref encoding=''utf-8'' mod; '; + put ' filename _sjs4 temp lrecl=131068 encoding=''utf-8''; '; + put ' data _null_; '; + put ' file _sjs4; '; put ' put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; '; put ' do i=1 to &numcols; '; put ' name=quote(trim(symget(cats(''name'',i)))); '; @@ -445,6 +470,15 @@ data _null_; put ' end; '; put ' put ''}}''; '; put ' run; '; + put ' /* send back to webout */ '; + put ' data _null_; '; + put ' infile _sjs4 lrecl=1 recfm=n; '; + put ' file &jref mod lrecl=1 recfm=n; '; + put ' input sourcechar $char1. @@; '; + put ' format sourcechar hex2.; '; + put ' put sourcechar char1. @@; '; + put ' run; '; + put ' filename _sjs4 clear; '; put ' %end; '; put '%end; '; put ' ';