From b525b4171dd4152da589c933da9161cc6c774438 Mon Sep 17 00:00:00 2001 From: munja Date: Thu, 28 Apr 2022 21:06:28 +0100 Subject: [PATCH] fix: updating ms_runstp to accept parameters and file inputs. Explicitly setting lrecl everywhere. Adding lrecl=80 as default in testinit.sas --- base/mp_gsubfile.sas | 2 +- base/mp_jsonout.sas | 2 +- meta/mm_createwebservice.sas | 2 +- server/ms_createwebservice.sas | 64 +++++----- server/ms_runstp.sas | 135 +++++++++++++++++++-- server/ms_webout.sas | 17 +-- tests/crossplatform/mp_base64copy.test.sas | 2 +- tests/crossplatform/mp_ds2cards.test.sas | 4 +- tests/crossplatform/mp_gsubfile.test.sas | 2 +- tests/crossplatform/mp_replace.test.sas | 8 +- tests/serveronly/mfs_httpheader.test.sas | 5 + tests/serveronly/ms_runstp.test.sas | 1 + tests/testinit.sas | 6 +- tests/testterm.sas | 2 +- viya/mv_createwebservice.sas | 2 +- 15 files changed, 196 insertions(+), 58 deletions(-) diff --git a/base/mp_gsubfile.sas b/base/mp_gsubfile.sas index 8aa02ea..52dd887 100644 --- a/base/mp_gsubfile.sas +++ b/base/mp_gsubfile.sas @@ -48,7 +48,7 @@ outfile=0 )/*/STORE SOURCE*/; - %if "%substr(&sysver,1,4)"="V.04" %then %do; + %if "%substr(&sysver.XX,1,4)"="V.04" %then %do; %put %str(ERR)OR: Viya 4 does not support the IO library in lua; %return; %end; diff --git a/base/mp_jsonout.sas b/base/mp_jsonout.sas index bcb0a25..be2c7bc 100644 --- a/base/mp_jsonout.sas +++ b/base/mp_jsonout.sas @@ -70,7 +70,7 @@ %if &action=OPEN %then %do; options nobomfile; - data _null_;file &jref encoding='utf-8' ; + data _null_;file &jref encoding='utf-8' lrecl=200; put '{"PROCESSED_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '"'; run; %end; diff --git a/meta/mm_createwebservice.sas b/meta/mm_createwebservice.sas index 049d606..68ba65b 100644 --- a/meta/mm_createwebservice.sas +++ b/meta/mm_createwebservice.sas @@ -103,7 +103,7 @@ data _null_; put ' '; put '%if &action=OPEN %then %do; '; put ' options nobomfile; '; - put ' data _null_;file &jref encoding=''utf-8'' ; '; + put ' data _null_;file &jref encoding=''utf-8'' lrecl=200; '; put ' put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; '; put ' run; '; put '%end; '; diff --git a/server/ms_createwebservice.sas b/server/ms_createwebservice.sas index e1394d4..e6d6d93 100644 --- a/server/ms_createwebservice.sas +++ b/server/ms_createwebservice.sas @@ -31,7 +31,7 @@ For more examples of using these web services with the SASjs Adapter, see: https://github.com/sasjs/adapter#readme - @param [in] path= The full SASjs Drive path in which to create the service + @param [in] path= (0) The full SASjs Drive path in which to create the service @param [in] name= Stored Program name @param [in] desc= The description of the service (not implemented yet) @param [in] precode= Space separated list of filerefs, pointing to the code @@ -44,6 +44,8 @@ @li ms_createfile.sas @li mf_getuser.sas @li mf_getuniquename.sas + @li mf_getuniquefileref.sas + @li mp_abort.sas

Related Files

@li ms_createwebservice.test.sas @@ -53,7 +55,7 @@ **/ -%macro ms_createwebservice(path= +%macro ms_createwebservice(path=0 ,name=initService ,precode= ,code=ft15f001 @@ -61,18 +63,23 @@ ,mDebug=0 )/*/STORE SOURCE*/; -%if &syscc ge 4 %then %do; - %put &=syscc - &sysmacroname will not execute in this state; - %return; +%local dbg; +%if &mdebug=1 %then %do; + %put &sysmacroname entry vars:; + %put _local_; %end; +%else %let dbg=*; -%local mD; -%if &mDebug=1 %then %let mD=; -%else %let mD=%str(*); -%&mD.put Executing ms_createwebservice.sas; -%&mD.put _local_; +%mp_abort(iftrue=(&syscc ge 4 ) + ,mac=ms_createwebservice + ,msg=%str(syscc=&syscc on macro entry) +) +%mp_abort(iftrue=("&path"="0") + ,mac=ms_createwebservice + ,msg=%str(Path not provided) +) -* remove any trailing slash ; +/* remove any trailing slash */ %if "%substr(&path,%length(&path),1)" = "/" %then %let path=%substr(&path,1,%length(&path)-1); @@ -81,9 +88,10 @@ * These put statements are auto generated - to change the macro, change the * source (ms_webout) and run `build.py` */ -filename sasjs temp; +%local sasjsref; +%let sasjsref=%mf_getuniquefileref(); data _null_; - file sasjs lrecl=3000 ; + file &sasjsref termstr=crlf lrecl=512; put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */"; /* WEBOUT BEGIN */ put ' '; @@ -97,7 +105,7 @@ data _null_; put ' '; put '%if &action=OPEN %then %do; '; put ' options nobomfile; '; - put ' data _null_;file &jref encoding=''utf-8'' ; '; + put ' data _null_;file &jref encoding=''utf-8'' lrecl=200; '; put ' put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; '; put ' run; '; put '%end; '; @@ -349,13 +357,14 @@ data _null_; put ' %let _webin_name1=&_webin_name; '; put ' %end; '; put ' data _null_; '; - put ' infile &&_webin_fileref&i termstr=crlf; '; + put ' infile &&_webin_fileref&i termstr=crlf lrecl=32767; '; put ' input; '; put ' call symputx(''input_statement'',_infile_); '; put ' putlog "&&_webin_name&i input statement: " _infile_; '; put ' stop; '; put ' data &&_webin_name&i; '; - put ' infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding=''utf-8''; '; + put ' infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding=''utf-8'' '; + put ' lrecl=32767; '; put ' input &input_statement; '; put ' %if %str(&_debug) ge 131 %then %do; '; put ' if _n_<20 then putlog _infile_; '; @@ -366,14 +375,14 @@ data _null_; put '%end; '; put ' '; put '%else %if &action=OPEN %then %do; '; - put ' /* fix encoding */ '; - put ' OPTIONS NOBOMFILE; '; + put ' /* fix encoding and ensure enough lrecl */ '; + put ' OPTIONS NOBOMFILE lrecl=32767; '; put ' '; put ' /* set the header */ '; put ' %mfs_httpheader(Content-type,application/json) '; put ' '; - put ' /* setup json */ '; - put ' data _null_;file &fref encoding=''utf-8'' termstr=lf; '; + put ' /* setup json. */ '; + put ' data _null_;file &fref encoding=''utf-8'' termstr=lf ; '; put ' put ''{"SYSDATE" : "'' "&SYSDATE" ''"''; '; put ' put '',"SYSTIME" : "'' "&SYSTIME" ''"''; '; put ' run; '; @@ -417,12 +426,12 @@ data _null_; put ' data _null_; file &fref mod encoding=''utf-8'' termstr=lf; '; put ' put "}"; '; put ' %end; '; - put ' data _null_; file &fref mod encoding=''utf-8'' termstr=lf termstr=lf; '; + put ' data _null_; file &fref mod encoding=''utf-8'' termstr=lf; '; put ' put "}"; '; put ' run; '; put ' %end; '; put ' /* close off json */ '; - put ' data _null_;file &fref mod encoding=''utf-8'' termstr=lf; '; + put ' data _null_;file &fref mod encoding=''utf-8'' termstr=lf lrecl=32767; '; put ' _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); '; put ' put ",""SYSUSERID"" : ""&sysuserid"" "; '; put ' put ",""MF_GETUSER"" : ""%mf_getuser()"" "; '; @@ -495,24 +504,21 @@ data _null_; run; /* add precode and code */ -%local tmpref x fref freflist mod; -%let tmpref=%mf_getuniquefileref(); +%local x fref freflist; %let freflist=&precode &code ; %do x=1 %to %sysfunc(countw(&freflist)); - %if &x>1 %then %let mod=mod; - %let fref=%scan(&freflist,&x); %put &sysmacroname: adding &fref; data _null_; - file &tmpref lrecl=3000 &mod; - infile &fref; + file &sasjsref lrecl=3000 termstr=crlf mod; + infile &fref lrecl=3000; input; put _infile_; run; %end; /* create the web service */ -%ms_createfile(&path/&name..sas, inref=&tmpref,mdebug=&mdebug) +%ms_createfile(&path/&name..sas, inref=&sasjsref, mdebug=&mdebug) %put ;%put ;%put ;%put ;%put ;%put ; %put &sysmacroname: STP &name successfully created in &path; diff --git a/server/ms_runstp.sas b/server/ms_runstp.sas index e8b2107..32e4d46 100644 --- a/server/ms_runstp.sas +++ b/server/ms_runstp.sas @@ -15,48 +15,154 @@ parameter) @param [in] debug= (131) The value to supply to the _debug URL parameter @param [in] mdebug= (0) Set to 1 to enable DEBUG messages + @param [in] inputparams=(_null_) A dataset containing name/value pairs in the + following format: + |name:$32|value:$10000| + |---|---| + |stpmacname|some value| + |mustbevalidname|can be anything, oops, %abort!!| + @param [in] inputfiles= (_null_) A dataset containing fileref/name/filename in + the following format: + |fileref:$8|name:$32|filename:$256| + |---|---|--| + |someref|some_name|some_filename.xls| + |fref2|another_file|zyx_v2.csv| + @param [out] outref= (outweb) The output fileref to contain the response JSON (will be created using temp engine) + @param [out] outlogds= (_null_) Set to the name of a dataset to contain the + log. Table format: + |line:$2000| + |---| + |log line 1| + |log line 2|

SAS Macros

@li mf_getuniquefileref.sas + @li mf_getuniquelibref.sas @li mp_abort.sas **/ %macro ms_runstp(pgm ,debug=131 + ,inputparams=_null_ + ,inputfiles=_null_ ,outref=outweb + ,outlogds=_null_ ,mdebug=0 ); -%local dbg fname1; +%local dbg mainref authref boundary; +%let mainref=%mf_getuniquefileref(); +%let authref=%mf_getuniquefileref(); +%let boundary=%mf_getuniquename(); +%if &inputparams=0 %then %let inputparams=_null_; + %if &mdebug=1 %then %do; %put &sysmacroname entry vars:; %put _local_; %end; %else %let dbg=*; -%let fname1=%mf_getuniquefileref(); + %mp_abort(iftrue=("&pgm"="") ,mac=&sysmacroname ,msg=%str(Program not provided) ) +/* avoid sending bom marker to API */ +%local optval; +%let optval=%sysfunc(getoption(bomfile)); +options nobomfile; + +/* add params */ data _null_; - file &fname1 lrecl=1000; + file &mainref termstr=crlf lrecl=32767 mod; + length line $1000 name $32 value $32767; + if _n_=1 then call missing(of _all_); + set &inputparams; + put "--&boundary"; + line=cats('Content-Disposition: form-data; name="',name,'"'); + put line; + put ; + put value; +run; + +/* parse input file list */ +%local webcount; +%let webcount=0; +data _null_; + set &inputfiles end=last; + length fileref $8 name $32 filename $256; + call symputx(cats('webref',_n_),fileref,'l'); + call symputx(cats('webname',_n_),name,'l'); + call symputx(cats('webfilename',_n_),filename,'l'); + if last then do; + call symputx('webcount',_n_); + call missing(of _all_); + end; +run; + +/* write out the input files */ +%local i; +%do i=1 %to &webcount; + data _null_; + file &mainref termstr=crlf lrecl=32767 mod; + infile &&webref&i lrecl=32767; + if _n_ = 1 then do; + length line $32767; + line=cats( + 'Content-Disposition: form-data; name="' + ,"&&webname&i" + ,'"; filename="' + ,"&&webfilename&i" + ,'"' + ); + put "--&boundary"; + put line; + put "Content-Type: text/plain"; + put ; + end; + input; + put _infile_; /* add the actual file to be sent */ + run; +%end; + +data _null_; + file &mainref termstr=crlf mod; + put "--&boundary--"; +run; + +data _null_; + file &authref lrecl=1000; infile "&_sasjs_tokenfile" lrecl=1000; input; put 'Authorization: Bearer ' _infile_; + put "Content-Type: multipart/form-data; boundary=&boundary"; run; -filename &outref temp; +%if &mdebug=1 %then %do; + data _null_; + infile &authref; + input; + put _infile_; + data _null_; + infile &mainref; + input; + put _infile_; + run; +%end; +filename &outref temp lrecl=32767; /* prepare request*/ -proc http method='POST' headerin=&fname1 out=&outref +proc http method='POST' headerin=&authref in=&mainref out=&outref url="&_sasjs_apiserverurl.&_sasjs_apipath?_program=&pgm%str(&)_debug=131"; +%if &mdebug=1 %then %do; + debug level=2; +%end; run; %if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201) - or &mdebug=1 %then %do; +%then %do; data _null_;infile &outref;input;putlog _infile_;run; %end; %mp_abort( @@ -65,6 +171,20 @@ run; ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) ) +/* reset options */ +options &optval; + +%if &outlogds ne _null_ or &mdebug=1 %then %do; + %local dumplib; + %let dumplib=%mf_getuniquelibref(); + libname &dumplib json (&outref); + data &outlogds; + set &dumplib..log; + %if &mdebug=1 %then %do; + putlog line=; + %end; + run; +%end; %if &mdebug=1 %then %do; %put &sysmacroname exit vars:; @@ -72,6 +192,7 @@ run; %end; %else %do; /* clear refs */ - filename &fname1 clear; + filename &authref; + filename &mainref; %end; %mend ms_runstp; \ No newline at end of file diff --git a/server/ms_webout.sas b/server/ms_webout.sas index e9a31da..504ec6b 100644 --- a/server/ms_webout.sas +++ b/server/ms_webout.sas @@ -67,13 +67,14 @@ %let _webin_name1=&_webin_name; %end; data _null_; - infile &&_webin_fileref&i termstr=crlf; + infile &&_webin_fileref&i termstr=crlf lrecl=32767; input; call symputx('input_statement',_infile_); putlog "&&_webin_name&i input statement: " _infile_; stop; data &&_webin_name&i; - infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding='utf-8'; + infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding='utf-8' + lrecl=32767; input &input_statement; %if %str(&_debug) ge 131 %then %do; if _n_<20 then putlog _infile_; @@ -84,14 +85,14 @@ %end; %else %if &action=OPEN %then %do; - /* fix encoding */ - OPTIONS NOBOMFILE; + /* fix encoding and ensure enough lrecl */ + OPTIONS NOBOMFILE lrecl=32767; /* set the header */ %mfs_httpheader(Content-type,application/json) - /* setup json */ - data _null_;file &fref encoding='utf-8' termstr=lf; + /* setup json. */ + data _null_;file &fref encoding='utf-8' termstr=lf ; put '{"SYSDATE" : "' "&SYSDATE" '"'; put ',"SYSTIME" : "' "&SYSTIME" '"'; run; @@ -135,12 +136,12 @@ data _null_; file &fref mod encoding='utf-8' termstr=lf; put "}"; %end; - data _null_; file &fref mod encoding='utf-8' termstr=lf termstr=lf; + data _null_; file &fref mod encoding='utf-8' termstr=lf; put "}"; run; %end; /* close off json */ - data _null_;file &fref mod encoding='utf-8' termstr=lf; + data _null_;file &fref mod encoding='utf-8' termstr=lf lrecl=32767; _PROGRAM=quote(trim(resolve(symget('_PROGRAM')))); put ",""SYSUSERID"" : ""&sysuserid"" "; put ",""MF_GETUSER"" : ""%mf_getuser()"" "; diff --git a/tests/crossplatform/mp_base64copy.test.sas b/tests/crossplatform/mp_base64copy.test.sas index ec74803..680ed35 100644 --- a/tests/crossplatform/mp_base64copy.test.sas +++ b/tests/crossplatform/mp_base64copy.test.sas @@ -41,7 +41,7 @@ run; /* multibyte string check */ -filename tmp2 temp; +filename tmp2 temp lrecl=500; data _null_; file tmp2; put "'╤', '╔', '╗', '═', '╧', '╚', '╝', '║', '╟', '─', '┼', '║', '╢', '│'"; diff --git a/tests/crossplatform/mp_ds2cards.test.sas b/tests/crossplatform/mp_ds2cards.test.sas index e4ae3bc..e2f7f10 100644 --- a/tests/crossplatform/mp_ds2cards.test.sas +++ b/tests/crossplatform/mp_ds2cards.test.sas @@ -18,7 +18,7 @@ , cards_file= "%sysfunc(pathname(work))/cars.sas" , showlog=NO ) -%inc "%sysfunc(pathname(work))/cars.sas"/source2; +%inc "%sysfunc(pathname(work))/cars.sas"/source2 lrecl=32767; proc compare base=sashelp.cars compare=work.test; quit; @@ -48,7 +48,7 @@ run; , append= ) -%inc "%sysfunc(pathname(work))/c2.sas"/source2; +%inc "%sysfunc(pathname(work))/c2.sas"/source2 lrecl=32767; proc compare base=work.binarybase compare=work.binarycompare; run; diff --git a/tests/crossplatform/mp_gsubfile.test.sas b/tests/crossplatform/mp_gsubfile.test.sas index 407f269..77edc16 100644 --- a/tests/crossplatform/mp_gsubfile.test.sas +++ b/tests/crossplatform/mp_gsubfile.test.sas @@ -10,7 +10,7 @@ %macro gsubtest(); -%if "%substr(&sysver,1,4)"="V.04" %then %do; +%if "%substr(&sysver.XX,1,4)"="V.04" %then %do; %put %str(ERR)OR: Viya 4 does not support the IO library in lua; %return; %end; diff --git a/tests/crossplatform/mp_replace.test.sas b/tests/crossplatform/mp_replace.test.sas index 2c0d1b7..2ba2d9c 100644 --- a/tests/crossplatform/mp_replace.test.sas +++ b/tests/crossplatform/mp_replace.test.sas @@ -39,7 +39,7 @@ run; %let str=%str(replacewith trailing spaces ); %let rep=%str( with more spaces ); data _null_; - file &test2; + file &test2 lrecl=500; put 'blahblah'; put "blahblah&str.blah&str. replace &str.X"; put "blahbreplacewith&str.spacesahblah"; @@ -47,7 +47,7 @@ run; %mp_replace(&test2, findvar=str, replacevar=rep) data _null_; - infile &test2; + infile &test2 lrecl=500; input; if _n_=2 then call symputx('test2resulta',_infile_); if _n_=3 then call symputx('test2resultb',_infile_); @@ -69,7 +69,7 @@ run; %let str=%str(replace.string.with.dots ); %let rep=%str( more.dots); data _null_; - file &test3; + file &test3 lrecl=500; put 'blahblah'; put "blahblah&str.blah&str. replace &str.X"; put "blahbreplacewith&str.spacesahblah"; @@ -77,7 +77,7 @@ run; %mp_replace(&test3, findvar=str, replacevar=rep) data _null_; - infile &test3; + infile &test3 lrecl=500; input; if _n_=2 then call symputx('test3resulta',_infile_); if _n_=3 then call symputx('test3resultb',_infile_); diff --git a/tests/serveronly/mfs_httpheader.test.sas b/tests/serveronly/mfs_httpheader.test.sas index baf5ddc..2874907 100644 --- a/tests/serveronly/mfs_httpheader.test.sas +++ b/tests/serveronly/mfs_httpheader.test.sas @@ -50,3 +50,8 @@ run; desc=Checking line was created, outds=work.test_results ) + + +/* reset header so the test will pass */ + +%mfs_httpheader(Content-type,application/json) \ No newline at end of file diff --git a/tests/serveronly/ms_runstp.test.sas b/tests/serveronly/ms_runstp.test.sas index 68e1e79..00fc38f 100644 --- a/tests/serveronly/ms_runstp.test.sas +++ b/tests/serveronly/ms_runstp.test.sas @@ -16,6 +16,7 @@ filename stpcode temp; data _null_; file stpcode; put '%put hello world;'; + put '%put _all_;'; run; options mprint; diff --git a/tests/testinit.sas b/tests/testinit.sas index 16009c5..d9344ee 100644 --- a/tests/testinit.sas +++ b/tests/testinit.sas @@ -5,6 +5,7 @@

SAS Macros

@li mf_uid.sas @li mp_init.sas + @li ms_webout.sas **/ @@ -14,12 +15,15 @@ /* set defaults */ %mp_init() +options lrecl=80; + %global _debug sasjs_mdebug; %let sasjs_mdebug=0; %macro loglevel(); - %if "&_debug"="2477" or "&_debug"="fields,log,trace" %then %do; + %if "&_debug"="2477" or "&_debug"="fields,log,trace" or "&_debug"="131" + %then %do; %put debug mode activated; options mprint mprintnest; %let sasjs_mdebug=1; diff --git a/tests/testterm.sas b/tests/testterm.sas index bc0e432..1fd9136 100644 --- a/tests/testterm.sas +++ b/tests/testterm.sas @@ -9,7 +9,7 @@ %mp_assert( iftrue=(&syscc=0), - desc=Checking final error condition, + desc=Checking final err condition, outds=work.test_results ) diff --git a/viya/mv_createwebservice.sas b/viya/mv_createwebservice.sas index d17d7cf..c2c44a0 100644 --- a/viya/mv_createwebservice.sas +++ b/viya/mv_createwebservice.sas @@ -247,7 +247,7 @@ data _null_; put ' '; put '%if &action=OPEN %then %do; '; put ' options nobomfile; '; - put ' data _null_;file &jref encoding=''utf-8'' ; '; + put ' data _null_;file &jref encoding=''utf-8'' lrecl=200; '; put ' put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; '; put ' run; '; put '%end; ';