fix: updating ms_runstp to accept parameters and file inputs. Explicitly setting lrecl everywhere. Adding lrecl=80 as default in testinit.sas

2025-12-10 14:04:36 +00:00 · 2022-04-28 21:06:28 +01:00
parent f2d80b3b63
commit b525b4171d
15 changed files with 196 additions and 58 deletions
--- a/base/mp_gsubfile.sas
+++ b/base/mp_gsubfile.sas
@@ -48,7 +48,7 @@
  outfile=0
 )/*/STORE SOURCE*/;

-  %if "%substr(&sysver,1,4)"="V.04" %then %do;
+  %if "%substr(&sysver.XX,1,4)"="V.04" %then %do;
    %put %str(ERR)OR: Viya 4 does not support the IO library in lua;
    %return;
  %end;
--- a/base/mp_jsonout.sas
+++ b/base/mp_jsonout.sas
@@ -70,7 +70,7 @@

 %if &action=OPEN %then %do;
  options nobomfile;
-  data _null_;file &jref encoding='utf-8' ;
+  data _null_;file &jref encoding='utf-8' lrecl=200;
    put '{"PROCESSED_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '"';
  run;
 %end;
--- a/meta/mm_createwebservice.sas
+++ b/meta/mm_createwebservice.sas
@@ -103,7 +103,7 @@ data _null_;
  put ' ';
  put '%if &action=OPEN %then %do; ';
  put '  options nobomfile; ';
-  put '  data _null_;file &jref encoding=''utf-8'' ; ';
+  put '  data _null_;file &jref encoding=''utf-8'' lrecl=200; ';
  put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
  put '  run; ';
  put '%end; ';
--- a/server/ms_createwebservice.sas
+++ b/server/ms_createwebservice.sas
@@ -31,7 +31,7 @@
  For more examples of using these web services with the SASjs Adapter, see:
  https://github.com/sasjs/adapter#readme

-  @param [in] path= The full SASjs Drive path in which to create the service
+  @param [in] path= (0) The full SASjs Drive path in which to create the service
  @param [in] name= Stored Program name
  @param [in] desc= The description of the service (not implemented yet)
  @param [in] precode= Space separated list of filerefs, pointing to the code
@@ -44,6 +44,8 @@
  @li ms_createfile.sas
  @li mf_getuser.sas
  @li mf_getuniquename.sas
+  @li mf_getuniquefileref.sas
+  @li mp_abort.sas

  <h4> Related Files </h4>
  @li ms_createwebservice.test.sas
@@ -53,7 +55,7 @@

 **/

-%macro ms_createwebservice(path=
+%macro ms_createwebservice(path=0
    ,name=initService
    ,precode=
    ,code=ft15f001
@@ -61,18 +63,23 @@
    ,mDebug=0
 )/*/STORE SOURCE*/;

-%if &syscc ge 4 %then %do;
-  %put &=syscc - &sysmacroname will not execute in this state;
-  %return;
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
 %end;
+%else %let dbg=*;

-%local mD;
-%if &mDebug=1 %then %let mD=;
-%else %let mD=%str(*);
-%&mD.put Executing ms_createwebservice.sas;
-%&mD.put _local_;
+%mp_abort(iftrue=(&syscc ge 4 )
+  ,mac=ms_createwebservice
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+%mp_abort(iftrue=("&path"="0")
+  ,mac=ms_createwebservice
+  ,msg=%str(Path not provided)
+)

-* remove any trailing slash ;
+/* remove any trailing slash */
 %if "%substr(&path,%length(&path),1)" = "/" %then
  %let path=%substr(&path,1,%length(&path)-1);

@@ -81,9 +88,10 @@
  * These put statements are auto generated - to change the macro, change the
  * source (ms_webout) and run `build.py`
  */
-filename sasjs temp;
+%local sasjsref;
+%let sasjsref=%mf_getuniquefileref();
 data _null_;
-  file sasjs lrecl=3000 ;
+  file &sasjsref termstr=crlf lrecl=512;
  put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
  put ' ';
@@ -97,7 +105,7 @@ data _null_;
  put ' ';
  put '%if &action=OPEN %then %do; ';
  put '  options nobomfile; ';
-  put '  data _null_;file &jref encoding=''utf-8'' ; ';
+  put '  data _null_;file &jref encoding=''utf-8'' lrecl=200; ';
  put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
  put '  run; ';
  put '%end; ';
@@ -349,13 +357,14 @@ data _null_;
  put '      %let _webin_name1=&_webin_name; ';
  put '    %end; ';
  put '    data _null_; ';
-  put '      infile &&_webin_fileref&i termstr=crlf; ';
+  put '      infile &&_webin_fileref&i termstr=crlf lrecl=32767; ';
  put '      input; ';
  put '      call symputx(''input_statement'',_infile_); ';
  put '      putlog "&&_webin_name&i input statement: "  _infile_; ';
  put '      stop; ';
  put '    data &&_webin_name&i; ';
-  put '      infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding=''utf-8''; ';
+  put '      infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding=''utf-8'' ';
+  put '        lrecl=32767; ';
  put '      input &input_statement; ';
  put '      %if %str(&_debug) ge 131 %then %do; ';
  put '        if _n_<20 then putlog _infile_; ';
@@ -366,14 +375,14 @@ data _null_;
  put '%end; ';
  put ' ';
  put '%else %if &action=OPEN %then %do; ';
-  put '  /* fix encoding */ ';
-  put '  OPTIONS NOBOMFILE; ';
+  put '  /* fix encoding and ensure enough lrecl */ ';
+  put '  OPTIONS NOBOMFILE lrecl=32767; ';
  put ' ';
  put '  /* set the header */ ';
  put '  %mfs_httpheader(Content-type,application/json) ';
  put ' ';
-  put '  /* setup json */ ';
-  put '  data _null_;file &fref encoding=''utf-8'' termstr=lf; ';
+  put '  /* setup json.  */ ';
+  put '  data _null_;file &fref encoding=''utf-8'' termstr=lf ; ';
  put '    put ''{"SYSDATE" : "'' "&SYSDATE" ''"''; ';
  put '    put '',"SYSTIME" : "'' "&SYSTIME" ''"''; ';
  put '  run; ';
@@ -417,12 +426,12 @@ data _null_;
  put '      data _null_; file &fref mod encoding=''utf-8'' termstr=lf; ';
  put '        put "}"; ';
  put '    %end; ';
-  put '    data _null_; file &fref mod encoding=''utf-8'' termstr=lf termstr=lf; ';
+  put '    data _null_; file &fref mod encoding=''utf-8'' termstr=lf; ';
  put '      put "}"; ';
  put '    run; ';
  put '  %end; ';
  put '  /* close off json */ ';
-  put '  data _null_;file &fref mod encoding=''utf-8'' termstr=lf; ';
+  put '  data _null_;file &fref mod encoding=''utf-8'' termstr=lf lrecl=32767; ';
  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
@@ -495,24 +504,21 @@ data _null_;
 run;

 /* add precode and code */
-%local tmpref x fref freflist mod;
-%let tmpref=%mf_getuniquefileref();
+%local x fref freflist;
 %let freflist=&precode &code ;
 %do x=1 %to %sysfunc(countw(&freflist));
-  %if &x>1 %then %let mod=mod;
-
  %let fref=%scan(&freflist,&x);
  %put &sysmacroname: adding &fref;
  data _null_;
-    file &tmpref lrecl=3000 &mod;
-    infile &fref;
+    file &sasjsref lrecl=3000 termstr=crlf mod;
+    infile &fref lrecl=3000;
    input;
    put _infile_;
  run;
 %end;

 /* create the web service */
-%ms_createfile(&path/&name..sas, inref=&tmpref,mdebug=&mdebug)
+%ms_createfile(&path/&name..sas, inref=&sasjsref, mdebug=&mdebug)

 %put ;%put ;%put ;%put ;%put ;%put ;
 %put &sysmacroname: STP &name successfully created in &path;
--- a/server/ms_runstp.sas
+++ b/server/ms_runstp.sas
@@ -15,48 +15,154 @@
    parameter)
  @param [in] debug= (131) The value to supply to the _debug URL parameter
  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] inputparams=(_null_) A dataset containing name/value pairs in the
+    following format:
+    |name:$32|value:$10000|
+    |---|---|
+    |stpmacname|some value|
+    |mustbevalidname|can be anything, oops, %abort!!|
+  @param [in] inputfiles= (_null_) A dataset containing fileref/name/filename in
+    the following format:
+    |fileref:$8|name:$32|filename:$256|
+    |---|---|--|
+    |someref|some_name|some_filename.xls|
+    |fref2|another_file|zyx_v2.csv|
+
  @param [out] outref= (outweb) The output fileref to contain the response JSON
    (will be created using temp engine)
+  @param [out] outlogds= (_null_) Set to the name of a dataset to contain the
+    log. Table format:
+    |line:$2000|
+    |---|
+    |log line 1|
+    |log line 2|

  <h4> SAS Macros </h4>
  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
  @li mp_abort.sas

 **/

 %macro ms_runstp(pgm
    ,debug=131
+    ,inputparams=_null_
+    ,inputfiles=_null_
    ,outref=outweb
+    ,outlogds=_null_
    ,mdebug=0
  );
-%local dbg fname1;
+%local dbg mainref authref boundary;
+%let mainref=%mf_getuniquefileref();
+%let authref=%mf_getuniquefileref();
+%let boundary=%mf_getuniquename();
+%if &inputparams=0 %then %let inputparams=_null_;
+
 %if &mdebug=1 %then %do;
  %put &sysmacroname entry vars:;
  %put _local_;
 %end;
 %else %let dbg=*;
-%let fname1=%mf_getuniquefileref();
+

 %mp_abort(iftrue=("&pgm"="")
  ,mac=&sysmacroname
  ,msg=%str(Program not provided)
 )

+/* avoid sending bom marker to API */
+%local optval;
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
+/* add params */
 data _null_;
-  file &fname1 lrecl=1000;
+  file &mainref termstr=crlf lrecl=32767 mod;
+  length line $1000 name $32 value $32767;
+  if _n_=1 then call missing(of _all_);
+  set &inputparams;
+  put "--&boundary";
+  line=cats('Content-Disposition: form-data; name="',name,'"');
+  put line;
+  put ;
+  put value;
+run;
+
+/* parse input file list */
+%local webcount;
+%let webcount=0;
+data _null_;
+  set &inputfiles end=last;
+  length fileref $8 name $32 filename $256;
+  call symputx(cats('webref',_n_),fileref,'l');
+  call symputx(cats('webname',_n_),name,'l');
+  call symputx(cats('webfilename',_n_),filename,'l');
+  if last then do;
+    call symputx('webcount',_n_);
+    call missing(of _all_);
+  end;
+run;
+
+/* write out the input files */
+%local i;
+%do i=1 %to &webcount;
+  data _null_;
+    file &mainref termstr=crlf lrecl=32767 mod;
+    infile &&webref&i lrecl=32767;
+    if _n_ = 1 then do;
+      length line $32767;
+      line=cats(
+        'Content-Disposition: form-data; name="'
+        ,"&&webname&i"
+        ,'"; filename="'
+        ,"&&webfilename&i"
+        ,'"'
+      );
+      put "--&boundary";
+      put line;
+      put "Content-Type: text/plain";
+      put ;
+    end;
+    input;
+    put _infile_; /* add the actual file to be sent */
+  run;
+%end;
+
+data _null_;
+  file &mainref termstr=crlf mod;
+  put "--&boundary--";
+run;
+
+data _null_;
+  file &authref lrecl=1000;
  infile "&_sasjs_tokenfile" lrecl=1000;
  input;
  put 'Authorization: Bearer ' _infile_;
+  put "Content-Type: multipart/form-data; boundary=&boundary";
 run;

-filename &outref temp;
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &authref;
+    input;
+    put _infile_;
+  data _null_;
+    infile &mainref;
+    input;
+    put _infile_;
+  run;
+%end;

+filename &outref temp lrecl=32767;
 /* prepare request*/
-proc http method='POST' headerin=&fname1 out=&outref
+proc http method='POST' headerin=&authref in=&mainref out=&outref
  url="&_sasjs_apiserverurl.&_sasjs_apipath?_program=&pgm%str(&)_debug=131";
+%if &mdebug=1 %then %do;
+  debug level=2;
+%end;
 run;
 %if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
-  or &mdebug=1 %then %do;
+%then %do;
  data _null_;infile &outref;input;putlog _infile_;run;
 %end;
 %mp_abort(
@@ -65,6 +171,20 @@ run;
  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
 )

+/* reset options */
+options &optval;
+
+%if &outlogds ne _null_ or &mdebug=1 %then %do;
+  %local dumplib;
+  %let dumplib=%mf_getuniquelibref();
+  libname &dumplib json (&outref);
+  data &outlogds;
+    set &dumplib..log;
+  %if &mdebug=1 %then %do;
+    putlog line=;
+  %end;
+  run;
+%end;

 %if &mdebug=1 %then %do;
  %put &sysmacroname exit vars:;
@@ -72,6 +192,7 @@ run;
 %end;
 %else %do;
  /* clear refs */
-  filename &fname1 clear;
+  filename &authref;
+  filename &mainref;
 %end;
 %mend ms_runstp;
--- a/server/ms_webout.sas
+++ b/server/ms_webout.sas
@@ -67,13 +67,14 @@
      %let _webin_name1=&_webin_name;
    %end;
    data _null_;
-      infile &&_webin_fileref&i termstr=crlf;
+      infile &&_webin_fileref&i termstr=crlf lrecl=32767;
      input;
      call symputx('input_statement',_infile_);
      putlog "&&_webin_name&i input statement: "  _infile_;
      stop;
    data &&_webin_name&i;
-      infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding='utf-8';
+      infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding='utf-8'
+        lrecl=32767;
      input &input_statement;
      %if %str(&_debug) ge 131 %then %do;
        if _n_<20 then putlog _infile_;
@@ -84,14 +85,14 @@
 %end;

 %else %if &action=OPEN %then %do;
-  /* fix encoding */
-  OPTIONS NOBOMFILE;
+  /* fix encoding and ensure enough lrecl */
+  OPTIONS NOBOMFILE lrecl=32767;

  /* set the header */
  %mfs_httpheader(Content-type,application/json)

-  /* setup json */
-  data _null_;file &fref encoding='utf-8' termstr=lf;
+  /* setup json.  */
+  data _null_;file &fref encoding='utf-8' termstr=lf ;
    put '{"SYSDATE" : "' "&SYSDATE" '"';
    put ',"SYSTIME" : "' "&SYSTIME" '"';
  run;
@@ -135,12 +136,12 @@
      data _null_; file &fref mod encoding='utf-8' termstr=lf;
        put "}";
    %end;
-    data _null_; file &fref mod encoding='utf-8' termstr=lf termstr=lf;
+    data _null_; file &fref mod encoding='utf-8' termstr=lf;
      put "}";
    run;
  %end;
  /* close off json */
-  data _null_;file &fref mod encoding='utf-8' termstr=lf;
+  data _null_;file &fref mod encoding='utf-8' termstr=lf lrecl=32767;
    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
    put ",""SYSUSERID"" : ""&sysuserid"" ";
    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
--- a/tests/crossplatform/mp_base64copy.test.sas
+++ b/tests/crossplatform/mp_base64copy.test.sas
@@ -41,7 +41,7 @@ run;

 /* multibyte string check */

-filename tmp2 temp;
+filename tmp2 temp lrecl=500;
 data _null_;
  file tmp2;
  put "'╤', '╔', '╗', '═', '╧', '╚', '╝', '║', '╟', '─', '┼', '║', '╢', '│'";
--- a/tests/crossplatform/mp_ds2cards.test.sas
+++ b/tests/crossplatform/mp_ds2cards.test.sas
@@ -18,7 +18,7 @@
  , cards_file= "%sysfunc(pathname(work))/cars.sas"
  , showlog=NO
 )
-%inc "%sysfunc(pathname(work))/cars.sas"/source2;
+%inc "%sysfunc(pathname(work))/cars.sas"/source2 lrecl=32767;

 proc compare base=sashelp.cars compare=work.test;
 quit;
@@ -48,7 +48,7 @@ run;
  , append=
 )

-%inc "%sysfunc(pathname(work))/c2.sas"/source2;
+%inc "%sysfunc(pathname(work))/c2.sas"/source2 lrecl=32767;

 proc compare base=work.binarybase compare=work.binarycompare;
 run;
--- a/tests/crossplatform/mp_gsubfile.test.sas
+++ b/tests/crossplatform/mp_gsubfile.test.sas
@@ -10,7 +10,7 @@


 %macro gsubtest();
-%if "%substr(&sysver,1,4)"="V.04" %then %do;
+%if "%substr(&sysver.XX,1,4)"="V.04" %then %do;
  %put %str(ERR)OR: Viya 4 does not support the IO library in lua;
  %return;
 %end;
--- a/tests/crossplatform/mp_replace.test.sas
+++ b/tests/crossplatform/mp_replace.test.sas
@@ -39,7 +39,7 @@ run;
 %let str=%str(replacewith trailing spaces   );
 %let rep=%str( with more spaces  );
 data _null_;
-  file &test2;
+  file &test2 lrecl=500;
  put 'blahblah';
  put "blahblah&str.blah&str. replace &str.X";
  put "blahbreplacewith&str.spacesahblah";
@@ -47,7 +47,7 @@ run;
 %mp_replace(&test2, findvar=str, replacevar=rep)

 data _null_;
-  infile &test2;
+  infile &test2 lrecl=500;
  input;
  if _n_=2 then call symputx('test2resulta',_infile_);
  if _n_=3 then call symputx('test2resultb',_infile_);
@@ -69,7 +69,7 @@ run;
 %let str=%str(replace.string.with.dots   );
 %let rep=%str( more.dots);
 data _null_;
-  file &test3;
+  file &test3 lrecl=500;
  put 'blahblah';
  put "blahblah&str.blah&str. replace &str.X";
  put "blahbreplacewith&str.spacesahblah";
@@ -77,7 +77,7 @@ run;
 %mp_replace(&test3, findvar=str, replacevar=rep)

 data _null_;
-  infile &test3;
+  infile &test3 lrecl=500;
  input;
  if _n_=2 then call symputx('test3resulta',_infile_);
  if _n_=3 then call symputx('test3resultb',_infile_);
--- a/tests/serveronly/mfs_httpheader.test.sas
+++ b/tests/serveronly/mfs_httpheader.test.sas
@@ -50,3 +50,8 @@ run;
  desc=Checking line was created,
  outds=work.test_results
 )
+
+
+/* reset header so the test will pass */
+
+%mfs_httpheader(Content-type,application/json)
--- a/tests/serveronly/ms_runstp.test.sas
+++ b/tests/serveronly/ms_runstp.test.sas
@@ -16,6 +16,7 @@ filename stpcode temp;
 data _null_;
  file stpcode;
  put '%put hello world;';
+  put '%put _all_;';
 run;

 options mprint;
--- a/tests/testinit.sas
+++ b/tests/testinit.sas
@@ -5,6 +5,7 @@
  <h4> SAS Macros </h4>
  @li mf_uid.sas
  @li mp_init.sas
+  @li ms_webout.sas

 **/

@@ -14,12 +15,15 @@
 /* set defaults */
 %mp_init()

+options lrecl=80;
+
 %global _debug sasjs_mdebug;

 %let sasjs_mdebug=0;

 %macro loglevel();
-  %if "&_debug"="2477" or "&_debug"="fields,log,trace" %then %do;
+  %if "&_debug"="2477" or "&_debug"="fields,log,trace" or "&_debug"="131"
+  %then %do;
    %put debug mode activated;
    options mprint mprintnest;
    %let sasjs_mdebug=1;
--- a/tests/testterm.sas
+++ b/tests/testterm.sas
@@ -9,7 +9,7 @@

 %mp_assert(
  iftrue=(&syscc=0),
-  desc=Checking final error condition,
+  desc=Checking final err condition,
  outds=work.test_results
 )

--- a/viya/mv_createwebservice.sas
+++ b/viya/mv_createwebservice.sas
@@ -247,7 +247,7 @@ data _null_;
  put ' ';
  put '%if &action=OPEN %then %do; ';
  put '  options nobomfile; ';
-  put '  data _null_;file &jref encoding=''utf-8'' ; ';
+  put '  data _null_;file &jref encoding=''utf-8'' lrecl=200; ';
  put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
  put '  run; ';
  put '%end; ';