From 04a3189a89dd7fdd4f94aa58b5972779170dc4e1 Mon Sep 17 00:00:00 2001 From: Allan Bowe Date: Thu, 6 May 2021 01:07:25 +0300 Subject: [PATCH] feat: new mv_getjobresult.sas macro, corresponding test, and additional fixes --- base/mp_filtercheck.sas | 6 +- base/mp_hashdataset.sas | 2 +- base/mp_testservice.sas | 189 ++++++++++++++++++++++ tests/testinit.sas | 2 +- tests/viya/mv_createwebservice.test.sas | 16 +- tests/viya/mv_getjobresult.test.sas | 74 +++++++++ viya/mv_getjoblog.sas | 7 +- viya/mv_getjobresult.sas | 207 ++++++++++++++++++++++++ 8 files changed, 489 insertions(+), 14 deletions(-) create mode 100644 base/mp_testservice.sas create mode 100644 tests/viya/mv_getjobresult.test.sas create mode 100644 viya/mv_getjobresult.sas diff --git a/base/mp_filtercheck.sas b/base/mp_filtercheck.sas index 00748ae..96c2c79 100644 --- a/base/mp_filtercheck.sas +++ b/base/mp_filtercheck.sas @@ -70,7 +70,7 @@ * quotes, commas, periods and spaces. * Only numeric values should remain */ - +%local reason_cd; data &outds; set &inds; length reason_cd $32; @@ -147,9 +147,9 @@ data _null_; stop; run; -%mp_abort(iftrue=(&abort=YES), +%mp_abort(iftrue=(&abort=YES and %mf_nobs(&outds)>0), mac=&sysmacroname, - msg=%str(Filter issues in &inds, first was &reason_cd, details in &outds) + msg=%str(Filter issues in &inds, reason: &reason_cd, details in &outds) ) %if %mf_nobs(&outds)>0 %then %do; diff --git a/base/mp_hashdataset.sas b/base/mp_hashdataset.sas index 264cfc2..79ee366 100644 --- a/base/mp_hashdataset.sas +++ b/base/mp_hashdataset.sas @@ -56,7 +56,7 @@ retain &prevkeyvar; set &libds end=&lastvar; /* hash should include previous row */ - if _n_>1 then &keyvar=put(md5(&prevkeyvar + &keyvar=put(md5(&prevkeyvar /* loop every column, hashing every individual value */ %do i=1 %to %sysfunc(countw(&varlist)); %let var=%scan(&varlist,&i,%str( )); diff --git a/base/mp_testservice.sas b/base/mp_testservice.sas new file mode 100644 index 0000000..4876db9 --- /dev/null +++ b/base/mp_testservice.sas @@ -0,0 +1,189 @@ +/** + @file mp_testservice.sas + @brief Will execute a test against a SASjs web service on SAS 9 or Viya + @details Prepares the input files and retrieves the resulting datasets from + the response JSON. + + %mp_testjob( + duration=60*5 + ) + + Note - the _webout fileref should NOT be assigned prior to running this macro. + + @param [in] program The _PROGRAM endpoint to test + @param [in] inputfiles=(0) A list of space seperated fileref:filename pairs as + follows: + inputfiles=inref:filename inref2:filename2 + @param [in] inputparams=(0) A dataset containing name/value pairs in the + following format: + |name:$32|value:$1000| + |---|---| + |stpmacname|some value| + |mustbevalidname|can be anything, oops, %abort!!| + + @param [in] debug= (log) Provide the _debug value + @param [out] outlib= (0) Output libref to contain the final tables. Set to + 0 if the service output is not in JSON format. + @param [out] outref= (0) Output fileref to create, to contain the full _webout + response. + +

SAS Macros

+ @li mf_getplatform.sas + @li mf_getuniquefileref.sas + @li mf_getuniquename.sas + @li mp_abort.sas + @li mp_binarycopy.sas + @li mv_getjobresult.sas + @li mv_jobflow.sas + + @version 9.4 + @author Allan Bowe + +**/ + +%macro mp_testservice(program, + inputfiles=0, + inputparams=0, + debug=log, + outlib=0, + outref=0 +)/*/STORE SOURCE*/; + +/* sanitise inputparams */ +%local pcnt; +%let pcnt=0; +%if &inputparams ne 0 %then %do; + data _null_; + set &inputparams; + if not nvalid(name,'v7') then putlog (_all_)(=); + else if name in ( + 'program','inputfiles','inputparams','debug','outlib','outref' + ) then putlog (_all_)(=); + else do; + x+1; + call symputx(name,quote(cats(value)),'l'); + call symputx('pval'!!left(x),name,'l'); + call symputx('pcnt',x,'l'); + end; + run; + %mp_abort(iftrue= (%mf_nobs(&inputparams) ne &pcnt) + ,mac=&sysmacroname + ,msg=%str(Invalid values in &inputparams) + ) +%end; + +/* parse the input files */ +%local webcount i var; +%if %quote(&inputfiles) ne 0 %then %do; + %let webcount=%sysfunc(countw(&inputfiles)); + %put &=webcount; + %do i=1 %to &webcount; + %let var=%scan(&inputfiles,&i,%str( )); + %local webfref&i webname&i; + %let webref&i=%scan(&var,1,%str(:)); + %let webname&i=%scan(&var,2,%str(:)); + %put webref&i=&&webref&i; + %put webname&i=&&webname&i; + %end; +%end; +%else %let webcount=0; + + +%local fref1 webref; +%let fref1=%mf_getuniquefileref(); +%let webref=%mf_getuniquefileref(); + +%local platform; +%let platform=%mf_getplatform(); +%if &platform=SASMETA %then %do; + proc stp program="&program"; + inputparam _program="&program" + %do i=1 %to &webcount; + %if &webcount=1 %then %do; + _webin_fileref="&&webref&i" + _webin_name="&&webname&i" + %end; + %else %do; + _webin_fileref&i="&&webref&i" + _webin_name&i="&&webname&i" + %end; + %end; + _webin_file_count="&webcount" + _debug="&debug" + %do i=1 %to &pcnt; + /* resolve name only, proc stp fetches value */ + &&pval&i=&&&&&&pval&i + %end; + ; + %do i=1 %to &webcount; + inputfile &&webref&i; + %end; + outputfile _webout=&webref; + run; + + data _null_; + infile &webref; + file &fref1; + input; + length line $10000; + if index(_infile_,'>>weboutBEGIN<<') then do; + line=tranwrd(_infile_,'>>weboutBEGIN<<',''); + put line; + end; + else if index(_infile_,'>>weboutEND<<') then do; + line=tranwrd(_infile_,'>>weboutEND<<',''); + put line; + stop; + end; + else put _infile_; + run; + data _null_; + infile &fref1; + input; + put _infile_; + run; + %if &outlib ne 0 %then %do; + libname &outlib json (&fref1); + %end; + %if &outref ne 0 %then %do; + filename &outref temp; + %mp_binarycopy(inref=&webref,outref=&outref) + %end; + +%end; +%else %if &platform=SASVIYA %then %do; + data ; + _program="&program"; + run; + + %mv_jobflow(inds=&syslast + ,maxconcurrency=1 + ,outds=work.results + ,outref=&fref1 + ) + /* show the log */ + data _null_; + infile &fref1; + input; + putlog _infile_; + run; + /* get the uri to fetch results */ + data _null_; + set work.results; + call symputx('uri',uri); + run; + /* fetch results from webout.json */ + %mv_getjobresult(uri=&uri, + result=WEBOUT_JSON, + outref=&outref, + outlib=&outlib + ) + +%end; +%else %do; + %put %str(ERR)OR: Unrecognised platform: &platform; +%end; + +filename &webref clear; + +%mend; \ No newline at end of file diff --git a/tests/testinit.sas b/tests/testinit.sas index 4daa961..611a982 100644 --- a/tests/testinit.sas +++ b/tests/testinit.sas @@ -5,4 +5,4 @@ **/ /* location in metadata or SAS Drive for temporary files */ -%let mcTestAppLoc=/Public/temp/test; \ No newline at end of file +%let mcTestAppLoc=/Public/temp/macrocore; \ No newline at end of file diff --git a/tests/viya/mv_createwebservice.test.sas b/tests/viya/mv_createwebservice.test.sas index 2c8f8e7..215e76f 100644 --- a/tests/viya/mv_createwebservice.test.sas +++ b/tests/viya/mv_createwebservice.test.sas @@ -19,23 +19,27 @@ data _null_; put '01'x; run; %mv_createwebservice( - path=&mcTestAppLoc/tests/macros, + path=&mcTestAppLoc/temp/macros, code=testref, name=mv_createwebservice ) filename compare temp; %mv_getjobcode( - path=&mcTestAppLoc/tests/macros + path=&mcTestAppLoc/temp/macros ,name=mv_createwebservice ,outref=compare; ) data test_results; length test_description $256 test_result $4 test_comments $256; - infile compare; + infile compare end=eof; input; - if _infile_='01'x then test_result='PASS'; - else test_result='FAIL'; - test_description="Creating web service with invisible character"; + if eof then do; + if _infile_='01'x then test_result='PASS'; + else test_result='FAIL'; + test_description="Creating web service with invisible character"; + output; + stop; + end; run; \ No newline at end of file diff --git a/tests/viya/mv_getjobresult.test.sas b/tests/viya/mv_getjobresult.test.sas new file mode 100644 index 0000000..943b8c2 --- /dev/null +++ b/tests/viya/mv_getjobresult.test.sas @@ -0,0 +1,74 @@ +/** + @file + @brief Testing mv_createwebservice macro + +

SAS Macros

+ @li mp_assertdsobs.sas + @li mv_createwebservice.sas + @li mv_getjobresult.sas + @li mv_jobflow.sas + +**/ + +/** + * Test Case 1 + */ + +/* create a service */ +filename testref temp; +data _null_; + file testref; + put 'data test; set sashelp.class;run;'; + put '%webout(OPEN)'; + put '%webout(OBJ,test)'; + put '%webout(CLOSE)'; +run; +%mv_createwebservice( + path=&mcTestAppLoc/services/temp, + code=testref, + name=testsvc +) + +/* trigger and wait for it to finish */ +data work.inputjobs; + _program="&mcTestAppLoc/services/temp/testsvc"; +run; +%mv_jobflow(inds=work.inputjobs + ,maxconcurrency=4 + ,outds=work.results + ,outref=myjoblog +) +/* stream the log */ +data _null_; + infile myjoblog; + input; + put _infile_; +run; + +/* fetch the uri */ +data _null_; + set work.results; + call symputx('uri',uri); + put (_all_)(=); +run; + +/* now get the results */ +%mv_getjobresult(uri=&uri + ,result=WEBOUT_JSON + ,outref=myweb + ,outlib=myweblib +) +data _null_; + infile myweb; + input; + putlog _infile_; +run; +data work.out; + set myweblib.test; + put (_all_)(=); +run; +%mp_assertdsobs(work.out, + desc=Test1 - 19 obs from sashelp.class in service result, + test=EQUALS 19, + outds=work.test_results +) \ No newline at end of file diff --git a/viya/mv_getjoblog.sas b/viya/mv_getjoblog.sas index b72c553..a5cf25a 100644 --- a/viya/mv_getjoblog.sas +++ b/viya/mv_getjoblog.sas @@ -54,13 +54,14 @@ convenient way to wait for the job to finish before fetching the log. - @param [in] access_token_var= The global macro variable to contain the access token + @param [in] access_token_var= The global macro variable to contain the access + token @param [in] mdebug= set to 1 to enable DEBUG messages @param [in] grant_type= valid values: @li password @li authorization_code - @li detect - will check if access_token exists, if not will use sas_services if - a SASStudioV session else authorization_code. Default option. + @li detect - will check if access_token exists, if not will use sas_services + if a SASStudioV session else authorization_code. Default option. @li sas_services - will use oauth_bearer=sas_services. @param [in] uri= The uri of the running job for which to fetch the status, in the format `/jobExecution/jobs/$UUID/state` (unquoted). diff --git a/viya/mv_getjobresult.sas b/viya/mv_getjobresult.sas new file mode 100644 index 0000000..107e51d --- /dev/null +++ b/viya/mv_getjobresult.sas @@ -0,0 +1,207 @@ +/** + @file + @brief Extract the result from a completed SAS Viya Job + @details Extracts result from a Viya job and writes it out to a fileref + and/or a JSON-engine library. + + To query the job, you need the URI. Sample code for achieving this + is provided below. + + ## Example + + First, compile the macros: + + filename mc url + "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; + %inc mc; + + Next, create a job (in this case, a web service): + + filename ft15f001 temp; + parmcards4; + data test; + rand=ranuni(0)*1000; + do x=1 to rand; + y=rand*4; + output; + end; + run; + proc sort data=&syslast + by descending y; + run; + %webout(OPEN) + %webout(OBJ, test) + %webout(CLOSE) + ;;;; + %mv_createwebservice(path=/Public/temp,name=demo) + + Execute it: + + %mv_jobexecute(path=/Public/temp + ,name=demo + ,outds=work.info + ) + + Wait for it to finish, and grab the uri: + + data _null_; + set work.info; + if method='GET' and rel='self'; + call symputx('uri',uri); + run; + + Finally, fetch the result (In this case, WEBOUT): + + %mv_getjobresult(uri=&uri,result=WEBOUT_JSON,outref=myweb,outlib=myweblib) + + + @param [in] access_token_var= The global macro variable containing the access + token + @param [in] mdebug= set to 1 to enable DEBUG messages + @param [in] grant_type= valid values: + @li password + @li authorization_code + @li detect - will check if access_token exists, if not will use sas_services + if a SASStudioV session else authorization_code. Default option. + @li sas_services - will use oauth_bearer=sas_services. + @param [in] uri= The uri of the running job for which to fetch the status, + in the format `/jobExecution/jobs/$UUID` (unquoted). + + @param [out] result= (WEBOUT_JSON) The result type to capture. Resolves + to "_[column name]" from the results table when parsed with the JSON libname + engine. + + @param [out] outref= (0) The output fileref to which to write the results + @param [out] outlib= (0) The output library to which to assign the results + (assumes the data is in JSON format) + + + @version VIYA V.03.05 + @author Allan Bowe, source: https://github.com/sasjs/core + +

SAS Macros

+ @li mp_abort.sas + @li mp_binarycopy.sas + @li mf_getplatform.sas + @li mf_existfileref.sas + +**/ + +%macro mv_getjobresult(uri=0 + ,contextName=SAS Job Execution compute context + ,access_token_var=ACCESS_TOKEN + ,grant_type=sas_services + ,mdebug=0 + ,result=WEBOUT_JSON + ,outref=0 + ,outlib=0 + ); +%local oauth_bearer; +%if &grant_type=detect %then %do; + %if %symexist(&access_token_var) %then %let grant_type=authorization_code; + %else %let grant_type=sas_services; +%end; +%if &grant_type=sas_services %then %do; + %let oauth_bearer=oauth_bearer=sas_services; + %let &access_token_var=; +%end; + +%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password + and &grant_type ne sas_services + ) + ,mac=&sysmacroname + ,msg=%str(Invalid value for grant_type: &grant_type) +) + + +/* validation in datastep for better character safety */ +%local errmsg errflg; +data _null_; + uri=symget('uri'); + if length(uri)<12 then do; + call symputx('errflg',1); + call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l'); + end; + if scan(uri,-1)='state' or scan(uri,1) ne 'jobExecution' then do; + call symputx('errflg',1); + call symputx('errmsg', + "URI should be in format /jobExecution/jobs/$$$$UUID$$$$" + !!" but is actually like: &uri",'l'); + end; +run; + +%mp_abort(iftrue=(&errflg=1) + ,mac=&sysmacroname + ,msg=%str(&errmsg) +) + +%if &outref ne 0 and %mf_existfileref(&outref) ne 1 %then %do; + filename &outref temp; +%end; + +options noquotelenmax; +%local base_uri; /* location of rest apis */ +%let base_uri=%mf_getplatform(VIYARESTAPI); + +/* fetch job info */ +%local fname1; +%let fname1=%mf_getuniquefileref(); +proc http method='GET' out=&fname1 &oauth_bearer + url="&base_uri&uri"; + headers "Accept"="application/json" + %if &grant_type=authorization_code %then %do; + "Authorization"="Bearer &&&access_token_var" + %end; + ; +run; +%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then +%do; + data _null_;infile &fname1;input;putlog _infile_;run; + %mp_abort(mac=&sysmacroname + ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) + ) +%end; + +/* extract results link */ +%local lib1 resuri; +%let lib1=%mf_getuniquelibref(); +libname &lib1 JSON fileref=&fname1; +data _null_; + set &lib1..results; + call symputx('resuri',_&result,'l'); + putlog (_all_)(=); +run; +%mp_abort(iftrue=("&resuri"=".") + ,mac=&sysmacroname + ,msg=%str(Variable _&result did not exist in the response json) +) + +/* extract results */ +%local fname2; +%let fname2=%mf_getuniquefileref(); +proc http method='GET' out=&fname2 &oauth_bearer + url="&base_uri&resuri/content?limit=10000"; + headers "Accept"="application/json" + %if &grant_type=authorization_code %then %do; + "Authorization"="Bearer &&&access_token_var" + %end; + ; +run; + +%if &outref ne 0 %then %do; + filename &outref temp; + %mp_binarycopy(inref=&fname2,outref=&outref) +%end; +%if &outlib ne 0 %then %do; + libname &outlib JSON fileref=&fname2; +%end; + +%if &mdebug=0 %then %do; + filename &fname1 clear; + filename &fname2 clear; + libname &lib1 clear; +%end; +%else %do; + %put _local_; +%end; +%mend;