From 4b34322d9453941bbd6c98ba3b2b73277ae1d00c Mon Sep 17 00:00:00 2001 From: Allan Bowe Date: Wed, 27 Jan 2021 00:14:21 +0100 Subject: [PATCH] feat: mv_getjoblog.sas macro - will fetch a SAS log from an executed SAS Viya log and append it to a fileref. mv_jobwaitfor is updated to allow the log to be fetched for all the submitted jobs. --- all.sas | 112 +++++++++++++++-- viya/mv_getjobcode.sas | 5 +- viya/mv_getjoblog.sas | 267 +++++++++++++++++++++++++++++++++++++++++ viya/mv_jobwaitfor.sas | 14 ++- 4 files changed, 382 insertions(+), 16 deletions(-) create mode 100644 viya/mv_getjoblog.sas diff --git a/all.sas b/all.sas index f7c2e56..dc10711 100644 --- a/all.sas +++ b/all.sas @@ -12487,7 +12487,7 @@ run; %let fname3=%mf_getuniquefileref(); %let fpath1=%sysfunc(pathname(&fname1)); %let fpath2=%sysfunc(pathname(&fname2)); -%let fpath3=%sysfunc(pathname(&fname2)); +%let fpath3=%sysfunc(pathname(&fname3)); /* compile the lua JSON module */ %ml_json() @@ -12520,6 +12520,7 @@ data _null_; run; filename &fname1 clear; filename &fname2 clear; +filename &fname3 clear; %mend; /** @file @@ -12577,6 +12578,7 @@ filename &fname2 clear; @param [in] access_token_var= The global macro variable to contain the access token + @param [in] mdebug= set to 1 to enable DEBUG messages @param [in] grant_type= valid values: @li password @li authorization_code @@ -12604,6 +12606,7 @@ filename &fname2 clear; ,contextName=SAS Job Execution compute context ,access_token_var=ACCESS_TOKEN ,grant_type=sas_services + ,mdebug=0 ); %local oauth_bearer; %if &grant_type=detect %then %do; @@ -12660,7 +12663,7 @@ options noquotelenmax; %local fname1; %let fname1=%mf_getuniquefileref(); proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri&joburi"; + url="&base_uri&uri"; headers %if &grant_type=authorization_code %then %do; "Authorization"="Bearer &&&access_token_var" @@ -12679,7 +12682,7 @@ run; %let fname3=%mf_getuniquefileref(); %let fpath1=%sysfunc(pathname(&fname1)); %let fpath2=%sysfunc(pathname(&fname2)); -%let fpath3=%sysfunc(pathname(&fname2)); +%let fpath3=%sysfunc(pathname(&fname3)); /* compile the lua JSON module */ %ml_json() @@ -12699,15 +12702,94 @@ data _null_; run; %inc "&fpath3..lua"; /* get log path*/ +%let errflg=1; +%let errmsg=No entry in &fname2 fileref; data _null_; infile &fname2; input; - call symputx('logloc',_infile_,'l'); + uri=_infile_; + if length(uri)<12 then do; + call symputx('errflg',1); + call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l'); + end; + if scan(uri,1) ne 'files' or scan(uri,2) ne 'files' then do; + call symputx('errflg',1); + call symputx('errmsg', + "URI should be in format /files/files/$$$$UUID$$$$" + !!" but is actually like: &uri",'l'); + end; + call symputx('errflg',0,'l'); + call symputx('logloc',uri,'l'); run; -%put &=logloc; -filename &fname1 clear; -filename &fname2 clear; -%mend;/** + +%mp_abort(iftrue=(&errflg=1) + ,mac=&sysmacroname + ,msg=%str(&errmsg) +) + +/* we have a log uri - now fetch the log */ +proc http method='GET' out=&fname1 &oauth_bearer + url="&base_uri&logloc/content"; + headers + %if &grant_type=authorization_code %then %do; + "Authorization"="Bearer &&&access_token_var" + %end; + ; +run; +%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then +%do; + data _null_;infile &fname1;input;putlog _infile_;run; + %mp_abort(mac=&sysmacroname + ,msg=%str(logfetch: &SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) + ) +%end; + +data _null_; + file "&fpath3..lua"; + put ' + infile = io.open (sas.symget("fpath1"), "r") + outfile = io.open (sas.symget("fpath2"), "w") + io.input(infile) + local resp=json.decode(io.read()) + for i, v in pairs(resp["items"]) do + outfile:write(v.line,"\n") + end + io.close(infile) + io.close(outfile) + '; +run; +%inc "&fpath3..lua"; + +/* write log out to the specified fileref */ +data _null_; + infile &fname2 end=last; + file &outref mod; + if _n_=1 then do; + put "/** SASJS Viya Job Log Extract start: &uri **/"; + end; + input; + put _infile_; + %if &mdebug=0 %then %do; + putlog _infile_; + %end; + if last then do; + put "/** SASJS Viya Job Log Extract end: &uri **/"; + end; +run; + +%if &mdebug=0 %then %do; + filename &fname1 clear; + filename &fname2 clear; + filename &fname3 clear; +%end; +%else %do; + %put _local_; +%end; +%mend; + + + +/** @file @brief Extract the status from a running SAS Viya job @details Extracts the status from a running job and appends it to an output @@ -13661,7 +13743,7 @@ data;run;%let jdswaitfor=&syslast; should be in a `_program` variable. @param [out] outds= The output dataset containing the list of states by job (default=work.mv_jobexecute) - + @param [out] outref= A fileref to which the spawned job logs should be appended. @version VIYA V.03.04 @author Allan Bowe, source: https://github.com/sasjs/core @@ -13672,6 +13754,7 @@ data;run;%let jdswaitfor=&syslast; @li mf_getuniquefileref.sas @li mf_existvar.sas @li mf_nobs.sas + @li mv_getjoblog.sas **/ @@ -13680,6 +13763,7 @@ data;run;%let jdswaitfor=&syslast; ,grant_type=sas_services ,inds=0 ,outds=work.mv_jobwaitfor + ,outref=0 ); %local oauth_bearer; %if &grant_type=detect %then %do; @@ -13723,7 +13807,7 @@ options noquotelenmax; data _null_; length jobparams $32767; set &inds end=last; - call symputx(cats('joburi',_n_),uri,'l'); + call symputx(cats('joburi',_n_),substr(uri,1,55)!!'/state','l'); call symputx(cats('jobname',_n_),_program,'l'); call symputx(cats('jobparams',_n_),jobparams,'l'); if last then call symputx('uricnt',_n_,'l'); @@ -13767,14 +13851,20 @@ run; run; %if &status=completed or &status=failed or &status=canceled %then %do; + %local plainuri; + %let plainuri=%substr(&&joburi&i,1,55); proc sql; insert into &outds set _program="&&jobname&i", - uri="&&joburi&i", + uri="&plainuri", state="&status", timestamp=datetime(), jobparams=symget("jobparams&i"); %let joburi&i=0; /* do not re-check */ + /* fetch log */ + %if %str(&outref) ne 0 %then %do; + %mv_getjoblog(uri=&plainuri,outref=&outref) + %end; %end; %else %if &status=idle or &status=pending or &status=running %then %do; data _null_; diff --git a/viya/mv_getjobcode.sas b/viya/mv_getjobcode.sas index 7ac7d83..996168e 100644 --- a/viya/mv_getjobcode.sas +++ b/viya/mv_getjobcode.sas @@ -114,10 +114,10 @@ run; %let fname3=%mf_getuniquefileref(); %let fpath1=%sysfunc(pathname(&fname1)); %let fpath2=%sysfunc(pathname(&fname2)); -%let fpath3=%sysfunc(pathname(&fname2)); +%let fpath3=%sysfunc(pathname(&fname3)); /* compile the lua JSON module */ -%ml_json() +%ml_json() /* read using LUA - this allows the code to be of any length */ data _null_; file "&fpath3..lua"; @@ -147,4 +147,5 @@ data _null_; run; filename &fname1 clear; filename &fname2 clear; +filename &fname3 clear; %mend; diff --git a/viya/mv_getjoblog.sas b/viya/mv_getjoblog.sas new file mode 100644 index 0000000..7a558f7 --- /dev/null +++ b/viya/mv_getjoblog.sas @@ -0,0 +1,267 @@ +/** + @file + @brief Extract the log from a completed SAS Viya Job + @details Extracts log from a Viya job and writes it out to a fileref + + To query the job, you need the URI. Sample code for achieving this + is provided below. + + ## Example + + First, compile the macros: + + filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; + %inc mc; + + Next, create a job (in this case, a web service): + + filename ft15f001 temp; + parmcards4; + data ; + rand=ranuni(0)*1000; + do x=1 to rand; + y=rand*4; + output; + end; + run; + proc sort data=&syslast + by descending y; + run; + ;;;; + %mv_createwebservice(path=/Public/temp,name=demo) + + Execute it: + + %mv_jobexecute(path=/Public/temp + ,name=demo + ,outds=work.info + ) + + Wait for it to finish, and grab the uri: + + data _null_; + set work.info; + if method='GET' and rel='self'; + call symputx('uri',uri); + run; + + Finally, fetch the log: + + %mv_getjoblog(uri=&uri,outref=mylog) + + This macro is used by the mv_jobwaitfor macro, which is generally a more + convenient way to wait for the job to finish before fetching the log. + + + @param [in] access_token_var= The global macro variable to contain the access token + @param [in] mdebug= set to 1 to enable DEBUG messages + @param [in] grant_type= valid values: + @li password + @li authorization_code + @li detect - will check if access_token exists, if not will use sas_services if + a SASStudioV session else authorization_code. Default option. + @li sas_services - will use oauth_bearer=sas_services. + @param [in] uri= The uri of the running job for which to fetch the status, + in the format `/jobExecution/jobs/$UUID/state` (unquoted). + @param [out] outref= The output fileref to which to APPEND the log (is always + appended). + + + @version VIYA V.03.04 + @author Allan Bowe, source: https://github.com/sasjs/core + +

SAS Macros

+ @li mp_abort.sas + @li mf_getplatform.sas + @li mf_existfileref.sas + @li ml_json.sas + +**/ + +%macro mv_getjoblog(uri=0,outref=0 + ,contextName=SAS Job Execution compute context + ,access_token_var=ACCESS_TOKEN + ,grant_type=sas_services + ,mdebug=0 + ); +%local oauth_bearer; +%if &grant_type=detect %then %do; + %if %symexist(&access_token_var) %then %let grant_type=authorization_code; + %else %let grant_type=sas_services; +%end; +%if &grant_type=sas_services %then %do; + %let oauth_bearer=oauth_bearer=sas_services; + %let &access_token_var=; +%end; +%put &sysmacroname: grant_type=&grant_type; +%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password + and &grant_type ne sas_services + ) + ,mac=&sysmacroname + ,msg=%str(Invalid value for grant_type: &grant_type) +) + +/* validation in datastep for better character safety */ +%local errmsg errflg; +data _null_; + uri=symget('uri'); + if length(uri)<12 then do; + call symputx('errflg',1); + call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l'); + end; + if scan(uri,-1)='state' or scan(uri,1) ne 'jobExecution' then do; + call symputx('errflg',1); + call symputx('errmsg', + "URI should be in format /jobExecution/jobs/$$$$UUID$$$$" + !!" but is actually like: &uri",'l'); + end; +run; + +%mp_abort(iftrue=(&errflg=1) + ,mac=&sysmacroname + ,msg=%str(&errmsg) +) + +%mp_abort(iftrue=(&outref=0) + ,mac=&sysmacroname + ,msg=%str(Output fileref should be provided) +) + +%if %mf_existfileref(&outref) ne 1 %then %do; + filename &outref temp; +%end; + +options noquotelenmax; +%local base_uri; /* location of rest apis */ +%let base_uri=%mf_getplatform(VIYARESTAPI); + +/* prepare request*/ +%local fname1; +%let fname1=%mf_getuniquefileref(); +proc http method='GET' out=&fname1 &oauth_bearer + url="&base_uri&uri"; + headers + %if &grant_type=authorization_code %then %do; + "Authorization"="Bearer &&&access_token_var" + %end; + ; +run; +%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then +%do; + data _null_;infile &fname1;input;putlog _infile_;run; + %mp_abort(mac=&sysmacroname + ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) + ) +%end; +%local fname2 fname3 fpath1 fpath2 fpath3; +%let fname2=%mf_getuniquefileref(); +%let fname3=%mf_getuniquefileref(); +%let fpath1=%sysfunc(pathname(&fname1)); +%let fpath2=%sysfunc(pathname(&fname2)); +%let fpath3=%sysfunc(pathname(&fname3)); + +/* compile the lua JSON module */ +%ml_json() +/* read using LUA - this allows the code to be of any length */ +data _null_; + file "&fpath3..lua"; + put ' + infile = io.open (sas.symget("fpath1"), "r") + outfile = io.open (sas.symget("fpath2"), "w") + io.input(infile) + local resp=json.decode(io.read()) + local logloc=resp["logLocation"] + outfile:write(logloc) + io.close(infile) + io.close(outfile) + '; +run; +%inc "&fpath3..lua"; +/* get log path*/ +%let errflg=1; +%let errmsg=No entry in &fname2 fileref; +data _null_; + infile &fname2; + input; + uri=_infile_; + if length(uri)<12 then do; + call symputx('errflg',1); + call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l'); + end; + if scan(uri,1) ne 'files' or scan(uri,2) ne 'files' then do; + call symputx('errflg',1); + call symputx('errmsg', + "URI should be in format /files/files/$$$$UUID$$$$" + !!" but is actually like: &uri",'l'); + end; + call symputx('errflg',0,'l'); + call symputx('logloc',uri,'l'); +run; + +%mp_abort(iftrue=(&errflg=1) + ,mac=&sysmacroname + ,msg=%str(&errmsg) +) + +/* we have a log uri - now fetch the log */ +proc http method='GET' out=&fname1 &oauth_bearer + url="&base_uri&logloc/content"; + headers + %if &grant_type=authorization_code %then %do; + "Authorization"="Bearer &&&access_token_var" + %end; + ; +run; +%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then +%do; + data _null_;infile &fname1;input;putlog _infile_;run; + %mp_abort(mac=&sysmacroname + ,msg=%str(logfetch: &SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) + ) +%end; + +data _null_; + file "&fpath3..lua"; + put ' + infile = io.open (sas.symget("fpath1"), "r") + outfile = io.open (sas.symget("fpath2"), "w") + io.input(infile) + local resp=json.decode(io.read()) + for i, v in pairs(resp["items"]) do + outfile:write(v.line,"\n") + end + io.close(infile) + io.close(outfile) + '; +run; +%inc "&fpath3..lua"; + +/* write log out to the specified fileref */ +data _null_; + infile &fname2 end=last; + file &outref mod; + if _n_=1 then do; + put "/** SASJS Viya Job Log Extract start: &uri **/"; + end; + input; + put _infile_; + %if &mdebug=0 %then %do; + putlog _infile_; + %end; + if last then do; + put "/** SASJS Viya Job Log Extract end: &uri **/"; + end; +run; + +%if &mdebug=0 %then %do; + filename &fname1 clear; + filename &fname2 clear; + filename &fname3 clear; +%end; +%else %do; + %put _local_; +%end; +%mend; + + + diff --git a/viya/mv_jobwaitfor.sas b/viya/mv_jobwaitfor.sas index eaad1b1..5875368 100644 --- a/viya/mv_jobwaitfor.sas +++ b/viya/mv_jobwaitfor.sas @@ -72,7 +72,7 @@ should be in a `_program` variable. @param [out] outds= The output dataset containing the list of states by job (default=work.mv_jobexecute) - + @param [out] outref= A fileref to which the spawned job logs should be appended. @version VIYA V.03.04 @author Allan Bowe, source: https://github.com/sasjs/core @@ -83,6 +83,7 @@ @li mf_getuniquefileref.sas @li mf_existvar.sas @li mf_nobs.sas + @li mv_getjoblog.sas **/ @@ -91,6 +92,7 @@ ,grant_type=sas_services ,inds=0 ,outds=work.mv_jobwaitfor + ,outref=0 ); %local oauth_bearer; %if &grant_type=detect %then %do; @@ -134,7 +136,7 @@ options noquotelenmax; data _null_; length jobparams $32767; set &inds end=last; - call symputx(cats('joburi',_n_),uri,'l'); + call symputx(cats('joburi',_n_),substr(uri,1,55)!!'/state','l'); call symputx(cats('jobname',_n_),_program,'l'); call symputx(cats('jobparams',_n_),jobparams,'l'); if last then call symputx('uricnt',_n_,'l'); @@ -178,14 +180,20 @@ run; run; %if &status=completed or &status=failed or &status=canceled %then %do; + %local plainuri; + %let plainuri=%substr(&&joburi&i,1,55); proc sql; insert into &outds set _program="&&jobname&i", - uri="&&joburi&i", + uri="&plainuri", state="&status", timestamp=datetime(), jobparams=symget("jobparams&i"); %let joburi&i=0; /* do not re-check */ + /* fetch log */ + %if %str(&outref) ne 0 %then %do; + %mv_getjoblog(uri=&plainuri,outref=&outref) + %end; %end; %else %if &status=idle or &status=pending or &status=running %then %do; data _null_;