sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2026-01-10 10:50:04 +00:00
Code Issues Projects Releases Wiki Activity

feat: early exit, with syscc, when submitted jobs fail within a flow

Browse Source
This commit is contained in:
Trevor Moody
2021-05-12 12:06:02 +01:00
parent 6fc8408988
commit 90d69af7ee
1 changed files with 143 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
viya/mv_jobflow.sas
View File

@@ -13,7 +13,7 @@
@li FLOW_ID - Numeric value, provides sequential ordering capability. Is @li FLOW_ID - Numeric value, provides sequential ordering capability. Is
optional, will default to 0 if not provided. optional, will default to 0 if not provided.
@li _CONTEXTNAME - Dictates which context should be used to run the job. If @li _CONTEXTNAME - Dictates which context should be used to run the job. If
blank (or not provided), will default to `SAS Job Execution compute context`. blank (or not provided), it defaults to `SAS Job Execution compute context`.
Any additional variables provided in this table are converted into macro Any additional variables provided in this table are converted into macro
variables and passed into the relevant job. variables and passed into the relevant job.
@@ -97,17 +97,21 @@
run; run;
@param [in] access_token_var= The global macro variable to contain the access token @param [in] access_token_var= The global macro variable to contain the
access token
@param [in] grant_type= valid values: @param [in] grant_type= valid values:
@li password @li password
@li authorization_code @li authorization_code
@li detect - will check if access_token exists, if not will use sas_services if @li detect - will check if access_token exists, if not will use
a SASStudioV session else authorization_code. Default option. sas_services if a SASStudioV session else authorization_code.
Default option.
@li sas_services - will use oauth_bearer=sas_services @li sas_services - will use oauth_bearer=sas_services
@param [in] inds= The input dataset containing a list of jobs and parameters @param [in] inds= The input dataset containing a list of jobs and
@param [in] maxconcurrency= The max number of parallel jobs to run. Default=8. parameters
@param [in] raise_err=0 Set to 1 to raise SYSCC when a job does not complete @param [in] maxconcurrency= The max number of parallel jobs to run.
succcessfully Default=8.
@param [in] raise_err=0 Set to 1 to raise SYSCC when a job does not
complete succcessfully
@param [in] mdebug= set to 1 to enable DEBUG messages @param [in] mdebug= set to 1 to enable DEBUG messages
@param [out] outds= The output dataset containing the results @param [out] outds= The output dataset containing the results
@param [out] outref= The output fileref to which to append the log file(s). @param [out] outref= The output fileref to which to append the log file(s).
@@ -228,6 +232,9 @@ data;run;%let jdswaitfor=&syslast;
/* start loop */ /* start loop */
%do fid=1 %to &flowcnt; %do fid=1 %to &flowcnt;
%if not ( &raise_err and &syscc ) %then %do;
%put preparing job attributes for flow &&flow&fid; %put preparing job attributes for flow &&flow&fid;
%local jds jcnt; %local jds jcnt;
data &jds(drop=_contextName _program); data &jds(drop=_contextName _program);
@@ -280,11 +287,18 @@ data;run;%let jdswaitfor=&syslast;
%end; %end;
%end; %end;
/* check if job was triggered and if so, if we have enough slots to run */ /* check if job was triggered and, if
%if "&&jobuid&jid"="0" and &concurrency<&maxconcurrency %then %do; so, if we have enough slots to run? */
%if ("&&jobuid&jid"="0") and (&concurrency<&maxconcurrency) %then %do;
/* But only start if no issues detected so far */
%if not ( &raise_err and &syscc ) %then %do;
%local jobname jobpath; %local jobname jobpath;
%let jobname=%scan(&&job&jid,-1,/); %let jobname=%scan(&&job&jid,-1,/);
%let jobpath=%substr(&&job&jid,1,%length(&&job&jid)-%length(&jobname)-1); %let jobpath=
%substr(&&job&jid,1,%length(&&job&jid)-%length(&jobname)-1);
%put executing &jobpath/&jobname with paramstring &&jparams&jid; %put executing &jobpath/&jobname with paramstring &&jparams&jid;
%mv_jobexecute(path=&jobpath %mv_jobexecute(path=&jobpath
,name=&jobname ,name=&jobname
@@ -306,12 +320,22 @@ data;run;%let jdswaitfor=&syslast;
data _null_; data _null_;
call sleep(1,1); call sleep(1,1);
run; run;
%end;
%else %do; /* Job was skipped due to problems */
%put jobid &&job&jid in flow &fid skipped due to SYSCC (&syscc);
%let completed = %eval(&completed+1);
%let job&jid=0; /* Indicate job has finished */
%end;
%end; %end;
%end; %end;
%if &jid=&jcnt %then %do; %if &jid=&jcnt %then %do;
/* we are at the end of the loop - time to see which jobs have finished */ /* we are at the end of the loop - check which jobs have finished */
%mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref
,raise_err=&raise_err) ,raise_err=&raise_err,mdebug=&mdebug)
%local done; %local done;
%let done=%mf_nobs(&jdswaitfor); %let done=%mf_nobs(&jdswaitfor);
%if &done>0 %then %do; %if &done>0 %then %do;
@@ -334,10 +358,18 @@ data;run;%let jdswaitfor=&syslast;
/* loop again if jobs are left */ /* loop again if jobs are left */
%if &completed < &jcnt %then %do; %if &completed < &jcnt %then %do;
%let jid=0; %let jid=0;
%put looping flow &fid again - &completed of &jcnt jobs completed, &concurrency jobs running; %put looping flow &fid again;
%put &completed of &jcnt jobs completed, &concurrency jobs running;
%end; %end;
%end; %end;
%end; %end;
%end;
%else %do;
%put Flow &&flow&fid skipped due to SYSCC (&syscc);
%end;
/* back up and execute the next flow */ /* back up and execute the next flow */
%end; %end;
@@ -345,4 +377,4 @@ data;run;%let jdswaitfor=&syslast;
%put _local_; %put _local_;
%end; %end;
%mend; %mend mv_jobflow;