Merge pull request #26 from tmoody/fix/clean_exit_mv_jobflow_on_syscc

fix: early exit, with syscc, when submitted jobs fail within a flow

tests/viya/mv_jobflow.test.1.sas

@@ -0,0 +1,78 @@
+/**
+
+  @file
+  @brief Testing mv_jobflow macro
+  @details One of the remote jobs aborts with syscc>0 - test to
+  make sure this comes back to the calling session
+​
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mv_createjob.sas
+  @li mv_jobflow.sas
+
+**/
+
+/**
+  * Test Case 1
+  */
+
+filename testprog temp;
+data _null_;
+  file testprog;
+  put '%put this is job: &_program;'
+  /   '%put this was run in flow &flow_id;'
+  /   'data ;'
+  /   '  rval=rand("uniform");'
+  /   '  rand=rval*&macrovar1;'
+  /   '  do x=1 to rand;'
+  /   '    y=rand*&macrovar2;'
+  /   '    if (rval>0.50) then abort;'
+  /   '    else output;'
+  /   '  end;'
+  /   'run;'
+  ;
+run;
+
+%mv_createjob(path=/Public/temp,name=demo1,code=testprog)
+%mv_createjob(path=/Public/temp,name=demo2,code=testprog)
+
+data work.inputjobs;
+  _contextName='SAS Job Execution compute context';
+  do flow_id=1 to 2;
+    do i=1 to 4;
+      _program='/Public/temp/demo1';
+      macrovar1=10*i;
+      macrovar2=4*i;
+      output;
+      i+1;
+      _program='/Public/temp/demo2';
+      macrovar1=40*i;
+      macrovar2=44*i;
+      output;
+    end;
+  end;
+run;
+
+* Trigger the flow ;
+
+%put NOTE: &=syscc;
+
+%mv_jobflow(inds=work.inputjobs
+  ,maxconcurrency=2
+  ,outds=work.results
+  ,outref=myjoblog
+  ,raise_err=1
+  ,mdebug=1
+)
+
+%put NOTE: &=syscc;
+
+data _null_;
+  infile myjoblog;
+  input; put _infile_;
+run;
+
+%mp_assert(
+  iftrue=(&syscc ne 0),
+  desc=Check that non zero return code is returned if called job fails
+)

tests/viya/mv_jobflow.test.2.sas

@@ -0,0 +1,74 @@
+/**
+  @file
+  @brief Testing mv_jobflow macro
+  @details All jobs complete successfully with syscc = 0 - test to
+  make sure this comes back to the calling session
+​
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mv_createjob.sas
+  @li mv_jobflow.sas
+**/
+
+/**
+  * Test Case 1
+  */
+filename testprog temp;
+data _null_;
+  file testprog;
+  put '%put this is job: &_program;'
+  /   '%put this was run in flow &flow_id;'
+  /   'data ;'
+  /   '  rval=rand("uniform");'
+  /   '  rand=rval*&macrovar1;'
+  /   '  do x=1 to rand;'
+  /   '    y=rand*&macrovar2;'
+  /   '    output;'
+  /   '  end;'
+  /   'run;'
+  ;
+run;
+
+%mv_createjob(path=/Public/temp,name=demo1,code=testprog)
+%mv_createjob(path=/Public/temp,name=demo2,code=testprog)
+
+data work.inputjobs;
+  _contextName='SAS Job Execution compute context';
+  do flow_id=1 to 2;
+    do i=1 to 4;
+      _program='/Public/temp/demo1';
+      macrovar1=10*i;
+      macrovar2=4*i;
+      output;
+      i+1;
+      _program='/Public/temp/demo2';
+      macrovar1=40*i;
+      macrovar2=44*i;
+      output;
+    end;
+  end;
+run;
+
+* Trigger the flow ;
+
+%put NOTE: &=syscc;
+
+%mv_jobflow(inds=work.inputjobs
+  ,maxconcurrency=2
+  ,outds=work.results
+  ,outref=myjoblog
+  ,raise_err=1
+  ,mdebug=1
+)
+
+%put NOTE: &=syscc;
+
+data _null_;
+  infile myjoblog;
+  input; put _infile_;
+run;
+
+%mp_assert(
+  iftrue=(&syscc eq 0),
+  desc=Check that a zero return code is returned if no called job fails
+)

viya/mv_jobflow.sas

@@ -97,8 +97,8 @@
       run;
 
 
-  @param [in] access_token_var= The global macro variable to contain the access
+  @param [in] access_token_var= The global macro variable to contain the
-    token
+              access token
   @param [in] grant_type= valid values:
       @li password
       @li authorization_code
@@ -237,6 +237,9 @@ data;run;%let jdswaitfor=&syslast;
 
 /* start loop */
 %do fid=1 %to &flowcnt;
+
+  %if not ( &raise_err and &syscc ) %then %do;
+
     %put preparing job attributes for flow &&flow&fid;
     %local jds jcnt;
     data &jds(drop=_contextName _program);
@@ -289,18 +292,23 @@ data;run;%let jdswaitfor=&syslast;
           %end;
         %end;
 
-      /* check if job was triggered and if so, if we have enough slots to run */
+        /* check if job was triggered and, if
-      %if "&&jobuid&jid"="0" and &concurrency<&maxconcurrency %then %do;
+            so, if we have enough slots to run? */
+        %if ("&&jobuid&jid"="0") and (&concurrency<&maxconcurrency) %then %do;
+
+          /* But only start if no issues detected so far */
+          %if not ( &raise_err and &syscc ) %then %do;
+
             %local jobname jobpath;
             %let jobname=%scan(&&job&jid,-1,/);
             %let jobpath=
                   %substr(&&job&jid,1,%length(&&job&jid)-%length(&jobname)-1);
+
             %put executing &jobpath/&jobname with paramstring &&jparams&jid;
             %mv_jobexecute(path=&jobpath
               ,name=&jobname
               ,paramstring=%superq(jparams&jid)
               ,outds=&jdsapp
-          ,mdebug=&mdebug
             )
             data &jdsapp;
               format jobparams $32767.;
@@ -317,17 +325,22 @@ data;run;%let jdswaitfor=&syslast;
             data _null_;
               call sleep(1,1);
             run;
+
+          %end;
+          %else %do; /* Job was skipped due to problems */
+
+            %put jobid &&job&jid in flow &fid skipped due to SYSCC (&syscc);
+            %let completed = %eval(&completed+1);
+            %let job&jid=0; /* Indicate job has finished */
+
+          %end;
+
         %end;
       %end;
       %if &jid=&jcnt %then %do;
-      /* we are at the end of the loop - time to see which jobs have finished */
+        /* we are at the end of the loop - check which jobs have finished */
-      %mv_jobwaitfor(ANY
+        %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref
-        ,inds=&jdsrunning
+                      ,raise_err=&raise_err,mdebug=&mdebug)
-        ,outds=&jdswaitfor
-        ,outref=&outref
-        ,raise_err=&raise_err
-        ,mdebug=&mdebug
-      )
         %local done;
         %let done=%mf_nobs(&jdswaitfor);
         %if &done>0 %then %do;
@@ -350,11 +363,18 @@ data;run;%let jdswaitfor=&syslast;
         /* loop again if jobs are left */
         %if &completed < &jcnt %then %do;
           %let jid=0;
-        %put looping flow &fid again - &completed of &jcnt jobs completed,
+          %put looping flow &fid again;
-          &concurrency jobs running;
+          %put &completed of &jcnt jobs completed, &concurrency jobs running;
         %end;
       %end;
     %end;
+
+  %end;
+  %else %do;
+
+    %put Flow &&flow&fid skipped due to SYSCC (&syscc);
+
+  %end;
   /* back up and execute the next flow */
 %end;