chore: automated commit

2026-04-18 22:23:12 +00:00 · 2021-05-12 16:32:24 +03:00
parent 779e4942c7
commit de67cd329b
3 changed files with 171 additions and 139 deletions
--- a/all.sas
+++ b/all.sas
@@ -1694,7 +1694,7 @@ Usage:
          input;
          i=1;
          stoploop=0;
-          if _n_ ge &logline-5 and stoploop=0 then do until (i>12);
+          if _n_ ge &logline-15 and stoploop=0 then do until (i>22);
            call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
            input;
            i+1;
@@ -3412,7 +3412,8 @@ run;
  @returns The &outds table containing any bad rows, plus a REASON_CD column.

  @param [in] inds The table to be checked, with the format above
-  @param [in] targetds= The target dataset against which to verify VARIABLE_NM
+  @param [in] targetds= The target dataset against which to verify VARIABLE_NM.
+    This must be available (ie, the library must be assigned).
  @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
  @param [out] outds= The output table, which is a copy of the &inds. table
    plus a REASON_CD column, containing only bad records.  If bad records found,
@@ -3463,41 +3464,52 @@ run;
  * quotes, commas, periods and spaces.
  * Only numeric values should remain
  */
-%local reason_cd;
+%local reason_cd nobs;
+%let nobs=0;
 data &outds;
  /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
    OPERATOR_NM $10 RAW_VALUE $4000;*/
  set &inds;
-  length reason_cd $32;
+  length reason_cd $4032;

  /* closed list checks */
  if GROUP_LOGIC not in ('AND','OR') then do;
-    REASON_CD='GROUP_LOGIC should be either AND or OR';
+    REASON_CD='GROUP_LOGIC should be AND/OR, not:'!!cats(GROUP_LOGIC);
    putlog REASON_CD= GROUP_LOGIC=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
    output;
  end;
  if SUBGROUP_LOGIC not in ('AND','OR') then do;
-    REASON_CD='SUBGROUP_LOGIC should be either AND or OR';
+    REASON_CD='SUBGROUP_LOGIC should be AND/OR, not:'!!cats(SUBGROUP_LOGIC);
    putlog REASON_CD= SUBGROUP_LOGIC=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
    output;
  end;
  if mod(SUBGROUP_ID,1) ne 0 then do;
-    REASON_CD='SUBGROUP_ID should be integer';
+    REASON_CD='SUBGROUP_ID should be integer, not '!!left(subgroup_id);
    putlog REASON_CD= SUBGROUP_ID=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
    output;
  end;
  if upcase(VARIABLE_NM) not in
  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
  then do;
-    REASON_CD="VARIABLE_NM not in &targetds";
+    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
    putlog REASON_CD= VARIABLE_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
    output;
  end;
  if OPERATOR_NM not in
  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
  then do;
-    REASON_CD='Invalid OPERATOR_NM';
+    REASON_CD='Invalid OPERATOR_NM: '!!left(OPERATOR_NM);
    putlog REASON_CD= OPERATOR_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
    output;
  end;

@@ -3507,8 +3519,10 @@ data &outds;
    if substr(raw_value,1,1) ne '('
    or substr(cats(reverse(raw_value)),1,1) ne ')'
    then do;
-      REASON_CD='Missing brackets in RAW_VALUE';
+      REASON_CD='Missing start/end bracket in RAW_VALUE';
      putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
      output;
    end;
    else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
@@ -3529,27 +3543,24 @@ data &outds;
  /* output records that contain values other than digits and spaces */
  if notdigit(compress(raw_value3,' '))>0 then do;
    putlog raw_value3= $hex32.;
-    REASON_CD='Invalid RAW_VALUE';
+    REASON_CD=cats('Invalid RAW_VALUE:',raw_value);
    putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
    output;
  end;

 run;

-%local nobs;
-%let nobs=0;
+
 data _null_;
  set &outds end=last;
  putlog (_all_)(=);
-  if last then do;
-    call symputx('REASON_CD',reason_cd,'l');
-    call symputx('nobs',_n_,'l');
-  end;
 run;

 %mp_abort(iftrue=(&abort=YES and &nobs>0),
  mac=&sysmacroname,
-  msg=%str(&nobs filter issues in &inds, reason: &reason_cd, details in &outds)
+  msg=%str(Data issue: %superq(reason_cd))
 )

 %if &nobs>0 %then %do;
@@ -3768,7 +3779,8 @@ filename &fref1 clear;
    run;
    %mp_abort(
      mac=&sysmacroname,
-      msg=%str(Filter issues in &inref: %quote(&reason_cd))
+      msg=%str(Filter validation issues. ERR=%superq(SYSERRORTEXT)
+        , WARN=%superq(SYSWARNINGTEXT) )
    )
  %end;
  %let syscc=1008;
@@ -5060,7 +5072,7 @@ create table &outds (rename=(
 )/*/STORE SOURCE*/;
 %put output location=&jref;
 %if &action=OPEN %then %do;
-  OPTIONS NOBOMFILE;
+  options nobomfile;
  data _null_;file &jref encoding='utf-8';
    put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
  run;
@@ -8890,7 +8902,7 @@ data _null_;
  put ')/*/STORE SOURCE*/; ';
  put '%put output location=&jref; ';
  put '%if &action=OPEN %then %do; ';
-  put '  OPTIONS NOBOMFILE; ';
+  put '  options nobomfile; ';
  put '  data _null_;file &jref encoding=''utf-8''; ';
  put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
  put '  run; ';
@@ -13504,7 +13516,7 @@ data _null_;
  put ')/*/STORE SOURCE*/; ';
  put '%put output location=&jref; ';
  put '%if &action=OPEN %then %do; ';
-  put '  OPTIONS NOBOMFILE; ';
+  put '  options nobomfile; ';
  put '  data _null_;file &jref encoding=''utf-8''; ';
  put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
  put '  run; ';
@@ -16334,8 +16346,8 @@ run;
      run;


-  @param [in] access_token_var= The global macro variable to contain the access
-    token
+  @param [in] access_token_var= The global macro variable to contain the
+              access token
  @param [in] grant_type= valid values:
      @li password
      @li authorization_code
@@ -16474,6 +16486,9 @@ data;run;%let jdswaitfor=&syslast;

 /* start loop */
 %do fid=1 %to &flowcnt;
+
+  %if not ( &raise_err and &syscc ) %then %do;
+
    %put preparing job attributes for flow &&flow&fid;
    %local jds jcnt;
    data &jds(drop=_contextName _program);
@@ -16526,18 +16541,23 @@ data;run;%let jdswaitfor=&syslast;
          %end;
        %end;

-      /* check if job was triggered and if so, if we have enough slots to run */
-      %if "&&jobuid&jid"="0" and &concurrency<&maxconcurrency %then %do;
+        /* check if job was triggered and, if
+            so, if we have enough slots to run? */
+        %if ("&&jobuid&jid"="0") and (&concurrency<&maxconcurrency) %then %do;
+
+          /* But only start if no issues detected so far */
+          %if not ( &raise_err and &syscc ) %then %do;
+
            %local jobname jobpath;
            %let jobname=%scan(&&job&jid,-1,/);
            %let jobpath=
                  %substr(&&job&jid,1,%length(&&job&jid)-%length(&jobname)-1);
+
            %put executing &jobpath/&jobname with paramstring &&jparams&jid;
            %mv_jobexecute(path=&jobpath
              ,name=&jobname
              ,paramstring=%superq(jparams&jid)
              ,outds=&jdsapp
-          ,mdebug=&mdebug
            )
            data &jdsapp;
              format jobparams $32767.;
@@ -16554,17 +16574,22 @@ data;run;%let jdswaitfor=&syslast;
            data _null_;
              call sleep(1,1);
            run;
+
+          %end;
+          %else %do; /* Job was skipped due to problems */
+
+            %put jobid &&job&jid in flow &fid skipped due to SYSCC (&syscc);
+            %let completed = %eval(&completed+1);
+            %let job&jid=0; /* Indicate job has finished */
+
+          %end;
+
        %end;
      %end;
      %if &jid=&jcnt %then %do;
-      /* we are at the end of the loop - time to see which jobs have finished */
-      %mv_jobwaitfor(ANY
-        ,inds=&jdsrunning
-        ,outds=&jdswaitfor
-        ,outref=&outref
-        ,raise_err=&raise_err
-        ,mdebug=&mdebug
-      )
+        /* we are at the end of the loop - check which jobs have finished */
+        %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref
+                      ,raise_err=&raise_err,mdebug=&mdebug)
        %local done;
        %let done=%mf_nobs(&jdswaitfor);
        %if &done>0 %then %do;
@@ -16587,11 +16612,18 @@ data;run;%let jdswaitfor=&syslast;
        /* loop again if jobs are left */
        %if &completed < &jcnt %then %do;
          %let jid=0;
-        %put looping flow &fid again - &completed of &jcnt jobs completed,
-          &concurrency jobs running;
+          %put looping flow &fid again;
+          %put &completed of &jcnt jobs completed, &concurrency jobs running;
        %end;
      %end;
    %end;
+
+  %end;
+  %else %do;
+
+    %put Flow &&flow&fid skipped due to SYSCC (&syscc);
+
+  %end;
  /* back up and execute the next flow */
 %end;

--- a/meta/mm_createwebservice.sas
+++ b/meta/mm_createwebservice.sas
@@ -90,7 +90,7 @@ data _null_;
  put ')/*/STORE SOURCE*/; ';
  put '%put output location=&jref; ';
  put '%if &action=OPEN %then %do; ';
-  put '  OPTIONS NOBOMFILE; ';
+  put '  options nobomfile; ';
  put '  data _null_;file &jref encoding=''utf-8''; ';
  put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
  put '  run; ';
--- a/viya/mv_createwebservice.sas
+++ b/viya/mv_createwebservice.sas
@@ -241,7 +241,7 @@ data _null_;
  put ')/*/STORE SOURCE*/; ';
  put '%put output location=&jref; ';
  put '%if &action=OPEN %then %do; ';
-  put '  OPTIONS NOBOMFILE; ';
+  put '  options nobomfile; ';
  put '  data _null_;file &jref encoding=''utf-8''; ';
  put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
  put '  run; ';