Merge pull request #28 from sasjs/issue7

fix: adding licence info.  Closes #7

all.sas

@@ -1694,7 +1694,7 @@ Usage:
           input;
           i=1;
           stoploop=0;
-          if _n_ ge &logline-5 and stoploop=0 then do until (i>12);
+          if _n_ ge &logline-15 and stoploop=0 then do until (i>22);
             call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
             input;
             i+1;
@@ -2936,20 +2936,24 @@ run;
 %mend;/**
   @file
   @brief Drops tables / views (if they exist) without warnings in the log
-  @details
+  @details Useful for dropping tables when you're not sure they exist, or if
+  you are not sure whether they are a dataset or view.  Also efficient for
+  dropping multiple tables / views.
+
   Example usage:
 
       proc sql;
       create table data1 as select * from sashelp.class;
       create view view2 as select * from sashelp.class;
-      %mp_dropmembers(list=data1 view2)
+      %mp_dropmembers(data1 view2, libref=WORK)
+
 
   <h4> SAS Macros </h4>
   @li mf_isblank.sas
 
 
-  @param list space separated list of datasets / views
-  @param libref= can only drop from a single library at a time
+  @param list space separated list of datasets / views, WITHOUT libref
+  @param libref= (WORK) Note - you can only drop from a single library at a time
 
   @version 9.2
   @author Allan Bowe
@@ -2970,7 +2974,7 @@ run;
     delete &list;
     delete &list /mtype=view;
   run;
-%mend;/**
+%mend mp_dropmembers;/**
   @file
   @brief Create a CARDS file from a SAS dataset.
   @details Uses dataset attributes to convert all data into datalines.
@@ -3412,7 +3416,8 @@ run;
   @returns The &outds table containing any bad rows, plus a REASON_CD column.
 
   @param [in] inds The table to be checked, with the format above
-  @param [in] targetds= The target dataset against which to verify VARIABLE_NM
+  @param [in] targetds= The target dataset against which to verify VARIABLE_NM.
+    This must be available (ie, the library must be assigned).
   @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
   @param [out] outds= The output table, which is a copy of the &inds. table
     plus a REASON_CD column, containing only bad records.  If bad records found,
@@ -3463,41 +3468,52 @@ run;
   * quotes, commas, periods and spaces.
   * Only numeric values should remain
   */
-%local reason_cd;
+%local reason_cd nobs;
+%let nobs=0;
 data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
-  length reason_cd $32;
+  length reason_cd $4032;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
-    REASON_CD='GROUP_LOGIC should be either AND or OR';
+    REASON_CD='GROUP_LOGIC should be AND/OR, not:'!!cats(GROUP_LOGIC);
     putlog REASON_CD= GROUP_LOGIC=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if SUBGROUP_LOGIC not in ('AND','OR') then do;
-    REASON_CD='SUBGROUP_LOGIC should be either AND or OR';
+    REASON_CD='SUBGROUP_LOGIC should be AND/OR, not:'!!cats(SUBGROUP_LOGIC);
     putlog REASON_CD= SUBGROUP_LOGIC=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if mod(SUBGROUP_ID,1) ne 0 then do;
-    REASON_CD='SUBGROUP_ID should be integer';
+    REASON_CD='SUBGROUP_ID should be integer, not '!!left(subgroup_id);
     putlog REASON_CD= SUBGROUP_ID=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if upcase(VARIABLE_NM) not in
   (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
   then do;
-    REASON_CD="VARIABLE_NM not in &targetds";
+    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
     putlog REASON_CD= VARIABLE_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if OPERATOR_NM not in
   ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
   then do;
-    REASON_CD='Invalid OPERATOR_NM';
+    REASON_CD='Invalid OPERATOR_NM: '!!left(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
 
@@ -3507,8 +3523,10 @@ data &outds;
     if substr(raw_value,1,1) ne '('
     or substr(cats(reverse(raw_value)),1,1) ne ')'
     then do;
-      REASON_CD='Missing brackets in RAW_VALUE';
+      REASON_CD='Missing start/end bracket in RAW_VALUE';
       putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
       output;
     end;
     else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
@@ -3529,27 +3547,24 @@ data &outds;
   /* output records that contain values other than digits and spaces */
   if notdigit(compress(raw_value3,' '))>0 then do;
     putlog raw_value3= $hex32.;
-    REASON_CD='Invalid RAW_VALUE';
+    REASON_CD=cats('Invalid RAW_VALUE:',raw_value);
     putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
 
 run;
 
-%local nobs;
-%let nobs=0;
+
 data _null_;
   set &outds end=last;
   putlog (_all_)(=);
-  if last then do;
-    call symputx('REASON_CD',reason_cd,'l');
-    call symputx('nobs',_n_,'l');
-  end;
 run;
 
 %mp_abort(iftrue=(&abort=YES and &nobs>0),
   mac=&sysmacroname,
-  msg=%str(&nobs filter issues in &inds, reason: &reason_cd, details in &outds)
+  msg=%str(Data issue: %superq(reason_cd))
 )
 
 %if &nobs>0 %then %do;
@@ -3768,7 +3783,8 @@ filename &fref1 clear;
     run;
     %mp_abort(
       mac=&sysmacroname,
-      msg=%str(Filter issues in &inref: %quote(&reason_cd))
+      msg=%str(Filter validation issues. ERR=%superq(SYSERRORTEXT)
+        , WARN=%superq(SYSWARNINGTEXT) )
     )
   %end;
   %let syscc=1008;
@@ -5060,7 +5076,7 @@ create table &outds (rename=(
 )/*/STORE SOURCE*/;
 %put output location=&jref;
 %if &action=OPEN %then %do;
-  OPTIONS NOBOMFILE;
+  options nobomfile;
   data _null_;file &jref encoding='utf-8';
     put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
   run;
@@ -5713,7 +5729,15 @@ run;
 
       %mp_searchcols(libs=sashelp work, cols=name sex age)
 
-  @param libs=
+  @param libs=(SASHELP) Space separated list of libraries to search for columns
+  @param cols= Space separated list of column names to search for (not case
+    sensitive)
+  @param outds=(mp_searchcols) the table to create with the results.  Will have
+    one line per table match.
+  @param match=(ANY) The match type. Valid values:
+    @li ANY - The table contains at least one of the columns
+    @li WILD - The table contains a column with a name that partially matches
+
   @version 9.2
   @author Allan Bowe
 **/
@@ -5721,6 +5745,7 @@ run;
 %macro mp_searchcols(libs=sashelp
   ,cols=
   ,outds=mp_searchcols
+  ,match=ANY
 )/*/STORE SOURCE*/;
 
 %put &sysmacroname process began at %sysfunc(datetime(),datetime19.);
@@ -5742,8 +5767,10 @@ create table _data_ as
 %end;
   order by 1,2,3;
 
+%local tempds;
+%let tempds=&syslast;
 data &outds;
-  set &syslast;
+  set &tempds;
   length cols matchcols $32767;
   cols=upcase(symget('cols'));
   colcount=countw(cols);
@@ -5757,10 +5784,29 @@ data &outds;
     retain matchcols;
     matchcols='';
   end;
+%if &match=ANY %then %do;
   if findw(cols,name,,'spit') then do;
     sumcols+1;
     matchcols=cats(matchcols)!!' '!!cats(name);
   end;
+%end;
+%else %if &match=WILD %then %do;
+  if _n_=1 then do;
+    retain wcount;
+    wcount=countw(cols);
+    drop wcount;
+  end;
+  do i=1 to wcount;
+    length curword $32;
+    curword=scan(cols,i,' ');
+    drop curword;
+    if index(name,cats(curword)) then do;
+      sumcols+1;
+      matchcols=cats(matchcols)!!' '!!cats(curword);
+    end;
+  end;
+%end;
+
   if last.memname then do;
     if sumcols>0 then output;
     if sumcols=colcount then putlog "Full Match: " libname memname;
@@ -5770,10 +5816,11 @@ run;
 
 proc sort; by descending sumcols memname libname; run;
 
+proc sql;
+drop table &tempds;
 %put &sysmacroname process finished at %sysfunc(datetime(),datetime19.);
 
-%mend;
-/**
+%mend mp_searchcols;/**
   @file
   @brief Searches all data in a library
   @details
@@ -8890,7 +8937,7 @@ data _null_;
   put ')/*/STORE SOURCE*/; ';
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
-  put '  OPTIONS NOBOMFILE; ';
+  put '  options nobomfile; ';
   put '  data _null_;file &jref encoding=''utf-8''; ';
   put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
   put '  run; ';
@@ -13504,7 +13551,7 @@ data _null_;
   put ')/*/STORE SOURCE*/; ';
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
-  put '  OPTIONS NOBOMFILE; ';
+  put '  options nobomfile; ';
   put '  data _null_;file &jref encoding=''utf-8''; ';
   put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
   put '  run; ';
@@ -16334,8 +16381,8 @@ run;
       run;
 
 
-  @param [in] access_token_var= The global macro variable to contain the access
-    token
+  @param [in] access_token_var= The global macro variable to contain the
+              access token
   @param [in] grant_type= valid values:
       @li password
       @li authorization_code
@@ -16474,6 +16521,9 @@ data;run;%let jdswaitfor=&syslast;
 
 /* start loop */
 %do fid=1 %to &flowcnt;
+
+  %if not ( &raise_err and &syscc ) %then %do;
+
     %put preparing job attributes for flow &&flow&fid;
     %local jds jcnt;
     data &jds(drop=_contextName _program);
@@ -16526,18 +16576,23 @@ data;run;%let jdswaitfor=&syslast;
           %end;
         %end;
 
-      /* check if job was triggered and if so, if we have enough slots to run */
-      %if "&&jobuid&jid"="0" and &concurrency<&maxconcurrency %then %do;
+        /* check if job was triggered and, if
+            so, if we have enough slots to run? */
+        %if ("&&jobuid&jid"="0") and (&concurrency<&maxconcurrency) %then %do;
+
+          /* But only start if no issues detected so far */
+          %if not ( &raise_err and &syscc ) %then %do;
+
             %local jobname jobpath;
             %let jobname=%scan(&&job&jid,-1,/);
             %let jobpath=
                   %substr(&&job&jid,1,%length(&&job&jid)-%length(&jobname)-1);
+
             %put executing &jobpath/&jobname with paramstring &&jparams&jid;
             %mv_jobexecute(path=&jobpath
               ,name=&jobname
               ,paramstring=%superq(jparams&jid)
               ,outds=&jdsapp
-          ,mdebug=&mdebug
             )
             data &jdsapp;
               format jobparams $32767.;
@@ -16554,17 +16609,22 @@ data;run;%let jdswaitfor=&syslast;
             data _null_;
               call sleep(1,1);
             run;
+
+          %end;
+          %else %do; /* Job was skipped due to problems */
+
+            %put jobid &&job&jid in flow &fid skipped due to SYSCC (&syscc);
+            %let completed = %eval(&completed+1);
+            %let job&jid=0; /* Indicate job has finished */
+
+          %end;
+
         %end;
       %end;
       %if &jid=&jcnt %then %do;
-      /* we are at the end of the loop - time to see which jobs have finished */
-      %mv_jobwaitfor(ANY
-        ,inds=&jdsrunning
-        ,outds=&jdswaitfor
-        ,outref=&outref
-        ,raise_err=&raise_err
-        ,mdebug=&mdebug
-      )
+        /* we are at the end of the loop - check which jobs have finished */
+        %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref
+                      ,raise_err=&raise_err,mdebug=&mdebug)
         %local done;
         %let done=%mf_nobs(&jdswaitfor);
         %if &done>0 %then %do;
@@ -16587,11 +16647,18 @@ data;run;%let jdswaitfor=&syslast;
         /* loop again if jobs are left */
         %if &completed < &jcnt %then %do;
           %let jid=0;
-        %put looping flow &fid again - &completed of &jcnt jobs completed,
-          &concurrency jobs running;
+          %put looping flow &fid again;
+          %put &completed of &jcnt jobs completed, &concurrency jobs running;
         %end;
       %end;
     %end;
+
+  %end;
+  %else %do;
+
+    %put Flow &&flow&fid skipped due to SYSCC (&syscc);
+
+  %end;
   /* back up and execute the next flow */
 %end;
 
@@ -17626,6 +17693,8 @@ filename &fref1 clear;
 %macro ml_json();
 data _null_;
   file "%sysfunc(pathname(work))/ml_json.lua";
+  put '-- NOTE - THE COPYRIGHT BELOW IS IN RELATION TO THE JSON.LUA FILE ONLY ';
+  put '-- THIS FILE STARTS ON THE NEXT LINE AND WILL FINISH WITH "JSON.LUA ENDS HERE" ';
   put '-- ';
   put '-- json.lua ';
   put '-- ';
@@ -17997,6 +18066,8 @@ data _null_;
   put 'end ';
   put ' ';
   put 'return json ';
+  put ' ';
+  put '-- JSON.LUA ENDS HERE ';
 run;
 
 %inc "%sysfunc(pathname(work))/ml_json.lua";

base/mp_abort.sas

@@ -74,7 +74,7 @@
           input;
           i=1;
           stoploop=0;
-          if _n_ ge &logline-5 and stoploop=0 then do until (i>12);
+          if _n_ ge &logline-15 and stoploop=0 then do until (i>22);
             call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
             input;
             i+1;

base/mp_dropmembers.sas

@@ -1,20 +1,24 @@
 /**
   @file
   @brief Drops tables / views (if they exist) without warnings in the log
-  @details
+  @details Useful for dropping tables when you're not sure they exist, or if
+  you are not sure whether they are a dataset or view.  Also efficient for
+  dropping multiple tables / views.
+
   Example usage:
 
       proc sql;
       create table data1 as select * from sashelp.class;
       create view view2 as select * from sashelp.class;
-      %mp_dropmembers(list=data1 view2)
+      %mp_dropmembers(data1 view2, libref=WORK)
+
 
   <h4> SAS Macros </h4>
   @li mf_isblank.sas
 
 
-  @param list space separated list of datasets / views
-  @param libref= can only drop from a single library at a time
+  @param list space separated list of datasets / views, WITHOUT libref
+  @param libref= (WORK) Note - you can only drop from a single library at a time
 
   @version 9.2
   @author Allan Bowe
@@ -35,4 +39,4 @@
     delete &list;
     delete &list /mtype=view;
   run;
-%mend;
+%mend mp_dropmembers;

base/mp_filtercheck.sas

@@ -33,7 +33,8 @@
   @returns The &outds table containing any bad rows, plus a REASON_CD column.
 
   @param [in] inds The table to be checked, with the format above
-  @param [in] targetds= The target dataset against which to verify VARIABLE_NM
+  @param [in] targetds= The target dataset against which to verify VARIABLE_NM.
+    This must be available (ie, the library must be assigned).
   @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
   @param [out] outds= The output table, which is a copy of the &inds. table
     plus a REASON_CD column, containing only bad records.  If bad records found,
@@ -84,41 +85,52 @@
   * quotes, commas, periods and spaces.
   * Only numeric values should remain
   */
-%local reason_cd;
+%local reason_cd nobs;
+%let nobs=0;
 data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
-  length reason_cd $32;
+  length reason_cd $4032;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
-    REASON_CD='GROUP_LOGIC should be either AND or OR';
+    REASON_CD='GROUP_LOGIC should be AND/OR, not:'!!cats(GROUP_LOGIC);
     putlog REASON_CD= GROUP_LOGIC=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if SUBGROUP_LOGIC not in ('AND','OR') then do;
-    REASON_CD='SUBGROUP_LOGIC should be either AND or OR';
+    REASON_CD='SUBGROUP_LOGIC should be AND/OR, not:'!!cats(SUBGROUP_LOGIC);
     putlog REASON_CD= SUBGROUP_LOGIC=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if mod(SUBGROUP_ID,1) ne 0 then do;
-    REASON_CD='SUBGROUP_ID should be integer';
+    REASON_CD='SUBGROUP_ID should be integer, not '!!left(subgroup_id);
     putlog REASON_CD= SUBGROUP_ID=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if upcase(VARIABLE_NM) not in
   (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
   then do;
-    REASON_CD="VARIABLE_NM not in &targetds";
+    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
     putlog REASON_CD= VARIABLE_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if OPERATOR_NM not in
   ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
   then do;
-    REASON_CD='Invalid OPERATOR_NM';
+    REASON_CD='Invalid OPERATOR_NM: '!!left(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
 
@@ -128,8 +140,10 @@ data &outds;
     if substr(raw_value,1,1) ne '('
     or substr(cats(reverse(raw_value)),1,1) ne ')'
     then do;
-      REASON_CD='Missing brackets in RAW_VALUE';
+      REASON_CD='Missing start/end bracket in RAW_VALUE';
       putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
       output;
     end;
     else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
@@ -150,27 +164,24 @@ data &outds;
   /* output records that contain values other than digits and spaces */
   if notdigit(compress(raw_value3,' '))>0 then do;
     putlog raw_value3= $hex32.;
-    REASON_CD='Invalid RAW_VALUE';
+    REASON_CD=cats('Invalid RAW_VALUE:',raw_value);
     putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
 
 run;
 
-%local nobs;
-%let nobs=0;
+
 data _null_;
   set &outds end=last;
   putlog (_all_)(=);
-  if last then do;
-    call symputx('REASON_CD',reason_cd,'l');
-    call symputx('nobs',_n_,'l');
-  end;
 run;
 
 %mp_abort(iftrue=(&abort=YES and &nobs>0),
   mac=&sysmacroname,
-  msg=%str(&nobs filter issues in &inds, reason: &reason_cd, details in &outds)
+  msg=%str(Data issue: %superq(reason_cd))
 )
 
 %if &nobs>0 %then %do;

base/mp_filtervalidate.sas

@@ -95,7 +95,8 @@ filename &fref1 clear;
     run;
     %mp_abort(
       mac=&sysmacroname,
-      msg=%str(Filter issues in &inref: %quote(&reason_cd))
+      msg=%str(Filter validation issues. ERR=%superq(SYSERRORTEXT)
+        , WARN=%superq(SYSWARNINGTEXT) )
     )
   %end;
   %let syscc=1008;

base/mp_jsonout.sas

@@ -61,7 +61,7 @@
 )/*/STORE SOURCE*/;
 %put output location=&jref;
 %if &action=OPEN %then %do;
-  OPTIONS NOBOMFILE;
+  options nobomfile;
   data _null_;file &jref encoding='utf-8';
     put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
   run;

base/mp_searchcols.sas

@@ -9,7 +9,15 @@
 
       %mp_searchcols(libs=sashelp work, cols=name sex age)
 
-  @param libs=
+  @param libs=(SASHELP) Space separated list of libraries to search for columns
+  @param cols= Space separated list of column names to search for (not case
+    sensitive)
+  @param outds=(mp_searchcols) the table to create with the results.  Will have
+    one line per table match.
+  @param match=(ANY) The match type. Valid values:
+    @li ANY - The table contains at least one of the columns
+    @li WILD - The table contains a column with a name that partially matches
+
   @version 9.2
   @author Allan Bowe
 **/
@@ -17,6 +25,7 @@
 %macro mp_searchcols(libs=sashelp
   ,cols=
   ,outds=mp_searchcols
+  ,match=ANY
 )/*/STORE SOURCE*/;
 
 %put &sysmacroname process began at %sysfunc(datetime(),datetime19.);
@@ -38,8 +47,10 @@ create table _data_ as
 %end;
   order by 1,2,3;
 
+%local tempds;
+%let tempds=&syslast;
 data &outds;
-  set &syslast;
+  set &tempds;
   length cols matchcols $32767;
   cols=upcase(symget('cols'));
   colcount=countw(cols);
@@ -53,10 +64,29 @@ data &outds;
     retain matchcols;
     matchcols='';
   end;
+%if &match=ANY %then %do;
   if findw(cols,name,,'spit') then do;
     sumcols+1;
     matchcols=cats(matchcols)!!' '!!cats(name);
   end;
+%end;
+%else %if &match=WILD %then %do;
+  if _n_=1 then do;
+    retain wcount;
+    wcount=countw(cols);
+    drop wcount;
+  end;
+  do i=1 to wcount;
+    length curword $32;
+    curword=scan(cols,i,' ');
+    drop curword;
+    if index(name,cats(curword)) then do;
+      sumcols+1;
+      matchcols=cats(matchcols)!!' '!!cats(curword);
+    end;
+  end;
+%end;
+
   if last.memname then do;
     if sumcols>0 then output;
     if sumcols=colcount then putlog "Full Match: " libname memname;
@@ -66,6 +96,8 @@ run;
 
 proc sort; by descending sumcols memname libname; run;
 
+proc sql;
+drop table &tempds;
 %put &sysmacroname process finished at %sysfunc(datetime(),datetime19.);
 
-%mend;
+%mend mp_searchcols;

lua/json.lua

@@ -1,3 +1,5 @@
+-- NOTE - THE COPYRIGHT BELOW IS IN RELATION TO THE JSON.LUA FILE ONLY
+-- THIS FILE STARTS ON THE NEXT LINE AND WILL FINISH WITH "JSON.LUA ENDS HERE"
 --
 -- json.lua
 --
@@ -369,3 +371,5 @@ function json.decode(str)
 end
 
 return json
+
+-- JSON.LUA ENDS HERE

lua/ml_json.sas

@@ -12,6 +12,8 @@
 %macro ml_json();
 data _null_;
   file "%sysfunc(pathname(work))/ml_json.lua";
+  put '-- NOTE - THE COPYRIGHT BELOW IS IN RELATION TO THE JSON.LUA FILE ONLY ';
+  put '-- THIS FILE STARTS ON THE NEXT LINE AND WILL FINISH WITH "JSON.LUA ENDS HERE" ';
   put '-- ';
   put '-- json.lua ';
   put '-- ';
@@ -383,6 +385,8 @@ data _null_;
   put 'end ';
   put ' ';
   put 'return json ';
+  put ' ';
+  put '-- JSON.LUA ENDS HERE ';
 run;
 
 %inc "%sysfunc(pathname(work))/ml_json.lua";

meta/mm_createwebservice.sas

@@ -90,7 +90,7 @@ data _null_;
   put ')/*/STORE SOURCE*/; ';
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
-  put '  OPTIONS NOBOMFILE; ';
+  put '  options nobomfile; ';
   put '  data _null_;file &jref encoding=''utf-8''; ';
   put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
   put '  run; ';

tests/base/mp_searchcols.test.sas

@@ -0,0 +1,50 @@
+/**
+  @file
+  @brief Testing mp_searchcols.sas
+
+  <h4> SAS Macros </h4>
+  @li mp_searchcols.sas
+  @li mp_assertdsobs.sas
+
+
+**/
+
+
+/** Test 1 - full col match  */
+data example1;
+  var1=1;
+  var2=2;
+  var3=3;
+data example2;
+  var1=1;
+  var2=2;
+data example3;
+  var2=2;
+  var3=3;
+data example4;
+  matchmehere=1;
+data example5;
+  hereyoucan_matchme_also=1;
+data example6;
+  do_not_forget_me=1;
+data example7;
+  we_shall_not_forget=1;
+run;
+
+%mp_searchcols(libs=work,cols=var1 var2,outds=testme)
+
+%mp_assertdsobs(work.testme,
+  desc=Test1 - check exact variables are found,
+  test=EQUALS 3,
+  outds=work.test_results
+)
+
+/* test 2 - wildcard match */
+
+%mp_searchcols(libs=work,cols=matchme forget,match=WILD, outds=testme2)
+
+%mp_assertdsobs(work.testme2,
+  desc=Test1 - check fuzzy matches are found,
+  test=EQUALS 4,
+  outds=work.test_results
+)

tests/viya/mv_getjoblog.test.sas

@@ -19,6 +19,7 @@
 filename testref temp;
 data _null_;
   file testref;
+  put 'data;run;';
   put 'endsas;';
 run;
 %mv_createjob(
@@ -31,7 +32,7 @@ run;
 %mv_jobexecute(
   path=&mcTestAppLoc/jobs/temp,
   name=testjob,
-  outds=work.info,
+  outds=work.info
 )
 
 %* Wait for it to finish;
@@ -52,10 +53,15 @@ run;
 
 
 data _null_;
-  infile mylog;
+  infile mylog end=eof;
   input;
-  if index(_infile_,'endsas;') then call symputx('found',1);
-  else call symputx('found',0);
+  putlog _infile_;
+  retain found 0;
+  if index(_infile_,'endsas;') then do;
+    found=1;
+    call symputx('found',found);
+  end;
+  else if eof and found ne 1 then call symputx('found',0);
 run;
 
 %mp_assert(

tests/viya/mv_jobflow.test.1.sas

@@ -0,0 +1,78 @@
+/**
+
+  @file
+  @brief Testing mv_jobflow macro
+  @details One of the remote jobs aborts with syscc>0 - test to
+  make sure this comes back to the calling session
+​
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mv_createjob.sas
+  @li mv_jobflow.sas
+
+**/
+
+/**
+  * Test Case 1
+  */
+
+filename testprog temp;
+data _null_;
+  file testprog;
+  put '%put this is job: &_program;'
+  /   '%put this was run in flow &flow_id;'
+  /   'data ;'
+  /   '  rval=rand("uniform");'
+  /   '  rand=rval*&macrovar1;'
+  /   '  do x=1 to rand;'
+  /   '    y=rand*&macrovar2;'
+  /   '    if (rval>0.50) then abort;'
+  /   '    else output;'
+  /   '  end;'
+  /   'run;'
+  ;
+run;
+
+%mv_createjob(path=/Public/temp,name=demo1,code=testprog)
+%mv_createjob(path=/Public/temp,name=demo2,code=testprog)
+
+data work.inputjobs;
+  _contextName='SAS Job Execution compute context';
+  do flow_id=1 to 2;
+    do i=1 to 4;
+      _program='/Public/temp/demo1';
+      macrovar1=10*i;
+      macrovar2=4*i;
+      output;
+      i+1;
+      _program='/Public/temp/demo2';
+      macrovar1=40*i;
+      macrovar2=44*i;
+      output;
+    end;
+  end;
+run;
+
+* Trigger the flow ;
+
+%put NOTE: &=syscc;
+
+%mv_jobflow(inds=work.inputjobs
+  ,maxconcurrency=2
+  ,outds=work.results
+  ,outref=myjoblog
+  ,raise_err=1
+  ,mdebug=1
+)
+
+%put NOTE: &=syscc;
+
+data _null_;
+  infile myjoblog;
+  input; put _infile_;
+run;
+
+%mp_assert(
+  iftrue=(&syscc ne 0),
+  desc=Check that non zero return code is returned if called job fails
+)

tests/viya/mv_jobflow.test.2.sas

@@ -0,0 +1,74 @@
+/**
+  @file
+  @brief Testing mv_jobflow macro
+  @details All jobs complete successfully with syscc = 0 - test to
+  make sure this comes back to the calling session
+​
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mv_createjob.sas
+  @li mv_jobflow.sas
+**/
+
+/**
+  * Test Case 1
+  */
+filename testprog temp;
+data _null_;
+  file testprog;
+  put '%put this is job: &_program;'
+  /   '%put this was run in flow &flow_id;'
+  /   'data ;'
+  /   '  rval=rand("uniform");'
+  /   '  rand=rval*&macrovar1;'
+  /   '  do x=1 to rand;'
+  /   '    y=rand*&macrovar2;'
+  /   '    output;'
+  /   '  end;'
+  /   'run;'
+  ;
+run;
+
+%mv_createjob(path=/Public/temp,name=demo1,code=testprog)
+%mv_createjob(path=/Public/temp,name=demo2,code=testprog)
+
+data work.inputjobs;
+  _contextName='SAS Job Execution compute context';
+  do flow_id=1 to 2;
+    do i=1 to 4;
+      _program='/Public/temp/demo1';
+      macrovar1=10*i;
+      macrovar2=4*i;
+      output;
+      i+1;
+      _program='/Public/temp/demo2';
+      macrovar1=40*i;
+      macrovar2=44*i;
+      output;
+    end;
+  end;
+run;
+
+* Trigger the flow ;
+
+%put NOTE: &=syscc;
+
+%mv_jobflow(inds=work.inputjobs
+  ,maxconcurrency=2
+  ,outds=work.results
+  ,outref=myjoblog
+  ,raise_err=1
+  ,mdebug=1
+)
+
+%put NOTE: &=syscc;
+
+data _null_;
+  infile myjoblog;
+  input; put _infile_;
+run;
+
+%mp_assert(
+  iftrue=(&syscc eq 0),
+  desc=Check that a zero return code is returned if no called job fails
+)

viya/mv_createwebservice.sas

@@ -241,7 +241,7 @@ data _null_;
   put ')/*/STORE SOURCE*/; ';
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
-  put '  OPTIONS NOBOMFILE; ';
+  put '  options nobomfile; ';
   put '  data _null_;file &jref encoding=''utf-8''; ';
   put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
   put '  run; ';

viya/mv_jobflow.sas

@@ -97,8 +97,8 @@
       run;
 
 
-  @param [in] access_token_var= The global macro variable to contain the access
-    token
+  @param [in] access_token_var= The global macro variable to contain the
+              access token
   @param [in] grant_type= valid values:
       @li password
       @li authorization_code
@@ -237,6 +237,9 @@ data;run;%let jdswaitfor=&syslast;
 
 /* start loop */
 %do fid=1 %to &flowcnt;
+
+  %if not ( &raise_err and &syscc ) %then %do;
+
     %put preparing job attributes for flow &&flow&fid;
     %local jds jcnt;
     data &jds(drop=_contextName _program);
@@ -289,18 +292,23 @@ data;run;%let jdswaitfor=&syslast;
           %end;
         %end;
 
-      /* check if job was triggered and if so, if we have enough slots to run */
-      %if "&&jobuid&jid"="0" and &concurrency<&maxconcurrency %then %do;
+        /* check if job was triggered and, if
+            so, if we have enough slots to run? */
+        %if ("&&jobuid&jid"="0") and (&concurrency<&maxconcurrency) %then %do;
+
+          /* But only start if no issues detected so far */
+          %if not ( &raise_err and &syscc ) %then %do;
+
             %local jobname jobpath;
             %let jobname=%scan(&&job&jid,-1,/);
             %let jobpath=
                   %substr(&&job&jid,1,%length(&&job&jid)-%length(&jobname)-1);
+
             %put executing &jobpath/&jobname with paramstring &&jparams&jid;
             %mv_jobexecute(path=&jobpath
               ,name=&jobname
               ,paramstring=%superq(jparams&jid)
               ,outds=&jdsapp
-          ,mdebug=&mdebug
             )
             data &jdsapp;
               format jobparams $32767.;
@@ -317,17 +325,22 @@ data;run;%let jdswaitfor=&syslast;
             data _null_;
               call sleep(1,1);
             run;
+
+          %end;
+          %else %do; /* Job was skipped due to problems */
+
+            %put jobid &&job&jid in flow &fid skipped due to SYSCC (&syscc);
+            %let completed = %eval(&completed+1);
+            %let job&jid=0; /* Indicate job has finished */
+
+          %end;
+
         %end;
       %end;
       %if &jid=&jcnt %then %do;
-      /* we are at the end of the loop - time to see which jobs have finished */
-      %mv_jobwaitfor(ANY
-        ,inds=&jdsrunning
-        ,outds=&jdswaitfor
-        ,outref=&outref
-        ,raise_err=&raise_err
-        ,mdebug=&mdebug
-      )
+        /* we are at the end of the loop - check which jobs have finished */
+        %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref
+                      ,raise_err=&raise_err,mdebug=&mdebug)
         %local done;
         %let done=%mf_nobs(&jdswaitfor);
         %if &done>0 %then %do;
@@ -350,11 +363,18 @@ data;run;%let jdswaitfor=&syslast;
         /* loop again if jobs are left */
         %if &completed < &jcnt %then %do;
           %let jid=0;
-        %put looping flow &fid again - &completed of &jcnt jobs completed,
-          &concurrency jobs running;
+          %put looping flow &fid again;
+          %put &completed of &jcnt jobs completed, &concurrency jobs running;
         %end;
       %end;
     %end;
+
+  %end;
+  %else %do;
+
+    %put Flow &&flow&fid skipped due to SYSCC (&syscc);
+
+  %end;
   /* back up and execute the next flow */
 %end;