feat: mp_filtervalidate.sas - to run a proc sql validate against the target table to ensure validity

all.sas

@@ -3077,7 +3077,7 @@ run;
   @li SUBGROUP_LOGIC - only AND/OR
   @li SUBGROUP_ID - only integers
   @li VARIABLE_NM - must be in the target table
-  @li OPERATOR_NM - only =/>/</<=/>=/BETWEEN/IN/NOT IN/NOT EQUAL/CONTAINS
+  @li OPERATOR_NM - only =/>/</<=/>=/BETWEEN/IN/NOT IN/NE/CONTAINS
   @li RAW_VALUE - no unquoted values except integers, commas and spaces.
 
   @returns The &outds table containing any bad rows, plus a REASON_CD column.
@@ -3092,11 +3092,15 @@ run;
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas
+  @li mf_getuniquefileref.sas
   @li mf_getvarlist.sas
   @li mf_nobs.sas
+  @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
 
   <h4> Related Macros </h4>
   @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
 
   @version 9.3
   @author Allan Bowe
@@ -3200,9 +3204,19 @@ run;
     )
   %end;
   %let syscc=1008;
+  %return;
 %end;
 
+/**
+  * syntax checking passed but it does not mean the filter is valid
+  * for that we can run a proc sql validate query
+  */
+%local fref1;
+%let fref1=%mf_getuniquefileref();
+%mp_filtergenerate(&inds,outref=&fref1)
 
+/* this macro will also set syscc to 1008 if any issues found */
+%mp_filtervalidate(&fref1,&targetds,outds=&outds,abort=&abort)
 
 %mend;
 /**
@@ -3228,7 +3242,7 @@ run;
       data work.filtertable;
         infile datalines4 dsd;
         input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-          OPERATOR_NM:$10. RAW_VALUE:$32767.;
+          OPERATOR_NM:$10. RAW_VALUE:$4000.;
       datalines4;
       AND,AND,1,AGE,=,12
       AND,AND,1,SEX,<=,"'M'"
@@ -3263,6 +3277,7 @@ run;
 
   <h4> Related Macros </h4>
   @li mp_filtercheck.sas
+  @li mp_filtervalidate.sas
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas
@@ -3304,6 +3319,110 @@ filename &outref temp;
   run;
 %end;
 
+%mend;
+/**
+  @file
+  @brief Checks a generated filter query for validity
+  @details Runs a generated filter in proc sql with the validate option.
+  Used in mp_filtercheck.sas in an fcmp container.
+
+  Built to support dynamic filtering in
+  [Data Controller for SAS&reg;](https://datacontroller.io).
+
+  Usage:
+
+      data work.filtertable;
+        infile datalines4 dsd;
+        input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+          OPERATOR_NM:$10. RAW_VALUE:$4000.;
+      datalines4;
+      AND,AND,1,AGE,=,12
+      AND,AND,1,SEX,<=,"'M'"
+      AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
+      AND,OR,2,Weight,>=,7
+      ;;;;
+      run;
+
+      %mp_filtergenerate(work.filtertable,outref=myfilter)
+
+      %mp_filtervalidate(myfilter,sashelp.class)
+
+
+  @returns The SYSCC value will be 1008 if there are validation issues.
+
+  @param [in] inref The input fileref to validate (generated by
+    mp_filtergenerate.sas)
+  @param [in] targetds The target dataset against which to verify the query
+  @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
+  @param [out] outds= (work.mp_filtervalidate) Output dataset containing the
+  error / warning message, if one exists.  If this table contains any rows,
+  there are problems!
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filtercheck.sas
+  @li mp_filtergenerate.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mp_filtervalidate(inref,targetds,abort=YES,outds=work.mp_filtervalidate);
+
+%mp_abort(iftrue= (&syscc ne 0 or &syserr ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(syscc=&syscc / syserr=&syserr - on macro entry)
+)
+
+%local fref1;
+%let fref1=%mf_getuniquefileref();
+
+data _null_;
+  file &fref1;
+  infile &inref end=eof;
+  if _n_=1 then do;
+    put "proc sql;";
+    put "validate select * from &targetds";
+    put "where " ;
+  end;
+  input;
+  put _infile_;
+  putlog _infile_;
+  if eof then put ";quit;";
+run;
+
+%inc &fref1;
+
+data &outds;
+  if &sqlrc or &syscc or &syserr then do;
+    REASON_CD=coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
+    output;
+  end;
+  else stop;
+run;
+
+filename &fref1 clear;
+
+%if %mf_nobs(&outds)>0 %then %do;
+  %if &abort=YES %then %do;
+    data _null_;
+      set &outds;
+      call symputx('REASON_CD',reason_cd,'l');
+      stop;
+    run;
+    %mp_abort(
+      mac=&sysmacroname,
+      msg=%str(Filter issues in &inref: %quote(&reason_cd))
+    )
+  %end;
+  %let syscc=1008;
+%end;
+
 %mend;
 /**
   @file mp_getconstraints.sas

base/mp_filtercheck.sas

@@ -27,7 +27,7 @@
   @li SUBGROUP_LOGIC - only AND/OR
   @li SUBGROUP_ID - only integers
   @li VARIABLE_NM - must be in the target table
-  @li OPERATOR_NM - only =/>/</<=/>=/BETWEEN/IN/NOT IN/NOT EQUAL/CONTAINS
+  @li OPERATOR_NM - only =/>/</<=/>=/BETWEEN/IN/NOT IN/NE/CONTAINS
   @li RAW_VALUE - no unquoted values except integers, commas and spaces.
 
   @returns The &outds table containing any bad rows, plus a REASON_CD column.
@@ -42,11 +42,15 @@
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas
+  @li mf_getuniquefileref.sas
   @li mf_getvarlist.sas
   @li mf_nobs.sas
+  @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
 
   <h4> Related Macros </h4>
   @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
 
   @version 9.3
   @author Allan Bowe
@@ -150,8 +154,18 @@ run;
     )
   %end;
   %let syscc=1008;
+  %return;
 %end;
 
+/**
+  * syntax checking passed but it does not mean the filter is valid
+  * for that we can run a proc sql validate query
+  */
+%local fref1;
+%let fref1=%mf_getuniquefileref();
+%mp_filtergenerate(&inds,outref=&fref1)
 
+/* this macro will also set syscc to 1008 if any issues found */
+%mp_filtervalidate(&fref1,&targetds,outds=&outds,abort=&abort)
 
 %mend;

base/mp_filtergenerate.sas

@@ -21,7 +21,7 @@
       data work.filtertable;
         infile datalines4 dsd;
         input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-          OPERATOR_NM:$10. RAW_VALUE:$32767.;
+          OPERATOR_NM:$10. RAW_VALUE:$4000.;
       datalines4;
       AND,AND,1,AGE,=,12
       AND,AND,1,SEX,<=,"'M'"
@@ -56,6 +56,7 @@
 
   <h4> Related Macros </h4>
   @li mp_filtercheck.sas
+  @li mp_filtervalidate.sas
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas

base/mp_filtervalidate.sas

@@ -0,0 +1,104 @@
+/**
+  @file
+  @brief Checks a generated filter query for validity
+  @details Runs a generated filter in proc sql with the validate option.
+  Used in mp_filtercheck.sas in an fcmp container.
+
+  Built to support dynamic filtering in
+  [Data Controller for SAS®](https://datacontroller.io).
+
+  Usage:
+
+      data work.filtertable;
+        infile datalines4 dsd;
+        input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+          OPERATOR_NM:$10. RAW_VALUE:$4000.;
+      datalines4;
+      AND,AND,1,AGE,=,12
+      AND,AND,1,SEX,<=,"'M'"
+      AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
+      AND,OR,2,Weight,>=,7
+      ;;;;
+      run;
+
+      %mp_filtergenerate(work.filtertable,outref=myfilter)
+
+      %mp_filtervalidate(myfilter,sashelp.class)
+
+
+  @returns The SYSCC value will be 1008 if there are validation issues.
+
+  @param [in] inref The input fileref to validate (generated by
+    mp_filtergenerate.sas)
+  @param [in] targetds The target dataset against which to verify the query
+  @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
+  @param [out] outds= (work.mp_filtervalidate) Output dataset containing the
+  error / warning message, if one exists.  If this table contains any rows,
+  there are problems!
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filtercheck.sas
+  @li mp_filtergenerate.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mp_filtervalidate(inref,targetds,abort=YES,outds=work.mp_filtervalidate);
+
+%mp_abort(iftrue= (&syscc ne 0 or &syserr ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(syscc=&syscc / syserr=&syserr - on macro entry)
+)
+
+%local fref1;
+%let fref1=%mf_getuniquefileref();
+
+data _null_;
+  file &fref1;
+  infile &inref end=eof;
+  if _n_=1 then do;
+    put "proc sql;";
+    put "validate select * from &targetds";
+    put "where " ;
+  end;
+  input;
+  put _infile_;
+  putlog _infile_;
+  if eof then put ";quit;";
+run;
+
+%inc &fref1;
+
+data &outds;
+  if &sqlrc or &syscc or &syserr then do;
+    REASON_CD=coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
+    output;
+  end;
+  else stop;
+run;
+
+filename &fref1 clear;
+
+%if %mf_nobs(&outds)>0 %then %do;
+  %if &abort=YES %then %do;
+    data _null_;
+      set &outds;
+      call symputx('REASON_CD',reason_cd,'l');
+      stop;
+    run;
+    %mp_abort(
+      mac=&sysmacroname,
+      msg=%str(Filter issues in &inref: %quote(&reason_cd))
+    )
+  %end;
+  %let syscc=1008;
+%end;
+
+%mend;

tests/base/mp_filtercheck.test.sas

@@ -63,7 +63,7 @@ run;
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
 AND,OR,2,Name,NOT IN,"(''''Jane','Alfred')"
 ;;;;
@@ -85,7 +85,7 @@ run;
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
 AND,AND,1,%abort,=,12
 AND,OR,2,Weight,>=,7
@@ -108,7 +108,7 @@ run;
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
 AND,AND,1,age,=,;;%abort
 ;;;;

tests/base/mp_filtervalidate.test.sas

@@ -0,0 +1,72 @@
+/**
+  @file
+  @brief Testing mp_filtervalidate macro
+
+  <h4> SAS Macros </h4>
+  @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
+  @li mp_assertdsobs.sas
+
+**/
+
+
+/* valid filter */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,AGE,>,5
+AND,AND,1,SEX,NE,"'M'"
+AND,OR,2,Name,NOT IN,"('Jane','Janet')"
+AND,OR,2,Weight,>=,84.6
+;;;;
+run;
+%mp_filtergenerate(work.inds,outref=myfilter)
+%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%mp_assertdsobs(work.results,
+  desc=Valid filter,
+  test=EMPTY,
+  outds=work.test_results
+)
+
+/* empty filter (return all records) */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+;;;;
+run;
+%mp_filtergenerate(work.inds,outref=myfilter)
+%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%mp_assertdsobs(work.results,
+  desc=Valid filter,
+  test=EMPTY,
+  outds=work.test_results
+)
+
+
+
+/* invalid filter*/
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,SEX,NE,2
+;;;;
+run;
+%mp_filtergenerate(work.inds,outref=myfilter)
+%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%let syscc=0;
+%mp_assertdsobs(work.results,
+  desc=Valid filter,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+
+%webout(OPEN)
+%webout(OBJ, TEST_RESULTS)
+%webout(CLOSE)