diff --git a/all.sas b/all.sas
index f14330e..f391764 100644
--- a/all.sas
+++ b/all.sas
@@ -3077,7 +3077,7 @@ run;
@li SUBGROUP_LOGIC - only AND/OR
@li SUBGROUP_ID - only integers
@li VARIABLE_NM - must be in the target table
- @li OPERATOR_NM - only =/>/<=/>=/BETWEEN/IN/NOT IN/NOT EQUAL/CONTAINS
+ @li OPERATOR_NM - only =/>/<=/>=/BETWEEN/IN/NOT IN/NE/CONTAINS
@li RAW_VALUE - no unquoted values except integers, commas and spaces.
@returns The &outds table containing any bad rows, plus a REASON_CD column.
@@ -3092,11 +3092,15 @@ run;
SAS Macros
@li mp_abort.sas
+ @li mf_getuniquefileref.sas
@li mf_getvarlist.sas
@li mf_nobs.sas
+ @li mp_filtergenerate.sas
+ @li mp_filtervalidate.sas
Related Macros
@li mp_filtergenerate.sas
+ @li mp_filtervalidate.sas
@version 9.3
@author Allan Bowe
@@ -3200,9 +3204,19 @@ run;
)
%end;
%let syscc=1008;
+ %return;
%end;
+/**
+ * syntax checking passed but it does not mean the filter is valid
+ * for that we can run a proc sql validate query
+ */
+%local fref1;
+%let fref1=%mf_getuniquefileref();
+%mp_filtergenerate(&inds,outref=&fref1)
+/* this macro will also set syscc to 1008 if any issues found */
+%mp_filtervalidate(&fref1,&targetds,outds=&outds,abort=&abort)
%mend;
/**
@@ -3228,7 +3242,7 @@ run;
data work.filtertable;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
- OPERATOR_NM:$10. RAW_VALUE:$32767.;
+ OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,AGE,=,12
AND,AND,1,SEX,<=,"'M'"
@@ -3263,6 +3277,7 @@ run;
Related Macros
@li mp_filtercheck.sas
+ @li mp_filtervalidate.sas
SAS Macros
@li mp_abort.sas
@@ -3304,6 +3319,110 @@ filename &outref temp;
run;
%end;
+%mend;
+/**
+ @file
+ @brief Checks a generated filter query for validity
+ @details Runs a generated filter in proc sql with the validate option.
+ Used in mp_filtercheck.sas in an fcmp container.
+
+ Built to support dynamic filtering in
+ [Data Controller for SAS®](https://datacontroller.io).
+
+ Usage:
+
+ data work.filtertable;
+ infile datalines4 dsd;
+ input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+ OPERATOR_NM:$10. RAW_VALUE:$4000.;
+ datalines4;
+ AND,AND,1,AGE,=,12
+ AND,AND,1,SEX,<=,"'M'"
+ AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
+ AND,OR,2,Weight,>=,7
+ ;;;;
+ run;
+
+ %mp_filtergenerate(work.filtertable,outref=myfilter)
+
+ %mp_filtervalidate(myfilter,sashelp.class)
+
+
+ @returns The SYSCC value will be 1008 if there are validation issues.
+
+ @param [in] inref The input fileref to validate (generated by
+ mp_filtergenerate.sas)
+ @param [in] targetds The target dataset against which to verify the query
+ @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
+ @param [out] outds= (work.mp_filtervalidate) Output dataset containing the
+ error / warning message, if one exists. If this table contains any rows,
+ there are problems!
+
+ SAS Macros
+ @li mf_getuniquefileref.sas
+ @li mf_nobs.sas
+ @li mp_abort.sas
+
+ Related Macros
+ @li mp_filtercheck.sas
+ @li mp_filtergenerate.sas
+
+ @version 9.3
+ @author Allan Bowe
+
+**/
+
+%macro mp_filtervalidate(inref,targetds,abort=YES,outds=work.mp_filtervalidate);
+
+%mp_abort(iftrue= (&syscc ne 0 or &syserr ne 0)
+ ,mac=&sysmacroname
+ ,msg=%str(syscc=&syscc / syserr=&syserr - on macro entry)
+)
+
+%local fref1;
+%let fref1=%mf_getuniquefileref();
+
+data _null_;
+ file &fref1;
+ infile &inref end=eof;
+ if _n_=1 then do;
+ put "proc sql;";
+ put "validate select * from &targetds";
+ put "where " ;
+ end;
+ input;
+ put _infile_;
+ putlog _infile_;
+ if eof then put ";quit;";
+run;
+
+%inc &fref1;
+
+data &outds;
+ if &sqlrc or &syscc or &syserr then do;
+ REASON_CD=coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
+ output;
+ end;
+ else stop;
+run;
+
+filename &fref1 clear;
+
+%if %mf_nobs(&outds)>0 %then %do;
+ %if &abort=YES %then %do;
+ data _null_;
+ set &outds;
+ call symputx('REASON_CD',reason_cd,'l');
+ stop;
+ run;
+ %mp_abort(
+ mac=&sysmacroname,
+ msg=%str(Filter issues in &inref: %quote(&reason_cd))
+ )
+ %end;
+ %let syscc=1008;
+%end;
+
%mend;
/**
@file mp_getconstraints.sas
diff --git a/base/mp_filtercheck.sas b/base/mp_filtercheck.sas
index 62312d1..28d2873 100644
--- a/base/mp_filtercheck.sas
+++ b/base/mp_filtercheck.sas
@@ -27,7 +27,7 @@
@li SUBGROUP_LOGIC - only AND/OR
@li SUBGROUP_ID - only integers
@li VARIABLE_NM - must be in the target table
- @li OPERATOR_NM - only =/>/<=/>=/BETWEEN/IN/NOT IN/NOT EQUAL/CONTAINS
+ @li OPERATOR_NM - only =/>/<=/>=/BETWEEN/IN/NOT IN/NE/CONTAINS
@li RAW_VALUE - no unquoted values except integers, commas and spaces.
@returns The &outds table containing any bad rows, plus a REASON_CD column.
@@ -42,11 +42,15 @@
SAS Macros
@li mp_abort.sas
+ @li mf_getuniquefileref.sas
@li mf_getvarlist.sas
@li mf_nobs.sas
+ @li mp_filtergenerate.sas
+ @li mp_filtervalidate.sas
Related Macros
@li mp_filtergenerate.sas
+ @li mp_filtervalidate.sas
@version 9.3
@author Allan Bowe
@@ -150,8 +154,18 @@ run;
)
%end;
%let syscc=1008;
+ %return;
%end;
+/**
+ * syntax checking passed but it does not mean the filter is valid
+ * for that we can run a proc sql validate query
+ */
+%local fref1;
+%let fref1=%mf_getuniquefileref();
+%mp_filtergenerate(&inds,outref=&fref1)
+/* this macro will also set syscc to 1008 if any issues found */
+%mp_filtervalidate(&fref1,&targetds,outds=&outds,abort=&abort)
%mend;
diff --git a/base/mp_filtergenerate.sas b/base/mp_filtergenerate.sas
index 32f820a..a0005e7 100644
--- a/base/mp_filtergenerate.sas
+++ b/base/mp_filtergenerate.sas
@@ -21,7 +21,7 @@
data work.filtertable;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
- OPERATOR_NM:$10. RAW_VALUE:$32767.;
+ OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,AGE,=,12
AND,AND,1,SEX,<=,"'M'"
@@ -56,6 +56,7 @@
Related Macros
@li mp_filtercheck.sas
+ @li mp_filtervalidate.sas
SAS Macros
@li mp_abort.sas
diff --git a/base/mp_filtervalidate.sas b/base/mp_filtervalidate.sas
new file mode 100644
index 0000000..cd8e0c4
--- /dev/null
+++ b/base/mp_filtervalidate.sas
@@ -0,0 +1,104 @@
+/**
+ @file
+ @brief Checks a generated filter query for validity
+ @details Runs a generated filter in proc sql with the validate option.
+ Used in mp_filtercheck.sas in an fcmp container.
+
+ Built to support dynamic filtering in
+ [Data Controller for SAS®](https://datacontroller.io).
+
+ Usage:
+
+ data work.filtertable;
+ infile datalines4 dsd;
+ input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+ OPERATOR_NM:$10. RAW_VALUE:$4000.;
+ datalines4;
+ AND,AND,1,AGE,=,12
+ AND,AND,1,SEX,<=,"'M'"
+ AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
+ AND,OR,2,Weight,>=,7
+ ;;;;
+ run;
+
+ %mp_filtergenerate(work.filtertable,outref=myfilter)
+
+ %mp_filtervalidate(myfilter,sashelp.class)
+
+
+ @returns The SYSCC value will be 1008 if there are validation issues.
+
+ @param [in] inref The input fileref to validate (generated by
+ mp_filtergenerate.sas)
+ @param [in] targetds The target dataset against which to verify the query
+ @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
+ @param [out] outds= (work.mp_filtervalidate) Output dataset containing the
+ error / warning message, if one exists. If this table contains any rows,
+ there are problems!
+
+ SAS Macros
+ @li mf_getuniquefileref.sas
+ @li mf_nobs.sas
+ @li mp_abort.sas
+
+ Related Macros
+ @li mp_filtercheck.sas
+ @li mp_filtergenerate.sas
+
+ @version 9.3
+ @author Allan Bowe
+
+**/
+
+%macro mp_filtervalidate(inref,targetds,abort=YES,outds=work.mp_filtervalidate);
+
+%mp_abort(iftrue= (&syscc ne 0 or &syserr ne 0)
+ ,mac=&sysmacroname
+ ,msg=%str(syscc=&syscc / syserr=&syserr - on macro entry)
+)
+
+%local fref1;
+%let fref1=%mf_getuniquefileref();
+
+data _null_;
+ file &fref1;
+ infile &inref end=eof;
+ if _n_=1 then do;
+ put "proc sql;";
+ put "validate select * from &targetds";
+ put "where " ;
+ end;
+ input;
+ put _infile_;
+ putlog _infile_;
+ if eof then put ";quit;";
+run;
+
+%inc &fref1;
+
+data &outds;
+ if &sqlrc or &syscc or &syserr then do;
+ REASON_CD=coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
+ output;
+ end;
+ else stop;
+run;
+
+filename &fref1 clear;
+
+%if %mf_nobs(&outds)>0 %then %do;
+ %if &abort=YES %then %do;
+ data _null_;
+ set &outds;
+ call symputx('REASON_CD',reason_cd,'l');
+ stop;
+ run;
+ %mp_abort(
+ mac=&sysmacroname,
+ msg=%str(Filter issues in &inref: %quote(&reason_cd))
+ )
+ %end;
+ %let syscc=1008;
+%end;
+
+%mend;
diff --git a/tests/base/mp_filtercheck.test.sas b/tests/base/mp_filtercheck.test.sas
index 8ce727a..ce65277 100644
--- a/tests/base/mp_filtercheck.test.sas
+++ b/tests/base/mp_filtercheck.test.sas
@@ -63,7 +63,7 @@ run;
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
- OPERATOR_NM:$10. RAW_VALUE:$32767.;
+ OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,OR,2,Name,NOT IN,"(''''Jane','Alfred')"
;;;;
@@ -85,7 +85,7 @@ run;
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
- OPERATOR_NM:$10. RAW_VALUE:$32767.;
+ OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,%abort,=,12
AND,OR,2,Weight,>=,7
@@ -108,7 +108,7 @@ run;
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
- OPERATOR_NM:$10. RAW_VALUE:$32767.;
+ OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,age,=,;;%abort
;;;;
diff --git a/tests/base/mp_filtervalidate.test.sas b/tests/base/mp_filtervalidate.test.sas
new file mode 100644
index 0000000..7f1c4a6
--- /dev/null
+++ b/tests/base/mp_filtervalidate.test.sas
@@ -0,0 +1,72 @@
+/**
+ @file
+ @brief Testing mp_filtervalidate macro
+
+ SAS Macros
+ @li mp_filtergenerate.sas
+ @li mp_filtervalidate.sas
+ @li mp_assertdsobs.sas
+
+**/
+
+
+/* valid filter */
+data work.inds;
+ infile datalines4 dsd;
+ input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+ OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,AGE,>,5
+AND,AND,1,SEX,NE,"'M'"
+AND,OR,2,Name,NOT IN,"('Jane','Janet')"
+AND,OR,2,Weight,>=,84.6
+;;;;
+run;
+%mp_filtergenerate(work.inds,outref=myfilter)
+%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%mp_assertdsobs(work.results,
+ desc=Valid filter,
+ test=EMPTY,
+ outds=work.test_results
+)
+
+/* empty filter (return all records) */
+data work.inds;
+ infile datalines4 dsd;
+ input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+ OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+;;;;
+run;
+%mp_filtergenerate(work.inds,outref=myfilter)
+%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%mp_assertdsobs(work.results,
+ desc=Valid filter,
+ test=EMPTY,
+ outds=work.test_results
+)
+
+
+
+/* invalid filter*/
+data work.inds;
+ infile datalines4 dsd;
+ input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+ OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,SEX,NE,2
+;;;;
+run;
+%mp_filtergenerate(work.inds,outref=myfilter)
+%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%let syscc=0;
+%mp_assertdsobs(work.results,
+ desc=Valid filter,
+ test=EQUALS 1,
+ outds=work.test_results
+)
+
+
+%webout(OPEN)
+%webout(OBJ, TEST_RESULTS)
+%webout(CLOSE)
\ No newline at end of file