diff --git a/all.sas b/all.sas index f14330e..f391764 100644 --- a/all.sas +++ b/all.sas @@ -3077,7 +3077,7 @@ run; @li SUBGROUP_LOGIC - only AND/OR @li SUBGROUP_ID - only integers @li VARIABLE_NM - must be in the target table - @li OPERATOR_NM - only =/>/=/BETWEEN/IN/NOT IN/NOT EQUAL/CONTAINS + @li OPERATOR_NM - only =/>/=/BETWEEN/IN/NOT IN/NE/CONTAINS @li RAW_VALUE - no unquoted values except integers, commas and spaces. @returns The &outds table containing any bad rows, plus a REASON_CD column. @@ -3092,11 +3092,15 @@ run;

SAS Macros

@li mp_abort.sas + @li mf_getuniquefileref.sas @li mf_getvarlist.sas @li mf_nobs.sas + @li mp_filtergenerate.sas + @li mp_filtervalidate.sas

Related Macros

@li mp_filtergenerate.sas + @li mp_filtervalidate.sas @version 9.3 @author Allan Bowe @@ -3200,9 +3204,19 @@ run; ) %end; %let syscc=1008; + %return; %end; +/** + * syntax checking passed but it does not mean the filter is valid + * for that we can run a proc sql validate query + */ +%local fref1; +%let fref1=%mf_getuniquefileref(); +%mp_filtergenerate(&inds,outref=&fref1) +/* this macro will also set syscc to 1008 if any issues found */ +%mp_filtervalidate(&fref1,&targetds,outds=&outds,abort=&abort) %mend; /** @@ -3228,7 +3242,7 @@ run; data work.filtertable; infile datalines4 dsd; input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. - OPERATOR_NM:$10. RAW_VALUE:$32767.; + OPERATOR_NM:$10. RAW_VALUE:$4000.; datalines4; AND,AND,1,AGE,=,12 AND,AND,1,SEX,<=,"'M'" @@ -3263,6 +3277,7 @@ run;

Related Macros

@li mp_filtercheck.sas + @li mp_filtervalidate.sas

SAS Macros

@li mp_abort.sas @@ -3304,6 +3319,110 @@ filename &outref temp; run; %end; +%mend; +/** + @file + @brief Checks a generated filter query for validity + @details Runs a generated filter in proc sql with the validate option. + Used in mp_filtercheck.sas in an fcmp container. + + Built to support dynamic filtering in + [Data Controller for SAS®](https://datacontroller.io). + + Usage: + + data work.filtertable; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$4000.; + datalines4; + AND,AND,1,AGE,=,12 + AND,AND,1,SEX,<=,"'M'" + AND,OR,2,Name,NOT IN,"('Jane','Alfred')" + AND,OR,2,Weight,>=,7 + ;;;; + run; + + %mp_filtergenerate(work.filtertable,outref=myfilter) + + %mp_filtervalidate(myfilter,sashelp.class) + + + @returns The SYSCC value will be 1008 if there are validation issues. + + @param [in] inref The input fileref to validate (generated by + mp_filtergenerate.sas) + @param [in] targetds The target dataset against which to verify the query + @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions + @param [out] outds= (work.mp_filtervalidate) Output dataset containing the + error / warning message, if one exists. If this table contains any rows, + there are problems! + +

SAS Macros

+ @li mf_getuniquefileref.sas + @li mf_nobs.sas + @li mp_abort.sas + +

Related Macros

+ @li mp_filtercheck.sas + @li mp_filtergenerate.sas + + @version 9.3 + @author Allan Bowe + +**/ + +%macro mp_filtervalidate(inref,targetds,abort=YES,outds=work.mp_filtervalidate); + +%mp_abort(iftrue= (&syscc ne 0 or &syserr ne 0) + ,mac=&sysmacroname + ,msg=%str(syscc=&syscc / syserr=&syserr - on macro entry) +) + +%local fref1; +%let fref1=%mf_getuniquefileref(); + +data _null_; + file &fref1; + infile &inref end=eof; + if _n_=1 then do; + put "proc sql;"; + put "validate select * from &targetds"; + put "where " ; + end; + input; + put _infile_; + putlog _infile_; + if eof then put ";quit;"; +run; + +%inc &fref1; + +data &outds; + if &sqlrc or &syscc or &syserr then do; + REASON_CD=coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT')); + output; + end; + else stop; +run; + +filename &fref1 clear; + +%if %mf_nobs(&outds)>0 %then %do; + %if &abort=YES %then %do; + data _null_; + set &outds; + call symputx('REASON_CD',reason_cd,'l'); + stop; + run; + %mp_abort( + mac=&sysmacroname, + msg=%str(Filter issues in &inref: %quote(&reason_cd)) + ) + %end; + %let syscc=1008; +%end; + %mend; /** @file mp_getconstraints.sas diff --git a/base/mp_filtercheck.sas b/base/mp_filtercheck.sas index 62312d1..28d2873 100644 --- a/base/mp_filtercheck.sas +++ b/base/mp_filtercheck.sas @@ -27,7 +27,7 @@ @li SUBGROUP_LOGIC - only AND/OR @li SUBGROUP_ID - only integers @li VARIABLE_NM - must be in the target table - @li OPERATOR_NM - only =/>/=/BETWEEN/IN/NOT IN/NOT EQUAL/CONTAINS + @li OPERATOR_NM - only =/>/=/BETWEEN/IN/NOT IN/NE/CONTAINS @li RAW_VALUE - no unquoted values except integers, commas and spaces. @returns The &outds table containing any bad rows, plus a REASON_CD column. @@ -42,11 +42,15 @@

SAS Macros

@li mp_abort.sas + @li mf_getuniquefileref.sas @li mf_getvarlist.sas @li mf_nobs.sas + @li mp_filtergenerate.sas + @li mp_filtervalidate.sas

Related Macros

@li mp_filtergenerate.sas + @li mp_filtervalidate.sas @version 9.3 @author Allan Bowe @@ -150,8 +154,18 @@ run; ) %end; %let syscc=1008; + %return; %end; +/** + * syntax checking passed but it does not mean the filter is valid + * for that we can run a proc sql validate query + */ +%local fref1; +%let fref1=%mf_getuniquefileref(); +%mp_filtergenerate(&inds,outref=&fref1) +/* this macro will also set syscc to 1008 if any issues found */ +%mp_filtervalidate(&fref1,&targetds,outds=&outds,abort=&abort) %mend; diff --git a/base/mp_filtergenerate.sas b/base/mp_filtergenerate.sas index 32f820a..a0005e7 100644 --- a/base/mp_filtergenerate.sas +++ b/base/mp_filtergenerate.sas @@ -21,7 +21,7 @@ data work.filtertable; infile datalines4 dsd; input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. - OPERATOR_NM:$10. RAW_VALUE:$32767.; + OPERATOR_NM:$10. RAW_VALUE:$4000.; datalines4; AND,AND,1,AGE,=,12 AND,AND,1,SEX,<=,"'M'" @@ -56,6 +56,7 @@

Related Macros

@li mp_filtercheck.sas + @li mp_filtervalidate.sas

SAS Macros

@li mp_abort.sas diff --git a/base/mp_filtervalidate.sas b/base/mp_filtervalidate.sas new file mode 100644 index 0000000..cd8e0c4 --- /dev/null +++ b/base/mp_filtervalidate.sas @@ -0,0 +1,104 @@ +/** + @file + @brief Checks a generated filter query for validity + @details Runs a generated filter in proc sql with the validate option. + Used in mp_filtercheck.sas in an fcmp container. + + Built to support dynamic filtering in + [Data Controller for SAS®](https://datacontroller.io). + + Usage: + + data work.filtertable; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$4000.; + datalines4; + AND,AND,1,AGE,=,12 + AND,AND,1,SEX,<=,"'M'" + AND,OR,2,Name,NOT IN,"('Jane','Alfred')" + AND,OR,2,Weight,>=,7 + ;;;; + run; + + %mp_filtergenerate(work.filtertable,outref=myfilter) + + %mp_filtervalidate(myfilter,sashelp.class) + + + @returns The SYSCC value will be 1008 if there are validation issues. + + @param [in] inref The input fileref to validate (generated by + mp_filtergenerate.sas) + @param [in] targetds The target dataset against which to verify the query + @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions + @param [out] outds= (work.mp_filtervalidate) Output dataset containing the + error / warning message, if one exists. If this table contains any rows, + there are problems! + +

SAS Macros

+ @li mf_getuniquefileref.sas + @li mf_nobs.sas + @li mp_abort.sas + +

Related Macros

+ @li mp_filtercheck.sas + @li mp_filtergenerate.sas + + @version 9.3 + @author Allan Bowe + +**/ + +%macro mp_filtervalidate(inref,targetds,abort=YES,outds=work.mp_filtervalidate); + +%mp_abort(iftrue= (&syscc ne 0 or &syserr ne 0) + ,mac=&sysmacroname + ,msg=%str(syscc=&syscc / syserr=&syserr - on macro entry) +) + +%local fref1; +%let fref1=%mf_getuniquefileref(); + +data _null_; + file &fref1; + infile &inref end=eof; + if _n_=1 then do; + put "proc sql;"; + put "validate select * from &targetds"; + put "where " ; + end; + input; + put _infile_; + putlog _infile_; + if eof then put ";quit;"; +run; + +%inc &fref1; + +data &outds; + if &sqlrc or &syscc or &syserr then do; + REASON_CD=coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT')); + output; + end; + else stop; +run; + +filename &fref1 clear; + +%if %mf_nobs(&outds)>0 %then %do; + %if &abort=YES %then %do; + data _null_; + set &outds; + call symputx('REASON_CD',reason_cd,'l'); + stop; + run; + %mp_abort( + mac=&sysmacroname, + msg=%str(Filter issues in &inref: %quote(&reason_cd)) + ) + %end; + %let syscc=1008; +%end; + +%mend; diff --git a/tests/base/mp_filtercheck.test.sas b/tests/base/mp_filtercheck.test.sas index 8ce727a..ce65277 100644 --- a/tests/base/mp_filtercheck.test.sas +++ b/tests/base/mp_filtercheck.test.sas @@ -63,7 +63,7 @@ run; data work.inds; infile datalines4 dsd; input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. - OPERATOR_NM:$10. RAW_VALUE:$32767.; + OPERATOR_NM:$10. RAW_VALUE:$4000.; datalines4; AND,OR,2,Name,NOT IN,"(''''Jane','Alfred')" ;;;; @@ -85,7 +85,7 @@ run; data work.inds; infile datalines4 dsd; input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. - OPERATOR_NM:$10. RAW_VALUE:$32767.; + OPERATOR_NM:$10. RAW_VALUE:$4000.; datalines4; AND,AND,1,%abort,=,12 AND,OR,2,Weight,>=,7 @@ -108,7 +108,7 @@ run; data work.inds; infile datalines4 dsd; input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. - OPERATOR_NM:$10. RAW_VALUE:$32767.; + OPERATOR_NM:$10. RAW_VALUE:$4000.; datalines4; AND,AND,1,age,=,;;%abort ;;;; diff --git a/tests/base/mp_filtervalidate.test.sas b/tests/base/mp_filtervalidate.test.sas new file mode 100644 index 0000000..7f1c4a6 --- /dev/null +++ b/tests/base/mp_filtervalidate.test.sas @@ -0,0 +1,72 @@ +/** + @file + @brief Testing mp_filtervalidate macro + +

SAS Macros

+ @li mp_filtergenerate.sas + @li mp_filtervalidate.sas + @li mp_assertdsobs.sas + +**/ + + +/* valid filter */ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$4000.; +datalines4; +AND,AND,1,AGE,>,5 +AND,AND,1,SEX,NE,"'M'" +AND,OR,2,Name,NOT IN,"('Jane','Janet')" +AND,OR,2,Weight,>=,84.6 +;;;; +run; +%mp_filtergenerate(work.inds,outref=myfilter) +%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO) +%mp_assertdsobs(work.results, + desc=Valid filter, + test=EMPTY, + outds=work.test_results +) + +/* empty filter (return all records) */ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$4000.; +datalines4; +;;;; +run; +%mp_filtergenerate(work.inds,outref=myfilter) +%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO) +%mp_assertdsobs(work.results, + desc=Valid filter, + test=EMPTY, + outds=work.test_results +) + + + +/* invalid filter*/ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$4000.; +datalines4; +AND,AND,1,SEX,NE,2 +;;;; +run; +%mp_filtergenerate(work.inds,outref=myfilter) +%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO) +%let syscc=0; +%mp_assertdsobs(work.results, + desc=Valid filter, + test=EQUALS 1, + outds=work.test_results +) + + +%webout(OPEN) +%webout(OBJ, TEST_RESULTS) +%webout(CLOSE) \ No newline at end of file