diff --git a/all.sas b/all.sas index 298df2e..20cce9a 100644 --- a/all.sas +++ b/all.sas @@ -1793,13 +1793,15 @@ Usage:

SAS Macros

@li mf_nobs.sas + @li mp_abort.sas @param [in] inds input dataset to test for presence of observations @param [in] desc= (Testing observations) The user provided test description @param [in] test= (HASOBS) The test to apply. Valid values are: - @li HASOBS Test is a PASS if the input dataset has any observations - @li EMPTY Test is a PASS if input dataset is empty + @li HASOBS - Test is a PASS if the input dataset has any observations + @li EMPTY - Test is a PASS if input dataset is empty + @li EQUALS [integer] - Test passes if obs count matches the provided integer @param [out] outds= (work.test_results) The output dataset to contain the results. If it does not exist, it will be created, with the following format: |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256| @@ -1822,6 +1824,21 @@ Usage: %let nobs=%mf_nobs(&inds); %let test=%upcase(&test); + %if %substr(&test.xxxxx,1,6)=EQUALS %then %do; + %let val=%scan(&test,2,%str( )); + %mp_abort(iftrue= (%DATATYP(&val)=CHAR) + ,mac=&sysmacroname + ,msg=%str(Invalid test - &test, expected EQUALS [integer]) + ) + %let test=EQUALS; + %end; + %else %if &test ne HASOBS and &test ne EMPTY %then %do; + %mp_abort( + mac=&sysmacroname, + msg=%str(Invalid test - &test) + ) + %end; + data; length test_description $256 test_result $4 test_comments $256; test_description=symget('desc'); @@ -1833,6 +1850,9 @@ Usage: %else %if &test=EMPTY %then %do; if &nobs=0 then test_result='PASS'; %end; + %else %if &test=EQUALS %then %do; + if &nobs=&val then test_result='PASS'; + %end; %else %do; test_comments="&sysmacroname: Unsatisfied test condition - &test"; %end; @@ -2885,7 +2905,8 @@ run; @brief Checks an input filter table for validity @details Performs checks on the input table to ensure it arrives in the correct format. This is necessary to prevent code injection. Will update - SYSCC to 1008 if bad records are found. + SYSCC to 1008 if bad records are found, and call mp_abort.sas for a + graceful service exit (configurable). Used for dynamic filtering in [Data Controller for SAS®](https://datacontroller.io). @@ -2915,6 +2936,7 @@ run; @param [in] inds The table to be checked, with the format above @param [in] targetds= The target dataset against which to verify VARIABLE_NM + @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions @param [out] outds= The output table, which is a copy of the &inds. table plus a REASON_CD column, containing only bad records. If bad records found, the SYSCC value will be set to 1008 (general data problem). Downstream @@ -2934,7 +2956,7 @@ run; @todo Support date / hex / name literals and exponents in RAW_VALUE field **/ -%macro mp_filtercheck(inds,targetds=,outds=work.badrecords); +%macro mp_filtercheck(inds,targetds=,outds=work.badrecords,abort=YES); %mp_abort(iftrue= (&syscc ne 0) ,mac=&sysmacroname @@ -2975,7 +2997,7 @@ data &outds; output; end; if OPERATOR_NM not in - ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NOT EQUAL','CONTAINS') + ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS') then do; REASON_CD='Invalid OPERATOR_NM'; putlog REASON_CD= OPERATOR_NM=; @@ -3004,11 +3026,8 @@ data &outds; regex = prxparse("s/(\').*?(\')//"); call prxchange(regex,-1,raw_value2); - /* remove commas */ - raw_value3=compress(raw_value2,','); - - - + /* remove commas and periods*/ + raw_value3=compress(raw_value2,',.'); /* output records that contain values other than digits and spaces */ if notdigit(compress(raw_value3,' '))>0 then do; @@ -3020,7 +3039,22 @@ data &outds; run; -%if %mf_nobs(&outds)>0 %then %let syscc=1008; +%if %mf_nobs(&outds)>0 %then %do; + %if &abort=YES %then %do; + data _null_; + set &outds; + call symputx('REASON_CD',reason_cd,'l'); + stop; + run; + %mp_abort( + mac=&sysmacroname, + msg=%str(Filter issues in &inds, first was &reason_cd, details in &outds) + ) + %end; + %let syscc=1008; +%end; + + %mend; /** @@ -3084,6 +3118,7 @@ run;

SAS Macros

@li mp_abort.sas + @li mf_nobs.sas @version 9.3 @author Allan Bowe @@ -3099,18 +3134,27 @@ run; filename &outref temp; -data _null_; - file &outref lrecl=32800; - set &inds end=last; - by SUBGROUP_ID; - if _n_=1 then put '('; - else if first.SUBGROUP_ID then put +1 GROUP_LOGIC '('; - else put +2 SUBGROUP_LOGIC; +%if %mf_nobs(&inds)=0 %then %do; + /* ensure we have a default filter */ + data _null_; + file &outref; + put '1=1'; + run; +%end; +%else %do; + data _null_; + file &outref lrecl=32800; + set &inds end=last; + by SUBGROUP_ID; + if _n_=1 then put '('; + else if first.SUBGROUP_ID then put +1 GROUP_LOGIC '('; + else put +2 SUBGROUP_LOGIC; - put +4 VARIABLE_NM OPERATOR_NM RAW_VALUE; + put +4 VARIABLE_NM OPERATOR_NM RAW_VALUE; - if last.SUBGROUP_ID then put ')'@; -run; + if last.SUBGROUP_ID then put ')'@; + run; +%end; %mend; /** diff --git a/base/mp_assertdsobs.sas b/base/mp_assertdsobs.sas index 7726a5a..1769ffe 100644 --- a/base/mp_assertdsobs.sas +++ b/base/mp_assertdsobs.sas @@ -10,13 +10,15 @@

SAS Macros

@li mf_nobs.sas + @li mp_abort.sas @param [in] inds input dataset to test for presence of observations @param [in] desc= (Testing observations) The user provided test description @param [in] test= (HASOBS) The test to apply. Valid values are: - @li HASOBS Test is a PASS if the input dataset has any observations - @li EMPTY Test is a PASS if input dataset is empty + @li HASOBS - Test is a PASS if the input dataset has any observations + @li EMPTY - Test is a PASS if input dataset is empty + @li EQUALS [integer] - Test passes if obs count matches the provided integer @param [out] outds= (work.test_results) The output dataset to contain the results. If it does not exist, it will be created, with the following format: |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256| @@ -39,6 +41,21 @@ %let nobs=%mf_nobs(&inds); %let test=%upcase(&test); + %if %substr(&test.xxxxx,1,6)=EQUALS %then %do; + %let val=%scan(&test,2,%str( )); + %mp_abort(iftrue= (%DATATYP(&val)=CHAR) + ,mac=&sysmacroname + ,msg=%str(Invalid test - &test, expected EQUALS [integer]) + ) + %let test=EQUALS; + %end; + %else %if &test ne HASOBS and &test ne EMPTY %then %do; + %mp_abort( + mac=&sysmacroname, + msg=%str(Invalid test - &test) + ) + %end; + data; length test_description $256 test_result $4 test_comments $256; test_description=symget('desc'); @@ -50,6 +67,9 @@ %else %if &test=EMPTY %then %do; if &nobs=0 then test_result='PASS'; %end; + %else %if &test=EQUALS %then %do; + if &nobs=&val then test_result='PASS'; + %end; %else %do; test_comments="&sysmacroname: Unsatisfied test condition - &test"; %end; diff --git a/base/mp_filtercheck.sas b/base/mp_filtercheck.sas index 49ab1dc..79343ac 100644 --- a/base/mp_filtercheck.sas +++ b/base/mp_filtercheck.sas @@ -3,7 +3,8 @@ @brief Checks an input filter table for validity @details Performs checks on the input table to ensure it arrives in the correct format. This is necessary to prevent code injection. Will update - SYSCC to 1008 if bad records are found. + SYSCC to 1008 if bad records are found, and call mp_abort.sas for a + graceful service exit (configurable). Used for dynamic filtering in [Data Controller for SAS®](https://datacontroller.io). @@ -33,6 +34,7 @@ @param [in] inds The table to be checked, with the format above @param [in] targetds= The target dataset against which to verify VARIABLE_NM + @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions @param [out] outds= The output table, which is a copy of the &inds. table plus a REASON_CD column, containing only bad records. If bad records found, the SYSCC value will be set to 1008 (general data problem). Downstream @@ -52,7 +54,7 @@ @todo Support date / hex / name literals and exponents in RAW_VALUE field **/ -%macro mp_filtercheck(inds,targetds=,outds=work.badrecords); +%macro mp_filtercheck(inds,targetds=,outds=work.badrecords,abort=YES); %mp_abort(iftrue= (&syscc ne 0) ,mac=&sysmacroname @@ -93,7 +95,7 @@ data &outds; output; end; if OPERATOR_NM not in - ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NOT EQUAL','CONTAINS') + ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS') then do; REASON_CD='Invalid OPERATOR_NM'; putlog REASON_CD= OPERATOR_NM=; @@ -122,11 +124,8 @@ data &outds; regex = prxparse("s/(\').*?(\')//"); call prxchange(regex,-1,raw_value2); - /* remove commas */ - raw_value3=compress(raw_value2,','); - - - + /* remove commas and periods*/ + raw_value3=compress(raw_value2,',.'); /* output records that contain values other than digits and spaces */ if notdigit(compress(raw_value3,' '))>0 then do; @@ -138,6 +137,21 @@ data &outds; run; -%if %mf_nobs(&outds)>0 %then %let syscc=1008; +%if %mf_nobs(&outds)>0 %then %do; + %if &abort=YES %then %do; + data _null_; + set &outds; + call symputx('REASON_CD',reason_cd,'l'); + stop; + run; + %mp_abort( + mac=&sysmacroname, + msg=%str(Filter issues in &inds, first was &reason_cd, details in &outds) + ) + %end; + %let syscc=1008; +%end; + + %mend; diff --git a/base/mp_filtergenerate.sas b/base/mp_filtergenerate.sas index ac7c202..d6ef9f7 100644 --- a/base/mp_filtergenerate.sas +++ b/base/mp_filtergenerate.sas @@ -59,6 +59,7 @@

SAS Macros

@li mp_abort.sas + @li mf_nobs.sas @version 9.3 @author Allan Bowe @@ -74,17 +75,26 @@ filename &outref temp; -data _null_; - file &outref lrecl=32800; - set &inds end=last; - by SUBGROUP_ID; - if _n_=1 then put '('; - else if first.SUBGROUP_ID then put +1 GROUP_LOGIC '('; - else put +2 SUBGROUP_LOGIC; +%if %mf_nobs(&inds)=0 %then %do; + /* ensure we have a default filter */ + data _null_; + file &outref; + put '1=1'; + run; +%end; +%else %do; + data _null_; + file &outref lrecl=32800; + set &inds end=last; + by SUBGROUP_ID; + if _n_=1 then put '('; + else if first.SUBGROUP_ID then put +1 GROUP_LOGIC '('; + else put +2 SUBGROUP_LOGIC; - put +4 VARIABLE_NM OPERATOR_NM RAW_VALUE; + put +4 VARIABLE_NM OPERATOR_NM RAW_VALUE; - if last.SUBGROUP_ID then put ')'@; -run; + if last.SUBGROUP_ID then put ')'@; + run; +%end; %mend; diff --git a/tests/base/mp_filtercheck.test.sas b/tests/base/mp_filtercheck.test.sas index 6c957ff..5cce1b4 100644 --- a/tests/base/mp_filtercheck.test.sas +++ b/tests/base/mp_filtercheck.test.sas @@ -18,13 +18,15 @@ datalines4; AND,AND,1,AGE,=,12 AND,AND,1,SEX,<=,"'M'" AND,OR,2,Name,NOT IN,"('Jane','Alfred')" -AND,OR,2,Weight,>=,7 +AND,OR,2,Weight,>=,77.7 +AND,OR,2,Weight,NE,77.7 ;;;; run; %mp_filtercheck(work.inds, targetds=sashelp.class, - outds=work.badrecords + outds=work.badrecords, + abort=NO ) %let syscc=0; %mp_assertdsobs(work.badrecords, @@ -45,10 +47,10 @@ AND,OR,2,Name,NOT IN,"('Jane','Alfred')" AND,OR,2,Weight,>=,7 ;;;; run; - %mp_filtercheck(work.inds, targetds=sashelp.class, - outds=work.badrecords + outds=work.badrecords, + abort=NO ) %let syscc=0; %mp_assertdsobs(work.badrecords, @@ -69,7 +71,8 @@ run; %mp_filtercheck(work.inds, targetds=sashelp.class, - outds=work.badrecords + outds=work.badrecords, + abort=NO ) %let syscc=0; %mp_assertdsobs(work.badrecords, @@ -91,7 +94,8 @@ run; %mp_filtercheck(work.inds, targetds=sashelp.class, - outds=work.badrecords + outds=work.badrecords, + abort=NO ) %let syscc=0; %mp_assertdsobs(work.badrecords, @@ -109,10 +113,10 @@ datalines4; AND,AND,1,age,=,;;%abort ;;;; run; - %mp_filtercheck(work.inds, targetds=sashelp.class, - outds=work.badrecords + outds=work.badrecords, + abort=NO ) %let syscc=0; %mp_assertdsobs(work.badrecords, diff --git a/tests/base/mp_filtergenerate.test.sas b/tests/base/mp_filtergenerate.test.sas new file mode 100644 index 0000000..be9e1c3 --- /dev/null +++ b/tests/base/mp_filtergenerate.test.sas @@ -0,0 +1,126 @@ +/** + @file + @brief Testing mp_filtergenerate macro + +

SAS Macros

+ @li mp_filtergenerate.sas + @li mp_filtercheck.sas + @li mp_assertdsobs.sas + +**/ + +options source2; + +/* valid filter */ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$32767.; +datalines4; +AND,AND,1,AGE,>,5 +AND,AND,1,SEX,NE,"'M'" +AND,OR,2,Name,NOT IN,"('Jane','Janet')" +AND,OR,2,Weight,>=,84.6 +;;;; +run; +%mp_filtercheck(work.inds,targetds=sashelp.class) +%mp_filtergenerate(work.inds,outref=myfilter) +data work.test; + set sashelp.class; + where %inc myfilter;; +run; +%mp_assertdsobs(work.test, + desc=Valid filter, + test=EQUALS 8, + outds=work.test_results +) + +/* empty filter (return all records) */ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$32767.; +datalines4; +;;;; +run; +%mp_filtercheck(work.inds,targetds=sashelp.class) +%mp_filtergenerate(work.inds,outref=myfilter) +data work.test; + set sashelp.class; + where %inc myfilter;; +run; +%mp_assertdsobs(work.test, + desc=Empty filter (return all records) , + test=EQUALS 19, + outds=work.test_results +) + +/* single line filter */ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$32767.; +datalines4; +AND,OR,2,Name,IN,"('Jane','Janet')" +;;;; +run; +%mp_filtercheck(work.inds,targetds=sashelp.class) +%mp_filtergenerate(work.inds,outref=myfilter) +data work.test; + set sashelp.class; + where %inc myfilter;; +run; +%mp_assertdsobs(work.test, + desc=Single line filter , + test=EQUALS 2, + outds=work.test_results +) + +/* single line 2 group filter */ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$32767.; +datalines4; +OR,OR,2,Name,IN,"('Jane','Janet')" +OR,OR,3,Name,IN,"('James')" +;;;; +run; +%mp_filtercheck(work.inds,targetds=sashelp.class) +%mp_filtergenerate(work.inds,outref=myfilter) +data work.test; + set sashelp.class; + where %inc myfilter;; +run; +%mp_assertdsobs(work.test, + desc=Single line 2 group filter , + test=EQUALS 3, + outds=work.test_results +) + +/* filter with nothing returned */ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$32767.; +datalines4; +AND,OR,2,Name,IN,"('Jane','Janet')" +AND,OR,3,Name,IN,"('James')" +;;;; +run; +%mp_filtercheck(work.inds,targetds=sashelp.class) +%mp_filtergenerate(work.inds,outref=myfilter) +data work.test; + set sashelp.class; + where %inc myfilter;; +run; +%mp_assertdsobs(work.test, + desc=Filter with nothing returned, + test=EQUALS 0, + outds=work.test_results +) + + +%webout(OPEN) +%webout(OBJ, TEST_RESULTS) +%webout(CLOSE) \ No newline at end of file