sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2025-12-10 22:14:35 +00:00
Code Issues Projects Releases Wiki Activity

fix: adding tests for mp_filtergenerate, also updating the corresponding macros following test results

Browse Source
This commit is contained in:
Allan Bowe
2021-05-02 22:11:44 +03:00
parent 7d7608f06c
commit 369c4412f3
6 changed files with 268 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
all.sas
View File

@@ -1793,13 +1793,15 @@ Usage:
<h4> SAS Macros </h4>
@li mf_nobs.sas
@li mp_abort.sas
@param [in] inds input dataset to test for presence of observations
@param [in] desc= (Testing observations) The user provided test description
@param [in] test= (HASOBS) The test to apply. Valid values are:
@li HASOBS Test is a PASS if the input dataset has any observations
@li EMPTY Test is a PASS if input dataset is empty
@li HASOBS - Test is a PASS if the input dataset has any observations
@li EMPTY - Test is a PASS if input dataset is empty
@li EQUALS [integer] - Test passes if obs count matches the provided integer
@param [out] outds= (work.test_results) The output dataset to contain the
results. If it does not exist, it will be created, with the following format:
|TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
@@ -1822,6 +1824,21 @@ Usage:
%let nobs=%mf_nobs(&inds);
%let test=%upcase(&test);
%if %substr(&test.xxxxx,1,6)=EQUALS %then %do;
%let val=%scan(&test,2,%str( ));
%mp_abort(iftrue= (%DATATYP(&val)=CHAR)
,mac=&sysmacroname
,msg=%str(Invalid test - &test, expected EQUALS [integer])
)
%let test=EQUALS;
%end;
%else %if &test ne HASOBS and &test ne EMPTY %then %do;
%mp_abort(
mac=&sysmacroname,
msg=%str(Invalid test - &test)
)
%end;
data;
length test_description $256 test_result $4 test_comments $256;
test_description=symget('desc');
@@ -1833,6 +1850,9 @@ Usage:
%else %if &test=EMPTY %then %do;
if &nobs=0 then test_result='PASS';
%end;
%else %if &test=EQUALS %then %do;
if &nobs=&val then test_result='PASS';
%end;
%else %do;
test_comments="&sysmacroname: Unsatisfied test condition - &test";
%end;
@@ -2885,7 +2905,8 @@ run;
@brief Checks an input filter table for validity
@details Performs checks on the input table to ensure it arrives in the
correct format. This is necessary to prevent code injection. Will update
SYSCC to 1008 if bad records are found.
SYSCC to 1008 if bad records are found, and call mp_abort.sas for a
graceful service exit (configurable).
Used for dynamic filtering in [Data Controller for SAS&reg;](https://datacontroller.io).
@@ -2915,6 +2936,7 @@ run;
@param [in] inds The table to be checked, with the format above
@param [in] targetds= The target dataset against which to verify VARIABLE_NM
@param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
@param [out] outds= The output table, which is a copy of the &inds. table
plus a REASON_CD column, containing only bad records. If bad records found,
the SYSCC value will be set to 1008 (general data problem). Downstream
@@ -2934,7 +2956,7 @@ run;
@todo Support date / hex / name literals and exponents in RAW_VALUE field
**/
%macro mp_filtercheck(inds,targetds=,outds=work.badrecords);
%macro mp_filtercheck(inds,targetds=,outds=work.badrecords,abort=YES);
%mp_abort(iftrue= (&syscc ne 0)
,mac=&sysmacroname
@@ -2975,7 +2997,7 @@ data &outds;
output;
end;
if OPERATOR_NM not in
('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NOT EQUAL','CONTAINS')
('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
then do;
REASON_CD='Invalid OPERATOR_NM';
putlog REASON_CD= OPERATOR_NM=;
@@ -3004,11 +3026,8 @@ data &outds;
regex = prxparse("s/(\').*?(\')//");
call prxchange(regex,-1,raw_value2);
/* remove commas */
raw_value3=compress(raw_value2,',');
/* remove commas and periods*/
raw_value3=compress(raw_value2,',.');
/* output records that contain values other than digits and spaces */
if notdigit(compress(raw_value3,' '))>0 then do;
@@ -3020,7 +3039,22 @@ data &outds;
run;
%if %mf_nobs(&outds)>0 %then %let syscc=1008;
%if %mf_nobs(&outds)>0 %then %do;
%if &abort=YES %then %do;
data _null_;
set &outds;
call symputx('REASON_CD',reason_cd,'l');
stop;
run;
%mp_abort(
mac=&sysmacroname,
msg=%str(Filter issues in &inds, first was &reason_cd, details in &outds)
)
%end;
%let syscc=1008;
%end;
%mend;
/**
@@ -3084,6 +3118,7 @@ run;
<h4> SAS Macros </h4>
@li mp_abort.sas
@li mf_nobs.sas
@version 9.3
@author Allan Bowe
@@ -3099,18 +3134,27 @@ run;
filename &outref temp;
data _null_;
file &outref lrecl=32800;
set &inds end=last;
by SUBGROUP_ID;
if _n_=1 then put '(';
else if first.SUBGROUP_ID then put +1 GROUP_LOGIC '(';
else put +2 SUBGROUP_LOGIC;
%if %mf_nobs(&inds)=0 %then %do;
/* ensure we have a default filter */
data _null_;
file &outref;
put '1=1';
run;
%end;
%else %do;
data _null_;
file &outref lrecl=32800;
set &inds end=last;
by SUBGROUP_ID;
if _n_=1 then put '(';
else if first.SUBGROUP_ID then put +1 GROUP_LOGIC '(';
else put +2 SUBGROUP_LOGIC;
put +4 VARIABLE_NM OPERATOR_NM RAW_VALUE;
put +4 VARIABLE_NM OPERATOR_NM RAW_VALUE;
if last.SUBGROUP_ID then put ')'@;
run;
if last.SUBGROUP_ID then put ')'@;
run;
%end;
%mend;
/**

24
base/mp_assertdsobs.sas
View File

@@ -10,13 +10,15 @@
<h4> SAS Macros </h4>
@li mf_nobs.sas
@li mp_abort.sas
@param [in] inds input dataset to test for presence of observations
@param [in] desc= (Testing observations) The user provided test description
@param [in] test= (HASOBS) The test to apply. Valid values are:
@li HASOBS Test is a PASS if the input dataset has any observations
@li EMPTY Test is a PASS if input dataset is empty
@li HASOBS - Test is a PASS if the input dataset has any observations
@li EMPTY - Test is a PASS if input dataset is empty
@li EQUALS [integer] - Test passes if obs count matches the provided integer
@param [out] outds= (work.test_results) The output dataset to contain the
results. If it does not exist, it will be created, with the following format:
|TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
@@ -39,6 +41,21 @@
%let nobs=%mf_nobs(&inds);
%let test=%upcase(&test);
%if %substr(&test.xxxxx,1,6)=EQUALS %then %do;
%let val=%scan(&test,2,%str( ));
%mp_abort(iftrue= (%DATATYP(&val)=CHAR)
,mac=&sysmacroname
,msg=%str(Invalid test - &test, expected EQUALS [integer])
)
%let test=EQUALS;
%end;
%else %if &test ne HASOBS and &test ne EMPTY %then %do;
%mp_abort(
mac=&sysmacroname,
msg=%str(Invalid test - &test)
)
%end;
data;
length test_description $256 test_result $4 test_comments $256;
test_description=symget('desc');
@@ -50,6 +67,9 @@
%else %if &test=EMPTY %then %do;
if &nobs=0 then test_result='PASS';
%end;
%else %if &test=EQUALS %then %do;
if &nobs=&val then test_result='PASS';
%end;
%else %do;
test_comments="&sysmacroname: Unsatisfied test condition - &test";
%end;

32
base/mp_filtercheck.sas
View File

@@ -3,7 +3,8 @@
@brief Checks an input filter table for validity
@details Performs checks on the input table to ensure it arrives in the
correct format. This is necessary to prevent code injection. Will update
SYSCC to 1008 if bad records are found.
SYSCC to 1008 if bad records are found, and call mp_abort.sas for a
graceful service exit (configurable).
Used for dynamic filtering in [Data Controller for SAS&reg;](https://datacontroller.io).
@@ -33,6 +34,7 @@
@param [in] inds The table to be checked, with the format above
@param [in] targetds= The target dataset against which to verify VARIABLE_NM
@param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
@param [out] outds= The output table, which is a copy of the &inds. table
plus a REASON_CD column, containing only bad records. If bad records found,
the SYSCC value will be set to 1008 (general data problem). Downstream
@@ -52,7 +54,7 @@
@todo Support date / hex / name literals and exponents in RAW_VALUE field
**/
%macro mp_filtercheck(inds,targetds=,outds=work.badrecords);
%macro mp_filtercheck(inds,targetds=,outds=work.badrecords,abort=YES);
%mp_abort(iftrue= (&syscc ne 0)
,mac=&sysmacroname
@@ -93,7 +95,7 @@ data &outds;
output;
end;
if OPERATOR_NM not in
('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NOT EQUAL','CONTAINS')
('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
then do;
REASON_CD='Invalid OPERATOR_NM';
putlog REASON_CD= OPERATOR_NM=;
@@ -122,11 +124,8 @@ data &outds;
regex = prxparse("s/(\').*?(\')//");
call prxchange(regex,-1,raw_value2);
/* remove commas */
raw_value3=compress(raw_value2,',');
/* remove commas and periods*/
raw_value3=compress(raw_value2,',.');
/* output records that contain values other than digits and spaces */
if notdigit(compress(raw_value3,' '))>0 then do;
@@ -138,6 +137,21 @@ data &outds;
run;
%if %mf_nobs(&outds)>0 %then %let syscc=1008;
%if %mf_nobs(&outds)>0 %then %do;
%if &abort=YES %then %do;
data _null_;
set &outds;
call symputx('REASON_CD',reason_cd,'l');
stop;
run;
%mp_abort(
mac=&sysmacroname,
msg=%str(Filter issues in &inds, first was &reason_cd, details in &outds)
)
%end;
%let syscc=1008;
%end;
%mend;

30
base/mp_filtergenerate.sas
View File

@@ -59,6 +59,7 @@
<h4> SAS Macros </h4>
@li mp_abort.sas
@li mf_nobs.sas
@version 9.3
@author Allan Bowe
@@ -74,17 +75,26 @@
filename &outref temp;
data _null_;
file &outref lrecl=32800;
set &inds end=last;
by SUBGROUP_ID;
if _n_=1 then put '(';
else if first.SUBGROUP_ID then put +1 GROUP_LOGIC '(';
else put +2 SUBGROUP_LOGIC;
%if %mf_nobs(&inds)=0 %then %do;
/* ensure we have a default filter */
data _null_;
file &outref;
put '1=1';
run;
%end;
%else %do;
data _null_;
file &outref lrecl=32800;
set &inds end=last;
by SUBGROUP_ID;
if _n_=1 then put '(';
else if first.SUBGROUP_ID then put +1 GROUP_LOGIC '(';
else put +2 SUBGROUP_LOGIC;
put +4 VARIABLE_NM OPERATOR_NM RAW_VALUE;
put +4 VARIABLE_NM OPERATOR_NM RAW_VALUE;
if last.SUBGROUP_ID then put ')'@;
run;
if last.SUBGROUP_ID then put ')'@;
run;
%end;
%mend;

20
tests/base/mp_filtercheck.test.sas
View File

@@ -18,13 +18,15 @@ datalines4;
AND,AND,1,AGE,=,12
AND,AND,1,SEX,<=,"'M'"
AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
AND,OR,2,Weight,>=,7
AND,OR,2,Weight,>=,77.7
AND,OR,2,Weight,NE,77.7
;;;;
run;
%mp_filtercheck(work.inds,
targetds=sashelp.class,
outds=work.badrecords
outds=work.badrecords,
abort=NO
)
%let syscc=0;
%mp_assertdsobs(work.badrecords,
@@ -45,10 +47,10 @@ AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
AND,OR,2,Weight,>=,7
;;;;
run;
%mp_filtercheck(work.inds,
targetds=sashelp.class,
outds=work.badrecords
outds=work.badrecords,
abort=NO
)
%let syscc=0;
%mp_assertdsobs(work.badrecords,
@@ -69,7 +71,8 @@ run;
%mp_filtercheck(work.inds,
targetds=sashelp.class,
outds=work.badrecords
outds=work.badrecords,
abort=NO
)
%let syscc=0;
%mp_assertdsobs(work.badrecords,
@@ -91,7 +94,8 @@ run;
%mp_filtercheck(work.inds,
targetds=sashelp.class,
outds=work.badrecords
outds=work.badrecords,
abort=NO
)
%let syscc=0;
%mp_assertdsobs(work.badrecords,
@@ -109,10 +113,10 @@ datalines4;
AND,AND,1,age,=,;;%abort
;;;;
run;
%mp_filtercheck(work.inds,
targetds=sashelp.class,
outds=work.badrecords
outds=work.badrecords,
abort=NO
)
%let syscc=0;
%mp_assertdsobs(work.badrecords,

126
tests/base/mp_filtergenerate.test.sas Normal file
View File

@@ -0,0 +1,126 @@
/**
@file
@brief Testing mp_filtergenerate macro
<h4> SAS Macros </h4>
@li mp_filtergenerate.sas
@li mp_filtercheck.sas
@li mp_assertdsobs.sas
**/
options source2;
/* valid filter */
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$32767.;
datalines4;
AND,AND,1,AGE,>,5
AND,AND,1,SEX,NE,"'M'"
AND,OR,2,Name,NOT IN,"('Jane','Janet')"
AND,OR,2,Weight,>=,84.6
;;;;
run;
%mp_filtercheck(work.inds,targetds=sashelp.class)
%mp_filtergenerate(work.inds,outref=myfilter)
data work.test;
set sashelp.class;
where %inc myfilter;;
run;
%mp_assertdsobs(work.test,
desc=Valid filter,
test=EQUALS 8,
outds=work.test_results
)
/* empty filter (return all records) */
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$32767.;
datalines4;
;;;;
run;
%mp_filtercheck(work.inds,targetds=sashelp.class)
%mp_filtergenerate(work.inds,outref=myfilter)
data work.test;
set sashelp.class;
where %inc myfilter;;
run;
%mp_assertdsobs(work.test,
desc=Empty filter (return all records) ,
test=EQUALS 19,
outds=work.test_results
)
/* single line filter */
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$32767.;
datalines4;
AND,OR,2,Name,IN,"('Jane','Janet')"
;;;;
run;
%mp_filtercheck(work.inds,targetds=sashelp.class)
%mp_filtergenerate(work.inds,outref=myfilter)
data work.test;
set sashelp.class;
where %inc myfilter;;
run;
%mp_assertdsobs(work.test,
desc=Single line filter ,
test=EQUALS 2,
outds=work.test_results
)
/* single line 2 group filter */
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$32767.;
datalines4;
OR,OR,2,Name,IN,"('Jane','Janet')"
OR,OR,3,Name,IN,"('James')"
;;;;
run;
%mp_filtercheck(work.inds,targetds=sashelp.class)
%mp_filtergenerate(work.inds,outref=myfilter)
data work.test;
set sashelp.class;
where %inc myfilter;;
run;
%mp_assertdsobs(work.test,
desc=Single line 2 group filter ,
test=EQUALS 3,
outds=work.test_results
)
/* filter with nothing returned */
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$32767.;
datalines4;
AND,OR,2,Name,IN,"('Jane','Janet')"
AND,OR,3,Name,IN,"('James')"
;;;;
run;
%mp_filtercheck(work.inds,targetds=sashelp.class)
%mp_filtergenerate(work.inds,outref=myfilter)
data work.test;
set sashelp.class;
where %inc myfilter;;
run;
%mp_assertdsobs(work.test,
desc=Filter with nothing returned,
test=EQUALS 0,
outds=work.test_results
)
%webout(OPEN)
%webout(OBJ, TEST_RESULTS)
%webout(CLOSE)