sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2025-12-10 22:14:35 +00:00
Code Issues Projects Releases Wiki Activity

feat: mp_filtervalidate.sas - to run a proc sql validate against the target table to ensure validity

Browse Source
This commit is contained in:
Allan Bowe
2021-05-03 13:48:24 +03:00
parent ff1eb54cc3
commit 7f2ad5fc66
6 changed files with 317 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
all.sas
View File

@@ -3077,7 +3077,7 @@ run;
@li SUBGROUP_LOGIC - only AND/OR
@li SUBGROUP_ID - only integers
@li VARIABLE_NM - must be in the target table
@li OPERATOR_NM - only =/>/</<=/>=/BETWEEN/IN/NOT IN/NOT EQUAL/CONTAINS
@li OPERATOR_NM - only =/>/</<=/>=/BETWEEN/IN/NOT IN/NE/CONTAINS
@li RAW_VALUE - no unquoted values except integers, commas and spaces.
@returns The &outds table containing any bad rows, plus a REASON_CD column.
@@ -3092,11 +3092,15 @@ run;
<h4> SAS Macros </h4>
@li mp_abort.sas
@li mf_getuniquefileref.sas
@li mf_getvarlist.sas
@li mf_nobs.sas
@li mp_filtergenerate.sas
@li mp_filtervalidate.sas
<h4> Related Macros </h4>
@li mp_filtergenerate.sas
@li mp_filtervalidate.sas
@version 9.3
@author Allan Bowe
@@ -3200,9 +3204,19 @@ run;
)
%end;
%let syscc=1008;
%return;
%end;
/**
* syntax checking passed but it does not mean the filter is valid
* for that we can run a proc sql validate query
*/
%local fref1;
%let fref1=%mf_getuniquefileref();
%mp_filtergenerate(&inds,outref=&fref1)
/* this macro will also set syscc to 1008 if any issues found */
%mp_filtervalidate(&fref1,&targetds,outds=&outds,abort=&abort)
%mend;
/**
@@ -3228,7 +3242,7 @@ run;
data work.filtertable;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$32767.;
OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,AGE,=,12
AND,AND,1,SEX,<=,"'M'"
@@ -3263,6 +3277,7 @@ run;
<h4> Related Macros </h4>
@li mp_filtercheck.sas
@li mp_filtervalidate.sas
<h4> SAS Macros </h4>
@li mp_abort.sas
@@ -3304,6 +3319,110 @@ filename &outref temp;
run;
%end;
%mend;
/**
@file
@brief Checks a generated filter query for validity
@details Runs a generated filter in proc sql with the validate option.
Used in mp_filtercheck.sas in an fcmp container.
Built to support dynamic filtering in
[Data Controller for SAS&reg;](https://datacontroller.io).
Usage:
data work.filtertable;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,AGE,=,12
AND,AND,1,SEX,<=,"'M'"
AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
AND,OR,2,Weight,>=,7
;;;;
run;
%mp_filtergenerate(work.filtertable,outref=myfilter)
%mp_filtervalidate(myfilter,sashelp.class)
@returns The SYSCC value will be 1008 if there are validation issues.
@param [in] inref The input fileref to validate (generated by
mp_filtergenerate.sas)
@param [in] targetds The target dataset against which to verify the query
@param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
@param [out] outds= (work.mp_filtervalidate) Output dataset containing the
error / warning message, if one exists. If this table contains any rows,
there are problems!
<h4> SAS Macros </h4>
@li mf_getuniquefileref.sas
@li mf_nobs.sas
@li mp_abort.sas
<h4> Related Macros </h4>
@li mp_filtercheck.sas
@li mp_filtergenerate.sas
@version 9.3
@author Allan Bowe
**/
%macro mp_filtervalidate(inref,targetds,abort=YES,outds=work.mp_filtervalidate);
%mp_abort(iftrue= (&syscc ne 0 or &syserr ne 0)
,mac=&sysmacroname
,msg=%str(syscc=&syscc / syserr=&syserr - on macro entry)
)
%local fref1;
%let fref1=%mf_getuniquefileref();
data _null_;
file &fref1;
infile &inref end=eof;
if _n_=1 then do;
put "proc sql;";
put "validate select * from &targetds";
put "where " ;
end;
input;
put _infile_;
putlog _infile_;
if eof then put ";quit;";
run;
%inc &fref1;
data &outds;
if &sqlrc or &syscc or &syserr then do;
REASON_CD=coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
output;
end;
else stop;
run;
filename &fref1 clear;
%if %mf_nobs(&outds)>0 %then %do;
%if &abort=YES %then %do;
data _null_;
set &outds;
call symputx('REASON_CD',reason_cd,'l');
stop;
run;
%mp_abort(
mac=&sysmacroname,
msg=%str(Filter issues in &inref: %quote(&reason_cd))
)
%end;
%let syscc=1008;
%end;
%mend;
/**
@file mp_getconstraints.sas

16
base/mp_filtercheck.sas
View File

@@ -27,7 +27,7 @@
@li SUBGROUP_LOGIC - only AND/OR
@li SUBGROUP_ID - only integers
@li VARIABLE_NM - must be in the target table
@li OPERATOR_NM - only =/>/</<=/>=/BETWEEN/IN/NOT IN/NOT EQUAL/CONTAINS
@li OPERATOR_NM - only =/>/</<=/>=/BETWEEN/IN/NOT IN/NE/CONTAINS
@li RAW_VALUE - no unquoted values except integers, commas and spaces.
@returns The &outds table containing any bad rows, plus a REASON_CD column.
@@ -42,11 +42,15 @@
<h4> SAS Macros </h4>
@li mp_abort.sas
@li mf_getuniquefileref.sas
@li mf_getvarlist.sas
@li mf_nobs.sas
@li mp_filtergenerate.sas
@li mp_filtervalidate.sas
<h4> Related Macros </h4>
@li mp_filtergenerate.sas
@li mp_filtervalidate.sas
@version 9.3
@author Allan Bowe
@@ -150,8 +154,18 @@ run;
)
%end;
%let syscc=1008;
%return;
%end;
/**
* syntax checking passed but it does not mean the filter is valid
* for that we can run a proc sql validate query
*/
%local fref1;
%let fref1=%mf_getuniquefileref();
%mp_filtergenerate(&inds,outref=&fref1)
/* this macro will also set syscc to 1008 if any issues found */
%mp_filtervalidate(&fref1,&targetds,outds=&outds,abort=&abort)
%mend;

3
base/mp_filtergenerate.sas
View File

@@ -21,7 +21,7 @@
data work.filtertable;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$32767.;
OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,AGE,=,12
AND,AND,1,SEX,<=,"'M'"
@@ -56,6 +56,7 @@
<h4> Related Macros </h4>
@li mp_filtercheck.sas
@li mp_filtervalidate.sas
<h4> SAS Macros </h4>
@li mp_abort.sas

104
base/mp_filtervalidate.sas Normal file
View File

@@ -0,0 +1,104 @@
/**
@file
@brief Checks a generated filter query for validity
@details Runs a generated filter in proc sql with the validate option.
Used in mp_filtercheck.sas in an fcmp container.
Built to support dynamic filtering in
[Data Controller for SAS&reg;](https://datacontroller.io).
Usage:
data work.filtertable;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,AGE,=,12
AND,AND,1,SEX,<=,"'M'"
AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
AND,OR,2,Weight,>=,7
;;;;
run;
%mp_filtergenerate(work.filtertable,outref=myfilter)
%mp_filtervalidate(myfilter,sashelp.class)
@returns The SYSCC value will be 1008 if there are validation issues.
@param [in] inref The input fileref to validate (generated by
mp_filtergenerate.sas)
@param [in] targetds The target dataset against which to verify the query
@param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
@param [out] outds= (work.mp_filtervalidate) Output dataset containing the
error / warning message, if one exists. If this table contains any rows,
there are problems!
<h4> SAS Macros </h4>
@li mf_getuniquefileref.sas
@li mf_nobs.sas
@li mp_abort.sas
<h4> Related Macros </h4>
@li mp_filtercheck.sas
@li mp_filtergenerate.sas
@version 9.3
@author Allan Bowe
**/
%macro mp_filtervalidate(inref,targetds,abort=YES,outds=work.mp_filtervalidate);
%mp_abort(iftrue= (&syscc ne 0 or &syserr ne 0)
,mac=&sysmacroname
,msg=%str(syscc=&syscc / syserr=&syserr - on macro entry)
)
%local fref1;
%let fref1=%mf_getuniquefileref();
data _null_;
file &fref1;
infile &inref end=eof;
if _n_=1 then do;
put "proc sql;";
put "validate select * from &targetds";
put "where " ;
end;
input;
put _infile_;
putlog _infile_;
if eof then put ";quit;";
run;
%inc &fref1;
data &outds;
if &sqlrc or &syscc or &syserr then do;
REASON_CD=coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
output;
end;
else stop;
run;
filename &fref1 clear;
%if %mf_nobs(&outds)>0 %then %do;
%if &abort=YES %then %do;
data _null_;
set &outds;
call symputx('REASON_CD',reason_cd,'l');
stop;
run;
%mp_abort(
mac=&sysmacroname,
msg=%str(Filter issues in &inref: %quote(&reason_cd))
)
%end;
%let syscc=1008;
%end;
%mend;

6
tests/base/mp_filtercheck.test.sas
View File

@@ -63,7 +63,7 @@ run;
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$32767.;
OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,OR,2,Name,NOT IN,"(''''Jane','Alfred')"
;;;;
@@ -85,7 +85,7 @@ run;
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$32767.;
OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,%abort,=,12
AND,OR,2,Weight,>=,7
@@ -108,7 +108,7 @@ run;
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$32767.;
OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,age,=,;;%abort
;;;;

72
tests/base/mp_filtervalidate.test.sas Normal file
View File

@@ -0,0 +1,72 @@
/**
@file
@brief Testing mp_filtervalidate macro
<h4> SAS Macros </h4>
@li mp_filtergenerate.sas
@li mp_filtervalidate.sas
@li mp_assertdsobs.sas
**/
/* valid filter */
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,AGE,>,5
AND,AND,1,SEX,NE,"'M'"
AND,OR,2,Name,NOT IN,"('Jane','Janet')"
AND,OR,2,Weight,>=,84.6
;;;;
run;
%mp_filtergenerate(work.inds,outref=myfilter)
%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
%mp_assertdsobs(work.results,
desc=Valid filter,
test=EMPTY,
outds=work.test_results
)
/* empty filter (return all records) */
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
;;;;
run;
%mp_filtergenerate(work.inds,outref=myfilter)
%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
%mp_assertdsobs(work.results,
desc=Valid filter,
test=EMPTY,
outds=work.test_results
)
/* invalid filter*/
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,SEX,NE,2
;;;;
run;
%mp_filtergenerate(work.inds,outref=myfilter)
%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
%let syscc=0;
%mp_assertdsobs(work.results,
desc=Valid filter,
test=EQUALS 1,
outds=work.test_results
)
%webout(OPEN)
%webout(OBJ, TEST_RESULTS)
%webout(CLOSE)