fix: update to mp_filtercheck & tests for special missings

base/mp_filtercheck.sas

@@ -92,7 +92,26 @@ data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
-  length reason_cd $4032;
+  length reason_cd $4032 vtype $1 vnum dsid 8;
+
+  /* get column attributes */
+  if dsid=open("&inds","i")<1 then do;
+    REASON_CD="Unable to assign &inds";
+    putlog REASON_CD= dsid=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
+    output;
+  end;
+  vnum=varnum(dsid,VARIABLE_NM);
+  if vnum<1 then do;
+    REASON_CD=cats("Variable (",VARIABLE_NM,") not found in &inds");
+    putlog REASON_CD= dsid=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
+    output;
+  end;
+
+  vtype=vartype(dsid,vnum);
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
@@ -126,7 +145,7 @@ data &outds;
     output;
   end;
   if OPERATOR_NM not in
-  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE')
+  ('=','>','<','<=','>=','NE','GE','LE','BETWEEN','IN','NOT IN','CONTAINS')
   then do;
     REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
@@ -135,6 +154,18 @@ data &outds;
     output;
   end;
 
+  /* special missing logic */
+  if vtype='N'
+    and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE')
+    and cats(upcase(raw_value)) in (
+      '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N'
+      '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._'
+    )
+  then do;
+    /* valid numeric - exit data step loop */
+    return;
+  end;
+
   /* special logic */
   if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ','');
   else if OPERATOR_NM in ('IN','NOT IN') then do;

tests/crossplatform/mp_filtercheck.test.sas

@@ -6,11 +6,39 @@
   @li mp_filtercheck.sas
   @li mp_assertdsobs.sas
   @li mp_assert.sas
+  @li mp_assertscope.sas
 
 **/
 
+/* set up test data */
+data work.class ;
+length name $8 sex $1 age height weight 8;
+infile cards dsd;
+input Name:$char. Sex :$char. Age Height Weight;
+datalines4;
+Alfred,M,14,69,112.5
+Alice,F,13,56.5,84
+Barbara,F,13,65.3,98
+Carol,F,14,62.8,102.5
+Henry,M,14,63.5,102.5
+James,M,12,57.3,83
+Jane,F,12,59.8,84.5
+Janet,F,15,62.5,112.5
+Jeffrey,M,13,62.5,84
+John,M,12,59,99.5
+Joyce,F,11,51.3,50.5
+Judy,F,14,64.3,90
+Louise,F,12,56.3,77
+Mary,F,15,66.5,112
+Philip,M,16,72,150
+Robert,M,12,64.8,128
+Ronald,M,15,67,133
+Thomas,M,11,57.5,85
+William,M,15,66.5,112
+;;;;
+run;
 
-/* valid filter */
+/* valid filter conditions */
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
@@ -21,14 +49,19 @@ AND,AND,1,SEX,<=,"'M'"
 AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
 AND,OR,2,Weight,>=,77.7
 AND,OR,2,Weight,NE,77.7
+AND,AND,1,age,=,.A
+AND,AND,1,height,<,.B
 ;;;;
 run;
 
+%mp_assertscope(SNAPSHOT)
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )
+%mp_assertscope(COMPARE)
+
 %let syscc=0;
 %mp_assertdsobs(work.badrecords,
   desc=Valid filter query,
@@ -49,7 +82,7 @@ AND,OR,2,Weight,>=,7
 ;;;;
 run;
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )
@@ -71,7 +104,7 @@ AND,OR,2,Name,NOT IN,"(''''Jane','Alfred')"
 run;
 
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )
@@ -94,7 +127,7 @@ AND,OR,2,Weight,>=,7
 run;
 
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )
@@ -115,7 +148,7 @@ AND,AND,1,age,=,;;%abort
 ;;;;
 run;
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )
@@ -137,7 +170,7 @@ AND,AND,1,age,=,0
 run;
 %let syscc=0;
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )

tests/crossplatform/mp_filtergenerate.test.sas

@@ -11,6 +11,34 @@
 
 options source2;
 
+/* set up test data */
+data work.class ;
+length name $8 sex $1 age height weight 8;
+infile cards dsd;
+input Name:$char. Sex :$char. Age Height Weight;
+datalines4;
+Alfred,M,14,69,112.5
+Alice,F,13,56.5,84
+Barbara,F,13,65.3,98
+Carol,F,14,62.8,102.5
+Henry,M,14,63.5,102.5
+James,M,12,57.3,83
+Jane,F,12,59.8,84.5
+Janet,F,15,62.5,112.5
+Jeffrey,M,13,62.5,84
+John,M,12,59,99.5
+Joyce,F,11,51.3,50.5
+Judy,F,14,64.3,90
+Louise,F,12,56.3,77
+Mary,F,15,66.5,112
+Philip,M,16,72,150
+Robert,M,12,64.8,128
+Ronald,M,15,67,133
+Thomas,M,11,57.5,85
+William,M,15,66.5,112
+;;;;
+run;
+
 /* valid filter */
 data work.inds;
   infile datalines4 dsd;
@@ -23,10 +51,10 @@ AND,OR,2,Name,NOT IN,"('Jane','Janet')"
 AND,OR,2,Weight,>=,84.6
 ;;;;
 run;
-%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtercheck(work.inds,targetds=work.class)
 %mp_filtergenerate(work.inds,outref=myfilter)
 data work.test;
-  set sashelp.class;
+  set work.class;
   where %inc myfilter;;
 run;
 %mp_assertdsobs(work.test,
@@ -43,10 +71,10 @@ data work.inds;
 datalines4;
 ;;;;
 run;
-%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtercheck(work.inds,targetds=work.class)
 %mp_filtergenerate(work.inds,outref=myfilter)
 data work.test;
-  set sashelp.class;
+  set work.class;
   where %inc myfilter;;
 run;
 %mp_assertdsobs(work.test,
@@ -64,10 +92,10 @@ datalines4;
 AND,OR,2,Name,IN,"('Jane','Janet')"
 ;;;;
 run;
-%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtercheck(work.inds,targetds=work.class)
 %mp_filtergenerate(work.inds,outref=myfilter)
 data work.test;
-  set sashelp.class;
+  set work.class;
   where %inc myfilter;;
 run;
 %mp_assertdsobs(work.test,
@@ -86,10 +114,10 @@ OR,OR,2,Name,IN,"('Jane','Janet')"
 OR,OR,3,Name,IN,"('James')"
 ;;;;
 run;
-%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtercheck(work.inds,targetds=work.class)
 %mp_filtergenerate(work.inds,outref=myfilter)
 data work.test;
-  set sashelp.class;
+  set work.class;
   where %inc myfilter;;
 run;
 %mp_assertdsobs(work.test,
@@ -108,10 +136,10 @@ AND,OR,2,Name,IN,"('Jane','Janet')"
 AND,OR,3,Name,IN,"('James')"
 ;;;;
 run;
-%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtercheck(work.inds,targetds=work.class)
 %mp_filtergenerate(work.inds,outref=myfilter)
 data work.test;
-  set sashelp.class;
+  set work.class;
   where %inc myfilter;;
 run;
 %mp_assertdsobs(work.test,

tests/crossplatform/mp_filterstore.test.1.sas

@@ -10,6 +10,34 @@
 
 **/
 
+/* set up test data */
+data work.class ;
+length name $8 sex $1 age height weight 8;
+infile cards dsd;
+input Name:$char. Sex :$char. Age Height Weight;
+datalines4;
+Alfred,M,14,69,112.5
+Alice,F,13,56.5,84
+Barbara,F,13,65.3,98
+Carol,F,14,62.8,102.5
+Henry,M,14,63.5,102.5
+James,M,12,57.3,83
+Jane,F,12,59.8,84.5
+Janet,F,15,62.5,112.5
+Jeffrey,M,13,62.5,84
+John,M,12,59,99.5
+Joyce,F,11,51.3,50.5
+Judy,F,14,64.3,90
+Louise,F,12,56.3,77
+Mary,F,15,66.5,112
+Philip,M,16,72,150
+Robert,M,12,64.8,128
+Ronald,M,15,67,133
+Thomas,M,11,57.5,85
+William,M,15,66.5,112
+;;;;
+run;
+
 libname permlib (work);
 
 %mp_coretable(LOCKTABLE,libds=permlib.locktable)
@@ -31,7 +59,7 @@ AND,OR,2,Weight,NE,77.7
 ;;;;
 run;
 
-%mp_filterstore(libds=sashelp.class,
+%mp_filterstore(libds=work.class,
   queryds=work.inds,
   filter_summary=permlib.filtsum,
   filter_detail=permlib.filtdet,
@@ -60,7 +88,7 @@ select max(filter_rk) into: test1 from work.result;
 )
 
 /* Test 2 - load same table again and ensure we get the same RK */
-%mp_filterstore(libds=sashelp.class,
+%mp_filterstore(libds=work.class,
   queryds=work.inds,
   filter_summary=permlib.filtsum,
   filter_detail=permlib.filtdet,

tests/crossplatform/mp_filtervalidate.test.sas

@@ -7,9 +7,36 @@
   @li mp_filtervalidate.sas
   @li mp_assertdsobs.sas
   @li mp_assert.sas
+  @li mp_assertscope.sas
 
 **/
 
+data work.class ;
+length name $8 sex $1 age height weight 8;
+infile cards dsd;
+input Name:$char. Sex :$char. Age Height Weight;
+datalines4;
+Alfred,M,14,69,112.5
+Alice,F,13,56.5,84
+Barbara,F,13,65.3,98
+Carol,F,14,62.8,102.5
+Henry,M,14,63.5,102.5
+James,M,12,57.3,83
+Jane,F,12,59.8,84.5
+Janet,F,15,62.5,112.5
+Jeffrey,M,13,62.5,84
+John,M,12,59,99.5
+Joyce,F,11,51.3,50.5
+Judy,F,14,64.3,90
+Louise,F,12,56.3,77
+Mary,F,15,66.5,112
+Philip,M,16,72,150
+Robert,M,12,64.8,128
+Ronald,M,15,67,133
+Thomas,M,11,57.5,85
+William,M,15,66.5,112
+;;;;
+run;
 
 /* valid filter */
 data work.inds;
@@ -21,10 +48,16 @@ AND,AND,1,AGE,>,5
 AND,AND,1,SEX,NE,"'M'"
 AND,OR,2,Name,NOT IN,"('Jane','Janet')"
 AND,OR,2,Weight,>=,84.6
+AND,AND,3,age,=,.a
+AND,AND,4,weight,NE,._
 ;;;;
 run;
 %mp_filtergenerate(work.inds,outref=myfilter)
-%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+
+%mp_assertscope(SNAPSHOT)
+%mp_filtervalidate(myfilter,work.class,outds=work.results,abort=NO)
+%mp_assertscope(COMPARE)
+
 %mp_assertdsobs(work.results,
   desc=Valid filter,
   test=EMPTY,
@@ -40,7 +73,7 @@ datalines4;
 ;;;;
 run;
 %mp_filtergenerate(work.inds,outref=myfilter)
-%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%mp_filtervalidate(myfilter,work.class,outds=work.results,abort=NO)
 %mp_assertdsobs(work.results,
   desc=Empty filter,
   test=EMPTY,
@@ -59,7 +92,7 @@ AND,AND,1,SEX,NE,2
 ;;;;
 run;
 %mp_filtergenerate(work.inds,outref=myfilter)
-%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%mp_filtervalidate(myfilter,work.class,outds=work.results,abort=NO)
 %let syscc=0;
 %let test3=0;
 data _null_;
@@ -84,7 +117,7 @@ AND,AND,1,age,NE,"'M'"
 ;;;;
 run;
 %mp_filtergenerate(work.inds,outref=myfilter)
-%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%mp_filtervalidate(myfilter,work.class,outds=work.results,abort=NO)
 %let syscc=0;
 %let test4=0;
 data _null_;