Merge pull request #372 from sasjs/issue371

Issue371

all.sas

@@ -4189,8 +4189,8 @@ data &cntlout/nonote2err;
   end;
 
   /* create row marker. Data cannot be sorted without it! */
-  if first.fmtname then fmtrow=0;
+  if first.fmtname then fmtrow=1;
-  fmtrow+1;
+  else fmtrow+1;
 
 run;
 proc sort;
@@ -5541,13 +5541,21 @@ data _null_;
     header = cats(coalescec(varlabel(dsid,i),varnm),dlm);
   %end;
   %else %if &headerformat=SASJS %then %do;
-    if vartype(dsid,i)='C' then header=cats(varnm,':$char',varlen(dsid,i),'.');
+    vlen=varlen(dsid,i);
+    if vartype(dsid,i)='C' then header=cats(varnm,':$char',vlen,'.');
     else do;
       vfmt=coalescec(varfmt(dsid,i),'0');
       fmttype=mcf_getfmttype(vfmt);
       if fmttype='DATE' then header=cats(varnm,':date9.');
       else if fmttype='DATETIME' then header=cats(varnm,':E8601DT26.6');
       else if fmttype='TIME' then header=cats(varnm,':TIME12.');
+      /**
+        * there is not much point importing a short length numeric like this,
+        * eg with best4., as the resulting variable will still be stored as
+        * length 8.  We need a length or format statement to ensure variable
+        * is creatd with the smaller length...
+        **/
+      else if vlen<8 then header=cats(varnm,':best',vlen,'.');
       else header=cats(varnm,':best.');
     end;
   %end;
@@ -5574,6 +5582,7 @@ data _null_;
   set &ds end=last;
 %do i=1 %to &vcnt;
   %let var=%scan(&varlist,&i);
+  %local vlen&i;
   %if %mf_getvartype(&ds,&var)=C %then %do;
     %let dsv1=%mf_getuniquename(prefix=csvcol1_);
     %let dsv2=%mf_getuniquename(prefix=csvcol2_);
@@ -6377,15 +6386,14 @@ drop table &ds1, &ds2;
 /**
   * Sanitise the values based on valid value lists, then strip out
   * quotes, commas, periods and spaces.
-  * Only numeric values should remain
   */
 %local reason_cd nobs;
 %let nobs=0;
 data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
-  set &inds;
+  set &inds end=last;
-  length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000;
+  length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000;
   drop tmp;
 
   /* quick check to ensure column exists */
@@ -6401,7 +6409,8 @@ data &outds;
   end;
 
   /* need to open the dataset to get the column type */
-  dsid=open("&targetds","i");
+  retain dsid;
+  if _n_=1 then dsid=open("&targetds","i");
   if dsid>0 then do;
     vnum=varnum(dsid,VARIABLE_NM);
     if vnum<1 then do;
@@ -6411,11 +6420,19 @@ data &outds;
       call symputx('reason_cd',reason_cd,'l');
       call symputx('nobs',_n_,'l');
       output;
-      return;
+      goto endstep;
     end;
     /* now we can get the type */
     else vtype=vartype(dsid,vnum);
   end;
+  else do;
+      REASON_CD=cats("Could not open &targetds");
+      putlog REASON_CD= dsid=;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
+      output;
+      stop;
+  end;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
@@ -6450,15 +6467,40 @@ data &outds;
   end;
 
   /* special missing logic */
-  if vtype='N'
+  if vtype='N' & OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE') then do;
-    and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE')
+    if cats(upcase(raw_value)) in (
-    and cats(upcase(raw_value)) in (
       '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N'
       '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._'
     )
-  then do;
+    then do;
-    /* valid numeric - exit data step loop */
+      /* valid numeric - exit data step loop */
-    return;
+      return;
+    end;
+    else if subpad(upcase(raw_value),1,1) in (
+      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+    )
+    then do;
+      /* check if the raw_value contains a valid variable NAME */
+      vnum=varnum(dsid,subpad(raw_value,1,32));
+      if vnum>0 then do;
+        /* now we can get the type */
+        vtype2=vartype(dsid,vnum);
+        /* check type matches */
+        if vtype2=vtype then do;
+          /* valid target var - exit loop */
+          return;
+        end;
+        else do;
+          REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
+          putlog REASON_CD= dsid=;
+          call symputx('reason_cd',reason_cd,'l');
+          call symputx('nobs',_n_,'l');
+          output;
+          goto endstep;
+        end;
+      end;
+    end;
   end;
 
   /* special logic */
@@ -6480,6 +6522,32 @@ data &outds;
     if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
       tmp=scan(raw_value1,i,',');
       if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
+        if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in (
+          'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+          'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+        )
+        then do;
+          /* check if the raw_value contains a valid variable NAME */
+          /* is not valid syntax for IN or NOT IN */
+          vnum=varnum(dsid,subpad(tmp,1,32));
+          if vnum>0 then do;
+            /* now we can get the type */
+            vtype2=vartype(dsid,vnum);
+            /* check type matches */
+            if vtype2=vtype then do;
+              /* valid target var - exit loop */
+              return;
+            end;
+            else do;
+              REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
+              putlog REASON_CD= dsid=;
+              call symputx('reason_cd',reason_cd,'l');
+              call symputx('nobs',_n_,'l');
+              output;
+              goto endstep;
+            end;
+          end;
+        end;
         REASON_CD='Non Numeric value provided';
         putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
         call symputx('reason_cd',reason_cd,'l');
@@ -6504,14 +6572,42 @@ data &outds;
 
   /* output records that contain values other than digits and spaces */
   if notdigit(compress(raw_value3,' '))>0 then do;
+    if vtype='C' and subpad(upcase(raw_value),1,1) in (
+      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+    )
+    then do;
+      /* check if the raw_value contains a valid variable NAME */
+      vnum=varnum(dsid,subpad(raw_value,1,32));
+      if vnum>0 then do;
+        /* now we can get the type */
+        vtype2=vartype(dsid,vnum);
+        /* check type matches */
+        if vtype2=vtype then do;
+          /* valid target var - exit loop */
+          return;
+        end;
+        else do;
+          REASON_CD=cats("Compared Char Type (",vtype2,") is not (",vtype,")");
+          putlog REASON_CD= dsid=;
+          call symputx('reason_cd',reason_cd,'l');
+          call symputx('nobs',_n_,'l');
+          output;
+          goto endstep;
+        end;
+      end;
+    end;
+
     putlog raw_value3= $hex32.;
     REASON_CD=cats('Invalid RAW_VALUE:',raw_value);
-    putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
+    putlog (_all_)(=);
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');
     output;
   end;
 
+  endstep:
+  if last then rc=close(dsid);
 run;
 
 
@@ -10153,6 +10249,9 @@ select distinct lowcase(memname)
   format, to prevent loss of data - UNLESS the input dataset contains a marker
   column, specifying that a particular row needs to be deleted (`delete_col=`).
 
+  Positions of formats are made using the FMTROW variable - this must be present
+  and unique (on TYPE / FMTNAME / FMTROW).
+
   This macro can also be used to identify which records would be (or were)
   considered new, modified or deleted (`loadtarget=`) by creating the following
   tables:
@@ -10161,7 +10260,7 @@ select distinct lowcase(memname)
   @li work.outds_del
   @li work.outds_mod
 
-  For example usage, see mp_loadformat.test.sas
+  For example usage, see test (under Related Macros)
 
   @param [in] libcat The format catalog to be loaded
   @param [in] libds The staging table to load
@@ -10178,6 +10277,8 @@ select distinct lowcase(memname)
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_existvar.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
@@ -10228,6 +10329,16 @@ select distinct lowcase(memname)
 %let libcat=%scan(&libcat,1,-);
 
 /* perform input validations */
+%mp_abort(
+  iftrue=(%mf_existds(&libds)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(&libds could not be found)
+)
+%mp_abort(
+  iftrue=(%mf_existvar(&libds,FMTROW)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(FMTROW not found in &libds)
+)
 %let err=0;
 %let msg=0;
 data _null_;
@@ -10248,13 +10359,6 @@ data _null_;
       stop;
     end;
   end;
-  else if name='LIBDS' then do;
-    if exist(value) le 0 then do;
-      call symputx('msg',"Unable to open staging table: "!!value);
-      call symputx('err',1);
-      stop;
-    end;
-  end;
   else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
   and missing(value) then do;
     call symputx('msg',"missing value in var: "!!name);
@@ -10262,6 +10366,14 @@ data _null_;
     stop;
   end;
 run;
+data _null_;
+  set &libds;
+  if missing(fmtrow) then do;
+    call symputx('msg',"missing fmtrow in format: "!!FMTNAME);
+    call symputx('err',1);
+    stop;
+  end;
+run;
 
 %mp_abort(
   iftrue=(&err ne 0)
@@ -10269,6 +10381,15 @@ run;
   ,msg=%str(&msg)
 )
 
+%local cnt;
+proc sql noprint;
+select count(distinct catx('|',type,fmtname,fmtrow)) into: cnt from &libds;
+%mp_abort(
+  iftrue=(&cnt ne %mf_nobs(&libds))
+  ,mac=&sysmacroname
+  ,msg=%str(Non-unique primary key on &libds)
+)
+
 /**
   * First, extract only relevant formats from the catalog
   */
@@ -10322,12 +10443,6 @@ data &inlibds/nonote2err;
     %mp_aligndecimal(end,width=16)
   end;
 
-  /* update row marker - retain new var as fmtrow may already be in libds */
-  if first.fmtname then row=1;
-  else row+1;
-  drop row;
-  fmtrow=row;
-
   fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
 run;
 

base/mp_cntlout.sas

@@ -79,8 +79,8 @@ data &cntlout/nonote2err;
   end;
 
   /* create row marker. Data cannot be sorted without it! */
-  if first.fmtname then fmtrow=0;
+  if first.fmtname then fmtrow=1;
-  fmtrow+1;
+  else fmtrow+1;
 
 run;
 proc sort;

base/mp_ds2csv.sas

@@ -118,13 +118,21 @@ data _null_;
     header = cats(coalescec(varlabel(dsid,i),varnm),dlm);
   %end;
   %else %if &headerformat=SASJS %then %do;
-    if vartype(dsid,i)='C' then header=cats(varnm,':$char',varlen(dsid,i),'.');
+    vlen=varlen(dsid,i);
+    if vartype(dsid,i)='C' then header=cats(varnm,':$char',vlen,'.');
     else do;
       vfmt=coalescec(varfmt(dsid,i),'0');
       fmttype=mcf_getfmttype(vfmt);
       if fmttype='DATE' then header=cats(varnm,':date9.');
       else if fmttype='DATETIME' then header=cats(varnm,':E8601DT26.6');
       else if fmttype='TIME' then header=cats(varnm,':TIME12.');
+      /**
+        * there is not much point importing a short length numeric like this,
+        * eg with best4., as the resulting variable will still be stored as
+        * length 8.  We need a length or format statement to ensure variable
+        * is creatd with the smaller length...
+        **/
+      else if vlen<8 then header=cats(varnm,':best',vlen,'.');
       else header=cats(varnm,':best.');
     end;
   %end;
@@ -151,6 +159,7 @@ data _null_;
   set &ds end=last;
 %do i=1 %to &vcnt;
   %let var=%scan(&varlist,&i);
+  %local vlen&i;
   %if %mf_getvartype(&ds,&var)=C %then %do;
     %let dsv1=%mf_getuniquename(prefix=csvcol1_);
     %let dsv2=%mf_getuniquename(prefix=csvcol2_);

base/mp_filtercheck.sas

@@ -86,15 +86,14 @@
 /**
   * Sanitise the values based on valid value lists, then strip out
   * quotes, commas, periods and spaces.
-  * Only numeric values should remain
   */
 %local reason_cd nobs;
 %let nobs=0;
 data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
-  set &inds;
+  set &inds end=last;
-  length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000;
+  length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000;
   drop tmp;
 
   /* quick check to ensure column exists */
@@ -110,7 +109,8 @@ data &outds;
   end;
 
   /* need to open the dataset to get the column type */
-  dsid=open("&targetds","i");
+  retain dsid;
+  if _n_=1 then dsid=open("&targetds","i");
   if dsid>0 then do;
     vnum=varnum(dsid,VARIABLE_NM);
     if vnum<1 then do;
@@ -120,11 +120,19 @@ data &outds;
       call symputx('reason_cd',reason_cd,'l');
       call symputx('nobs',_n_,'l');
       output;
-      return;
+      goto endstep;
     end;
     /* now we can get the type */
     else vtype=vartype(dsid,vnum);
   end;
+  else do;
+      REASON_CD=cats("Could not open &targetds");
+      putlog REASON_CD= dsid=;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
+      output;
+      stop;
+  end;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
@@ -159,15 +167,40 @@ data &outds;
   end;
 
   /* special missing logic */
-  if vtype='N'
+  if vtype='N' & OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE') then do;
-    and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE')
+    if cats(upcase(raw_value)) in (
-    and cats(upcase(raw_value)) in (
       '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N'
       '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._'
     )
-  then do;
+    then do;
-    /* valid numeric - exit data step loop */
+      /* valid numeric - exit data step loop */
-    return;
+      return;
+    end;
+    else if subpad(upcase(raw_value),1,1) in (
+      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+    )
+    then do;
+      /* check if the raw_value contains a valid variable NAME */
+      vnum=varnum(dsid,subpad(raw_value,1,32));
+      if vnum>0 then do;
+        /* now we can get the type */
+        vtype2=vartype(dsid,vnum);
+        /* check type matches */
+        if vtype2=vtype then do;
+          /* valid target var - exit loop */
+          return;
+        end;
+        else do;
+          REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
+          putlog REASON_CD= dsid=;
+          call symputx('reason_cd',reason_cd,'l');
+          call symputx('nobs',_n_,'l');
+          output;
+          goto endstep;
+        end;
+      end;
+    end;
   end;
 
   /* special logic */
@@ -189,6 +222,32 @@ data &outds;
     if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
       tmp=scan(raw_value1,i,',');
       if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
+        if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in (
+          'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+          'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+        )
+        then do;
+          /* check if the raw_value contains a valid variable NAME */
+          /* is not valid syntax for IN or NOT IN */
+          vnum=varnum(dsid,subpad(tmp,1,32));
+          if vnum>0 then do;
+            /* now we can get the type */
+            vtype2=vartype(dsid,vnum);
+            /* check type matches */
+            if vtype2=vtype then do;
+              /* valid target var - exit loop */
+              return;
+            end;
+            else do;
+              REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
+              putlog REASON_CD= dsid=;
+              call symputx('reason_cd',reason_cd,'l');
+              call symputx('nobs',_n_,'l');
+              output;
+              goto endstep;
+            end;
+          end;
+        end;
         REASON_CD='Non Numeric value provided';
         putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
         call symputx('reason_cd',reason_cd,'l');
@@ -213,14 +272,42 @@ data &outds;
 
   /* output records that contain values other than digits and spaces */
   if notdigit(compress(raw_value3,' '))>0 then do;
+    if vtype='C' and subpad(upcase(raw_value),1,1) in (
+      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
+      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
+    )
+    then do;
+      /* check if the raw_value contains a valid variable NAME */
+      vnum=varnum(dsid,subpad(raw_value,1,32));
+      if vnum>0 then do;
+        /* now we can get the type */
+        vtype2=vartype(dsid,vnum);
+        /* check type matches */
+        if vtype2=vtype then do;
+          /* valid target var - exit loop */
+          return;
+        end;
+        else do;
+          REASON_CD=cats("Compared Char Type (",vtype2,") is not (",vtype,")");
+          putlog REASON_CD= dsid=;
+          call symputx('reason_cd',reason_cd,'l');
+          call symputx('nobs',_n_,'l');
+          output;
+          goto endstep;
+        end;
+      end;
+    end;
+
     putlog raw_value3= $hex32.;
     REASON_CD=cats('Invalid RAW_VALUE:',raw_value);
-    putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
+    putlog (_all_)(=);
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');
     output;
   end;
 
+  endstep:
+  if last then rc=close(dsid);
 run;
 
 

base/mp_loadformat.sas

@@ -9,6 +9,9 @@
   format, to prevent loss of data - UNLESS the input dataset contains a marker
   column, specifying that a particular row needs to be deleted (`delete_col=`).
 
+  Positions of formats are made using the FMTROW variable - this must be present
+  and unique (on TYPE / FMTNAME / FMTROW).
+
   This macro can also be used to identify which records would be (or were)
   considered new, modified or deleted (`loadtarget=`) by creating the following
   tables:
@@ -17,7 +20,7 @@
   @li work.outds_del
   @li work.outds_mod
 
-  For example usage, see mp_loadformat.test.sas
+  For example usage, see test (under Related Macros)
 
   @param [in] libcat The format catalog to be loaded
   @param [in] libds The staging table to load
@@ -34,6 +37,8 @@
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_existvar.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
@@ -84,6 +89,16 @@
 %let libcat=%scan(&libcat,1,-);
 
 /* perform input validations */
+%mp_abort(
+  iftrue=(%mf_existds(&libds)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(&libds could not be found)
+)
+%mp_abort(
+  iftrue=(%mf_existvar(&libds,FMTROW)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(FMTROW not found in &libds)
+)
 %let err=0;
 %let msg=0;
 data _null_;
@@ -104,13 +119,6 @@ data _null_;
       stop;
     end;
   end;
-  else if name='LIBDS' then do;
-    if exist(value) le 0 then do;
-      call symputx('msg',"Unable to open staging table: "!!value);
-      call symputx('err',1);
-      stop;
-    end;
-  end;
   else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
   and missing(value) then do;
     call symputx('msg',"missing value in var: "!!name);
@@ -118,6 +126,14 @@ data _null_;
     stop;
   end;
 run;
+data _null_;
+  set &libds;
+  if missing(fmtrow) then do;
+    call symputx('msg',"missing fmtrow in format: "!!FMTNAME);
+    call symputx('err',1);
+    stop;
+  end;
+run;
 
 %mp_abort(
   iftrue=(&err ne 0)
@@ -125,6 +141,15 @@ run;
   ,msg=%str(&msg)
 )
 
+%local cnt;
+proc sql noprint;
+select count(distinct catx('|',type,fmtname,fmtrow)) into: cnt from &libds;
+%mp_abort(
+  iftrue=(&cnt ne %mf_nobs(&libds))
+  ,mac=&sysmacroname
+  ,msg=%str(Non-unique primary key on &libds)
+)
+
 /**
   * First, extract only relevant formats from the catalog
   */
@@ -178,12 +203,6 @@ data &inlibds/nonote2err;
     %mp_aligndecimal(end,width=16)
   end;
 
-  /* update row marker - retain new var as fmtrow may already be in libds */
-  if first.fmtname then row=1;
-  else row+1;
-  drop row;
-  fmtrow=row;
-
   fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
 run;
 

sasjs/sasjsconfig.json

@@ -67,7 +67,7 @@
     },
     {
       "name": "server",
-      "serverUrl": "https://sas9.4gl.io",
+      "serverUrl": "https://sas.4gl.io",
       "serverType": "SASJS",
       "httpsAgentOptions": {
         "allowInsecureRequests": false

tests/base/mp_ds2csv.test.2.sas

@@ -0,0 +1,43 @@
+/**
+  @file
+  @brief Testing mp_ds2csv.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2csv.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+data work.shortnum;
+  length a 3 b 4 c 8;
+  a=1;b=2;c=3;
+  output;
+  stop;
+run;
+
+/**
+  * Test 1 - default CSV
+  */
+
+%mp_ds2csv(work.shortnum,outfile="&sasjswork/test1.csv",headerformat=SASJS)
+
+%let test1b=FAIL;
+data _null_;
+  infile "&sasjswork/test1.csv";
+  input;
+  list;
+  if _n_=1 then call symputx('test1a',_infile_);
+  else if _infile_=:'1,2,3' then call symputx('test1b','PASS');
+run;
+
+%mp_assert(
+  iftrue=("&test1a"="A:best3. B:best4. C:best."),
+  desc=Checking header row Test 1,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test1b"="PASS"),
+  desc=Checking data row Test 1,
+  outds=work.test_results
+)

tests/base/mp_filtercheck.test.sas

@@ -53,7 +53,10 @@ AND,AND,1,age,=,.A
 AND,AND,1,height,<,.B
 AND,AND,1,age,IN,"(.a,.b,.)"
 AND,AND,1,age,IN,"(.A)"
-
+AND,AND,1,AGE,=,AGE
+AND,AND,1,AGE,<,Weight
+AND,AND,1,AGE,BETWEEN,"HEIGHT AND WEIGHT"
+AND,OR,2,Name,=,name
 ;;;;
 run;
 
@@ -204,3 +207,26 @@ run;
   outds=work.test_results
 )
 %let syscc=0;
+
+
+/* invalid IN value (cannot use var names) */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,AGE,NOT IN,"(height, age)"
+;;;;
+run;
+
+%mp_filtercheck(work.inds,
+  targetds=work.class,
+  outds=work.badrecords,
+  abort=NO
+)
+%let syscc=0;
+%mp_assertdsobs(work.badrecords,
+  desc=Invalid IN syntax,
+  test=HASOBS,
+  outds=work.test_results
+)

tests/base/mp_loadformat.test.1.sas

@@ -189,8 +189,10 @@ data work.stagedata3;
     if last.fmtname then do;
       output; /* 6 new records */
       x=_n_;
-      x+1;start=cats("mod",x);end=start;label='newlabel1';output;
+      x+1;start=cats("mod",x);end=start;label='newlabel1';fmtrow=fmtrow+1;
-      x+1;start=cats("mod",x);end=start;label='newlabel2';output;
+      output;
+      x+1;start=cats("mod",x);end=start;label='newlabel2';fmtrow=fmtrow+2;
+      output;
     end;
     else if fmtrow le 3 then do; /* 9 more changed values */
       start= cats("mod",_n_);

tests/base/mp_loadformat.test.2.sas

@@ -58,6 +58,9 @@ proc format library=&cat1;
   value agemlb (multilabel)
     19-120='Adults'
     1-18='Children'
+    0-1='Preschool'
+    1-2='Preschool'
+    2-3='Preschool'
     1-4='Preschool';
   value agemlc (multilabel notsorted)
     19-120='Adults'
@@ -67,16 +70,19 @@ run;
 
 %mp_cntlout(libcat=&cat1,cntlout=work.cntlout1)
 %mp_assertdsobs(work.cntlout1,
-  desc=Has 16 records,
+  desc=Has 19 records,
-  test=EQUALS 16
+  test=EQUALS 19
 )
 
 data work.stagedata3;
   set work.cntlout1;
   if fmtname='AGEMLA' and label ne 'Preschool' then deleteme='Yes';
   if fmtname='AGEMLB' and label = 'Preschool' then label='Kids';
-  if fmtname='GENDERML' and label='Farmale' then output;
+  if fmtname='GENDERML' and label='Farmale' then do;
-  output;
+    output;
+    fmtrow=101; output;
+  end;
+  else output;
 run;
 
 
@@ -113,14 +119,17 @@ run;
 
 %let check1=0;
 %let check2=0;
+%let check3=0;
 data test;
   set work.cntlout2;
   where fmtname='GENDERML';
+  putlog fmtrow= label=;
   if _n_=4 and label='Farmale' then call symputx('check1',1);
-  if _n_=5 and label='Farmale' then call symputx('check2',1);
+  if _n_=5 and label ne 'Farmale' then call symputx('check2',1);
+  if _n_=8 and label = 'Farmale' then call symputx('check3',1);
 run;
 %mp_assert(
-  iftrue=(&check1=1 and &check2=1),
+  iftrue=(&check1=1 and &check2=1 and &check3=1),
   desc=Ensuring Farmale values retain their order,
   outds=work.test_results
 )