Merge pull request #284 from sasjs/allanbowe/dictionary-table-constraints-283

fix: avoid exceptions from dictionary.table_constraints.

all.sas

@@ -3505,6 +3505,7 @@ run;
   @param [in] mode (CREATE) Valid values:
     @li CREATE - Create the file (even if it already exists)
     @li APPEND - Append to the file (don't overwrite)
+  @param iftrue= (1=1) Supply a condition for which the macro should be executed
 
   @returns nothing
 
@@ -3518,8 +3519,12 @@ run;
     ,inref=____in   /* override default to use own filerefs */
     ,outref=____out /* override default to use own filerefs */
     ,mode=CREATE
+    ,iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
   %local mod;
+
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
   %if &mode=APPEND %then %let mod=mod;
 
   /* these IN and OUT filerefs can point to anything */
@@ -5878,7 +5883,8 @@ data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
-  length reason_cd $4032 vtype $1 vnum dsid 8;
+  length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000;
+  drop tmp;
 
   /* quick check to ensure column exists */
   if upcase(VARIABLE_NM) not in
@@ -5954,18 +5960,32 @@ data &outds;
   end;
 
   /* special logic */
-  if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ','');
+  if OPERATOR_NM in ('IN','NOT IN','BETWEEN') then do;
-  else if OPERATOR_NM in ('IN','NOT IN') then do;
+    if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ',',');
-    if substr(raw_value,1,1) ne '('
+    else do;
-    or substr(cats(reverse(raw_value)),1,1) ne ')'
+      if substr(raw_value,1,1) ne '('
-    then do;
+      or substr(cats(reverse(raw_value)),1,1) ne ')'
-      REASON_CD='Missing start/end bracket in RAW_VALUE';
+      then do;
-      putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+        REASON_CD='Missing start/end bracket in RAW_VALUE';
-      call symputx('reason_cd',reason_cd,'l');
+        putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
-      call symputx('nobs',_n_,'l');
+        call symputx('reason_cd',reason_cd,'l');
-      output;
+        call symputx('nobs',_n_,'l');
+        output;
+      end;
+      else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
+    end;
+    /* we now have a comma seperated list of values */
+    if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
+      tmp=scan(raw_value1,i,',');
+      if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
+        REASON_CD='Non Numeric value provided';
+        putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+        call symputx('reason_cd',reason_cd,'l');
+        call symputx('nobs',_n_,'l');
+        output;
+      end;
+      return;
     end;
-    else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
   end;
   else raw_value1=raw_value;
 
@@ -6645,8 +6665,11 @@ create table &outds as
 /**
   * We cannot apply this clause to the underlying dictionary table.  See:
   * https://communities.sas.com/t5/SAS-Programming/Unexpected-Where-Clause-behaviour-in-dictionary-TABLE/m-p/771554#M244867
+  * cannot use`where calculated libref="&lib"` either as it will STILL execute
+  * all the underlying constraint queries, causing exception errors in some
+  * cases: https://github.com/sasjs/core/issues/283
   */
-  where calculated libref="&lib"
+  where a.TABLE_CATALOG="&lib"
   %if "&ds" ne "" %then %do;
     and upcase(a.TABLE_NAME)="&ds"
     and upcase(b.TABLE_NAME)="&ds"
@@ -11764,7 +11787,9 @@ run;
 %else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
 
 data &ds4;
-  length &inds_keep $41 tgtvar_nm $32;
+  length &inds_keep $41 tgtvar_nm $32 _label_ $256;
+  if _n_=1 then call missing(_label_);
+  drop _label_;
   set &ds2 &ds3 indsname=&inds_auto;
 
   tgtvar_nm=upcase(tgtvar_nm);

base/mp_binarycopy.sas

@@ -31,6 +31,7 @@
   @param [in] mode (CREATE) Valid values:
     @li CREATE - Create the file (even if it already exists)
     @li APPEND - Append to the file (don't overwrite)
+  @param iftrue= (1=1) Supply a condition for which the macro should be executed
 
   @returns nothing
 
@@ -44,8 +45,12 @@
     ,inref=____in   /* override default to use own filerefs */
     ,outref=____out /* override default to use own filerefs */
     ,mode=CREATE
+    ,iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
   %local mod;
+
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
   %if &mode=APPEND %then %let mod=mod;
 
   /* these IN and OUT filerefs can point to anything */

base/mp_filtercheck.sas

@@ -92,7 +92,8 @@ data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
-  length reason_cd $4032 vtype $1 vnum dsid 8;
+  length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000;
+  drop tmp;
 
   /* quick check to ensure column exists */
   if upcase(VARIABLE_NM) not in
@@ -168,18 +169,32 @@ data &outds;
   end;
 
   /* special logic */
-  if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ','');
+  if OPERATOR_NM in ('IN','NOT IN','BETWEEN') then do;
-  else if OPERATOR_NM in ('IN','NOT IN') then do;
+    if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ',',');
-    if substr(raw_value,1,1) ne '('
+    else do;
-    or substr(cats(reverse(raw_value)),1,1) ne ')'
+      if substr(raw_value,1,1) ne '('
-    then do;
+      or substr(cats(reverse(raw_value)),1,1) ne ')'
-      REASON_CD='Missing start/end bracket in RAW_VALUE';
+      then do;
-      putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+        REASON_CD='Missing start/end bracket in RAW_VALUE';
-      call symputx('reason_cd',reason_cd,'l');
+        putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
-      call symputx('nobs',_n_,'l');
+        call symputx('reason_cd',reason_cd,'l');
-      output;
+        call symputx('nobs',_n_,'l');
+        output;
+      end;
+      else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
+    end;
+    /* we now have a comma seperated list of values */
+    if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
+      tmp=scan(raw_value1,i,',');
+      if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
+        REASON_CD='Non Numeric value provided';
+        putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+        call symputx('reason_cd',reason_cd,'l');
+        call symputx('nobs',_n_,'l');
+        output;
+      end;
+      return;
     end;
-    else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
   end;
   else raw_value1=raw_value;
 

base/mp_getconstraints.sas

@@ -94,8 +94,11 @@ create table &outds as
 /**
   * We cannot apply this clause to the underlying dictionary table.  See:
   * https://communities.sas.com/t5/SAS-Programming/Unexpected-Where-Clause-behaviour-in-dictionary-TABLE/m-p/771554#M244867
+  * cannot use`where calculated libref="&lib"` either as it will STILL execute
+  * all the underlying constraint queries, causing exception errors in some
+  * cases: https://github.com/sasjs/core/issues/283
   */
-  where calculated libref="&lib"
+  where a.TABLE_CATALOG="&lib"
   %if "&ds" ne "" %then %do;
     and upcase(a.TABLE_NAME)="&ds"
     and upcase(b.TABLE_NAME)="&ds"

base/mp_storediffs.sas

@@ -154,7 +154,9 @@ run;
 %else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
 
 data &ds4;
-  length &inds_keep $41 tgtvar_nm $32;
+  length &inds_keep $41 tgtvar_nm $32 _label_ $256;
+  if _n_=1 then call missing(_label_);
+  drop _label_;
   set &ds2 &ds3 indsname=&inds_auto;
 
   tgtvar_nm=upcase(tgtvar_nm);

sasjs/sasjsconfig.json

@@ -67,7 +67,7 @@
     },
     {
       "name": "server",
-      "serverUrl": "",
+      "serverUrl": "https://sas.4gl.io",
       "serverType": "SASJS",
       "httpsAgentOptions": {
         "allowInsecureRequests": false
@@ -107,4 +107,4 @@
       "contextName": "SAS Job Execution compute context"
     }
   ]
-}
+}

tests/base/mp_filtercheck.test.sas

@@ -38,7 +38,7 @@ William,M,15,66.5,112
 ;;;;
 run;
 
-/* valid filter conditions */
+/* VALID filter conditions */
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
@@ -51,6 +51,9 @@ AND,OR,2,Weight,>=,77.7
 AND,OR,2,Weight,NE,77.7
 AND,AND,1,age,=,.A
 AND,AND,1,height,<,.B
+AND,AND,1,age,IN,"(.a,.b,.)"
+AND,AND,1,age,IN,"(.A)"
+
 ;;;;
 run;
 
@@ -115,6 +118,28 @@ run;
   outds=work.test_results
 )
 
+/* invalid IN value */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,OR,2,age,IN,"(.,.a,X)"
+;;;;
+run;
+
+%mp_filtercheck(work.inds,
+  targetds=work.class,
+  outds=work.badrecords,
+  abort=NO
+)
+%let syscc=0;
+%mp_assertdsobs(work.badrecords,
+  desc=Invalid IN value,
+  test=HASOBS,
+  outds=work.test_results
+)
+
 /* Code injection - column name */
 data work.inds;
   infile datalines4 dsd;
@@ -163,7 +188,7 @@ run;
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:8;
+    OPERATOR_NM:$10. RAW_VALUE:8.;
 datalines4;
 AND,AND,1,age,=,0
 ;;;;

tests/base/mp_filterstore.test.1.sas

@@ -56,6 +56,9 @@ AND,AND,1,SEX,<=,"'M'"
 AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
 AND,OR,2,Weight,>=,77.7
 AND,OR,2,Weight,NE,77.7
+AND,AND,3,age,NOT IN,"(.a,.b,.)"
+AND,AND,3,age,NOT IN,"(.A)"
+AND,AND,4,Name,=,"'Jeremiah'"
 ;;;;
 run;