fix: setting default to best. over 8. for mf_getformat with force option

feat: new (generic) mp_assert macro, and new feature (type filter) fo…

all.sas

@@ -932,9 +932,9 @@ options noquotelenmax;
       8.
       NOTE: Variable renegade does not exist in test
 
-  @param libds Two part dataset (or view) reference.
+  @param [in] libds Two part dataset (or view) reference.
-  @param var Variable name for which a format should be returned
+  @param [in] var Variable name for which a format should be returned
-  @param force Set to 1 to supply a default if the variable has no format
+  @param [in] force=(0) Set to 1 to supply a default if the variable has no format
   @returns outputs format
 
   @author Allan Bowe
@@ -969,7 +969,7 @@ options noquotelenmax;
     %let vlen = %sysfunc(varlen(&dsid, &vnum));
     %let vtype = %sysfunc(vartype(&dsid, &vnum.));
     %if &vtype=C %then %let vformat=$&vlen..;
-    %else %let vformat=8.;
+    %else %let vformat=best.;
   %end;
 
 
@@ -977,7 +977,7 @@ options noquotelenmax;
   %let rc = %sysfunc(close(&dsid));
   /* Return variable format */
   &vformat
-%mend;/**
+%mend mf_getVarFormat;/**
   @file
   @brief Returns the length of a variable
   @details Uses varlen function to identify the length of a particular variable.
@@ -1051,6 +1051,10 @@ options noquotelenmax;
   @param [in] dlm= ( ) Provide a delimiter (eg comma or space) to separate the
     variables
   @param [in] quote= (none) use either DOUBLE or SINGLE to quote the results
+  @param [in] typefilter= (A) Filter for certain types of column.  Valid values:
+    @li A Return All columns
+    @li C Return Character columns
+    @li N Return Numeric columns
 
   @version 9.2
   @author Allan Bowe
@@ -1060,9 +1064,10 @@ options noquotelenmax;
 %macro mf_getvarlist(libds
       ,dlm=%str( )
       ,quote=no
+      ,typefilter=A
 )/*/STORE SOURCE*/;
   /* declare local vars */
-  %local outvar dsid nvars x rc dlm q var;
+  %local outvar dsid nvars x rc dlm q var vtype;
 
   /* credit Rowland Hale  - byte34 is double quote, 39 is single quote */
   %if %upcase(&quote)=DOUBLE %then %let q=%qsysfunc(byte(34));
@@ -1070,21 +1075,22 @@ options noquotelenmax;
   /* open dataset in macro */
   %let dsid=%sysfunc(open(&libds));
 
-
   %if &dsid %then %do;
     %let nvars=%sysfunc(attrn(&dsid,NVARS));
     %if &nvars>0 %then %do;
-      /* add first dataset variable to global macro variable */
+      /* add variables with supplied delimeter */
-      %let outvar=&q.%sysfunc(varname(&dsid,1))&q.;
-      /* add remaining variables with supplied delimeter */
       %do x=1 %to &nvars;
-        %let var=&q.%sysfunc(varname(&dsid,&x))&q.;
+        /* get variable type */
-        %if &var=&q&q %then %do;
+        %let vtype=%sysfunc(vartype(&dsid,&x));
-          %put &sysmacroname: Empty column found in &libds!;
+        %if &vtype=&typefilter or &typefilter=A %then %do;
-          %let var=&q. &q.;
+          %let var=&q.%sysfunc(varname(&dsid,&x))&q.;
+          %if &var=&q&q %then %do;
+            %put &sysmacroname: Empty column found in &libds!;
+            %let var=&q. &q.;
+          %end;
+          %if %quote(&outvar)=%quote() %then %let outvar=&var;
+          %else %let outvar=&outvar.&dlm.&var.;
         %end;
-        %if &x=1 %then %let outvar=&var;
-        %else %let outvar=&outvar.&dlm.&var.;
       %end;
     %end;
     %let rc=%sysfunc(close(&dsid));
@@ -1094,7 +1100,7 @@ options noquotelenmax;
     %let rc=%sysfunc(close(&dsid));
   %end;
   &outvar
-%mend;/**
+%mend mf_getvarlist;/**
   @file
   @brief Returns the position of a variable in dataset (varnum attribute).
   @details Uses varnum function to determine position.
@@ -1194,7 +1200,7 @@ Usage:
   %let rc = %sysfunc(close(&dsid));
   /* Return variable type */
   &vtype
-%mend;/**
+%mend mf_getvartype;/**
   @file
   @brief Returns the engine type of a SAS fileref
   @details Queries sashelp.vextfl to get the xengine value.
@@ -1782,6 +1788,61 @@ Usage:
 %mend;
 
 /** @endcond *//**
+  @file
+  @brief Generic assertion
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      %mp_assert(iftrue=(1=1),
+        desc=Obviously true
+      )
+
+      %mp_assert(iftrue=(1=0),
+        desc=Will fail
+      )
+
+  @param [in] iftrue= (1=1) A condition where, if true, the test is a PASS.
+  Else, the test is a fail.
+
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Column &inds contained ALL columns|
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assert(iftrue=(1=1),
+  desc=0,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  data ;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_comments="&sysmacroname: Test result of "!!symget('iftrue');
+  %if %eval(%unquote(&iftrue)) %then %do;
+    test_result='PASS';
+  %end;
+  %else %do;
+    test_result='FAIL';
+  %end;
+  run;
+
+  %local ds ;
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+
+%mend mp_assert;/**
   @file
   @brief Asserts the existence (or not) of columns
   @details Useful in the context of writing sasjs tests.  The results of the
@@ -3238,6 +3299,7 @@ run;
   @li mp_abort.sas
   @li mf_getuniquefileref.sas
   @li mf_getvarlist.sas
+  @li mf_getvartype.sas
   @li mf_nobs.sas
   @li mp_filtergenerate.sas
   @li mp_filtervalidate.sas
@@ -3259,6 +3321,20 @@ run;
   ,msg=%str(syscc=&syscc - on macro entry)
 )
 
+/* Validate input column */
+%local vtype;
+%let vtype=%mf_getvartype(&inds,RAW_VALUE);
+%mp_abort(iftrue=(&abort=YES and &vtype ne C),
+  mac=&sysmacroname,
+  msg=%str(%str(ERR)OR: RAW_VALUE must be character)
+)
+%if &vtype ne C %then %do;
+  %put &sysmacroname: RAW_VALUE must be character;
+  %let syscc=42;
+  %return;
+%end;
+
+
 /**
   * Sanitise the values based on valid value lists, then strip out
   * quotes, commas, periods and spaces.
@@ -3266,6 +3342,8 @@ run;
   */
 %local reason_cd;
 data &outds;
+  /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
+    OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
   length reason_cd $32;
 
@@ -3362,7 +3440,7 @@ run;
 /* this macro will also set syscc to 1008 if any issues found */
 %mp_filtervalidate(&fref1,&targetds,outds=&outds,abort=&abort)
 
-%mend;
+%mend mp_filtercheck;
 /**
   @file
   @brief Generates a filter clause from an input table, to a fileref

base/mf_getvarformat.sas

@@ -23,9 +23,9 @@
       8.
       NOTE: Variable renegade does not exist in test
 
-  @param libds Two part dataset (or view) reference.
+  @param [in] libds Two part dataset (or view) reference.
-  @param var Variable name for which a format should be returned
+  @param [in] var Variable name for which a format should be returned
-  @param force Set to 1 to supply a default if the variable has no format
+  @param [in] force=(0) Set to 1 to supply a default if the variable has no format
   @returns outputs format
 
   @author Allan Bowe
@@ -60,7 +60,7 @@
     %let vlen = %sysfunc(varlen(&dsid, &vnum));
     %let vtype = %sysfunc(vartype(&dsid, &vnum.));
     %if &vtype=C %then %let vformat=$&vlen..;
-    %else %let vformat=8.;
+    %else %let vformat=best.;
   %end;
 
 
@@ -68,4 +68,4 @@
   %let rc = %sysfunc(close(&dsid));
   /* Return variable format */
   &vformat
-%mend;
+%mend mf_getVarFormat;

base/mf_getvarlist.sas

@@ -21,6 +21,10 @@
   @param [in] dlm= ( ) Provide a delimiter (eg comma or space) to separate the
     variables
   @param [in] quote= (none) use either DOUBLE or SINGLE to quote the results
+  @param [in] typefilter= (A) Filter for certain types of column.  Valid values:
+    @li A Return All columns
+    @li C Return Character columns
+    @li N Return Numeric columns
 
   @version 9.2
   @author Allan Bowe
@@ -30,9 +34,10 @@
 %macro mf_getvarlist(libds
       ,dlm=%str( )
       ,quote=no
+      ,typefilter=A
 )/*/STORE SOURCE*/;
   /* declare local vars */
-  %local outvar dsid nvars x rc dlm q var;
+  %local outvar dsid nvars x rc dlm q var vtype;
 
   /* credit Rowland Hale  - byte34 is double quote, 39 is single quote */
   %if %upcase(&quote)=DOUBLE %then %let q=%qsysfunc(byte(34));
@@ -40,21 +45,22 @@
   /* open dataset in macro */
   %let dsid=%sysfunc(open(&libds));
 
-
   %if &dsid %then %do;
     %let nvars=%sysfunc(attrn(&dsid,NVARS));
     %if &nvars>0 %then %do;
-      /* add first dataset variable to global macro variable */
+      /* add variables with supplied delimeter */
-      %let outvar=&q.%sysfunc(varname(&dsid,1))&q.;
-      /* add remaining variables with supplied delimeter */
       %do x=1 %to &nvars;
-        %let var=&q.%sysfunc(varname(&dsid,&x))&q.;
+        /* get variable type */
-        %if &var=&q&q %then %do;
+        %let vtype=%sysfunc(vartype(&dsid,&x));
-          %put &sysmacroname: Empty column found in &libds!;
+        %if &vtype=&typefilter or &typefilter=A %then %do;
-          %let var=&q. &q.;
+          %let var=&q.%sysfunc(varname(&dsid,&x))&q.;
+          %if &var=&q&q %then %do;
+            %put &sysmacroname: Empty column found in &libds!;
+            %let var=&q. &q.;
+          %end;
+          %if %quote(&outvar)=%quote() %then %let outvar=&var;
+          %else %let outvar=&outvar.&dlm.&var.;
         %end;
-        %if &x=1 %then %let outvar=&var;
-        %else %let outvar=&outvar.&dlm.&var.;
       %end;
     %end;
     %let rc=%sysfunc(close(&dsid));
@@ -64,4 +70,4 @@
     %let rc=%sysfunc(close(&dsid));
   %end;
   &outvar
-%mend;
+%mend mf_getvarlist;

base/mf_getvartype.sas

@@ -45,4 +45,4 @@ Usage:
   %let rc = %sysfunc(close(&dsid));
   /* Return variable type */
   &vtype
-%mend;
+%mend mf_getvartype;

base/mp_assert.sas

@@ -0,0 +1,56 @@
+/**
+  @file
+  @brief Generic assertion
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      %mp_assert(iftrue=(1=1),
+        desc=Obviously true
+      )
+
+      %mp_assert(iftrue=(1=0),
+        desc=Will fail
+      )
+
+  @param [in] iftrue= (1=1) A condition where, if true, the test is a PASS.
+  Else, the test is a fail.
+
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Column &inds contained ALL columns|
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assert(iftrue=(1=1),
+  desc=0,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  data ;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_comments="&sysmacroname: Test result of "!!symget('iftrue');
+  %if %eval(%unquote(&iftrue)) %then %do;
+    test_result='PASS';
+  %end;
+  %else %do;
+    test_result='FAIL';
+  %end;
+  run;
+
+  %local ds ;
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+
+%mend mp_assert;

base/mp_filtercheck.sas

@@ -44,6 +44,7 @@
   @li mp_abort.sas
   @li mf_getuniquefileref.sas
   @li mf_getvarlist.sas
+  @li mf_getvartype.sas
   @li mf_nobs.sas
   @li mp_filtergenerate.sas
   @li mp_filtervalidate.sas
@@ -65,6 +66,20 @@
   ,msg=%str(syscc=&syscc - on macro entry)
 )
 
+/* Validate input column */
+%local vtype;
+%let vtype=%mf_getvartype(&inds,RAW_VALUE);
+%mp_abort(iftrue=(&abort=YES and &vtype ne C),
+  mac=&sysmacroname,
+  msg=%str(%str(ERR)OR: RAW_VALUE must be character)
+)
+%if &vtype ne C %then %do;
+  %put &sysmacroname: RAW_VALUE must be character;
+  %let syscc=42;
+  %return;
+%end;
+
+
 /**
   * Sanitise the values based on valid value lists, then strip out
   * quotes, commas, periods and spaces.
@@ -72,6 +87,8 @@
   */
 %local reason_cd;
 data &outds;
+  /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
+    OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
   length reason_cd $32;
 
@@ -168,4 +185,4 @@ run;
 /* this macro will also set syscc to 1008 if any issues found */
 %mp_filtervalidate(&fref1,&targetds,outds=&outds,abort=&abort)
 
-%mend;
+%mend mp_filtercheck;

tests/base/mf_getvarlist.test.sas

@@ -0,0 +1,60 @@
+/**
+  @file
+  @brief Testing mf_getvarlist macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getvarlist.sas
+
+**/
+
+
+%let test1=%mf_getvarlist(sashelp.class);
+%let test2=%mf_getvarlist(sashelp.class,dlm=X);
+%let test3=%mf_getvarlist(sashelp.class,dlm=%str(,),quote=double);
+%let test4=%mf_getvarlist(sashelp.class,typefilter=N);
+%let test5=%mf_getvarlist(sashelp.class,typefilter=C);
+
+data work.test_results;
+  length test_description $256 test_result $4 test_comments base result  $256;
+  test_description="Basic test";
+  base=symget('test1');
+  result='Name Sex Age Height Weight';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  test_description="DLM test";
+  base=symget('test2');
+  result='NameXSexXAgeXHeightXWeight';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  test_description="DLM + quote test";
+  base=symget('test3');
+  result='"Name","Sex","Age","Height","Weight"';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  test_description="Numeric Filter";
+  base=symget('test4');
+  result='Age Height Weight';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  test_description="Char Filter";
+  base=symget('test5');
+  result='Name Sex';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  drop base result;
+run;

tests/base/mp_filtercheck.test.sas

@@ -5,6 +5,7 @@
   <h4> SAS Macros </h4>
   @li mp_filtercheck.sas
   @li mp_assertdsobs.sas
+  @li mp_assert.sas
 
 **/
 
@@ -125,3 +126,23 @@ run;
   outds=work.test_results
 )
 
+/* Supply variables with incorrect types */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:8;
+datalines4;
+AND,AND,1,age,=,0
+;;;;
+run;
+%let syscc=0;
+%mp_filtercheck(work.inds,
+  targetds=sashelp.class,
+  outds=work.badrecords,
+  abort=NO
+)
+%mp_assert(iftrue=(&syscc=42),
+  desc=Throw error if RAW_VALUE is incorrect,
+  outds=work.test_results
+)
+%let syscc=0;

tests/base/mp_validatecol.test.sas

@@ -17,7 +17,7 @@ data test1;
   input;
   libds=_infile_;
   %mp_validatecol(libds,LIBDS,is_libds)
-  if libds=1;
+  if is_libds=1;
 datalines4;
 some.libname
 !lib.blah