Merge pull request #165 from sasjs/fmtstore

feat: adding format catalog capability to mp_filterstore

README.md

@@ -192,11 +192,12 @@ When contributing to this library, it is therefore important to ensure that all
 
 ## Breaking Changes
 
-We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major (breaking) release becomes necessary:
+We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major/breaking release (v5) becomes necessary:
 
 * mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
 * `insert_cmplib` option of mcf_xxx macros will be deprecated (the option is now checked automatically with value inserted only if needed)
-* mcf_xxx macros to have `wrap=` option defaulted to YES for convenience
+* mcf_xxx macros to have `wrap=` option defaulted to YES for convenience.  Set this option explicitly to avoid issues.
+* mp_getddl.sas to be renamed to mp_ds2ddl.sas (consistent with other ds2xxx macros).  A wrapper macro is already in place, and you are able to use this immediately.  The default for SHOWLOG will also be YES instead of NO.
 
 ## Star Gazing
 

all.sas

@@ -3533,6 +3533,12 @@ run;
     make use of permanent tables.  To avoid duplication in definitions, this
     macro provides a central location for managing the corresponding DDL.
 
+  Note - this macro is likely to be deprecated in future in favour of a
+  dedicated "datamodel" folder (prefix mddl)
+
+  Any corresponding data would go in a seperate repo, to avoid this one
+  ballooning in size!
+
   Example usage:
 
       %mp_coretable(LOCKTABLE,libds=work.locktable)
@@ -4888,6 +4894,35 @@ data _null_;
 run;
 
 %mend mp_ds2csv;/**
+  @file
+  @brief A wrapper for mp_getddl.sas
+  @details In the next release, this will be the main version.
+
+
+  <h4> SAS Macros </h4>
+  @li mp_getddl.sas
+
+**/
+
+%macro mp_ds2ddl(libds,fref=getddl,flavour=SAS,showlog=YES,schema=
+  ,applydttm=NO
+)/*/STORE SOURCE*/;
+
+%local libref;
+%let libds=%upcase(&libds);
+%let libref=%scan(&libds,1,.);
+%if &libref=&libds %then %let libds=WORK.&libds;
+
+%mp_getddl(%scan(&libds,1,.)
+  ,%scan(&libds,2,.)
+  ,fref=&fref
+  ,flavour=SAS
+  ,showlog=&showlog
+  ,schema=&schema
+  ,applydttm=&applydttm
+)
+
+%mend mp_ds2ddl;/**
   @file
   @brief Converts every value in a dataset to formatted value
   @details Converts every value to it's formatted value.  All variables will
@@ -5394,7 +5429,8 @@ options varlenchk=&optval;
   SYSCC to 1008 if bad records are found, and call mp_abort.sas for a
   graceful service exit (configurable).
 
-  Used for dynamic filtering in [Data Controller for SAS&reg;](https://datacontroller.io).
+  Used for dynamic filtering in [Data Controller for SAS&reg;](
+  https://datacontroller.io).
 
   Usage:
 
@@ -5513,7 +5549,7 @@ data &outds;
     output;
   end;
   if OPERATOR_NM not in
-  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
+  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE')
   then do;
     REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
@@ -5701,8 +5737,13 @@ filename &outref temp;
   https://sasapps.io)).  This macro is also used in [Data Controller for SAS](
   https://datacontroller.io).
 
+  A more recent feature of this macro is the ability to support filter queries
+  on Format Catalogs.  This is achieved by adding a `-FC` suffix to the `libds`
+  parameter - where the "ds" in this case is the catalog name.
 
-  @param [in] libds= The target dataset to be filtered (lib should be assigned)
+
+  @param [in] libds= The target dataset to be filtered (lib should be assigned).
+    If filtering a format catalog, add the following suffix:  `-FC`.
   @param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to
     be validated.  Has the following format:
 |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
@@ -5771,7 +5812,10 @@ filename &outref temp;
 %put &sysmacroname entry vars:;
 %put _local_;
 
-%local ds1 ds2 ds3 ds4 filter_hash;
+%local ds0 ds1 ds2 ds3 ds4 filter_hash orig_libds;
+%let libds=%upcase(&libds);
+%let orig_libds=&libds;
+
 %mp_abort(iftrue= (&syscc ne 0)
   ,mac=mp_filterstore
   ,msg=%str(syscc=&syscc on macro entry)
@@ -5789,12 +5833,49 @@ filename &outref temp;
   ,msg=%str(Invalid lock_table value: &lock_table)
 )
 
-/* validate query */
+/**
+  * validate query
+  * use format catalog export, if a format
+  */
+%if "%substr(&libds,%length(&libds)-2,3)"="-FC" %then %do;
+  %let libds=%scan(&libds,1,-); /* chop off -FC extension */
+  %let ds0=%mf_getuniquename(prefix=fmtds_);
+  /*
+    There is no need to export the entire format catalog here - the validations
+    are done against the data model, not the data values.  So we can simply
+    hardcode the structure based on the cntlout dataset.
+  */
+  proc sql;
+  create table &ds0(
+    FMTNAME char(32)
+    ,START char(16)
+    ,END char(16)
+    ,LABEL char(256)
+    ,MIN num length=3
+    ,MAX num length=3
+    ,DEFAULT num length=3
+    ,LENGTH num length=3
+    ,FUZZ num
+    ,PREFIX char(2)
+    ,MULT num
+    ,FILL char(1)
+    ,NOEDIT num length=3
+    ,TYPE char(1)
+    ,SEXCL char(1)
+    ,EEXCL char(1)
+    ,HLO char(13)
+    ,DECSEP char(1)
+    ,DIG3SEP char(1)
+    ,DATATYPE char(8)
+    ,LANGUAGE char(8)
+  );
+  %let libds=&ds0;
+%end;
 %mp_filtercheck(&queryds,targetds=&libds,abort=YES)
 
 /* hash the result */
 %let ds1=%mf_getuniquename(prefix=hashds);
-%mp_hashdataset(&queryds,outds=&ds1,salt=&libds)
+%mp_hashdataset(&queryds,outds=&ds1,salt=&orig_libds)
 %let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
 %if &mdebug=1 %then %do;
   data _null_;
@@ -5825,7 +5906,7 @@ run;
   %let ds3=%mf_getuniquename(prefix=filtersum);
   data work.&ds3;
     if 0 then set &filter_summary;
-    filter_table=symget('libds');
+    filter_table="&orig_libds";
     filter_hash="&filter_hash";
     PROCESSED_DTTM=%sysfunc(datetime());
     output;
@@ -7055,6 +7136,10 @@ create table &outsummary as
     proc format library=&&fmtloc&i CNTLOUT=&tempds;
       select &&fmtname&i;
     run;
+    data &tempds;
+      length label $256;
+      set &tempds;
+    run;
     proc append base=&outdetail data=&tempds;
     run;
   %end;

base/mp_coretable.sas

@@ -5,6 +5,12 @@
     make use of permanent tables.  To avoid duplication in definitions, this
     macro provides a central location for managing the corresponding DDL.
 
+  Note - this macro is likely to be deprecated in future in favour of a
+  dedicated "datamodel" folder (prefix mddl)
+
+  Any corresponding data would go in a seperate repo, to avoid this one
+  ballooning in size!
+
   Example usage:
 
       %mp_coretable(LOCKTABLE,libds=work.locktable)

base/mp_ds2ddl.sas

@@ -0,0 +1,30 @@
+/**
+  @file
+  @brief A wrapper for mp_getddl.sas
+  @details In the next release, this will be the main version.
+
+
+  <h4> SAS Macros </h4>
+  @li mp_getddl.sas
+
+**/
+
+%macro mp_ds2ddl(libds,fref=getddl,flavour=SAS,showlog=YES,schema=
+  ,applydttm=NO
+)/*/STORE SOURCE*/;
+
+%local libref;
+%let libds=%upcase(&libds);
+%let libref=%scan(&libds,1,.);
+%if &libref=&libds %then %let libds=WORK.&libds;
+
+%mp_getddl(%scan(&libds,1,.)
+  ,%scan(&libds,2,.)
+  ,fref=&fref
+  ,flavour=SAS
+  ,showlog=&showlog
+  ,schema=&schema
+  ,applydttm=&applydttm
+)
+
+%mend mp_ds2ddl;

base/mp_filtercheck.sas

@@ -6,7 +6,8 @@
   SYSCC to 1008 if bad records are found, and call mp_abort.sas for a
   graceful service exit (configurable).
 
-  Used for dynamic filtering in [Data Controller for SAS®](https://datacontroller.io).
+  Used for dynamic filtering in [Data Controller for SAS®](
+  https://datacontroller.io).
 
   Usage:
 
@@ -125,7 +126,7 @@ data &outds;
     output;
   end;
   if OPERATOR_NM not in
-  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
+  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE')
   then do;
     REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;

base/mp_filterstore.sas

@@ -8,8 +8,13 @@
   https://sasapps.io)).  This macro is also used in [Data Controller for SAS](
   https://datacontroller.io).
 
+  A more recent feature of this macro is the ability to support filter queries
+  on Format Catalogs.  This is achieved by adding a `-FC` suffix to the `libds`
+  parameter - where the "ds" in this case is the catalog name.
 
-  @param [in] libds= The target dataset to be filtered (lib should be assigned)
+
+  @param [in] libds= The target dataset to be filtered (lib should be assigned).
+    If filtering a format catalog, add the following suffix:  `-FC`.
   @param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to
     be validated.  Has the following format:
 |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
@@ -78,7 +83,10 @@
 %put &sysmacroname entry vars:;
 %put _local_;
 
-%local ds1 ds2 ds3 ds4 filter_hash;
+%local ds0 ds1 ds2 ds3 ds4 filter_hash orig_libds;
+%let libds=%upcase(&libds);
+%let orig_libds=&libds;
+
 %mp_abort(iftrue= (&syscc ne 0)
   ,mac=mp_filterstore
   ,msg=%str(syscc=&syscc on macro entry)
@@ -96,12 +104,49 @@
   ,msg=%str(Invalid lock_table value: &lock_table)
 )
 
-/* validate query */
+/**
+  * validate query
+  * use format catalog export, if a format
+  */
+%if "%substr(&libds,%length(&libds)-2,3)"="-FC" %then %do;
+  %let libds=%scan(&libds,1,-); /* chop off -FC extension */
+  %let ds0=%mf_getuniquename(prefix=fmtds_);
+  /*
+    There is no need to export the entire format catalog here - the validations
+    are done against the data model, not the data values.  So we can simply
+    hardcode the structure based on the cntlout dataset.
+  */
+  proc sql;
+  create table &ds0(
+    FMTNAME char(32)
+    ,START char(16)
+    ,END char(16)
+    ,LABEL char(256)
+    ,MIN num length=3
+    ,MAX num length=3
+    ,DEFAULT num length=3
+    ,LENGTH num length=3
+    ,FUZZ num
+    ,PREFIX char(2)
+    ,MULT num
+    ,FILL char(1)
+    ,NOEDIT num length=3
+    ,TYPE char(1)
+    ,SEXCL char(1)
+    ,EEXCL char(1)
+    ,HLO char(13)
+    ,DECSEP char(1)
+    ,DIG3SEP char(1)
+    ,DATATYPE char(8)
+    ,LANGUAGE char(8)
+  );
+  %let libds=&ds0;
+%end;
 %mp_filtercheck(&queryds,targetds=&libds,abort=YES)
 
 /* hash the result */
 %let ds1=%mf_getuniquename(prefix=hashds);
-%mp_hashdataset(&queryds,outds=&ds1,salt=&libds)
+%mp_hashdataset(&queryds,outds=&ds1,salt=&orig_libds)
 %let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
 %if &mdebug=1 %then %do;
   data _null_;
@@ -132,7 +177,7 @@ run;
   %let ds3=%mf_getuniquename(prefix=filtersum);
   data work.&ds3;
     if 0 then set &filter_summary;
-    filter_table=symget('libds');
+    filter_table="&orig_libds";
     filter_hash="&filter_hash";
     PROCESSED_DTTM=%sysfunc(datetime());
     output;

base/mp_getformats.sas

@@ -134,6 +134,10 @@ create table &outsummary as
     proc format library=&&fmtloc&i CNTLOUT=&tempds;
       select &&fmtname&i;
     run;
+    data &tempds;
+      length label $256;
+      set &tempds;
+    run;
     proc append base=&outdetail data=&tempds;
     run;
   %end;

package-lock.json

@@ -8,7 +8,7 @@
       "license": "MIT",
       "devDependencies": {
         "@sasjs/cli": "3.6.0",
-        "@sasjs/core": "4.4.4"
+        "@sasjs/core": "4.4.7"
       }
     },
     "node_modules/@sasjs/adapter": {
@@ -110,9 +110,9 @@
       }
     },
     "node_modules/@sasjs/core": {
-      "version": "4.4.4",
+      "version": "4.4.7",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.4.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.7.tgz",
-      "integrity": "sha512-gN6d0fvhaofp7buemS5KIOo5Bz8lbqhsEQD7SuH5FZ02MQurmfu7A0Zg0lIEi0w2/ptI4M/sZdF4D2DRh1D5xA==",
+      "integrity": "sha512-9+HmwspvWu/gH9ElnmRaGdjOCspOidBRUYUfxLzgOy1Ya1JMZ2RErMklCAMg7XI0Us5jecd2FXdo8zlQDhRkWQ==",
       "dev": true
     },
     "node_modules/@sasjs/lint": {
@@ -2830,9 +2830,9 @@
       }
     },
     "@sasjs/core": {
-      "version": "4.4.4",
+      "version": "4.4.7",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.4.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.7.tgz",
-      "integrity": "sha512-gN6d0fvhaofp7buemS5KIOo5Bz8lbqhsEQD7SuH5FZ02MQurmfu7A0Zg0lIEi0w2/ptI4M/sZdF4D2DRh1D5xA==",
+      "integrity": "sha512-9+HmwspvWu/gH9ElnmRaGdjOCspOidBRUYUfxLzgOy1Ya1JMZ2RErMklCAMg7XI0Us5jecd2FXdo8zlQDhRkWQ==",
       "dev": true
     },
     "@sasjs/lint": {

package.json

@@ -34,6 +34,6 @@
   },
   "devDependencies": {
     "@sasjs/cli": "3.6.0",
-    "@sasjs/core": "4.4.4"
+    "@sasjs/core": "4.4.7"
   }
 }

tests/crossplatform/mp_filterstore.test.2.sas

@@ -0,0 +1,112 @@
+/**
+  @file
+  @brief Testing mp_filterstore macro with a format catalog
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
+  @li mp_coretable.sas
+  @li mp_filterstore.sas
+
+**/
+
+libname permlib (work);
+
+%mp_coretable(LOCKTABLE,libds=permlib.locktable)
+%mp_coretable(FILTER_SUMMARY,libds=permlib.filtsum)
+%mp_coretable(FILTER_DETAIL,libds=permlib.filtdet)
+%mp_coretable(MAXKEYTABLE,libds=permlib.maxkey)
+
+/* valid filter */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$12. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,Start,>,"'2'"
+AND,AND,1,Fmtname,=,"'MORDOR'"
+OR,OR,2,Label,IN,"('Dragon1','Dragon2')"
+OR,OR,2,End,=,"'6'"
+OR,OR,2,Start,GE,"'10'"
+;;;;
+run;
+
+/* make some formats */
+PROC FORMAT library=permlib.testfmts;
+  picture MyMSdt other='%0Y-%0m-%0dT%0H:%0M:%0S' (datatype=datetime);
+RUN;
+data work.fmts;
+  length fmtname $32;
+  do fmtname='SMAUG','MORDOR','GOLLUM';
+    do start=1 to 10;
+      label= cats('Dragon ',start);
+      output;
+    end;
+  end;
+run;
+proc sort data=work.fmts nodupkey;
+  by fmtname start;
+run;
+proc format cntlin=work.fmts library=permlib.testfmts;
+run;
+proc format library=permlib.testfmts;
+  invalue indays (default=13) other=42;
+run;
+
+
+%mp_assertscope(SNAPSHOT)
+%mp_filterstore(libds=permlib.testfmts-fc,
+  queryds=work.inds,
+  filter_summary=permlib.filtsum,
+  filter_detail=permlib.filtdet,
+  lock_table=permlib.locktable,
+  maxkeytable=permlib.maxkey,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+)
+%mp_assertscope(COMPARE)
+
+%mp_assert(iftrue=(&syscc=0),
+  desc=Ensure macro runs without errors,
+  outds=work.test_results
+)
+/* ensure only one record created */
+%mp_assertdsobs(permlib.filtsum,
+  desc=Initial query,
+  test=ATMOST 1,
+  outds=work.test_results
+)
+/* check RK is correct */
+proc sql noprint;
+select max(filter_rk) into: test1 from work.result;
+%mp_assert(iftrue=(&test1=1),
+  desc=Ensure filter rk is correct,
+  outds=work.test_results
+)
+
+/* Test 2 - load same table again and ensure we get the same RK */
+%mp_filterstore(libds=permlib.testfmts-fc,
+  queryds=work.inds,
+  filter_summary=permlib.filtsum,
+  filter_detail=permlib.filtdet,
+  lock_table=permlib.locktable,
+  maxkeytable=permlib.maxkey,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+)
+/* ensure only one record created */
+%mp_assertdsobs(permlib.filtsum,
+  desc=Initial query - same obs,
+  test=ATMOST 1,
+  outds=work.test_results
+)
+/* check RK is correct */
+proc sql noprint;
+select max(filter_rk) into: test2 from work.result;
+%mp_assert(iftrue=(&test2=1),
+  desc=Ensure filter rk is correct for second run,
+  outds=work.test_results
+)