From 8b0acf2eae66ce532fbfcb26d8196555bb1aaede Mon Sep 17 00:00:00 2001 From: munja Date: Sun, 6 Feb 2022 22:12:00 +0100 Subject: [PATCH] feat: adding format catalog capability to mp_filterstore --- README.md | 5 +- all.sas | 116 ++++++++++++++++-- base/mp_coretable.sas | 27 ++++ base/mp_ds2ddl.sas | 30 +++++ base/mp_filtercheck.sas | 5 +- base/mp_filterstore.sas | 55 ++++++++- package-lock.json | 14 +-- package.json | 2 +- ...ore.test.sas => mp_filterstore.test.1.sas} | 0 tests/crossplatform/mp_filterstore.test.2.sas | 112 +++++++++++++++++ 10 files changed, 342 insertions(+), 24 deletions(-) create mode 100644 base/mp_ds2ddl.sas rename tests/crossplatform/{mp_filterstore.test.sas => mp_filterstore.test.1.sas} (100%) create mode 100644 tests/crossplatform/mp_filterstore.test.2.sas diff --git a/README.md b/README.md index a51ff0f..42d1e30 100644 --- a/README.md +++ b/README.md @@ -192,11 +192,12 @@ When contributing to this library, it is therefore important to ensure that all ## Breaking Changes -We are currently on major release v4. Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag. The following changes are planned when the next major (breaking) release becomes necessary: +We are currently on major release v4. Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag. The following changes are planned when the next major/breaking release (v5) becomes necessary: * mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything) * `insert_cmplib` option of mcf_xxx macros will be deprecated (the option is now checked automatically with value inserted only if needed) -* mcf_xxx macros to have `wrap=` option defaulted to YES for convenience +* mcf_xxx macros to have `wrap=` option defaulted to YES for convenience. Set this option explicitly to avoid issues. +* mp_getddl.sas to be renamed to mp_ds2ddl.sas (consistent with other ds2xxx macros). A wrapper macro is already in place, and you are able to use this immediately. The default for SHOWLOG will also be YES instead of NO. ## Star Gazing diff --git a/all.sas b/all.sas index d56d0e9..eb2c07b 100644 --- a/all.sas +++ b/all.sas @@ -3538,6 +3538,8 @@ run; %mp_coretable(LOCKTABLE,libds=work.locktable) @param [in] table_ref The type of table to create. Example values: + @li CNTLOUT_DS - Mimics the structure of the table produced by `proc format` + with the `cntlout=` option. @li DIFFTABLE - Used to store changes to tables. Used by mp_storediffs.sas and mp_stackdiffs.sas @li FILTER_DETAIL - For storing detailed filter values. Used by @@ -3568,6 +3570,31 @@ run; %local outds ; %let outds=%sysfunc(ifc(&libds=0,_data_,&libds)); proc sql; +%if &table_ref=CNTLOUT_DS %then %do; + create table &outds( + FMTNAME char(32) label='Format name' + ,START char(16) label='Starting value for format' + ,END char(16) label='Ending value for format' + ,LABEL char(23) label='Format value label' + ,MIN num length=3 label='Minimum length' + ,MAX num length=3 label='Maximum length' + ,DEFAULT num length=3 label='Default length' + ,LENGTH num length=3 label='Format length' + ,FUZZ num label='Fuzz value' + ,PREFIX char(2) label='Prefix characters' + ,MULT num label='Multiplier' + ,FILL char(1) label='Fill character' + ,NOEDIT num length=3 label='Is picture string noedit?' + ,TYPE char(1) label='Type of format' + ,SEXCL char(1) label='Start exclusion' + ,EEXCL char(1) label='End exclusion' + ,HLO char(13) label='Additional information' + ,DECSEP char(1) label='Decimal separator' + ,DIG3SEP char(1) label='Three-digit separator' + ,DATATYPE char(8) label='Date/time/datetime?' + ,LANGUAGE char(8) label='Language for date strings' + ); +%end; %if &table_ref=DIFFTABLE %then %do; create table &outds( load_ref char(36) label='unique load reference', @@ -4888,6 +4915,35 @@ data _null_; run; %mend mp_ds2csv;/** + @file + @brief A wrapper for mp_getddl.sas + @details In the next release, this will be the main version. + + +

SAS Macros

+ @li mp_getddl.sas + +**/ + +%macro mp_ds2ddl(libds,fref=getddl,flavour=SAS,showlog=YES,schema= + ,applydttm=NO +)/*/STORE SOURCE*/; + +%local libref; +%let libds=%upcase(&libds); +%let libref=%scan(&libds,1,.); +%if &libref=&libds %then %let libds=WORK.&libds; + +%mp_getddl(%scan(&libds,1,.) + ,%scan(&libds,2,.) + ,fref=&fref + ,flavour=SAS + ,showlog=&showlog + ,schema=&schema + ,applydttm=&applydttm +) + +%mend mp_ds2ddl;/** @file @brief Converts every value in a dataset to formatted value @details Converts every value to it's formatted value. All variables will @@ -5394,7 +5450,8 @@ options varlenchk=&optval; SYSCC to 1008 if bad records are found, and call mp_abort.sas for a graceful service exit (configurable). - Used for dynamic filtering in [Data Controller for SAS®](https://datacontroller.io). + Used for dynamic filtering in [Data Controller for SAS®]( + https://datacontroller.io). Usage: @@ -5513,7 +5570,7 @@ data &outds; output; end; if OPERATOR_NM not in - ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS') + ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE') then do; REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM); putlog REASON_CD= OPERATOR_NM=; @@ -5701,8 +5758,13 @@ filename &outref temp; https://sasapps.io)). This macro is also used in [Data Controller for SAS]( https://datacontroller.io). + A more recent feature of this macro is the ability to support filter queries + on Format Catalogs. This is achieved by adding a `-FC` suffix to the `libds` + parameter - where the "ds" in this case is the catalog name. - @param [in] libds= The target dataset to be filtered (lib should be assigned) + + @param [in] libds= The target dataset to be filtered (lib should be assigned). + If filtering a format catalog, add the following suffix: `-FC`. @param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to be validated. Has the following format: |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767| @@ -5771,7 +5833,10 @@ filename &outref temp; %put &sysmacroname entry vars:; %put _local_; -%local ds1 ds2 ds3 ds4 filter_hash; +%local ds0 ds1 ds2 ds3 ds4 filter_hash orig_libds; +%let libds=%upcase(&libds); +%let orig_libds=&libds; + %mp_abort(iftrue= (&syscc ne 0) ,mac=mp_filterstore ,msg=%str(syscc=&syscc on macro entry) @@ -5789,12 +5854,49 @@ filename &outref temp; ,msg=%str(Invalid lock_table value: &lock_table) ) -/* validate query */ +/** + * validate query + * use format catalog export, if a format + */ +%if "%substr(&libds,%length(&libds)-2,3)"="-FC" %then %do; + %let libds=%scan(&libds,1,-); /* chop off -FC extension */ + %let ds0=%mf_getuniquename(prefix=fmtds_); + /* + There is no need to export the entire format catalog here - the validations + are done against the data model, not the data values. So we can simply + hardcode the structure based on the cntlout dataset. + */ + proc sql; + create table &ds0( + FMTNAME char(32) + ,START char(16) + ,END char(16) + ,LABEL char(23) + ,MIN num length=3 + ,MAX num length=3 + ,DEFAULT num length=3 + ,LENGTH num length=3 + ,FUZZ num + ,PREFIX char(2) + ,MULT num + ,FILL char(1) + ,NOEDIT num length=3 + ,TYPE char(1) + ,SEXCL char(1) + ,EEXCL char(1) + ,HLO char(13) + ,DECSEP char(1) + ,DIG3SEP char(1) + ,DATATYPE char(8) + ,LANGUAGE char(8) + ); + %let libds=&ds0; +%end; %mp_filtercheck(&queryds,targetds=&libds,abort=YES) /* hash the result */ %let ds1=%mf_getuniquename(prefix=hashds); -%mp_hashdataset(&queryds,outds=&ds1,salt=&libds) +%mp_hashdataset(&queryds,outds=&ds1,salt=&orig_libds) %let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey)); %if &mdebug=1 %then %do; data _null_; @@ -5825,7 +5927,7 @@ run; %let ds3=%mf_getuniquename(prefix=filtersum); data work.&ds3; if 0 then set &filter_summary; - filter_table=symget('libds'); + filter_table="&orig_libds"; filter_hash="&filter_hash"; PROCESSED_DTTM=%sysfunc(datetime()); output; diff --git a/base/mp_coretable.sas b/base/mp_coretable.sas index 1de4c43..0aab97f 100644 --- a/base/mp_coretable.sas +++ b/base/mp_coretable.sas @@ -10,6 +10,8 @@ %mp_coretable(LOCKTABLE,libds=work.locktable) @param [in] table_ref The type of table to create. Example values: + @li CNTLOUT_DS - Mimics the structure of the table produced by `proc format` + with the `cntlout=` option. @li DIFFTABLE - Used to store changes to tables. Used by mp_storediffs.sas and mp_stackdiffs.sas @li FILTER_DETAIL - For storing detailed filter values. Used by @@ -40,6 +42,31 @@ %local outds ; %let outds=%sysfunc(ifc(&libds=0,_data_,&libds)); proc sql; +%if &table_ref=CNTLOUT_DS %then %do; + create table &outds( + FMTNAME char(32) label='Format name' + ,START char(16) label='Starting value for format' + ,END char(16) label='Ending value for format' + ,LABEL char(23) label='Format value label' + ,MIN num length=3 label='Minimum length' + ,MAX num length=3 label='Maximum length' + ,DEFAULT num length=3 label='Default length' + ,LENGTH num length=3 label='Format length' + ,FUZZ num label='Fuzz value' + ,PREFIX char(2) label='Prefix characters' + ,MULT num label='Multiplier' + ,FILL char(1) label='Fill character' + ,NOEDIT num length=3 label='Is picture string noedit?' + ,TYPE char(1) label='Type of format' + ,SEXCL char(1) label='Start exclusion' + ,EEXCL char(1) label='End exclusion' + ,HLO char(13) label='Additional information' + ,DECSEP char(1) label='Decimal separator' + ,DIG3SEP char(1) label='Three-digit separator' + ,DATATYPE char(8) label='Date/time/datetime?' + ,LANGUAGE char(8) label='Language for date strings' + ); +%end; %if &table_ref=DIFFTABLE %then %do; create table &outds( load_ref char(36) label='unique load reference', diff --git a/base/mp_ds2ddl.sas b/base/mp_ds2ddl.sas new file mode 100644 index 0000000..7c30d86 --- /dev/null +++ b/base/mp_ds2ddl.sas @@ -0,0 +1,30 @@ +/** + @file + @brief A wrapper for mp_getddl.sas + @details In the next release, this will be the main version. + + +

SAS Macros

+ @li mp_getddl.sas + +**/ + +%macro mp_ds2ddl(libds,fref=getddl,flavour=SAS,showlog=YES,schema= + ,applydttm=NO +)/*/STORE SOURCE*/; + +%local libref; +%let libds=%upcase(&libds); +%let libref=%scan(&libds,1,.); +%if &libref=&libds %then %let libds=WORK.&libds; + +%mp_getddl(%scan(&libds,1,.) + ,%scan(&libds,2,.) + ,fref=&fref + ,flavour=SAS + ,showlog=&showlog + ,schema=&schema + ,applydttm=&applydttm +) + +%mend mp_ds2ddl; \ No newline at end of file diff --git a/base/mp_filtercheck.sas b/base/mp_filtercheck.sas index 7159dee..94e1120 100644 --- a/base/mp_filtercheck.sas +++ b/base/mp_filtercheck.sas @@ -6,7 +6,8 @@ SYSCC to 1008 if bad records are found, and call mp_abort.sas for a graceful service exit (configurable). - Used for dynamic filtering in [Data Controller for SAS®](https://datacontroller.io). + Used for dynamic filtering in [Data Controller for SAS®]( + https://datacontroller.io). Usage: @@ -125,7 +126,7 @@ data &outds; output; end; if OPERATOR_NM not in - ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS') + ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE') then do; REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM); putlog REASON_CD= OPERATOR_NM=; diff --git a/base/mp_filterstore.sas b/base/mp_filterstore.sas index d9ad9a9..f0ff2e5 100644 --- a/base/mp_filterstore.sas +++ b/base/mp_filterstore.sas @@ -8,8 +8,13 @@ https://sasapps.io)). This macro is also used in [Data Controller for SAS]( https://datacontroller.io). + A more recent feature of this macro is the ability to support filter queries + on Format Catalogs. This is achieved by adding a `-FC` suffix to the `libds` + parameter - where the "ds" in this case is the catalog name. - @param [in] libds= The target dataset to be filtered (lib should be assigned) + + @param [in] libds= The target dataset to be filtered (lib should be assigned). + If filtering a format catalog, add the following suffix: `-FC`. @param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to be validated. Has the following format: |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767| @@ -78,7 +83,10 @@ %put &sysmacroname entry vars:; %put _local_; -%local ds1 ds2 ds3 ds4 filter_hash; +%local ds0 ds1 ds2 ds3 ds4 filter_hash orig_libds; +%let libds=%upcase(&libds); +%let orig_libds=&libds; + %mp_abort(iftrue= (&syscc ne 0) ,mac=mp_filterstore ,msg=%str(syscc=&syscc on macro entry) @@ -96,12 +104,49 @@ ,msg=%str(Invalid lock_table value: &lock_table) ) -/* validate query */ +/** + * validate query + * use format catalog export, if a format + */ +%if "%substr(&libds,%length(&libds)-2,3)"="-FC" %then %do; + %let libds=%scan(&libds,1,-); /* chop off -FC extension */ + %let ds0=%mf_getuniquename(prefix=fmtds_); + /* + There is no need to export the entire format catalog here - the validations + are done against the data model, not the data values. So we can simply + hardcode the structure based on the cntlout dataset. + */ + proc sql; + create table &ds0( + FMTNAME char(32) + ,START char(16) + ,END char(16) + ,LABEL char(23) + ,MIN num length=3 + ,MAX num length=3 + ,DEFAULT num length=3 + ,LENGTH num length=3 + ,FUZZ num + ,PREFIX char(2) + ,MULT num + ,FILL char(1) + ,NOEDIT num length=3 + ,TYPE char(1) + ,SEXCL char(1) + ,EEXCL char(1) + ,HLO char(13) + ,DECSEP char(1) + ,DIG3SEP char(1) + ,DATATYPE char(8) + ,LANGUAGE char(8) + ); + %let libds=&ds0; +%end; %mp_filtercheck(&queryds,targetds=&libds,abort=YES) /* hash the result */ %let ds1=%mf_getuniquename(prefix=hashds); -%mp_hashdataset(&queryds,outds=&ds1,salt=&libds) +%mp_hashdataset(&queryds,outds=&ds1,salt=&orig_libds) %let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey)); %if &mdebug=1 %then %do; data _null_; @@ -132,7 +177,7 @@ run; %let ds3=%mf_getuniquename(prefix=filtersum); data work.&ds3; if 0 then set &filter_summary; - filter_table=symget('libds'); + filter_table="&orig_libds"; filter_hash="&filter_hash"; PROCESSED_DTTM=%sysfunc(datetime()); output; diff --git a/package-lock.json b/package-lock.json index caa5b92..0499f04 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,7 @@ "license": "MIT", "devDependencies": { "@sasjs/cli": "3.6.0", - "@sasjs/core": "4.4.4" + "@sasjs/core": "4.4.7" } }, "node_modules/@sasjs/adapter": { @@ -110,9 +110,9 @@ } }, "node_modules/@sasjs/core": { - "version": "4.4.4", - "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.4.tgz", - "integrity": "sha512-gN6d0fvhaofp7buemS5KIOo5Bz8lbqhsEQD7SuH5FZ02MQurmfu7A0Zg0lIEi0w2/ptI4M/sZdF4D2DRh1D5xA==", + "version": "4.4.7", + "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.7.tgz", + "integrity": "sha512-9+HmwspvWu/gH9ElnmRaGdjOCspOidBRUYUfxLzgOy1Ya1JMZ2RErMklCAMg7XI0Us5jecd2FXdo8zlQDhRkWQ==", "dev": true }, "node_modules/@sasjs/lint": { @@ -2830,9 +2830,9 @@ } }, "@sasjs/core": { - "version": "4.4.4", - "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.4.tgz", - "integrity": "sha512-gN6d0fvhaofp7buemS5KIOo5Bz8lbqhsEQD7SuH5FZ02MQurmfu7A0Zg0lIEi0w2/ptI4M/sZdF4D2DRh1D5xA==", + "version": "4.4.7", + "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.7.tgz", + "integrity": "sha512-9+HmwspvWu/gH9ElnmRaGdjOCspOidBRUYUfxLzgOy1Ya1JMZ2RErMklCAMg7XI0Us5jecd2FXdo8zlQDhRkWQ==", "dev": true }, "@sasjs/lint": { diff --git a/package.json b/package.json index 56b831a..557aa64 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,6 @@ }, "devDependencies": { "@sasjs/cli": "3.6.0", - "@sasjs/core": "4.4.4" + "@sasjs/core": "4.4.7" } } diff --git a/tests/crossplatform/mp_filterstore.test.sas b/tests/crossplatform/mp_filterstore.test.1.sas similarity index 100% rename from tests/crossplatform/mp_filterstore.test.sas rename to tests/crossplatform/mp_filterstore.test.1.sas diff --git a/tests/crossplatform/mp_filterstore.test.2.sas b/tests/crossplatform/mp_filterstore.test.2.sas new file mode 100644 index 0000000..21b293a --- /dev/null +++ b/tests/crossplatform/mp_filterstore.test.2.sas @@ -0,0 +1,112 @@ +/** + @file + @brief Testing mp_filterstore macro with a format catalog + +

SAS Macros

+ @li mp_assert.sas + @li mp_assertdsobs.sas + @li mp_assertscope.sas + @li mp_coretable.sas + @li mp_filterstore.sas + +**/ + +libname permlib (work); + +%mp_coretable(LOCKTABLE,libds=permlib.locktable) +%mp_coretable(FILTER_SUMMARY,libds=permlib.filtsum) +%mp_coretable(FILTER_DETAIL,libds=permlib.filtdet) +%mp_coretable(MAXKEYTABLE,libds=permlib.maxkey) + +/* valid filter */ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$12. RAW_VALUE:$4000.; +datalines4; +AND,AND,1,Start,>,"'2'" +AND,AND,1,Fmtname,=,"'MORDOR'" +OR,OR,2,Label,IN,"('Dragon1','Dragon2')" +OR,OR,2,End,=,"'6'" +OR,OR,2,Start,GE,"'10'" +;;;; +run; + +/* make some formats */ +PROC FORMAT library=permlib.testfmts; + picture MyMSdt other='%0Y-%0m-%0dT%0H:%0M:%0S' (datatype=datetime); +RUN; +data work.fmts; + length fmtname $32; + do fmtname='SMAUG','MORDOR','GOLLUM'; + do start=1 to 10; + label= cats('Dragon ',start); + output; + end; + end; +run; +proc sort data=work.fmts nodupkey; + by fmtname start; +run; +proc format cntlin=work.fmts library=permlib.testfmts; +run; +proc format library=permlib.testfmts; + invalue indays (default=13) other=42; +run; + + +%mp_assertscope(SNAPSHOT) +%mp_filterstore(libds=permlib.testfmts-fc, + queryds=work.inds, + filter_summary=permlib.filtsum, + filter_detail=permlib.filtdet, + lock_table=permlib.locktable, + maxkeytable=permlib.maxkey, + outresult=work.result, + outquery=work.query, + mdebug=1 +) +%mp_assertscope(COMPARE) + +%mp_assert(iftrue=(&syscc=0), + desc=Ensure macro runs without errors, + outds=work.test_results +) +/* ensure only one record created */ +%mp_assertdsobs(permlib.filtsum, + desc=Initial query, + test=ATMOST 1, + outds=work.test_results +) +/* check RK is correct */ +proc sql noprint; +select max(filter_rk) into: test1 from work.result; +%mp_assert(iftrue=(&test1=1), + desc=Ensure filter rk is correct, + outds=work.test_results +) + +/* Test 2 - load same table again and ensure we get the same RK */ +%mp_filterstore(libds=permlib.testfmts-fc, + queryds=work.inds, + filter_summary=permlib.filtsum, + filter_detail=permlib.filtdet, + lock_table=permlib.locktable, + maxkeytable=permlib.maxkey, + outresult=work.result, + outquery=work.query, + mdebug=1 +) +/* ensure only one record created */ +%mp_assertdsobs(permlib.filtsum, + desc=Initial query - same obs, + test=ATMOST 1, + outds=work.test_results +) +/* check RK is correct */ +proc sql noprint; +select max(filter_rk) into: test2 from work.result; +%mp_assert(iftrue=(&test2=1), + desc=Ensure filter rk is correct for second run, + outds=work.test_results +)