diff --git a/README.md b/README.md
index a51ff0f..42d1e30 100644
--- a/README.md
+++ b/README.md
@@ -192,11 +192,12 @@ When contributing to this library, it is therefore important to ensure that all
## Breaking Changes
-We are currently on major release v4. Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag. The following changes are planned when the next major (breaking) release becomes necessary:
+We are currently on major release v4. Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag. The following changes are planned when the next major/breaking release (v5) becomes necessary:
* mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
* `insert_cmplib` option of mcf_xxx macros will be deprecated (the option is now checked automatically with value inserted only if needed)
-* mcf_xxx macros to have `wrap=` option defaulted to YES for convenience
+* mcf_xxx macros to have `wrap=` option defaulted to YES for convenience. Set this option explicitly to avoid issues.
+* mp_getddl.sas to be renamed to mp_ds2ddl.sas (consistent with other ds2xxx macros). A wrapper macro is already in place, and you are able to use this immediately. The default for SHOWLOG will also be YES instead of NO.
## Star Gazing
diff --git a/all.sas b/all.sas
index d56d0e9..eb2c07b 100644
--- a/all.sas
+++ b/all.sas
@@ -3538,6 +3538,8 @@ run;
%mp_coretable(LOCKTABLE,libds=work.locktable)
@param [in] table_ref The type of table to create. Example values:
+ @li CNTLOUT_DS - Mimics the structure of the table produced by `proc format`
+ with the `cntlout=` option.
@li DIFFTABLE - Used to store changes to tables. Used by mp_storediffs.sas
and mp_stackdiffs.sas
@li FILTER_DETAIL - For storing detailed filter values. Used by
@@ -3568,6 +3570,31 @@ run;
%local outds ;
%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
proc sql;
+%if &table_ref=CNTLOUT_DS %then %do;
+ create table &outds(
+ FMTNAME char(32) label='Format name'
+ ,START char(16) label='Starting value for format'
+ ,END char(16) label='Ending value for format'
+ ,LABEL char(23) label='Format value label'
+ ,MIN num length=3 label='Minimum length'
+ ,MAX num length=3 label='Maximum length'
+ ,DEFAULT num length=3 label='Default length'
+ ,LENGTH num length=3 label='Format length'
+ ,FUZZ num label='Fuzz value'
+ ,PREFIX char(2) label='Prefix characters'
+ ,MULT num label='Multiplier'
+ ,FILL char(1) label='Fill character'
+ ,NOEDIT num length=3 label='Is picture string noedit?'
+ ,TYPE char(1) label='Type of format'
+ ,SEXCL char(1) label='Start exclusion'
+ ,EEXCL char(1) label='End exclusion'
+ ,HLO char(13) label='Additional information'
+ ,DECSEP char(1) label='Decimal separator'
+ ,DIG3SEP char(1) label='Three-digit separator'
+ ,DATATYPE char(8) label='Date/time/datetime?'
+ ,LANGUAGE char(8) label='Language for date strings'
+ );
+%end;
%if &table_ref=DIFFTABLE %then %do;
create table &outds(
load_ref char(36) label='unique load reference',
@@ -4888,6 +4915,35 @@ data _null_;
run;
%mend mp_ds2csv;/**
+ @file
+ @brief A wrapper for mp_getddl.sas
+ @details In the next release, this will be the main version.
+
+
+
SAS Macros
+ @li mp_getddl.sas
+
+**/
+
+%macro mp_ds2ddl(libds,fref=getddl,flavour=SAS,showlog=YES,schema=
+ ,applydttm=NO
+)/*/STORE SOURCE*/;
+
+%local libref;
+%let libds=%upcase(&libds);
+%let libref=%scan(&libds,1,.);
+%if &libref=&libds %then %let libds=WORK.&libds;
+
+%mp_getddl(%scan(&libds,1,.)
+ ,%scan(&libds,2,.)
+ ,fref=&fref
+ ,flavour=SAS
+ ,showlog=&showlog
+ ,schema=&schema
+ ,applydttm=&applydttm
+)
+
+%mend mp_ds2ddl;/**
@file
@brief Converts every value in a dataset to formatted value
@details Converts every value to it's formatted value. All variables will
@@ -5394,7 +5450,8 @@ options varlenchk=&optval;
SYSCC to 1008 if bad records are found, and call mp_abort.sas for a
graceful service exit (configurable).
- Used for dynamic filtering in [Data Controller for SAS®](https://datacontroller.io).
+ Used for dynamic filtering in [Data Controller for SAS®](
+ https://datacontroller.io).
Usage:
@@ -5513,7 +5570,7 @@ data &outds;
output;
end;
if OPERATOR_NM not in
- ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
+ ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE')
then do;
REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
putlog REASON_CD= OPERATOR_NM=;
@@ -5701,8 +5758,13 @@ filename &outref temp;
https://sasapps.io)). This macro is also used in [Data Controller for SAS](
https://datacontroller.io).
+ A more recent feature of this macro is the ability to support filter queries
+ on Format Catalogs. This is achieved by adding a `-FC` suffix to the `libds`
+ parameter - where the "ds" in this case is the catalog name.
- @param [in] libds= The target dataset to be filtered (lib should be assigned)
+
+ @param [in] libds= The target dataset to be filtered (lib should be assigned).
+ If filtering a format catalog, add the following suffix: `-FC`.
@param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to
be validated. Has the following format:
|GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
@@ -5771,7 +5833,10 @@ filename &outref temp;
%put &sysmacroname entry vars:;
%put _local_;
-%local ds1 ds2 ds3 ds4 filter_hash;
+%local ds0 ds1 ds2 ds3 ds4 filter_hash orig_libds;
+%let libds=%upcase(&libds);
+%let orig_libds=&libds;
+
%mp_abort(iftrue= (&syscc ne 0)
,mac=mp_filterstore
,msg=%str(syscc=&syscc on macro entry)
@@ -5789,12 +5854,49 @@ filename &outref temp;
,msg=%str(Invalid lock_table value: &lock_table)
)
-/* validate query */
+/**
+ * validate query
+ * use format catalog export, if a format
+ */
+%if "%substr(&libds,%length(&libds)-2,3)"="-FC" %then %do;
+ %let libds=%scan(&libds,1,-); /* chop off -FC extension */
+ %let ds0=%mf_getuniquename(prefix=fmtds_);
+ /*
+ There is no need to export the entire format catalog here - the validations
+ are done against the data model, not the data values. So we can simply
+ hardcode the structure based on the cntlout dataset.
+ */
+ proc sql;
+ create table &ds0(
+ FMTNAME char(32)
+ ,START char(16)
+ ,END char(16)
+ ,LABEL char(23)
+ ,MIN num length=3
+ ,MAX num length=3
+ ,DEFAULT num length=3
+ ,LENGTH num length=3
+ ,FUZZ num
+ ,PREFIX char(2)
+ ,MULT num
+ ,FILL char(1)
+ ,NOEDIT num length=3
+ ,TYPE char(1)
+ ,SEXCL char(1)
+ ,EEXCL char(1)
+ ,HLO char(13)
+ ,DECSEP char(1)
+ ,DIG3SEP char(1)
+ ,DATATYPE char(8)
+ ,LANGUAGE char(8)
+ );
+ %let libds=&ds0;
+%end;
%mp_filtercheck(&queryds,targetds=&libds,abort=YES)
/* hash the result */
%let ds1=%mf_getuniquename(prefix=hashds);
-%mp_hashdataset(&queryds,outds=&ds1,salt=&libds)
+%mp_hashdataset(&queryds,outds=&ds1,salt=&orig_libds)
%let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
%if &mdebug=1 %then %do;
data _null_;
@@ -5825,7 +5927,7 @@ run;
%let ds3=%mf_getuniquename(prefix=filtersum);
data work.&ds3;
if 0 then set &filter_summary;
- filter_table=symget('libds');
+ filter_table="&orig_libds";
filter_hash="&filter_hash";
PROCESSED_DTTM=%sysfunc(datetime());
output;
diff --git a/base/mp_coretable.sas b/base/mp_coretable.sas
index 1de4c43..0aab97f 100644
--- a/base/mp_coretable.sas
+++ b/base/mp_coretable.sas
@@ -10,6 +10,8 @@
%mp_coretable(LOCKTABLE,libds=work.locktable)
@param [in] table_ref The type of table to create. Example values:
+ @li CNTLOUT_DS - Mimics the structure of the table produced by `proc format`
+ with the `cntlout=` option.
@li DIFFTABLE - Used to store changes to tables. Used by mp_storediffs.sas
and mp_stackdiffs.sas
@li FILTER_DETAIL - For storing detailed filter values. Used by
@@ -40,6 +42,31 @@
%local outds ;
%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
proc sql;
+%if &table_ref=CNTLOUT_DS %then %do;
+ create table &outds(
+ FMTNAME char(32) label='Format name'
+ ,START char(16) label='Starting value for format'
+ ,END char(16) label='Ending value for format'
+ ,LABEL char(23) label='Format value label'
+ ,MIN num length=3 label='Minimum length'
+ ,MAX num length=3 label='Maximum length'
+ ,DEFAULT num length=3 label='Default length'
+ ,LENGTH num length=3 label='Format length'
+ ,FUZZ num label='Fuzz value'
+ ,PREFIX char(2) label='Prefix characters'
+ ,MULT num label='Multiplier'
+ ,FILL char(1) label='Fill character'
+ ,NOEDIT num length=3 label='Is picture string noedit?'
+ ,TYPE char(1) label='Type of format'
+ ,SEXCL char(1) label='Start exclusion'
+ ,EEXCL char(1) label='End exclusion'
+ ,HLO char(13) label='Additional information'
+ ,DECSEP char(1) label='Decimal separator'
+ ,DIG3SEP char(1) label='Three-digit separator'
+ ,DATATYPE char(8) label='Date/time/datetime?'
+ ,LANGUAGE char(8) label='Language for date strings'
+ );
+%end;
%if &table_ref=DIFFTABLE %then %do;
create table &outds(
load_ref char(36) label='unique load reference',
diff --git a/base/mp_ds2ddl.sas b/base/mp_ds2ddl.sas
new file mode 100644
index 0000000..7c30d86
--- /dev/null
+++ b/base/mp_ds2ddl.sas
@@ -0,0 +1,30 @@
+/**
+ @file
+ @brief A wrapper for mp_getddl.sas
+ @details In the next release, this will be the main version.
+
+
+ SAS Macros
+ @li mp_getddl.sas
+
+**/
+
+%macro mp_ds2ddl(libds,fref=getddl,flavour=SAS,showlog=YES,schema=
+ ,applydttm=NO
+)/*/STORE SOURCE*/;
+
+%local libref;
+%let libds=%upcase(&libds);
+%let libref=%scan(&libds,1,.);
+%if &libref=&libds %then %let libds=WORK.&libds;
+
+%mp_getddl(%scan(&libds,1,.)
+ ,%scan(&libds,2,.)
+ ,fref=&fref
+ ,flavour=SAS
+ ,showlog=&showlog
+ ,schema=&schema
+ ,applydttm=&applydttm
+)
+
+%mend mp_ds2ddl;
\ No newline at end of file
diff --git a/base/mp_filtercheck.sas b/base/mp_filtercheck.sas
index 7159dee..94e1120 100644
--- a/base/mp_filtercheck.sas
+++ b/base/mp_filtercheck.sas
@@ -6,7 +6,8 @@
SYSCC to 1008 if bad records are found, and call mp_abort.sas for a
graceful service exit (configurable).
- Used for dynamic filtering in [Data Controller for SAS®](https://datacontroller.io).
+ Used for dynamic filtering in [Data Controller for SAS®](
+ https://datacontroller.io).
Usage:
@@ -125,7 +126,7 @@ data &outds;
output;
end;
if OPERATOR_NM not in
- ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
+ ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE')
then do;
REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
putlog REASON_CD= OPERATOR_NM=;
diff --git a/base/mp_filterstore.sas b/base/mp_filterstore.sas
index d9ad9a9..f0ff2e5 100644
--- a/base/mp_filterstore.sas
+++ b/base/mp_filterstore.sas
@@ -8,8 +8,13 @@
https://sasapps.io)). This macro is also used in [Data Controller for SAS](
https://datacontroller.io).
+ A more recent feature of this macro is the ability to support filter queries
+ on Format Catalogs. This is achieved by adding a `-FC` suffix to the `libds`
+ parameter - where the "ds" in this case is the catalog name.
- @param [in] libds= The target dataset to be filtered (lib should be assigned)
+
+ @param [in] libds= The target dataset to be filtered (lib should be assigned).
+ If filtering a format catalog, add the following suffix: `-FC`.
@param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to
be validated. Has the following format:
|GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
@@ -78,7 +83,10 @@
%put &sysmacroname entry vars:;
%put _local_;
-%local ds1 ds2 ds3 ds4 filter_hash;
+%local ds0 ds1 ds2 ds3 ds4 filter_hash orig_libds;
+%let libds=%upcase(&libds);
+%let orig_libds=&libds;
+
%mp_abort(iftrue= (&syscc ne 0)
,mac=mp_filterstore
,msg=%str(syscc=&syscc on macro entry)
@@ -96,12 +104,49 @@
,msg=%str(Invalid lock_table value: &lock_table)
)
-/* validate query */
+/**
+ * validate query
+ * use format catalog export, if a format
+ */
+%if "%substr(&libds,%length(&libds)-2,3)"="-FC" %then %do;
+ %let libds=%scan(&libds,1,-); /* chop off -FC extension */
+ %let ds0=%mf_getuniquename(prefix=fmtds_);
+ /*
+ There is no need to export the entire format catalog here - the validations
+ are done against the data model, not the data values. So we can simply
+ hardcode the structure based on the cntlout dataset.
+ */
+ proc sql;
+ create table &ds0(
+ FMTNAME char(32)
+ ,START char(16)
+ ,END char(16)
+ ,LABEL char(23)
+ ,MIN num length=3
+ ,MAX num length=3
+ ,DEFAULT num length=3
+ ,LENGTH num length=3
+ ,FUZZ num
+ ,PREFIX char(2)
+ ,MULT num
+ ,FILL char(1)
+ ,NOEDIT num length=3
+ ,TYPE char(1)
+ ,SEXCL char(1)
+ ,EEXCL char(1)
+ ,HLO char(13)
+ ,DECSEP char(1)
+ ,DIG3SEP char(1)
+ ,DATATYPE char(8)
+ ,LANGUAGE char(8)
+ );
+ %let libds=&ds0;
+%end;
%mp_filtercheck(&queryds,targetds=&libds,abort=YES)
/* hash the result */
%let ds1=%mf_getuniquename(prefix=hashds);
-%mp_hashdataset(&queryds,outds=&ds1,salt=&libds)
+%mp_hashdataset(&queryds,outds=&ds1,salt=&orig_libds)
%let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
%if &mdebug=1 %then %do;
data _null_;
@@ -132,7 +177,7 @@ run;
%let ds3=%mf_getuniquename(prefix=filtersum);
data work.&ds3;
if 0 then set &filter_summary;
- filter_table=symget('libds');
+ filter_table="&orig_libds";
filter_hash="&filter_hash";
PROCESSED_DTTM=%sysfunc(datetime());
output;
diff --git a/package-lock.json b/package-lock.json
index caa5b92..0499f04 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -8,7 +8,7 @@
"license": "MIT",
"devDependencies": {
"@sasjs/cli": "3.6.0",
- "@sasjs/core": "4.4.4"
+ "@sasjs/core": "4.4.7"
}
},
"node_modules/@sasjs/adapter": {
@@ -110,9 +110,9 @@
}
},
"node_modules/@sasjs/core": {
- "version": "4.4.4",
- "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.4.tgz",
- "integrity": "sha512-gN6d0fvhaofp7buemS5KIOo5Bz8lbqhsEQD7SuH5FZ02MQurmfu7A0Zg0lIEi0w2/ptI4M/sZdF4D2DRh1D5xA==",
+ "version": "4.4.7",
+ "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.7.tgz",
+ "integrity": "sha512-9+HmwspvWu/gH9ElnmRaGdjOCspOidBRUYUfxLzgOy1Ya1JMZ2RErMklCAMg7XI0Us5jecd2FXdo8zlQDhRkWQ==",
"dev": true
},
"node_modules/@sasjs/lint": {
@@ -2830,9 +2830,9 @@
}
},
"@sasjs/core": {
- "version": "4.4.4",
- "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.4.tgz",
- "integrity": "sha512-gN6d0fvhaofp7buemS5KIOo5Bz8lbqhsEQD7SuH5FZ02MQurmfu7A0Zg0lIEi0w2/ptI4M/sZdF4D2DRh1D5xA==",
+ "version": "4.4.7",
+ "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.7.tgz",
+ "integrity": "sha512-9+HmwspvWu/gH9ElnmRaGdjOCspOidBRUYUfxLzgOy1Ya1JMZ2RErMklCAMg7XI0Us5jecd2FXdo8zlQDhRkWQ==",
"dev": true
},
"@sasjs/lint": {
diff --git a/package.json b/package.json
index 56b831a..557aa64 100644
--- a/package.json
+++ b/package.json
@@ -34,6 +34,6 @@
},
"devDependencies": {
"@sasjs/cli": "3.6.0",
- "@sasjs/core": "4.4.4"
+ "@sasjs/core": "4.4.7"
}
}
diff --git a/tests/crossplatform/mp_filterstore.test.sas b/tests/crossplatform/mp_filterstore.test.1.sas
similarity index 100%
rename from tests/crossplatform/mp_filterstore.test.sas
rename to tests/crossplatform/mp_filterstore.test.1.sas
diff --git a/tests/crossplatform/mp_filterstore.test.2.sas b/tests/crossplatform/mp_filterstore.test.2.sas
new file mode 100644
index 0000000..21b293a
--- /dev/null
+++ b/tests/crossplatform/mp_filterstore.test.2.sas
@@ -0,0 +1,112 @@
+/**
+ @file
+ @brief Testing mp_filterstore macro with a format catalog
+
+ SAS Macros
+ @li mp_assert.sas
+ @li mp_assertdsobs.sas
+ @li mp_assertscope.sas
+ @li mp_coretable.sas
+ @li mp_filterstore.sas
+
+**/
+
+libname permlib (work);
+
+%mp_coretable(LOCKTABLE,libds=permlib.locktable)
+%mp_coretable(FILTER_SUMMARY,libds=permlib.filtsum)
+%mp_coretable(FILTER_DETAIL,libds=permlib.filtdet)
+%mp_coretable(MAXKEYTABLE,libds=permlib.maxkey)
+
+/* valid filter */
+data work.inds;
+ infile datalines4 dsd;
+ input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+ OPERATOR_NM:$12. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,Start,>,"'2'"
+AND,AND,1,Fmtname,=,"'MORDOR'"
+OR,OR,2,Label,IN,"('Dragon1','Dragon2')"
+OR,OR,2,End,=,"'6'"
+OR,OR,2,Start,GE,"'10'"
+;;;;
+run;
+
+/* make some formats */
+PROC FORMAT library=permlib.testfmts;
+ picture MyMSdt other='%0Y-%0m-%0dT%0H:%0M:%0S' (datatype=datetime);
+RUN;
+data work.fmts;
+ length fmtname $32;
+ do fmtname='SMAUG','MORDOR','GOLLUM';
+ do start=1 to 10;
+ label= cats('Dragon ',start);
+ output;
+ end;
+ end;
+run;
+proc sort data=work.fmts nodupkey;
+ by fmtname start;
+run;
+proc format cntlin=work.fmts library=permlib.testfmts;
+run;
+proc format library=permlib.testfmts;
+ invalue indays (default=13) other=42;
+run;
+
+
+%mp_assertscope(SNAPSHOT)
+%mp_filterstore(libds=permlib.testfmts-fc,
+ queryds=work.inds,
+ filter_summary=permlib.filtsum,
+ filter_detail=permlib.filtdet,
+ lock_table=permlib.locktable,
+ maxkeytable=permlib.maxkey,
+ outresult=work.result,
+ outquery=work.query,
+ mdebug=1
+)
+%mp_assertscope(COMPARE)
+
+%mp_assert(iftrue=(&syscc=0),
+ desc=Ensure macro runs without errors,
+ outds=work.test_results
+)
+/* ensure only one record created */
+%mp_assertdsobs(permlib.filtsum,
+ desc=Initial query,
+ test=ATMOST 1,
+ outds=work.test_results
+)
+/* check RK is correct */
+proc sql noprint;
+select max(filter_rk) into: test1 from work.result;
+%mp_assert(iftrue=(&test1=1),
+ desc=Ensure filter rk is correct,
+ outds=work.test_results
+)
+
+/* Test 2 - load same table again and ensure we get the same RK */
+%mp_filterstore(libds=permlib.testfmts-fc,
+ queryds=work.inds,
+ filter_summary=permlib.filtsum,
+ filter_detail=permlib.filtdet,
+ lock_table=permlib.locktable,
+ maxkeytable=permlib.maxkey,
+ outresult=work.result,
+ outquery=work.query,
+ mdebug=1
+)
+/* ensure only one record created */
+%mp_assertdsobs(permlib.filtsum,
+ desc=Initial query - same obs,
+ test=ATMOST 1,
+ outds=work.test_results
+)
+/* check RK is correct */
+proc sql noprint;
+select max(filter_rk) into: test2 from work.result;
+%mp_assert(iftrue=(&test2=1),
+ desc=Ensure filter rk is correct for second run,
+ outds=work.test_results
+)