sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2025-12-10 22:14:35 +00:00
Code Issues Projects Releases Wiki Activity

feat: adding format catalog capability to mp_filterstore

Browse Source
This commit is contained in:
munja
2022-02-06 22:12:00 +01:00
parent d254870439
commit 8b0acf2eae
10 changed files with 342 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@@ -192,11 +192,12 @@ When contributing to this library, it is therefore important to ensure that all
## Breaking Changes
We are currently on major release v4. Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag. The following changes are planned when the next major (breaking) release becomes necessary:
We are currently on major release v4. Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag. The following changes are planned when the next major/breaking release (v5) becomes necessary:
* mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
* `insert_cmplib` option of mcf_xxx macros will be deprecated (the option is now checked automatically with value inserted only if needed)
* mcf_xxx macros to have `wrap=` option defaulted to YES for convenience
* mcf_xxx macros to have `wrap=` option defaulted to YES for convenience. Set this option explicitly to avoid issues.
* mp_getddl.sas to be renamed to mp_ds2ddl.sas (consistent with other ds2xxx macros). A wrapper macro is already in place, and you are able to use this immediately. The default for SHOWLOG will also be YES instead of NO.
## Star Gazing

116
all.sas
View File

@@ -3538,6 +3538,8 @@ run;
%mp_coretable(LOCKTABLE,libds=work.locktable)
@param [in] table_ref The type of table to create. Example values:
@li CNTLOUT_DS - Mimics the structure of the table produced by `proc format`
with the `cntlout=` option.
@li DIFFTABLE - Used to store changes to tables. Used by mp_storediffs.sas
and mp_stackdiffs.sas
@li FILTER_DETAIL - For storing detailed filter values. Used by
@@ -3568,6 +3570,31 @@ run;
%local outds ;
%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
proc sql;
%if &table_ref=CNTLOUT_DS %then %do;
create table &outds(
FMTNAME char(32) label='Format name'
,START char(16) label='Starting value for format'
,END char(16) label='Ending value for format'
,LABEL char(23) label='Format value label'
,MIN num length=3 label='Minimum length'
,MAX num length=3 label='Maximum length'
,DEFAULT num length=3 label='Default length'
,LENGTH num length=3 label='Format length'
,FUZZ num label='Fuzz value'
,PREFIX char(2) label='Prefix characters'
,MULT num label='Multiplier'
,FILL char(1) label='Fill character'
,NOEDIT num length=3 label='Is picture string noedit?'
,TYPE char(1) label='Type of format'
,SEXCL char(1) label='Start exclusion'
,EEXCL char(1) label='End exclusion'
,HLO char(13) label='Additional information'
,DECSEP char(1) label='Decimal separator'
,DIG3SEP char(1) label='Three-digit separator'
,DATATYPE char(8) label='Date/time/datetime?'
,LANGUAGE char(8) label='Language for date strings'
);
%end;
%if &table_ref=DIFFTABLE %then %do;
create table &outds(
load_ref char(36) label='unique load reference',
@@ -4888,6 +4915,35 @@ data _null_;
run;
%mend mp_ds2csv;/**
@file
@brief A wrapper for mp_getddl.sas
@details In the next release, this will be the main version.
<h4> SAS Macros </h4>
@li mp_getddl.sas
**/
%macro mp_ds2ddl(libds,fref=getddl,flavour=SAS,showlog=YES,schema=
,applydttm=NO
)/*/STORE SOURCE*/;
%local libref;
%let libds=%upcase(&libds);
%let libref=%scan(&libds,1,.);
%if &libref=&libds %then %let libds=WORK.&libds;
%mp_getddl(%scan(&libds,1,.)
,%scan(&libds,2,.)
,fref=&fref
,flavour=SAS
,showlog=&showlog
,schema=&schema
,applydttm=&applydttm
)
%mend mp_ds2ddl;/**
@file
@brief Converts every value in a dataset to formatted value
@details Converts every value to it's formatted value. All variables will
@@ -5394,7 +5450,8 @@ options varlenchk=&optval;
SYSCC to 1008 if bad records are found, and call mp_abort.sas for a
graceful service exit (configurable).
Used for dynamic filtering in [Data Controller for SAS&reg;](https://datacontroller.io).
Used for dynamic filtering in [Data Controller for SAS&reg;](
https://datacontroller.io).
Usage:
@@ -5513,7 +5570,7 @@ data &outds;
output;
end;
if OPERATOR_NM not in
('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE')
then do;
REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
putlog REASON_CD= OPERATOR_NM=;
@@ -5701,8 +5758,13 @@ filename &outref temp;
https://sasapps.io)). This macro is also used in [Data Controller for SAS](
https://datacontroller.io).
A more recent feature of this macro is the ability to support filter queries
on Format Catalogs. This is achieved by adding a `-FC` suffix to the `libds`
parameter - where the "ds" in this case is the catalog name.
@param [in] libds= The target dataset to be filtered (lib should be assigned)
@param [in] libds= The target dataset to be filtered (lib should be assigned).
If filtering a format catalog, add the following suffix: `-FC`.
@param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to
be validated. Has the following format:
|GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
@@ -5771,7 +5833,10 @@ filename &outref temp;
%put &sysmacroname entry vars:;
%put _local_;
%local ds1 ds2 ds3 ds4 filter_hash;
%local ds0 ds1 ds2 ds3 ds4 filter_hash orig_libds;
%let libds=%upcase(&libds);
%let orig_libds=&libds;
%mp_abort(iftrue= (&syscc ne 0)
,mac=mp_filterstore
,msg=%str(syscc=&syscc on macro entry)
@@ -5789,12 +5854,49 @@ filename &outref temp;
,msg=%str(Invalid lock_table value: &lock_table)
)
/* validate query */
/**
* validate query
* use format catalog export, if a format
*/
%if "%substr(&libds,%length(&libds)-2,3)"="-FC" %then %do;
%let libds=%scan(&libds,1,-); /* chop off -FC extension */
%let ds0=%mf_getuniquename(prefix=fmtds_);
/*
There is no need to export the entire format catalog here - the validations
are done against the data model, not the data values. So we can simply
hardcode the structure based on the cntlout dataset.
*/
proc sql;
create table &ds0(
FMTNAME char(32)
,START char(16)
,END char(16)
,LABEL char(23)
,MIN num length=3
,MAX num length=3
,DEFAULT num length=3
,LENGTH num length=3
,FUZZ num
,PREFIX char(2)
,MULT num
,FILL char(1)
,NOEDIT num length=3
,TYPE char(1)
,SEXCL char(1)
,EEXCL char(1)
,HLO char(13)
,DECSEP char(1)
,DIG3SEP char(1)
,DATATYPE char(8)
,LANGUAGE char(8)
);
%let libds=&ds0;
%end;
%mp_filtercheck(&queryds,targetds=&libds,abort=YES)
/* hash the result */
%let ds1=%mf_getuniquename(prefix=hashds);
%mp_hashdataset(&queryds,outds=&ds1,salt=&libds)
%mp_hashdataset(&queryds,outds=&ds1,salt=&orig_libds)
%let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
%if &mdebug=1 %then %do;
data _null_;
@@ -5825,7 +5927,7 @@ run;
%let ds3=%mf_getuniquename(prefix=filtersum);
data work.&ds3;
if 0 then set &filter_summary;
filter_table=symget('libds');
filter_table="&orig_libds";
filter_hash="&filter_hash";
PROCESSED_DTTM=%sysfunc(datetime());
output;

27
base/mp_coretable.sas
View File

@@ -10,6 +10,8 @@
%mp_coretable(LOCKTABLE,libds=work.locktable)
@param [in] table_ref The type of table to create. Example values:
@li CNTLOUT_DS - Mimics the structure of the table produced by `proc format`
with the `cntlout=` option.
@li DIFFTABLE - Used to store changes to tables. Used by mp_storediffs.sas
and mp_stackdiffs.sas
@li FILTER_DETAIL - For storing detailed filter values. Used by
@@ -40,6 +42,31 @@
%local outds ;
%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
proc sql;
%if &table_ref=CNTLOUT_DS %then %do;
create table &outds(
FMTNAME char(32) label='Format name'
,START char(16) label='Starting value for format'
,END char(16) label='Ending value for format'
,LABEL char(23) label='Format value label'
,MIN num length=3 label='Minimum length'
,MAX num length=3 label='Maximum length'
,DEFAULT num length=3 label='Default length'
,LENGTH num length=3 label='Format length'
,FUZZ num label='Fuzz value'
,PREFIX char(2) label='Prefix characters'
,MULT num label='Multiplier'
,FILL char(1) label='Fill character'
,NOEDIT num length=3 label='Is picture string noedit?'
,TYPE char(1) label='Type of format'
,SEXCL char(1) label='Start exclusion'
,EEXCL char(1) label='End exclusion'
,HLO char(13) label='Additional information'
,DECSEP char(1) label='Decimal separator'
,DIG3SEP char(1) label='Three-digit separator'
,DATATYPE char(8) label='Date/time/datetime?'
,LANGUAGE char(8) label='Language for date strings'
);
%end;
%if &table_ref=DIFFTABLE %then %do;
create table &outds(
load_ref char(36) label='unique load reference',

30
base/mp_ds2ddl.sas Normal file
View File

@@ -0,0 +1,30 @@
/**
@file
@brief A wrapper for mp_getddl.sas
@details In the next release, this will be the main version.
<h4> SAS Macros </h4>
@li mp_getddl.sas
**/
%macro mp_ds2ddl(libds,fref=getddl,flavour=SAS,showlog=YES,schema=
,applydttm=NO
)/*/STORE SOURCE*/;
%local libref;
%let libds=%upcase(&libds);
%let libref=%scan(&libds,1,.);
%if &libref=&libds %then %let libds=WORK.&libds;
%mp_getddl(%scan(&libds,1,.)
,%scan(&libds,2,.)
,fref=&fref
,flavour=SAS
,showlog=&showlog
,schema=&schema
,applydttm=&applydttm
)
%mend mp_ds2ddl;

5
base/mp_filtercheck.sas
View File

@@ -6,7 +6,8 @@
SYSCC to 1008 if bad records are found, and call mp_abort.sas for a
graceful service exit (configurable).
Used for dynamic filtering in [Data Controller for SAS&reg;](https://datacontroller.io).
Used for dynamic filtering in [Data Controller for SAS&reg;](
https://datacontroller.io).
Usage:
@@ -125,7 +126,7 @@ data &outds;
output;
end;
if OPERATOR_NM not in
('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE')
then do;
REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
putlog REASON_CD= OPERATOR_NM=;

55
base/mp_filterstore.sas
View File

@@ -8,8 +8,13 @@
https://sasapps.io)). This macro is also used in [Data Controller for SAS](
https://datacontroller.io).
A more recent feature of this macro is the ability to support filter queries
on Format Catalogs. This is achieved by adding a `-FC` suffix to the `libds`
parameter - where the "ds" in this case is the catalog name.
@param [in] libds= The target dataset to be filtered (lib should be assigned)
@param [in] libds= The target dataset to be filtered (lib should be assigned).
If filtering a format catalog, add the following suffix: `-FC`.
@param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to
be validated. Has the following format:
|GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
@@ -78,7 +83,10 @@
%put &sysmacroname entry vars:;
%put _local_;
%local ds1 ds2 ds3 ds4 filter_hash;
%local ds0 ds1 ds2 ds3 ds4 filter_hash orig_libds;
%let libds=%upcase(&libds);
%let orig_libds=&libds;
%mp_abort(iftrue= (&syscc ne 0)
,mac=mp_filterstore
,msg=%str(syscc=&syscc on macro entry)
@@ -96,12 +104,49 @@
,msg=%str(Invalid lock_table value: &lock_table)
)
/* validate query */
/**
* validate query
* use format catalog export, if a format
*/
%if "%substr(&libds,%length(&libds)-2,3)"="-FC" %then %do;
%let libds=%scan(&libds,1,-); /* chop off -FC extension */
%let ds0=%mf_getuniquename(prefix=fmtds_);
/*
There is no need to export the entire format catalog here - the validations
are done against the data model, not the data values. So we can simply
hardcode the structure based on the cntlout dataset.
*/
proc sql;
create table &ds0(
FMTNAME char(32)
,START char(16)
,END char(16)
,LABEL char(23)
,MIN num length=3
,MAX num length=3
,DEFAULT num length=3
,LENGTH num length=3
,FUZZ num
,PREFIX char(2)
,MULT num
,FILL char(1)
,NOEDIT num length=3
,TYPE char(1)
,SEXCL char(1)
,EEXCL char(1)
,HLO char(13)
,DECSEP char(1)
,DIG3SEP char(1)
,DATATYPE char(8)
,LANGUAGE char(8)
);
%let libds=&ds0;
%end;
%mp_filtercheck(&queryds,targetds=&libds,abort=YES)
/* hash the result */
%let ds1=%mf_getuniquename(prefix=hashds);
%mp_hashdataset(&queryds,outds=&ds1,salt=&libds)
%mp_hashdataset(&queryds,outds=&ds1,salt=&orig_libds)
%let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
%if &mdebug=1 %then %do;
data _null_;
@@ -132,7 +177,7 @@ run;
%let ds3=%mf_getuniquename(prefix=filtersum);
data work.&ds3;
if 0 then set &filter_summary;
filter_table=symget('libds');
filter_table="&orig_libds";
filter_hash="&filter_hash";
PROCESSED_DTTM=%sysfunc(datetime());
output;

14
package-lock.json generated
View File

@@ -8,7 +8,7 @@
"license": "MIT",
"devDependencies": {
"@sasjs/cli": "3.6.0",
"@sasjs/core": "4.4.4"
"@sasjs/core": "4.4.7"
}
},
"node_modules/@sasjs/adapter": {
@@ -110,9 +110,9 @@
}
},
"node_modules/@sasjs/core": {
"version": "4.4.4",
"resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.4.tgz",
"integrity": "sha512-gN6d0fvhaofp7buemS5KIOo5Bz8lbqhsEQD7SuH5FZ02MQurmfu7A0Zg0lIEi0w2/ptI4M/sZdF4D2DRh1D5xA==",
"version": "4.4.7",
"resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.7.tgz",
"integrity": "sha512-9+HmwspvWu/gH9ElnmRaGdjOCspOidBRUYUfxLzgOy1Ya1JMZ2RErMklCAMg7XI0Us5jecd2FXdo8zlQDhRkWQ==",
"dev": true
},
"node_modules/@sasjs/lint": {
@@ -2830,9 +2830,9 @@
}
},
"@sasjs/core": {
"version": "4.4.4",
"resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.4.tgz",
"integrity": "sha512-gN6d0fvhaofp7buemS5KIOo5Bz8lbqhsEQD7SuH5FZ02MQurmfu7A0Zg0lIEi0w2/ptI4M/sZdF4D2DRh1D5xA==",
"version": "4.4.7",
"resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.4.7.tgz",
"integrity": "sha512-9+HmwspvWu/gH9ElnmRaGdjOCspOidBRUYUfxLzgOy1Ya1JMZ2RErMklCAMg7XI0Us5jecd2FXdo8zlQDhRkWQ==",
"dev": true
},
"@sasjs/lint": {

2
package.json
View File

@@ -34,6 +34,6 @@
},
"devDependencies": {
"@sasjs/cli": "3.6.0",
"@sasjs/core": "4.4.4"
"@sasjs/core": "4.4.7"
}
}

0
tests/crossplatform/mp_filterstore.test.sas → tests/crossplatform/mp_filterstore.test.1.sas
View File

112
tests/crossplatform/mp_filterstore.test.2.sas Normal file
View File

@@ -0,0 +1,112 @@
/**
@file
@brief Testing mp_filterstore macro with a format catalog
<h4> SAS Macros </h4>
@li mp_assert.sas
@li mp_assertdsobs.sas
@li mp_assertscope.sas
@li mp_coretable.sas
@li mp_filterstore.sas
**/
libname permlib (work);
%mp_coretable(LOCKTABLE,libds=permlib.locktable)
%mp_coretable(FILTER_SUMMARY,libds=permlib.filtsum)
%mp_coretable(FILTER_DETAIL,libds=permlib.filtdet)
%mp_coretable(MAXKEYTABLE,libds=permlib.maxkey)
/* valid filter */
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$12. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,Start,>,"'2'"
AND,AND,1,Fmtname,=,"'MORDOR'"
OR,OR,2,Label,IN,"('Dragon1','Dragon2')"
OR,OR,2,End,=,"'6'"
OR,OR,2,Start,GE,"'10'"
;;;;
run;
/* make some formats */
PROC FORMAT library=permlib.testfmts;
picture MyMSdt other='%0Y-%0m-%0dT%0H:%0M:%0S' (datatype=datetime);
RUN;
data work.fmts;
length fmtname $32;
do fmtname='SMAUG','MORDOR','GOLLUM';
do start=1 to 10;
label= cats('Dragon ',start);
output;
end;
end;
run;
proc sort data=work.fmts nodupkey;
by fmtname start;
run;
proc format cntlin=work.fmts library=permlib.testfmts;
run;
proc format library=permlib.testfmts;
invalue indays (default=13) other=42;
run;
%mp_assertscope(SNAPSHOT)
%mp_filterstore(libds=permlib.testfmts-fc,
queryds=work.inds,
filter_summary=permlib.filtsum,
filter_detail=permlib.filtdet,
lock_table=permlib.locktable,
maxkeytable=permlib.maxkey,
outresult=work.result,
outquery=work.query,
mdebug=1
)
%mp_assertscope(COMPARE)
%mp_assert(iftrue=(&syscc=0),
desc=Ensure macro runs without errors,
outds=work.test_results
)
/* ensure only one record created */
%mp_assertdsobs(permlib.filtsum,
desc=Initial query,
test=ATMOST 1,
outds=work.test_results
)
/* check RK is correct */
proc sql noprint;
select max(filter_rk) into: test1 from work.result;
%mp_assert(iftrue=(&test1=1),
desc=Ensure filter rk is correct,
outds=work.test_results
)
/* Test 2 - load same table again and ensure we get the same RK */
%mp_filterstore(libds=permlib.testfmts-fc,
queryds=work.inds,
filter_summary=permlib.filtsum,
filter_detail=permlib.filtdet,
lock_table=permlib.locktable,
maxkeytable=permlib.maxkey,
outresult=work.result,
outquery=work.query,
mdebug=1
)
/* ensure only one record created */
%mp_assertdsobs(permlib.filtsum,
desc=Initial query - same obs,
test=ATMOST 1,
outds=work.test_results
)
/* check RK is correct */
proc sql noprint;
select max(filter_rk) into: test2 from work.result;
%mp_assert(iftrue=(&test2=1),
desc=Ensure filter rk is correct for second run,
outds=work.test_results
)