diff --git a/all.sas b/all.sas
index 83f0aea..d3fe7ad 100644
--- a/all.sas
+++ b/all.sas
@@ -3351,6 +3351,97 @@ run;
drop table work.&tempds;
%mend mp_copyfolder;/**
+ @file
+ @brief Create the permanent Core tables
+ @details Several macros in the [core](https://github.com/sasjs/core) library
+ make use of permanent tables. To avoid duplication in definitions, this
+ macro provides a central location for managing the corresponding DDL.
+
+ Example usage:
+
+ %mp_coretable(LOCKTABLE,libds=work.locktable)
+
+ @param [in] table_ref The type of table to create. Example values:
+ @li FILTER_DETAIL - For storing detailed filter values. Used by
+ mp_filterstore.sas.
+ @li FILTER_SUMMARY - For storing summary filter values. Used by
+ mp_filterstore.sas.
+ @li LOCKANYTABLE - For "locking" tables prior to multipass loads. Used by
+ mp_lockanytable.sas
+ @li MAXKEYTABLE - For storing the maximum retained key information. Used
+ by mp_retainedkey.sas
+ @param [in] libds= (0) The library.dataset reference used to create the table.
+ If not provided, then the DDL is simply printed to the log.
+
+ Related Macros
+ @li mp_filterstore.sas
+ @li mp_lockanytable.sas
+ @li mp_retainedkey.sas
+
+ @version 9.2
+ @author Allan Bowe
+
+**/
+
+%macro mp_coretable(table_ref,libds=0
+)/*/STORE SOURCE*/;
+%local outds ;
+%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
+proc sql;
+%if &table_ref=LOCKTABLE %then %do;
+ create table &outds(
+ lock_lib char(8),
+ lock_ds char(32),
+ lock_status_cd char(10) not null,
+ lock_user_nm char(100) not null ,
+ lock_ref char(200),
+ lock_pid char(10),
+ lock_start_dttm num format=E8601DT26.6,
+ lock_end_dttm num format=E8601DT26.6,
+ constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+%end;
+%else %if &table_ref=FILTER_SUMMARY %then %do;
+ create table &outds(
+ filter_rk num not null,
+ filter_hash char(32) not null,
+ filter_table char(41) not null,
+ processed_dttm num not null format=E8601DT26.6,
+ constraint pk_mpe_filteranytable
+ primary key(filter_rk));
+%end;
+%else %if &table_ref=FILTER_DETAIL %then %do;
+ create table &outds(
+ filter_hash char(32) not null,
+ filter_line num not null,
+ group_logic char(3) not null,
+ subgroup_logic char(3) not null,
+ subgroup_id num not null,
+ variable_nm varchar(32) not null,
+ operator_nm varchar(12) not null,
+ raw_value varchar(4000) not null,
+ processed_dttm num not null format=E8601DT26.6,
+ constraint pk_mpe_filteranytable
+ primary key(filter_hash,filter_line));
+%end;
+%else %if &table_ref=MAXKEYTABLE %then %do;
+ create table &outds(
+ keytable varchar(41) label='Base table in libref.dataset format',
+ keycolumn char(32) format=$32.
+ label='The Retained key field containing the key values.',
+ max_key num label=
+ 'Integer representing current max RK or SK value in the KEYTABLE',
+ processed_dttm num format=E8601DT26.6
+ label='Datetime this value was last updated',
+ constraint pk_mpe_maxkeyvalues
+ primary key(keytable));
+%end;
+
+
+%if &libds=0 %then %do;
+ describe table &syslast;
+ drop table &syslast;
+%end;
+%mend mp_coretable;/**
@file mp_createconstraints.sas
@brief Creates constraints
@details Takes the output from mp_getconstraints.sas as input
@@ -5098,6 +5189,218 @@ filename &outref temp;
%mend mp_filtergenerate;
/**
+ @file
+ @brief Checks & Stores an input filter table and returns the Filter Key
+ @details Used to generate a FILTER_RK from an input query dataset. This
+ process requires several permanent tables (names are configurable). The
+ benefit of storing query values at backend is to enable stored 'views' of
+ filtered tables at frontend (ie, when building [SAS-Powered Apps](
+ https://sasapps.io)). This macro is also used in [Data Controller for SAS](
+ https://datacontroller.io).
+
+
+ @param [in] libds= The target dataset to be filtered (lib should be assigned)
+ @param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to
+ be validated. Has the following format:
+|GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
+|---|---|---|---|---|---|
+|AND|AND|1|SOME_BESTNUM|>|1|
+|AND|AND|1|SOME_TIME|=|77333|
+ @param [in] filter_summary= (PERM.FILTER_SUMMARY) Permanent table containing
+ summary filter values. The definition is available by running
+ mp_coretable.sas as follows: `mp_coretable(FILTER_SUMMARY)`. Example
+ values:
+|FILTER_RK:best.|FILTER_HASH:$32.|FILTER_TABLE:$41.|PROCESSED_DTTM:datetime19.|
+|---|---|---|---|
+|`1 `|`540E96F566D194AB58DD4C413C99C9DB `|`VIYA6014.MPE_TABLES `|`1956084246 `|
+|`2 `|`87737DB9EEE2650F5C89956CEAD0A14F `|`VIYA6014.MPE_X_TEST `|`1956084452.1`|
+|`3 `|`8048BD908DBBD83D013560734E90D394 `|`VIYA6014.MPE_TABLES `|`1956093620.6`|
+ @param [in] filter_detail= (PERM.FILTER_DETAIL) Permanent table containing
+ detailed (raw) filter values. The definition is available by running
+ mp_coretable.sas as follows: `mp_coretable(FILTER_DETAIL)`. Example
+ values:
+|FILTER_HASH:$32.|FILTER_LINE:best.|GROUP_LOGIC:$3.|SUBGROUP_LOGIC:$3.|SUBGROUP_ID:best.|VARIABLE_NM:$32.|OPERATOR_NM:$12.|RAW_VALUE:$4000.|PROCESSED_DTTM:datetime19.|
+|---|---|---|---|---|---|---|---|---|
+|`540E96F566D194AB58DD4C413C99C9DB `|`1 `|`AND `|`AND `|`1 `|`LIBREF `|`CONTAINS `|`DC`|`1956084245.8 `|
+|`540E96F566D194AB58DD4C413C99C9DB `|`2 `|`AND `|`OR `|`2 `|`DSN `|`= `|` MPE_LOCK_ANYTABLE `|`1956084245.8 `|
+|`87737DB9EEE2650F5C89956CEAD0A14F `|`1 `|`AND `|`AND `|`1 `|`PRIMARY_KEY_FIELD `|`IN `|`(1,2,3) `|`1956084451.9 `|
+ @param [in] lock_table= (PERM.LOCK_TABLE) Permanent locking table. Used to
+ manage concurrent access. The definition is available by running
+ mp_coretable.sas as follows: `mp_coretable(LOCKTABLE)`.
+ @param [in] maxkeytable= (0) Optional permanent reference table used for
+ retained key tracking. Described in mp_retainedkey.sas.
+ @param [in] mdebug= set to 1 to enable DEBUG messages
+ @param [out] outresult= The result table with the FILTER_RK
+ @param [out] outquery= The original query, taken as extract after table load
+
+
+ SAS Macros
+ @li mf_getuniquename.sas
+ @li mf_getvalue.sas
+ @li mf_islibds.sas
+ @li mf_nobs.sas
+ @li mp_abort.sas
+ @li mp_filtercheck.sas
+ @li mp_hashdataset.sas
+ @li mp_retainedkey.sas
+
+ Related Macros
+ @li mp_filtercheck.sas
+ @li mp_filtergenerate.sas
+ @li mp_filtervalidate.sas
+ @li mp_filterstore.test.sas
+
+ @version 9.2
+ @author [Allan Bowe](https://www.linkedin.com/in/allanbowe)
+
+**/
+
+%macro mp_filterstore(libds=,
+ queryds=work.filterquery,
+ filter_summary=PERM.FILTER_SUMMARY,
+ filter_detail=PERM.FILTER_DETAIL,
+ lock_table=PERM.LOCK_TABLE,
+ maxkeytable=PERM.MAXKEYTABLE,
+ outresult=work.result,
+ outquery=work.query,
+ mdebug=1
+);
+%put &sysmacroname entry vars:;
+%put _local_;
+
+%local ds1 ds2 ds3 ds4 filter_hash;
+%mp_abort(iftrue= (&syscc ne 0)
+ ,mac=mp_filterstore
+ ,msg=%str(syscc=&syscc on macro entry)
+)
+%mp_abort(iftrue= (%mf_islibds(&filter_summary)=0)
+ ,mac=mp_filterstore
+ ,msg=%str(Invalid filter_summary value: &filter_summary)
+)
+%mp_abort(iftrue= (%mf_islibds(&filter_detail)=0)
+ ,mac=mp_filterstore
+ ,msg=%str(Invalid filter_detail value: &filter_detail)
+)
+%mp_abort(iftrue= (%mf_islibds(&lock_table)=0)
+ ,mac=mp_filterstore
+ ,msg=%str(Invalid lock_table value: &lock_table)
+)
+
+/* validate query */
+%mp_filtercheck(&queryds,targetds=&libds,abort=YES)
+
+/* hash the result */
+%let ds1=%mf_getuniquename(prefix=hashds);
+%mp_hashdataset(&queryds,outds=&ds1,salt=&libds)
+%let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
+%if &mdebug=1 %then %do;
+ data _null_;
+ putlog "filter_hash=&filter_hash";
+ set &ds1;
+ putlog (_all_)(=);
+ run;
+%end;
+
+/* check if data already exists for this hash */
+data &outresult;
+ set &filter_summary;
+ where filter_hash="&filter_hash";
+run;
+
+%mp_abort(iftrue= (&syscc ne 0)
+ ,mac=mp_filterstore
+ ,msg=%str(syscc=&syscc after hash check)
+)
+%mp_abort(iftrue= ("&filter_hash"=" ")
+ ,mac=mp_filterstore
+ ,msg=%str(problem with filter_hash generation)
+)
+
+%if %mf_nobs(&outresult)=0 %then %do;
+
+ /* update detail table first */
+ %let ds2=%mf_getuniquename(prefix=filterdetail);
+ data &ds2;
+ if 0 then set &filter_detail;
+ set &queryds;
+ format filter_hash $hex32. filter_line 8. processed_dttm E8601DT26.6;
+ filter_hash="&filter_hash";
+ filter_line=_n_;
+ PROCESSED_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt;
+ run;
+ %mp_lockanytable(LOCK,
+ lib=%scan(&filter_detail,1,.)
+ ,ds=%scan(&filter_detail,2,.)
+ ,ref=MP_FILTERSTORE update - &filter_hash
+ ,ctl_ds=&lock_table
+ )
+ proc append base=&filter_detail data=&ds2;
+ run;
+
+ %mp_lockanytable(UNLOCK,
+ lib=%scan(&filter_detail,1,.)
+ ,ds=%scan(&filter_detail,2,.)
+ ,ref=MP_FILTERSTORE update - &filter_hash
+ ,ctl_ds=&lock_table
+ )
+
+ /* now update summary table */
+ %let ds3=%mf_getuniquename(prefix=filtersum);
+ data &ds3;
+ if 0 then set &filter_summary;
+ filter_table=symget('libds');
+ filter_hash="&filter_hash";
+ PROCESSED_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt;
+ output;
+ stop;
+ run;
+
+ %mp_lockanytable(LOCK,
+ lib=%scan(&filter_summary,1,.)
+ ,ds=%scan(&filter_summary,2,.)
+ ,ref=MP_FILTERSTORE update - &filter_hash
+ ,ctl_ds=&lock_table
+ )
+
+ %let ds4=%mf_getuniquename(prefix=filtersumappend);
+ %mp_retainedkey(
+ base_lib=%scan(&filter_summary,1,.)
+ ,base_dsn=%scan(&filter_summary,2,.)
+ ,append_lib=work
+ ,append_dsn=&ds3
+ ,retained_key=filter_rk
+ ,business_key=filter_hash
+ ,maxkeytable=&maxkeytable
+ ,locktable=&lock_table
+ ,outds=work.&ds4
+ )
+ proc append base=&filter_summary data=&ds4;
+ run;
+
+ %mp_lockanytable(UNLOCK,
+ lib=%scan(&filter_summary,1,.)
+ ,ds=%scan(&filter_summary,2,.)
+ ,ref=MP_FILTERSTORE update - &filter_hash
+ ,ctl_ds=&lock_table
+ )
+
+ data &outresult;
+ set &filter_summary;
+ where filter_hash="&filter_hash";
+ run;
+
+%end;
+
+proc sort data=&filter_detail(where=(filter_hash="&filter_hash")) out=&outquery;
+ by filter_line;
+run;
+
+%mp_abort(iftrue= (&syscc ne 0)
+ ,mac=mp_filterstore
+ ,msg=%str(syscc=&syscc on macro exit)
+)
+
+%mend mp_filterstore;/**
@file
@brief Checks a generated filter query for validity
@details Runs a generated filter in proc sql with the validate option.
@@ -7584,19 +7887,8 @@ select distinct lowcase(memname)
@param [in] ref= A meaningful reference to enable the lock to be traced. Max
length is 200 characters.
@param [out] ctl_ds= (0) The control table which controls the actual locking.
- Should already be assigned and available. Definition as follows:
-
- proc sql;
- create table &ctl_ds(
- lock_lib char(8),
- lock_ds char(32),
- lock_status_cd char(10) not null,
- lock_user_nm char(100) not null ,
- lock_ref char(200),
- lock_pid char(10),
- lock_start_dttm num format=E8601DT26.6,
- lock_end_dttm num format=E8601DT26.6,
- constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+ Should already be assigned and available. The definition is available by
+ running mp_coretable.sas as follows: `mp_coretable(LOCKTABLE)`.
@param [in] loops= (25) Number of times to check for a lock.
@param [in] loop_secs= (1) Seconds to wait between each lock attempt
@@ -8326,19 +8618,12 @@ run;
@li permlib.base_table - the target table to be loaded (**not** loaded by this
macro)
@li permlib.maxkeytable - optional, used to store load metaadata.
- The structure is as follows:
+ The definition is available by running mp_coretable.sas as follows:
+ `mp_coretable(MAXKEYTABLE)`.
+ @li permlib.locktable - Necessary if maxkeytable is being populated. The
+ definition is available by running mp_coretable.sas as follows:
+ `mp_coretable(LOCKTABLE)`.
- proc sql;
- create table yourlib.maxkeytable(
- keytable varchar(41) label='Base table in libref.dataset format',
- keycolumn char(32) format=$32.
- label='The Retained key field containing the key values.',
- max_key num label=
- 'Integer representing current max RK or SK value in the KEYTABLE',
- processed_dttm num format=E8601DT26.6
- label='Datetime this value was last updated',
- constraint pk_mpe_maxkeyvalues
- primary key(keytable));
@param [in] base_lib= (WORK) Libref of the base (target) table.
@param [in] base_dsn= (BASETABLE) Name of the base (target) table.
@@ -8374,6 +8659,7 @@ run;
@li mp_lockanytable.sas
Related Macros
+ @li mp_filterstore.sas
@li mp_retainedkey.test.sas
@version 9.2
@@ -8403,7 +8689,7 @@ run;
%let tempds1=%mf_getuniquename();
%let tempds2=%mf_getuniquename();
%let comma_pk=%mf_getquotedstr(in_str=%str(&business_key),dlm=%str(,),quote=);
-
+%let outds=%sysfunc(ifc(%index(&outds,.)=0,work.&outds,&outds));
/* validation checks */
%let iserr=0;
%if &syscc>0 %then %do;
@@ -8434,14 +8720,18 @@ run;
%do x=1 %to %sysfunc(countw(&business_key));
/* check business key values exist */
%let key_field=%scan(&business_key,&x,%str( ));
- %if (not %mf_existvar(&app_libds,&key_field))
- or (not %mf_existvar(&base_libds,&key_field))
- %then %do;
+ %if not %mf_existvar(&app_libds,&key_field) %then %do;
%let iserr=1;
- %let msg=Business key (&key_field) not found!;
+ %let msg=Business key (&key_field) not found on &app_libds!;
+ %goto err;
+ %end;
+ %else %if not %mf_existvar(&base_libds,&key_field) %then %do;
+ %let iserr=1;
+ %let msg=Business key (&key_field) not found on &base_libds!;
+ %goto err;
%end;
%end;
-
+%err:
%if &iserr=1 %then %do;
/* err case so first perform an unlock of the base table before exiting */
%mp_lockanytable(
@@ -8505,7 +8795,7 @@ quit;
* Update maxkey table if link provided
*/
%if &maxkeytable ne 0 %then %do;
- proc sql;
+ proc sql noprint;
select count(*) into: check from &maxkeytable
where upcase(keytable)="&base_libds";
diff --git a/base/mp_coretable.sas b/base/mp_coretable.sas
new file mode 100644
index 0000000..d0eca56
--- /dev/null
+++ b/base/mp_coretable.sas
@@ -0,0 +1,92 @@
+/**
+ @file
+ @brief Create the permanent Core tables
+ @details Several macros in the [core](https://github.com/sasjs/core) library
+ make use of permanent tables. To avoid duplication in definitions, this
+ macro provides a central location for managing the corresponding DDL.
+
+ Example usage:
+
+ %mp_coretable(LOCKTABLE,libds=work.locktable)
+
+ @param [in] table_ref The type of table to create. Example values:
+ @li FILTER_DETAIL - For storing detailed filter values. Used by
+ mp_filterstore.sas.
+ @li FILTER_SUMMARY - For storing summary filter values. Used by
+ mp_filterstore.sas.
+ @li LOCKANYTABLE - For "locking" tables prior to multipass loads. Used by
+ mp_lockanytable.sas
+ @li MAXKEYTABLE - For storing the maximum retained key information. Used
+ by mp_retainedkey.sas
+ @param [in] libds= (0) The library.dataset reference used to create the table.
+ If not provided, then the DDL is simply printed to the log.
+
+ Related Macros
+ @li mp_filterstore.sas
+ @li mp_lockanytable.sas
+ @li mp_retainedkey.sas
+
+ @version 9.2
+ @author Allan Bowe
+
+**/
+
+%macro mp_coretable(table_ref,libds=0
+)/*/STORE SOURCE*/;
+%local outds ;
+%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
+proc sql;
+%if &table_ref=LOCKTABLE %then %do;
+ create table &outds(
+ lock_lib char(8),
+ lock_ds char(32),
+ lock_status_cd char(10) not null,
+ lock_user_nm char(100) not null ,
+ lock_ref char(200),
+ lock_pid char(10),
+ lock_start_dttm num format=E8601DT26.6,
+ lock_end_dttm num format=E8601DT26.6,
+ constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+%end;
+%else %if &table_ref=FILTER_SUMMARY %then %do;
+ create table &outds(
+ filter_rk num not null,
+ filter_hash char(32) not null,
+ filter_table char(41) not null,
+ processed_dttm num not null format=E8601DT26.6,
+ constraint pk_mpe_filteranytable
+ primary key(filter_rk));
+%end;
+%else %if &table_ref=FILTER_DETAIL %then %do;
+ create table &outds(
+ filter_hash char(32) not null,
+ filter_line num not null,
+ group_logic char(3) not null,
+ subgroup_logic char(3) not null,
+ subgroup_id num not null,
+ variable_nm varchar(32) not null,
+ operator_nm varchar(12) not null,
+ raw_value varchar(4000) not null,
+ processed_dttm num not null format=E8601DT26.6,
+ constraint pk_mpe_filteranytable
+ primary key(filter_hash,filter_line));
+%end;
+%else %if &table_ref=MAXKEYTABLE %then %do;
+ create table &outds(
+ keytable varchar(41) label='Base table in libref.dataset format',
+ keycolumn char(32) format=$32.
+ label='The Retained key field containing the key values.',
+ max_key num label=
+ 'Integer representing current max RK or SK value in the KEYTABLE',
+ processed_dttm num format=E8601DT26.6
+ label='Datetime this value was last updated',
+ constraint pk_mpe_maxkeyvalues
+ primary key(keytable));
+%end;
+
+
+%if &libds=0 %then %do;
+ describe table &syslast;
+ drop table &syslast;
+%end;
+%mend mp_coretable;
\ No newline at end of file
diff --git a/base/mp_filterstore.sas b/base/mp_filterstore.sas
index 0b12e48..9bc99c2 100644
--- a/base/mp_filterstore.sas
+++ b/base/mp_filterstore.sas
@@ -2,10 +2,11 @@
@file
@brief Checks & Stores an input filter table and returns the Filter Key
@details Used to generate a FILTER_RK from an input query dataset. This
- process requires several permanent tables (names are configurable):
-
- @li filterdetail (contains raw values)
- @li filtersummary (contains summary data about the filter)
+ process requires several permanent tables (names are configurable). The
+ benefit of storing query values at backend is to enable stored 'views' of
+ filtered tables at frontend (ie, when building [SAS-Powered Apps](
+ https://sasapps.io)). This macro is also used in [Data Controller for SAS](
+ https://datacontroller.io).
@param [in] libds= The target dataset to be filtered (lib should be assigned)
@@ -16,21 +17,26 @@
|AND|AND|1|SOME_BESTNUM|>|1|
|AND|AND|1|SOME_TIME|=|77333|
@param [in] filter_summary= (PERM.FILTER_SUMMARY) Permanent table containing
- summary filter values. Structure:
+ summary filter values. The definition is available by running
+ mp_coretable.sas as follows: `mp_coretable(FILTER_SUMMARY)`. Example
+ values:
|FILTER_RK:best.|FILTER_HASH:$32.|FILTER_TABLE:$41.|PROCESSED_DTTM:datetime19.|
|---|---|---|---|
|`1 `|`540E96F566D194AB58DD4C413C99C9DB `|`VIYA6014.MPE_TABLES `|`1956084246 `|
|`2 `|`87737DB9EEE2650F5C89956CEAD0A14F `|`VIYA6014.MPE_X_TEST `|`1956084452.1`|
|`3 `|`8048BD908DBBD83D013560734E90D394 `|`VIYA6014.MPE_TABLES `|`1956093620.6`|
@param [in] filter_detail= (PERM.FILTER_DETAIL) Permanent table containing
- detailed (raw) filter values. Structure:
+ detailed (raw) filter values. The definition is available by running
+ mp_coretable.sas as follows: `mp_coretable(FILTER_DETAIL)`. Example
+ values:
|FILTER_HASH:$32.|FILTER_LINE:best.|GROUP_LOGIC:$3.|SUBGROUP_LOGIC:$3.|SUBGROUP_ID:best.|VARIABLE_NM:$32.|OPERATOR_NM:$12.|RAW_VALUE:$4000.|PROCESSED_DTTM:datetime19.|
|---|---|---|---|---|---|---|---|---|
|`540E96F566D194AB58DD4C413C99C9DB `|`1 `|`AND `|`AND `|`1 `|`LIBREF `|`CONTAINS `|`DC`|`1956084245.8 `|
|`540E96F566D194AB58DD4C413C99C9DB `|`2 `|`AND `|`OR `|`2 `|`DSN `|`= `|` MPE_LOCK_ANYTABLE `|`1956084245.8 `|
|`87737DB9EEE2650F5C89956CEAD0A14F `|`1 `|`AND `|`AND `|`1 `|`PRIMARY_KEY_FIELD `|`IN `|`(1,2,3) `|`1956084451.9 `|
@param [in] lock_table= (PERM.LOCK_TABLE) Permanent locking table. Used to
- manage concurrent access. Described in mp_lockanytable.sas.
+ manage concurrent access. The definition is available by running
+ mp_coretable.sas as follows: `mp_coretable(LOCKTABLE)`.
@param [in] maxkeytable= (0) Optional permanent reference table used for
retained key tracking. Described in mp_retainedkey.sas.
@param [in] mdebug= set to 1 to enable DEBUG messages
@@ -94,6 +100,7 @@
%mp_filtercheck(&queryds,targetds=&libds,abort=YES)
/* hash the result */
+%let ds1=%mf_getuniquename(prefix=hashds);
%mp_hashdataset(&queryds,outds=&ds1,salt=&libds)
%let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
%if &mdebug=1 %then %do;
@@ -124,6 +131,7 @@ run;
/* update detail table first */
%let ds2=%mf_getuniquename(prefix=filterdetail);
data &ds2;
+ if 0 then set &filter_detail;
set &queryds;
format filter_hash $hex32. filter_line 8. processed_dttm E8601DT26.6;
filter_hash="&filter_hash";
@@ -168,13 +176,13 @@ run;
%mp_retainedkey(
base_lib=%scan(&filter_summary,1,.)
,base_dsn=%scan(&filter_summary,2,.)
- ,append_lib=%scan(&ds3,1,.)
- ,append_dsn=%scan(&ds3,2,.)
+ ,append_lib=work
+ ,append_dsn=&ds3
,retained_key=filter_rk
,business_key=filter_hash
,maxkeytable=&maxkeytable
,locktable=&lock_table
- ,outds=&ds4
+ ,outds=work.&ds4
)
proc append base=&filter_summary data=&ds4;
run;
diff --git a/base/mp_lockanytable.sas b/base/mp_lockanytable.sas
index adabb6e..afb0375 100644
--- a/base/mp_lockanytable.sas
+++ b/base/mp_lockanytable.sas
@@ -14,19 +14,8 @@
@param [in] ref= A meaningful reference to enable the lock to be traced. Max
length is 200 characters.
@param [out] ctl_ds= (0) The control table which controls the actual locking.
- Should already be assigned and available. Definition as follows:
-
- proc sql;
- create table &ctl_ds(
- lock_lib char(8),
- lock_ds char(32),
- lock_status_cd char(10) not null,
- lock_user_nm char(100) not null ,
- lock_ref char(200),
- lock_pid char(10),
- lock_start_dttm num format=E8601DT26.6,
- lock_end_dttm num format=E8601DT26.6,
- constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+ Should already be assigned and available. The definition is available by
+ running mp_coretable.sas as follows: `mp_coretable(LOCKTABLE)`.
@param [in] loops= (25) Number of times to check for a lock.
@param [in] loop_secs= (1) Seconds to wait between each lock attempt
diff --git a/base/mp_retainedkey.sas b/base/mp_retainedkey.sas
index 25c3d22..8d183bd 100644
--- a/base/mp_retainedkey.sas
+++ b/base/mp_retainedkey.sas
@@ -24,19 +24,12 @@
@li permlib.base_table - the target table to be loaded (**not** loaded by this
macro)
@li permlib.maxkeytable - optional, used to store load metaadata.
- The structure is as follows:
+ The definition is available by running mp_coretable.sas as follows:
+ `mp_coretable(MAXKEYTABLE)`.
+ @li permlib.locktable - Necessary if maxkeytable is being populated. The
+ definition is available by running mp_coretable.sas as follows:
+ `mp_coretable(LOCKTABLE)`.
- proc sql;
- create table yourlib.maxkeytable(
- keytable varchar(41) label='Base table in libref.dataset format',
- keycolumn char(32) format=$32.
- label='The Retained key field containing the key values.',
- max_key num label=
- 'Integer representing current max RK or SK value in the KEYTABLE',
- processed_dttm num format=E8601DT26.6
- label='Datetime this value was last updated',
- constraint pk_mpe_maxkeyvalues
- primary key(keytable));
@param [in] base_lib= (WORK) Libref of the base (target) table.
@param [in] base_dsn= (BASETABLE) Name of the base (target) table.
@@ -102,7 +95,7 @@
%let tempds1=%mf_getuniquename();
%let tempds2=%mf_getuniquename();
%let comma_pk=%mf_getquotedstr(in_str=%str(&business_key),dlm=%str(,),quote=);
-
+%let outds=%sysfunc(ifc(%index(&outds,.)=0,work.&outds,&outds));
/* validation checks */
%let iserr=0;
%if &syscc>0 %then %do;
@@ -133,14 +126,18 @@
%do x=1 %to %sysfunc(countw(&business_key));
/* check business key values exist */
%let key_field=%scan(&business_key,&x,%str( ));
- %if (not %mf_existvar(&app_libds,&key_field))
- or (not %mf_existvar(&base_libds,&key_field))
- %then %do;
+ %if not %mf_existvar(&app_libds,&key_field) %then %do;
%let iserr=1;
- %let msg=Business key (&key_field) not found!;
+ %let msg=Business key (&key_field) not found on &app_libds!;
+ %goto err;
+ %end;
+ %else %if not %mf_existvar(&base_libds,&key_field) %then %do;
+ %let iserr=1;
+ %let msg=Business key (&key_field) not found on &base_libds!;
+ %goto err;
%end;
%end;
-
+%err:
%if &iserr=1 %then %do;
/* err case so first perform an unlock of the base table before exiting */
%mp_lockanytable(
@@ -204,7 +201,7 @@ quit;
* Update maxkey table if link provided
*/
%if &maxkeytable ne 0 %then %do;
- proc sql;
+ proc sql noprint;
select count(*) into: check from &maxkeytable
where upcase(keytable)="&base_libds";
diff --git a/tests/crossplatform/mp_coretable.test.sas b/tests/crossplatform/mp_coretable.test.sas
new file mode 100644
index 0000000..cd19134
--- /dev/null
+++ b/tests/crossplatform/mp_coretable.test.sas
@@ -0,0 +1,31 @@
+/**
+ @file
+ @brief Testing mp_coretable.sas macro
+
+ SAS Macros
+ @li mf_existds.sas
+ @li mp_coretable.sas
+ @li mp_assert.sas
+
+**/
+
+
+%mp_coretable(LOCKTABLE,libds=work.lock)
+%mp_assert(
+ iftrue=(%mf_existds(work.lock)=1),
+ desc=Lock table created,
+ outds=work.test_results
+)
+%mp_coretable(LOCKTABLE)
+%mp_assert(
+ iftrue=("&syscc"="0"),
+ desc=DDL export ran without errors,
+ outds=work.test_results
+)
+
+%mp_coretable(FILTER_SUMMARY,libds=work.sum)
+%mp_assert(
+ iftrue=(%mf_existds(work.sum)=1),
+ desc=Filter summary table created,
+ outds=work.test_results
+)
\ No newline at end of file
diff --git a/tests/crossplatform/mp_filterstore.test.sas b/tests/crossplatform/mp_filterstore.test.sas
new file mode 100644
index 0000000..370f824
--- /dev/null
+++ b/tests/crossplatform/mp_filterstore.test.sas
@@ -0,0 +1,85 @@
+/**
+ @file
+ @brief Testing mp_filterstore macro
+
+ SAS Macros
+ @li mp_coretable.sas
+ @li mp_filterstore.sas
+ @li mp_assertdsobs.sas
+ @li mp_assert.sas
+
+**/
+
+libname permlib (work);
+
+%mp_coretable(LOCKTABLE,libds=permlib.locktable)
+%mp_coretable(FILTER_SUMMARY,libds=permlib.filtsum)
+%mp_coretable(FILTER_DETAIL,libds=permlib.filtdet)
+%mp_coretable(MAXKEYTABLE,libds=permlib.maxkey)
+
+/* valid filter */
+data work.inds;
+ infile datalines4 dsd;
+ input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+ OPERATOR_NM:$12. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,AGE,=,12
+AND,AND,1,SEX,<=,"'M'"
+AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
+AND,OR,2,Weight,>=,77.7
+AND,OR,2,Weight,NE,77.7
+;;;;
+run;
+
+%mp_filterstore(libds=sashelp.class,
+ queryds=work.inds,
+ filter_summary=permlib.filtsum,
+ filter_detail=permlib.filtdet,
+ lock_table=permlib.locktable,
+ maxkeytable=permlib.maxkey,
+ outresult=work.result,
+ outquery=work.query,
+ mdebug=1
+)
+%mp_assert(iftrue=(&syscc>0),
+ desc=Ensure macro runs without errors,
+ outds=work.test_results
+)
+/* ensure only one record created */
+%mp_assertdsobs(permlib.filtsum,
+ desc=Initial query,
+ test=ATMOST 1,
+ outds=work.test_results
+)
+/* check RK is correct */
+proc sql noprint;
+select max(filter_rk) into: test1 from work.result;
+%mp_assert(iftrue=(&test1=1),
+ desc=Ensure filter rk is correct,
+ outds=work.test_results
+)
+
+/* Test 2 - load same table again and ensure we get the same RK */
+%mp_filterstore(libds=sashelp.class,
+ queryds=work.inds,
+ filter_summary=permlib.filtsum,
+ filter_detail=permlib.filtdet,
+ lock_table=permlib.locktable,
+ maxkeytable=permlib.maxkey,
+ outresult=work.result,
+ outquery=work.query,
+ mdebug=1
+)
+/* ensure only one record created */
+%mp_assertdsobs(permlib.filtsum,
+ desc=Initial query - same obs,
+ test=ATMOST 1,
+ outds=work.test_results
+)
+/* check RK is correct */
+proc sql noprint;
+select max(filter_rk) into: test2 from work.result;
+%mp_assert(iftrue=(&test2=1),
+ desc=Ensure filter rk is correct for second run,
+ outds=work.test_results
+)