sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2026-01-08 10:00:04 +00:00
Code Issues Projects Releases Wiki Activity

feat: new mp_filterstore macro

Browse Source
This commit is contained in:
munja
2021-12-26 15:06:06 +00:00
parent 8e723d06b0
commit ce026f19b5
7 changed files with 566 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
base/mp_coretable.sas Normal file
View File

@@ -0,0 +1,92 @@
/**
@file
@brief Create the permanent Core tables
@details Several macros in the [core](https://github.com/sasjs/core) library
make use of permanent tables. To avoid duplication in definitions, this
macro provides a central location for managing the corresponding DDL.
Example usage:
%mp_coretable(LOCKTABLE,libds=work.locktable)
@param [in] table_ref The type of table to create. Example values:
@li FILTER_DETAIL - For storing detailed filter values. Used by
mp_filterstore.sas.
@li FILTER_SUMMARY - For storing summary filter values. Used by
mp_filterstore.sas.
@li LOCKANYTABLE - For "locking" tables prior to multipass loads. Used by
mp_lockanytable.sas
@li MAXKEYTABLE - For storing the maximum retained key information. Used
by mp_retainedkey.sas
@param [in] libds= (0) The library.dataset reference used to create the table.
If not provided, then the DDL is simply printed to the log.
<h4> Related Macros </h4>
@li mp_filterstore.sas
@li mp_lockanytable.sas
@li mp_retainedkey.sas
@version 9.2
@author Allan Bowe
**/
%macro mp_coretable(table_ref,libds=0
)/*/STORE SOURCE*/;
%local outds ;
%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
proc sql;
%if &table_ref=LOCKTABLE %then %do;
create table &outds(
lock_lib char(8),
lock_ds char(32),
lock_status_cd char(10) not null,
lock_user_nm char(100) not null ,
lock_ref char(200),
lock_pid char(10),
lock_start_dttm num format=E8601DT26.6,
lock_end_dttm num format=E8601DT26.6,
constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
%end;
%else %if &table_ref=FILTER_SUMMARY %then %do;
create table &outds(
filter_rk num not null,
filter_hash char(32) not null,
filter_table char(41) not null,
processed_dttm num not null format=E8601DT26.6,
constraint pk_mpe_filteranytable
primary key(filter_rk));
%end;
%else %if &table_ref=FILTER_DETAIL %then %do;
create table &outds(
filter_hash char(32) not null,
filter_line num not null,
group_logic char(3) not null,
subgroup_logic char(3) not null,
subgroup_id num not null,
variable_nm varchar(32) not null,
operator_nm varchar(12) not null,
raw_value varchar(4000) not null,
processed_dttm num not null format=E8601DT26.6,
constraint pk_mpe_filteranytable
primary key(filter_hash,filter_line));
%end;
%else %if &table_ref=MAXKEYTABLE %then %do;
create table &outds(
keytable varchar(41) label='Base table in libref.dataset format',
keycolumn char(32) format=$32.
label='The Retained key field containing the key values.',
max_key num label=
'Integer representing current max RK or SK value in the KEYTABLE',
processed_dttm num format=E8601DT26.6
label='Datetime this value was last updated',
constraint pk_mpe_maxkeyvalues
primary key(keytable));
%end;
%if &libds=0 %then %do;
describe table &syslast;
drop table &syslast;
%end;
%mend mp_coretable;

28
base/mp_filterstore.sas
View File

@@ -2,10 +2,11 @@
@file
@brief Checks & Stores an input filter table and returns the Filter Key
@details Used to generate a FILTER_RK from an input query dataset. This
process requires several permanent tables (names are configurable):
@li filterdetail (contains raw values)
@li filtersummary (contains summary data about the filter)
process requires several permanent tables (names are configurable). The
benefit of storing query values at backend is to enable stored 'views' of
filtered tables at frontend (ie, when building [SAS-Powered Apps](
https://sasapps.io)). This macro is also used in [Data Controller for SAS](
https://datacontroller.io).
@param [in] libds= The target dataset to be filtered (lib should be assigned)
@@ -16,21 +17,26 @@
|AND|AND|1|SOME_BESTNUM|>|1|
|AND|AND|1|SOME_TIME|=|77333|
@param [in] filter_summary= (PERM.FILTER_SUMMARY) Permanent table containing
summary filter values. Structure:
summary filter values. The definition is available by running
mp_coretable.sas as follows: `mp_coretable(FILTER_SUMMARY)`. Example
values:
|FILTER_RK:best.|FILTER_HASH:$32.|FILTER_TABLE:$41.|PROCESSED_DTTM:datetime19.|
|---|---|---|---|
|`1 `|`540E96F566D194AB58DD4C413C99C9DB `|`VIYA6014.MPE_TABLES `|`1956084246 `|
|`2 `|`87737DB9EEE2650F5C89956CEAD0A14F `|`VIYA6014.MPE_X_TEST `|`1956084452.1`|
|`3 `|`8048BD908DBBD83D013560734E90D394 `|`VIYA6014.MPE_TABLES `|`1956093620.6`|
@param [in] filter_detail= (PERM.FILTER_DETAIL) Permanent table containing
detailed (raw) filter values. Structure:
detailed (raw) filter values. The definition is available by running
mp_coretable.sas as follows: `mp_coretable(FILTER_DETAIL)`. Example
values:
|FILTER_HASH:$32.|FILTER_LINE:best.|GROUP_LOGIC:$3.|SUBGROUP_LOGIC:$3.|SUBGROUP_ID:best.|VARIABLE_NM:$32.|OPERATOR_NM:$12.|RAW_VALUE:$4000.|PROCESSED_DTTM:datetime19.|
|---|---|---|---|---|---|---|---|---|
|`540E96F566D194AB58DD4C413C99C9DB `|`1 `|`AND `|`AND `|`1 `|`LIBREF `|`CONTAINS `|`DC`|`1956084245.8 `|
|`540E96F566D194AB58DD4C413C99C9DB `|`2 `|`AND `|`OR `|`2 `|`DSN `|`= `|` MPE_LOCK_ANYTABLE `|`1956084245.8 `|
|`87737DB9EEE2650F5C89956CEAD0A14F `|`1 `|`AND `|`AND `|`1 `|`PRIMARY_KEY_FIELD `|`IN `|`(1,2,3) `|`1956084451.9 `|
@param [in] lock_table= (PERM.LOCK_TABLE) Permanent locking table. Used to
manage concurrent access. Described in mp_lockanytable.sas.
manage concurrent access. The definition is available by running
mp_coretable.sas as follows: `mp_coretable(LOCKTABLE)`.
@param [in] maxkeytable= (0) Optional permanent reference table used for
retained key tracking. Described in mp_retainedkey.sas.
@param [in] mdebug= set to 1 to enable DEBUG messages
@@ -94,6 +100,7 @@
%mp_filtercheck(&queryds,targetds=&libds,abort=YES)
/* hash the result */
%let ds1=%mf_getuniquename(prefix=hashds);
%mp_hashdataset(&queryds,outds=&ds1,salt=&libds)
%let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
%if &mdebug=1 %then %do;
@@ -124,6 +131,7 @@ run;
/* update detail table first */
%let ds2=%mf_getuniquename(prefix=filterdetail);
data &ds2;
if 0 then set &filter_detail;
set &queryds;
format filter_hash $hex32. filter_line 8. processed_dttm E8601DT26.6;
filter_hash="&filter_hash";
@@ -168,13 +176,13 @@ run;
%mp_retainedkey(
base_lib=%scan(&filter_summary,1,.)
,base_dsn=%scan(&filter_summary,2,.)
,append_lib=%scan(&ds3,1,.)
,append_dsn=%scan(&ds3,2,.)
,append_lib=work
,append_dsn=&ds3
,retained_key=filter_rk
,business_key=filter_hash
,maxkeytable=&maxkeytable
,locktable=&lock_table
,outds=&ds4
,outds=work.&ds4
)
proc append base=&filter_summary data=&ds4;
run;

15
base/mp_lockanytable.sas
View File

@@ -14,19 +14,8 @@
@param [in] ref= A meaningful reference to enable the lock to be traced. Max
length is 200 characters.
@param [out] ctl_ds= (0) The control table which controls the actual locking.
Should already be assigned and available. Definition as follows:
proc sql;
create table &ctl_ds(
lock_lib char(8),
lock_ds char(32),
lock_status_cd char(10) not null,
lock_user_nm char(100) not null ,
lock_ref char(200),
lock_pid char(10),
lock_start_dttm num format=E8601DT26.6,
lock_end_dttm num format=E8601DT26.6,
constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
Should already be assigned and available. The definition is available by
running mp_coretable.sas as follows: `mp_coretable(LOCKTABLE)`.
@param [in] loops= (25) Number of times to check for a lock.
@param [in] loop_secs= (1) Seconds to wait between each lock attempt

35
base/mp_retainedkey.sas
View File

@@ -24,19 +24,12 @@
@li permlib.base_table - the target table to be loaded (**not** loaded by this
macro)
@li permlib.maxkeytable - optional, used to store load metaadata.
The structure is as follows:
The definition is available by running mp_coretable.sas as follows:
`mp_coretable(MAXKEYTABLE)`.
@li permlib.locktable - Necessary if maxkeytable is being populated. The
definition is available by running mp_coretable.sas as follows:
`mp_coretable(LOCKTABLE)`.
proc sql;
create table yourlib.maxkeytable(
keytable varchar(41) label='Base table in libref.dataset format',
keycolumn char(32) format=$32.
label='The Retained key field containing the key values.',
max_key num label=
'Integer representing current max RK or SK value in the KEYTABLE',
processed_dttm num format=E8601DT26.6
label='Datetime this value was last updated',
constraint pk_mpe_maxkeyvalues
primary key(keytable));
@param [in] base_lib= (WORK) Libref of the base (target) table.
@param [in] base_dsn= (BASETABLE) Name of the base (target) table.
@@ -102,7 +95,7 @@
%let tempds1=%mf_getuniquename();
%let tempds2=%mf_getuniquename();
%let comma_pk=%mf_getquotedstr(in_str=%str(&business_key),dlm=%str(,),quote=);
%let outds=%sysfunc(ifc(%index(&outds,.)=0,work.&outds,&outds));
/* validation checks */
%let iserr=0;
%if &syscc>0 %then %do;
@@ -133,14 +126,18 @@
%do x=1 %to %sysfunc(countw(&business_key));
/* check business key values exist */
%let key_field=%scan(&business_key,&x,%str( ));
%if (not %mf_existvar(&app_libds,&key_field))
or (not %mf_existvar(&base_libds,&key_field))
%then %do;
%if not %mf_existvar(&app_libds,&key_field) %then %do;
%let iserr=1;
%let msg=Business key (&key_field) not found!;
%let msg=Business key (&key_field) not found on &app_libds!;
%goto err;
%end;
%else %if not %mf_existvar(&base_libds,&key_field) %then %do;
%let iserr=1;
%let msg=Business key (&key_field) not found on &base_libds!;
%goto err;
%end;
%end;
%err:
%if &iserr=1 %then %do;
/* err case so first perform an unlock of the base table before exiting */
%mp_lockanytable(
@@ -204,7 +201,7 @@ quit;
* Update maxkey table if link provided
*/
%if &maxkeytable ne 0 %then %do;
proc sql;
proc sql noprint;
select count(*) into: check from &maxkeytable
where upcase(keytable)="&base_libds";