diff --git a/base/mp_coretable.sas b/base/mp_coretable.sas index d0eca56..1de4c43 100644 --- a/base/mp_coretable.sas +++ b/base/mp_coretable.sas @@ -10,6 +10,8 @@ %mp_coretable(LOCKTABLE,libds=work.locktable) @param [in] table_ref The type of table to create. Example values: + @li DIFFTABLE - Used to store changes to tables. Used by mp_storediffs.sas + and mp_stackdiffs.sas @li FILTER_DETAIL - For storing detailed filter values. Used by mp_filterstore.sas. @li FILTER_SUMMARY - For storing summary filter values. Used by @@ -25,6 +27,8 @@ @li mp_filterstore.sas @li mp_lockanytable.sas @li mp_retainedkey.sas + @li mp_storediffs.sas + @li mp_stackdiffs.sas @version 9.2 @author Allan Bowe @@ -36,7 +40,29 @@ %local outds ; %let outds=%sysfunc(ifc(&libds=0,_data_,&libds)); proc sql; -%if &table_ref=LOCKTABLE %then %do; +%if &table_ref=DIFFTABLE %then %do; + create table &outds( + load_ref char(36) label='unique load reference', + processed_dttm num format=E8601DT26.6 label='Processed at timestamp', + libref char(8) label='Library Reference (8 chars)', + dsn char(32) label='Dataset Name (32 chars)', + key_hash char(32) label= + 'MD5 Hash of primary key values (pipe seperated)', + move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified', + is_pk num label='Is Primary Key Field? (1/0)', + is_diff num label= + 'Did value change? (1/0/-1). Always -1 for appends and deletes.', + tgtvar_type char(1) label='Either (C)haracter or (N)umeric', + tgtvar_nm char(32) label='Target variable name (32 chars)', + oldval_num num format=best32. label='Old (numeric) value', + newval_num num format=best32. label='New (numeric) value', + oldval_char char(32765) label='Old (character) value', + newval_char char(32765) label='New (character) value', + constraint pk_mpe_audit + primary key(load_ref,libref,dsn,key_hash,tgtvar_nm) + ); +%end; +%else %if &table_ref=LOCKTABLE %then %do; create table &outds( lock_lib char(8), lock_ds char(32), diff --git a/base/mp_stackdiffs.sas b/base/mp_stackdiffs.sas new file mode 100644 index 0000000..a6da2be --- /dev/null +++ b/base/mp_stackdiffs.sas @@ -0,0 +1,224 @@ +/** + @file + @brief Prepares an audit table for stacking (re-applying) the changes. + @details WORK IN PROGRESS!! + + When the underlying data from a Base Table is refreshed, it can be helpful + to have any previously-applied changes, re-applied. + + Such situation might arise if you are applying those changes using a tool + like [Data Controller for SASĀ®](https://datacontroller.io) - which records + all such changes in an audit table. + It may also apply if you are preparing a series of specific cell-level + transactions, that you would like to apply to multiple sets of (similarly + structured) Base Tables. + + In both cases, it is necessary that the transactions are stored using + the mp_storediffs.sas macro, or at least that the underlying table is + structured as per the definition in mp_coretable.sas (DIFFTABLE entry) + + This macro is used to convert the stored changes (tall format) into + staged changes (wide format), with base table values incorporated (in the + case of modified rows), ready for the subsequent load process. + + Essentially then, what this macro does, is turn a table like this: + + |LIBREF:$8.|DSN:$32.|MOVE_TYPE:$1.|TGTVAR_NM:$32.|IS_PK:best.|IS_DIFF:best.|TGTVAR_TYPE:$1.|OLDVAL_NUM:best32.|NEWVAL_NUM:best32.|OLDVAL_CHAR:$32765.|NEWVAL_CHAR:$32765.| + |---|---|---|---|---|---|---|---|---|---|---| + |`SASHELP `|`CLASS `|`A `|`NAME `|`1 `|`-1 `|`C `|`. `|`. `|` `|`Newbie `| + |`SASHELP `|`CLASS `|`A `|`AGE `|`0 `|`-1 `|`N `|`. `|`13 `|` `|` `| + |`SASHELP `|`CLASS `|`A `|`HEIGHT `|`0 `|`-1 `|`N `|`. `|`65.3 `|` `|` `| + |`SASHELP `|`CLASS `|`A `|`SEX `|`0 `|`-1 `|`C `|`. `|`. `|` `|`F `| + |`SASHELP `|`CLASS `|`A `|`WEIGHT `|`0 `|`-1 `|`N `|`. `|`98 `|` `|` `| + |`SASHELP `|`CLASS `|`D `|`NAME `|`1 `|`-1 `|`C `|`. `|`. `|`Alfred `|` `| + |`SASHELP `|`CLASS `|`D `|`AGE `|`0 `|`-1 `|`N `|`14 `|`. `|` `|` `| + |`SASHELP `|`CLASS `|`D `|`HEIGHT `|`0 `|`-1 `|`N `|`69 `|`. `|` `|` `| + |`SASHELP `|`CLASS `|`D `|`SEX `|`0 `|`-1 `|`C `|`. `|`. `|`M `|` `| + |`SASHELP `|`CLASS `|`D `|`WEIGHT `|`0 `|`-1 `|`N `|`112.5 `|`. `|` `|` `| + |`SASHELP `|`CLASS `|`M `|`NAME `|`1 `|`0 `|`C `|`. `|`. `|`Alice `|`Alice `| + |`SASHELP `|`CLASS `|`M `|`AGE `|`0 `|`1 `|`N `|`13 `|`99 `|` `|` `| + |`SASHELP `|`CLASS `|`M `|`HEIGHT `|`0 `|`0 `|`N `|`56.5 `|`56.5 `|` `|` `| + |`SASHELP `|`CLASS `|`M `|`SEX `|`0 `|`0 `|`C `|`. `|`. `|`F `|`F `| + |`SASHELP `|`CLASS `|`M `|`WEIGHT `|`0 `|`0 `|`N `|`84 `|`84 `|` `|` `| + + Into three tables like this: + + `work.outmod`: + |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.| + |---|---|---|---|---| + |`Alice `|`F `|`99 `|`56.5 `|`84 `| + + `work.outadd`: + |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.| + |---|---|---|---|---| + |`Newbie `|`F `|`13 `|`65.3 `|`98 `| + + `work.outdel`: + |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.| + |---|---|---|---|---| + |`Alfred `|`M `|`14 `|`69 `|`112.5 `| + + As you might expect, there are a bunch of extra features and checks. + + The macro supports both SCD2 (TXTEMPORAL) and UPDATE loadtypes. If the + base table contains a PROCESSED_DTTM column (or similar), this can be + ignored by declaring it in the `processed_dttm_var` parameter. + + If the structure of the Base Table has changed, the following rules apply: + + @li New Columns - Irrelevant for deletes. For inserts, the new column + values are set to missing. For updates, the base table values are used. + @li Deleted Columns - These will also be absent in the output tables. + @li Change in Primary Key - This is not (well, cannot be) supported!! + + Rules for stacking changes are as follows: + +
| Transaction Type | Behaviour |
|---|---|
| Deletes | +
+ For previously deleted rows, the PK is added to the `outDEL` table + If the row no longer exists in the base table, the row is added to the + `errDS` table instead. + |
+
| Inserts | +
+ Previously newly added rows are added to the `outADD` table UNLESS they
+ are present in the Base table. In this case they are added to the + `errDS` table instead. + |
+
| Updates | +
+ Previously modified rows are merged with base table values such that
+ only the individual cells that were _previously_ changed are re-applied.
+ Where the row contains cells that were not marked as having changed in
+ the prior transaction, the 'blanks' are filled with base table values in
+ the `outMOD` table. + If the row no longer exists on the base table, then the row is added to + the `errDS` table instead. + |
+