Merge pull request #131 from sasjs/mp_filterstore

feat: mp_filterstore macro

all.sas

@@ -249,8 +249,12 @@ https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionex
 
         %put %mf_existvar(work.someds, somevar)
 
-  @param libds (positional) - 2 part dataset or view reference
-  @param var (positional) - variable name
+  @param [in] libds 2 part dataset or view reference
+  @param [in] var variable name
+
+  <h4> Related Macros </h4>
+  @li mf_existvar.test.sas
+
   @version 9.2
   @author Allan Bowe
 **/
@@ -484,9 +488,10 @@ https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionex
   contributors of Chris Hemedingers blog [post](
   http://blogs.sas.com/content/sasdummy/2013/06/04/find-a-sas-library-engine/)
 
-  @param libref Library reference (also accepts a 2 level libds ref).
+  @param [in] libref Library reference (also accepts a 2 level libds ref).
 
-  @return output returns the library engine for the FIRST library encountered.
+  @return output returns the library engine (uppercase) for the FIRST library
+    encountered.
 
   @warning will only return the FIRST library engine - for concatenated
     libraries, with different engines, inconsistent results may be encountered.
@@ -518,7 +523,7 @@ https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionex
     %let rc= %sysfunc(close(&dsid));
   %end;
 
-  &engine
+  %upcase(&engine)
 
 %mend mf_getengine;
 
@@ -1953,6 +1958,72 @@ Usage:
 %mend mf_wordsInStr1ButNotStr2;
 
 /**
+  @file
+  @brief Creates a text file using pure macro
+  @details Creates a text file of up to 10 lines.  If further lines are
+    desired, feel free to [create an issue](
+    https://github.com/sasjs/core/issues/new), or make a pull request!
+
+    The use of PARMBUFF was considered for this macro, but it would have made
+    things problematic for writing lines containing commas.
+
+    Usage:
+
+        %mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=more content)
+        data _null_;
+          infile "&sasjswork/myfile.txt";
+          input;
+          list;
+        run;
+
+  @param [in] fpath Full path to file to be created or appended to
+  @param [in] mode= (O) Available options are A or O as follows:
+    @li A APPEND mode, writes new records after the current end of the file.
+    @li O OUTPUT mode, writes new records from the beginning of the file.
+  @param [in] l1= First line
+  @param [in] l2= Second line (etc through to l10)
+
+  <h4> Related Macros </h4>
+  @li mf_writefile.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mf_writefile(fpath,mode=O,l1=,l2=,l3=,l4=,l5=,l6=,l7=,l8=,l9=,l10=
+)/*/STORE SOURCE*/;
+%local fref rc fid i total_lines;
+
+/* find number of lines by reference to first non-blank param */
+%do i=10 %to 1 %by -1;
+  %if %str(&&l&i) ne %str() %then %goto continue;
+%end;
+%continue:
+%let total_lines=&i;
+
+%if %sysfunc(filename(fref,&fpath)) ne 0 %then %do;
+  %put &=fref &=fpath;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let fid=%sysfunc(fopen(&fref,&mode));
+
+%if &fid=0 %then %do;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%do i=1 %to &total_lines;
+  %let rc=%sysfunc(fput(&fid, &&l&i));
+  %let rc=%sysfunc(fwrite(&fid));
+%end;
+%let rc=%sysfunc(fclose(&fid));
+%let rc=%sysfunc(filename(&fref));
+
+%mend mf_writefile;
+/** @endcond *//**
   @file
   @brief abort gracefully according to context
   @details Configures an abort mechanism according to site specific policies or
@@ -2264,6 +2335,186 @@ Usage:
 %end;
 
 %mend mp_appendfile;/**
+  @file
+  @brief Apply a set of formats to a table
+  @details Applies a set of formats to the metadata of one or more SAS datasets.
+    Can be used to migrate formats from one table to another.  The input table
+    must contain the following columns:
+
+    @li lib - the libref of the table to be updated
+    @li ds - the dataset to be updated
+    @li var - the variable to be updated
+    @li fmt - the format to apply.  Missing or default ($CHAR, 8.) formats are
+      ignored.
+
+  The macro will abort in the following scenarios:
+
+    @li Libref not assigned
+    @li Dataset does not exist
+    @li Input table contains null or invalid values
+
+  Example usage:
+
+      data work.example;
+        set sashelp.prdsale;
+        format _all_ clear;
+      run;
+
+      %mp_getcols(sashelp.prdsale,outds=work.cols)
+
+      data work.cols2;
+        set work.cols;
+        lib='WORK';
+        ds='EXAMPLE';
+        var=name;
+        fmt=format;
+        keep lib ds var fmt;
+      run;
+
+      %mp_applyformats(work.cols2)
+
+  @param [in] inds The input dataset containing the formats to apply (and where
+    to apply them).  Example structure:
+  |LIB:$8.|DS:$32.|VAR:$32.|FMT:$49.|
+  |---|---|---|---|
+  |`WORK `|`EXAMPLE `|`ACTUAL `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`COUNTRY `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`DIVISION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`MONTH `|`MONNAME3. `|
+  |`WORK `|`EXAMPLE `|`PREDICT `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`PRODTYPE `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`PRODUCT `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`QUARTER `|`8. `|
+  |`WORK `|`EXAMPLE `|`REGION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`YEAR `|`8. `|
+
+  @param [out] errds= (0) Provide a libds reference here to export the
+    error messages to a table.  In this case, they will not be printed to the
+    log.
+
+  <h4> SAS Macros </h4>
+  @li mf_getengine.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_validatecol.sas
+
+
+  <h4> Related Macros </h4>
+  @li mp_getformats.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_applyformats(inds,errds=0
+)/*/STORE SOURCE*/;
+%local outds liblist i engine lib msg ;
+
+/**
+  * Validations
+  */
+proc sort data=&inds;
+  by lib ds var fmt;
+run;
+
+%if &errds=0 %then %let outds=%mf_getuniquename(prefix=mp_applyformats);
+%else %let outds=&errds;
+
+data &outds;
+  set &inds;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  length msg $128;
+  by lib ds var fmt;
+  if libref(lib) ne 0 then do;
+    msg=catx(' ','libref',lib,'is not assigned!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  if exist(cats(lib,'.',ds)) ne 1 then do;
+    msg=catx(' ','libds',lib,'.',ds,'does not exist!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  %mp_validatecol(fmt,FORMAT,is_fmt)
+  if is_fmt=0 then do;
+    msg=catx(' ','format',fmt,'on libds',lib,'.',ds,'.',var,'is not valid!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+
+  if first.ds then do;
+    retain dsid;
+    dsid=open(cats(lib,'.',ds));
+    if dsid=0 then do;
+      msg=catx(' ','libds',lib,'.',ds,' could not be opened!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+      return;
+    end;
+    if varnum(dsid,var)<1 then do;
+      msg=catx(' ','Variable',lib,'.',ds,'.',var,' was not found!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+    end;
+  end;
+  if last.ds then rc=close(dsid);
+run;
+
+proc sql noprint;
+select distinct lib into: liblist separated by ' ' from &inds;
+%put &=liblist;
+%do i=1 %to %sysfunc(countw(&liblist));
+  %let lib=%scan(&liblist,1);
+  %let engine=%mf_getengine(&lib);
+  %if &engine ne V9 and &engine ne BASE %then %do;
+    %let msg=&lib has &engine engine - formats cannot be applied;
+    proc sql;
+    insert into &outds set lib="&lib",ds="_all_",var="_all", msg="&msg" ;
+    %if &errds=0 %then %put %str(ERR)OR: &msg;
+  %end;
+%end;
+
+%if %mf_nobs(&outds)>0 %then %return;
+
+/**
+  * Validations complete - now apply the actual formats!
+  */
+%let fref=%mf_getuniquefileref();
+data _null_;
+  set &inds;
+  by lib ds var fmt;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  file &fref;
+  if first.lib then put 'proc datasets nolist lib=' lib ';';
+  if first.ds then put '  modify ' ds ';';
+  put '    format ' var fmt ';';
+  if last.ds then put '  run;';
+  if last.lib then put 'quit;';
+run;
+
+%inc &fref/source2;
+
+%if &errds=0 %then %do;
+  proc sql;
+  drop table &outds;
+%end;
+
+%mend mp_applyformats;/**
   @file
   @brief Generic assertion
   @details Useful in the context of writing sasjs tests.  The results of the
@@ -3100,6 +3351,97 @@ run;
   drop table work.&tempds;
 
 %mend mp_copyfolder;/**
+  @file
+  @brief Create the permanent Core tables
+  @details Several macros in the [core](https://github.com/sasjs/core) library
+    make use of permanent tables.  To avoid duplication in definitions, this
+    macro provides a central location for managing the corresponding DDL.
+
+  Example usage:
+
+      %mp_coretable(LOCKTABLE,libds=work.locktable)
+
+  @param [in] table_ref The type of table to create.  Example values:
+    @li FILTER_DETAIL - For storing detailed filter values.  Used by
+      mp_filterstore.sas.
+    @li FILTER_SUMMARY - For storing summary filter values.  Used by
+      mp_filterstore.sas.
+    @li LOCKANYTABLE - For "locking" tables prior to multipass loads. Used by
+      mp_lockanytable.sas
+    @li MAXKEYTABLE - For storing the maximum retained key information.  Used
+      by mp_retainedkey.sas
+  @param [in] libds= (0) The library.dataset reference used to create the table.
+    If not provided, then the DDL is simply printed to the log.
+
+  <h4> Related Macros </h4>
+  @li mp_filterstore.sas
+  @li mp_lockanytable.sas
+  @li mp_retainedkey.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_coretable(table_ref,libds=0
+)/*/STORE SOURCE*/;
+%local outds ;
+%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
+proc sql;
+%if &table_ref=LOCKTABLE %then %do;
+  create table &outds(
+      lock_lib char(8),
+      lock_ds char(32),
+      lock_status_cd char(10) not null,
+      lock_user_nm char(100) not null ,
+      lock_ref char(200),
+      lock_pid char(10),
+      lock_start_dttm num format=E8601DT26.6,
+      lock_end_dttm num format=E8601DT26.6,
+    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+%end;
+%else %if &table_ref=FILTER_SUMMARY %then %do;
+  create table &outds(
+      filter_rk num not null,
+      filter_hash char(32) not null,
+      filter_table char(41) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_rk));
+%end;
+%else %if &table_ref=FILTER_DETAIL %then %do;
+  create table &outds(
+      filter_hash char(32) not null,
+      filter_line num not null,
+      group_logic char(3) not null,
+      subgroup_logic char(3) not null,
+      subgroup_id num not null,
+      variable_nm varchar(32) not null,
+      operator_nm varchar(12) not null,
+      raw_value varchar(4000) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_hash,filter_line));
+%end;
+%else %if &table_ref=MAXKEYTABLE %then %do;
+  create table &outds(
+      keytable varchar(41) label='Base table in libref.dataset format',
+      keycolumn char(32) format=$32.
+        label='The Retained key field containing the key values.',
+      max_key num label=
+        'Integer representing current max RK or SK value in the KEYTABLE',
+      processed_dttm num format=E8601DT26.6
+        label='Datetime this value was last updated',
+    constraint pk_mpe_maxkeyvalues
+        primary key(keytable));
+%end;
+
+
+%if &libds=0 %then %do;
+  describe table &syslast;
+  drop table &syslast;
+%end;
+%mend mp_coretable;/**
   @file mp_createconstraints.sas
   @brief Creates constraints
   @details Takes the output from mp_getconstraints.sas as input
@@ -4847,6 +5189,218 @@ filename &outref temp;
 
 %mend mp_filtergenerate;
 /**
+  @file
+  @brief Checks & Stores an input filter table and returns the Filter Key
+  @details Used to generate a FILTER_RK from an input query dataset.  This
+  process requires several permanent tables (names are configurable).  The
+  benefit of storing query values at backend is to enable stored 'views' of
+  filtered tables at frontend (ie, when building [SAS-Powered Apps](
+  https://sasapps.io)).  This macro is also used in [Data Controller for SAS](
+  https://datacontroller.io).
+
+
+  @param [in] libds= The target dataset to be filtered (lib should be assigned)
+  @param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to
+    be validated.  Has the following format:
+|GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
+|---|---|---|---|---|---|
+|AND|AND|1|SOME_BESTNUM|>|1|
+|AND|AND|1|SOME_TIME|=|77333|
+  @param [in] filter_summary= (PERM.FILTER_SUMMARY) Permanent table containing
+    summary filter values.  The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(FILTER_SUMMARY)`. Example
+    values:
+|FILTER_RK:best.|FILTER_HASH:$32.|FILTER_TABLE:$41.|PROCESSED_DTTM:datetime19.|
+|---|---|---|---|
+|`1 `|`540E96F566D194AB58DD4C413C99C9DB `|`VIYA6014.MPE_TABLES `|`1956084246 `|
+|`2 `|`87737DB9EEE2650F5C89956CEAD0A14F `|`VIYA6014.MPE_X_TEST `|`1956084452.1`|
+|`3 `|`8048BD908DBBD83D013560734E90D394 `|`VIYA6014.MPE_TABLES `|`1956093620.6`|
+  @param [in] filter_detail= (PERM.FILTER_DETAIL) Permanent table containing
+    detailed (raw) filter values. The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(FILTER_DETAIL)`. Example
+    values:
+|FILTER_HASH:$32.|FILTER_LINE:best.|GROUP_LOGIC:$3.|SUBGROUP_LOGIC:$3.|SUBGROUP_ID:best.|VARIABLE_NM:$32.|OPERATOR_NM:$12.|RAW_VALUE:$4000.|PROCESSED_DTTM:datetime19.|
+|---|---|---|---|---|---|---|---|---|
+|`540E96F566D194AB58DD4C413C99C9DB `|`1 `|`AND `|`AND `|`1 `|`LIBREF `|`CONTAINS `|`DC`|`1956084245.8 `|
+|`540E96F566D194AB58DD4C413C99C9DB `|`2 `|`AND `|`OR `|`2 `|`DSN `|`= `|` MPE_LOCK_ANYTABLE `|`1956084245.8 `|
+|`87737DB9EEE2650F5C89956CEAD0A14F `|`1 `|`AND `|`AND `|`1 `|`PRIMARY_KEY_FIELD `|`IN `|`(1,2,3) `|`1956084451.9 `|
+  @param [in] lock_table= (PERM.LOCK_TABLE) Permanent locking table.  Used to
+    manage concurrent access.  The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(LOCKTABLE)`.
+  @param [in] maxkeytable= (0) Optional permanent reference table used for
+    retained key tracking.  Described in mp_retainedkey.sas.
+  @param [in] mdebug= set to 1 to enable DEBUG messages
+  @param [out] outresult= The result table with the FILTER_RK
+  @param [out] outquery= The original query, taken as extract after table load
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_getvalue.sas
+  @li mf_islibds.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_filtercheck.sas
+  @li mp_hashdataset.sas
+  @li mp_retainedkey.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filtercheck.sas
+  @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
+  @li mp_filterstore.test.sas
+
+  @version 9.2
+  @author [Allan Bowe](https://www.linkedin.com/in/allanbowe)
+
+**/
+
+%macro mp_filterstore(libds=,
+  queryds=work.filterquery,
+  filter_summary=PERM.FILTER_SUMMARY,
+  filter_detail=PERM.FILTER_DETAIL,
+  lock_table=PERM.LOCK_TABLE,
+  maxkeytable=PERM.MAXKEYTABLE,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+);
+%put &sysmacroname entry vars:;
+%put _local_;
+
+%local ds1 ds2 ds3 ds4 filter_hash;
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+%mp_abort(iftrue= (%mf_islibds(&filter_summary)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid filter_summary value: &filter_summary)
+)
+%mp_abort(iftrue= (%mf_islibds(&filter_detail)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid filter_detail value: &filter_detail)
+)
+%mp_abort(iftrue= (%mf_islibds(&lock_table)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid lock_table value: &lock_table)
+)
+
+/* validate query */
+%mp_filtercheck(&queryds,targetds=&libds,abort=YES)
+
+/* hash the result */
+%let ds1=%mf_getuniquename(prefix=hashds);
+%mp_hashdataset(&queryds,outds=&ds1,salt=&libds)
+%let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
+%if &mdebug=1 %then %do;
+  data _null_;
+    putlog "filter_hash=&filter_hash";
+    set &ds1;
+    putlog (_all_)(=);
+  run;
+%end;
+
+/* check if data already exists for this hash */
+data &outresult;
+  set &filter_summary;
+  where filter_hash="&filter_hash";
+run;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc after hash check)
+)
+%mp_abort(iftrue= ("&filter_hash"=" ")
+  ,mac=mp_filterstore
+  ,msg=%str(problem with filter_hash generation)
+)
+
+%if %mf_nobs(&outresult)=0 %then %do;
+
+  /* update detail table first */
+  %let ds2=%mf_getuniquename(prefix=filterdetail);
+  data &ds2;
+    if 0 then set &filter_detail;
+    set &queryds;
+    format filter_hash $hex32. filter_line 8. processed_dttm E8601DT26.6;
+    filter_hash="&filter_hash";
+    filter_line=_n_;
+    PROCESSED_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt;
+  run;
+  %mp_lockanytable(LOCK,
+    lib=%scan(&filter_detail,1,.)
+    ,ds=%scan(&filter_detail,2,.)
+    ,ref=MP_FILTERSTORE update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+  proc append base=&filter_detail data=&ds2;
+  run;
+
+  %mp_lockanytable(UNLOCK,
+    lib=%scan(&filter_detail,1,.)
+    ,ds=%scan(&filter_detail,2,.)
+    ,ref=MP_FILTERSTORE update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  /* now update summary table */
+  %let ds3=%mf_getuniquename(prefix=filtersum);
+  data &ds3;
+    if 0 then set &filter_summary;
+    filter_table=symget('libds');
+    filter_hash="&filter_hash";
+    PROCESSED_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt;
+    output;
+    stop;
+  run;
+
+  %mp_lockanytable(LOCK,
+    lib=%scan(&filter_summary,1,.)
+    ,ds=%scan(&filter_summary,2,.)
+    ,ref=MP_FILTERSTORE update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  %let ds4=%mf_getuniquename(prefix=filtersumappend);
+  %mp_retainedkey(
+    base_lib=%scan(&filter_summary,1,.)
+    ,base_dsn=%scan(&filter_summary,2,.)
+    ,append_lib=work
+    ,append_dsn=&ds3
+    ,retained_key=filter_rk
+    ,business_key=filter_hash
+    ,maxkeytable=&maxkeytable
+    ,locktable=&lock_table
+    ,outds=work.&ds4
+  )
+  proc append base=&filter_summary data=&ds4;
+  run;
+
+  %mp_lockanytable(UNLOCK,
+    lib=%scan(&filter_summary,1,.)
+    ,ds=%scan(&filter_summary,2,.)
+    ,ref=MP_FILTERSTORE update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  data &outresult;
+    set &filter_summary;
+    where filter_hash="&filter_hash";
+  run;
+
+%end;
+
+proc sort data=&filter_detail(where=(filter_hash="&filter_hash")) out=&outquery;
+  by filter_line;
+run;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc on macro exit)
+)
+
+%mend mp_filterstore;/**
   @file
   @brief Checks a generated filter query for validity
   @details Runs a generated filter in proc sql with the validate option.
@@ -5894,6 +6448,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 
 
   <h4> Related Macros </h4>
+  @li mp_applyformats.sas
   @li mp_getformats.test.sas
 
   @version 9.2
@@ -7323,19 +7878,18 @@ select distinct lowcase(memname)
   Only useful if every update uses the macro!   Used heavily within
   [Data Controller for SAS](https://datacontroller.io).
 
-  The underlying table is structured as per the MAKETABLE action.
-
   @param [in] action The action to be performed.  Valid values:
     @li LOCK - Sets the lock flag, also confirms if a SAS lock is available
     @li UNLOCK - Unlocks the table
-    @li MAKETABLE - creates the control table (ctl_ds)
   @param [in] lib= (WORK) The libref of the table to lock.  Should already be
     assigned.
   @param [in] ds= The dataset to lock
   @param [in] ref= A meaningful reference to enable the lock to be traced. Max
     length is 200 characters.
   @param [out] ctl_ds= (0) The control table which controls the actual locking.
-    Should already be assigned and available.
+    Should already be assigned and available.  The definition is available by
+    running mp_coretable.sas as follows:  `mp_coretable(LOCKTABLE)`.
+
   @param [in] loops= (25) Number of times to check for a lock.
   @param [in] loop_secs= (1) Seconds to wait between each lock attempt
 
@@ -7539,19 +8093,6 @@ run;
     %let abortme=1;
   %end;
 %end;
-%else %if &action=MAKETABLE %then %do;
-  proc sql;
-  create table &ctl_ds(
-      lock_lib char(8),
-      lock_ds char(32),
-      lock_status_cd char(10) not null,
-      lock_user_nm char(100) not null ,
-      lock_ref char(200),
-      lock_pid char(10),
-      lock_start_dttm num format=E8601DT26.6,
-      lock_end_dttm num format=E8601DT26.6,
-    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
-%end;
 %else %do;
   %let msg=lock_anytable given unsupported action (&action);
   %let abortme=1;
@@ -7761,6 +8302,7 @@ data &ds2;
     %end;
     output;
   end;
+  stop;
 run;
 proc sort data=&ds2 nodupkey;
   by &pk_fields;
@@ -8051,6 +8593,252 @@ data _null_;
 run;
 
 %mend mp_resetoption;/**
+  @file
+  @brief Generate and apply retained key values to a staging table
+  @details This macro will populate a staging table with a Retained Key based on
+  a business key and a base (target) table.
+
+  Definition of retained key ([source](
+    http://bukhantsov.org/2012/04/what-is-data-vault/)):
+
+  > The retained key is a key which is mapped to business key one-to-one. In
+  > comparison,  the surrogate key includes time and there can be many surrogate
+  > keys corresponding to one business key. This explains the name of the key,
+  > it is retained with insertion of a new version of a row while surrogate key
+  > is increasing.
+
+  This macro is designed to be used as part of a wider load / ETL process (such
+  as the one in [Data Controller for SAS](https://datacontroller.io)).
+
+  Specifically, the macro assumes that the base table has already been 'locked'
+  (eg with the mp_lockanytable.sas macro) prior to invocation.  Also, several
+  tables are assumed to exist (names are configurable):
+
+  @li work.staging_table - the staged data, minus the retained key element
+  @li permlib.base_table - the target table to be loaded (**not** loaded by this
+    macro)
+  @li permlib.maxkeytable - optional, used to store load metaadata.
+    The definition is available by running mp_coretable.sas as follows:
+    `mp_coretable(MAXKEYTABLE)`.
+  @li permlib.locktable - Necessary if maxkeytable is being populated. The
+    definition is available by running mp_coretable.sas as follows:
+    `mp_coretable(LOCKTABLE)`.
+
+
+  @param [in] base_lib= (WORK) Libref of the base (target) table.
+  @param [in] base_dsn= (BASETABLE) Name of the base (target) table.
+  @param [in] append_lib= (WORK) Libref of the staging table
+  @param [in] append_dsn= (APPENDTABLE) Name of the staging table
+  @param [in] retained_key= (DEFAULT_RK) Name of RK to generate (should exist on
+    base table)
+  @param [in] business_key= (PK1 PK2) Business key against which to generate
+    RK values.  Should be unique and not null on the staging table.
+  @param [in] check_uniqueness=(NO) Set to yes to perform a uniqueness check.
+    Recommended if there is a chance that the staging data is not unique on the
+    business key.
+  @param [in] maxkeytable= (0) Provide a maxkeytable libds reference here, to
+    store load metadata (maxkey val, load time).  Set to zero if metadata is not
+    required, eg, when preparing a 'dummy' load. Structure is described above.
+    See below for sample data.
+    |KEYTABLE:$32.|KEYCOLUMN:$32.|MAX_KEY:best.|PROCESSED_DTTM:E8601DT26.6|
+    |---|---|---|---|
+    |`DC487173.MPE_SELECTBOX `|`SELECTBOX_RK `|`55 `|`1950427787.8 `|
+    |`DC487173.MPE_FILTERANYTABLE `|`filter_rk `|`14 `|`1951053886.8 `|
+  @param [in] locktable= (0) If updating the maxkeytable, provide the libds
+    reference to the lock table (per mp_lockanytable.sas macro)
+  @param [in] filter_str= Apply a filter - useful for SCD2 or BITEMPORAL loads.
+    Example: `filter_str=%str( (where=( &now < &tech_to)) )`
+  @param [out] outds= (WORK.APPEND) Output table (staging table + retained key)
+
+  <h4> SAS Macros </h4>
+  @li mf_existvar.sas
+  @li mf_getquotedstr.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_lockanytable.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filterstore.sas
+  @li mp_retainedkey.test.sas
+
+  @version 9.2
+
+**/
+
+%macro mp_retainedkey(
+  base_lib=WORK
+  ,base_dsn=BASETABLE
+  ,append_lib=WORK
+  ,append_dsn=APPENDTABLE
+  ,retained_key=DEFAULT_RK
+  ,business_key= PK1 PK2
+  ,check_uniqueness=NO
+  ,maxkeytable=0
+  ,locktable=0
+  ,outds=WORK.APPEND
+  ,filter_str=
+);
+%put &sysmacroname entry vars:;
+%put _local_;
+
+%local base_libds app_libds key_field check maxkey idx_pk newkey_cnt iserr
+  msg x tempds1 tempds2 comma_pk appnobs checknobs dropvar tempvar idx_val;
+%let base_libds=%upcase(&base_lib..&base_dsn);
+%let app_libds=%upcase(&append_lib..&append_dsn);
+%let tempds1=%mf_getuniquename();
+%let tempds2=%mf_getuniquename();
+%let comma_pk=%mf_getquotedstr(in_str=%str(&business_key),dlm=%str(,),quote=);
+%let outds=%sysfunc(ifc(%index(&outds,.)=0,work.&outds,&outds));
+/* validation checks */
+%let iserr=0;
+%if &syscc>0 %then %do;
+  %let iserr=1;
+  %let msg=%str(SYSCC=&syscc on macro entry);
+%end;
+%else %if %sysfunc(exist(&base_libds))=0 %then %do;
+  %let iserr=1;
+  %let msg=%str(Base LIBDS (&base_libds) expected but NOT FOUND);
+%end;
+%else %if %sysfunc(exist(&app_libds))=0 %then %do;
+  %let iserr=1;
+  %let msg=%str(Append LIBDS (&app_libds) expected but NOT FOUND);
+%end;
+%else %if &maxkeytable ne 0 and %sysfunc(exist(&maxkeytable))=0  %then %do;
+  %let iserr=1;
+  %let msg=%str(Maxkeytable (&maxkeytable) expected but NOT FOUND);
+%end;
+%else %if &maxkeytable ne 0 and %sysfunc(exist(&locktable))=0  %then %do;
+  %let iserr=1;
+  %let msg=%str(Locktable (&locktable) expected but NOT FOUND);
+%end;
+%else %if %length(&business_key)=0 %then %do;
+  %let iserr=1;
+  %let msg=%str(Business key (&business_key) expected but NOT FOUND);
+%end;
+
+%do x=1 %to %sysfunc(countw(&business_key));
+  /* check business key values exist */
+  %let key_field=%scan(&business_key,&x,%str( ));
+  %if not %mf_existvar(&app_libds,&key_field) %then %do;
+    %let iserr=1;
+    %let msg=Business key (&key_field) not found on &app_libds!;
+    %goto err;
+  %end;
+  %else %if not %mf_existvar(&base_libds,&key_field) %then %do;
+    %let iserr=1;
+    %let msg=Business key (&key_field) not found on &base_libds!;
+    %goto err;
+  %end;
+%end;
+%err:
+%if &iserr=1 %then %do;
+  /* err case so first perform an unlock of the base table before exiting */
+  %mp_lockanytable(
+    UNLOCK,lib=&base_lib,ds=&base_dsn,ref=%superq(msg),ctl_ds=&locktable
+  )
+%end;
+%mp_abort(iftrue=(&iserr=1),mac=mp_retainedkey,msg=%superq(msg))
+
+proc sql noprint;
+select sum(max(&retained_key),0) into: maxkey from &base_libds;
+
+/**
+  * get base table RK and bus field values for lookup
+  */
+proc sql noprint;
+create table &tempds1 as
+  select distinct &comma_pk,&retained_key
+  from &base_libds &filter_str
+  order by &comma_pk,&retained_key;
+
+%if &check_uniqueness=YES %then %do;
+  select count(*) into:checknobs
+    from (select distinct &comma_pk from &app_libds);
+  select count(*) into: appnobs from &app_libds; /* might be view */
+  %if &checknobs ne &appnobs %then %do;
+    %let msg=Source table &app_libds is not unique on (&business_key);
+    %let iserr=1;
+  %end;
+%end;
+%if &iserr=1 %then %do;
+  /* err case so first perform an unlock of the base table before exiting */
+  %mp_lockanytable(
+    UNLOCK,lib=&base_lib,ds=&base_dsn,ref=%superq(msg),ctl_ds=&locktable
+  )
+%end;
+%mp_abort(iftrue= (&iserr=1),mac=mp_retainedkey,msg=%superq(msg))
+
+%if %mf_existvar(&app_libds,&retained_key)
+%then %let dropvar=(drop=&retained_key);
+
+/* prepare interim table with retained key populated for matching keys */
+proc sql noprint;
+create table &tempds2 as
+  select b.&retained_key, a.*
+  from &app_libds &dropvar a
+  left join &tempds1 b
+  on 1
+  %do idx_pk=1 %to %sysfunc(countw(&business_key));
+    %let idx_val=%scan(&business_key,&idx_pk);
+    and a.&idx_val=b.&idx_val
+  %end;
+  order by &retained_key;
+
+/* identify the number of entries without retained keys (new records) */
+select count(*) into: newkey_cnt
+  from &tempds2
+  where missing(&retained_key);
+quit;
+
+/**
+  * Update maxkey table if link provided
+  */
+%if &maxkeytable ne 0 %then %do;
+  proc sql noprint;
+  select count(*) into: check from &maxkeytable
+    where upcase(keytable)="&base_libds";
+
+  %mp_lockanytable(LOCK
+    ,lib=%scan(&maxkeytable,1,.)
+    ,ds=%scan(&maxkeytable,2,.)
+    ,ref=Updating maxkeyvalues with mp_retainedkey
+    ,ctl_ds=&locktable
+  )
+  proc sql;
+  %if &check=0 %then %do;
+  insert into &maxkeytable
+    set keytable="&base_libds"
+      ,keycolumn="&retained_key"
+      ,max_key=%eval(&maxkey+&newkey_cnt)
+      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt;
+  %end;
+  %else %do;
+  update &maxkeytable
+    set max_key=%eval(&maxkey+&newkey_cnt)
+      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt
+    where keytable="&base_libds";
+  %end;
+  %mp_lockanytable(UNLOCK
+    ,lib=%scan(&maxkeytable,1,.)
+    ,ds=%scan(&maxkeytable,2,.)
+    ,ref=Updating maxkeyvalues with maxkey=%eval(&maxkey+&newkey_cnt)
+    ,ctl_ds=&locktable
+  )
+%end;
+
+/* fill in the missing retained key values */
+%let tempvar=%mf_getuniquename();
+data &outds(drop=&tempvar);
+  retain &tempvar %eval(&maxkey+1);
+  set &tempds2;
+  if &retained_key =. then &retained_key=&tempvar;
+  &tempvar=&tempvar+1;
+run;
+
+%mend mp_retainedkey;
+
+/**
   @file mp_runddl.sas
   @brief An opinionated way to execute DDL files in SAS.
   @details When delivering projects there should be seperation between the DDL
@@ -9639,15 +10427,24 @@ alter table &libds modify &var char(&len);
       ;;;;
       run;
 
+  For more examples, see mp_validatecol.test.sas
+
+  Tip - when contributing, use https://regex101.com to test the regex validity!
+
   @param [in] incol The column to be validated
   @param [in] rule The rule to apply.  Current rules:
+    @li ISINT - checks if the variable is an integer
     @li ISNUM - checks if the variable is numeric
     @li LIBDS - matches LIBREF.DATASET format
+    @li FORMAT - checks if the provided format is syntactically valid
   @param [out] outcol The variable to create, with the results of the match
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas
 
+  <h4> Related Macros </h4>
+  @li mp_validatecol.test.sas
+
   @version 9.3
 **/
 
@@ -9657,7 +10454,13 @@ alter table &libds modify &var char(&len);
 %local tempcol;
 %let tempcol=%mf_getuniquename();
 
-%if &rule=ISNUM %then %do;
+%if &rule=ISINT %then %do;
+  &tempcol=input(&incol,?? best32.);
+  &outcol=0;
+  if not missing(&tempcol) then if mod(&incol,1)=0 then &outcol=1;
+  drop &tempcol;
+%end;
+%else %if &rule=ISNUM %then %do;
   /*
     credit SØREN LASSEN
     https://sasmacro.blogspot.com/2009/06/welcome-isnum-macro.html
@@ -9681,6 +10484,20 @@ alter table &libds modify &var char(&len);
   if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
   else &outcol=0;
 %end;
+%else %if &rule=FORMAT %then %do;
+  /* match valid format - regex could probably be improved */
+  if _n_=1 then do;
+    retain &tempcol;
+    &tempcol=prxparse('/^[_a-z\$]\w{0,31}\.[0-9]*$/i');
+    if missing(&tempcol) then do;
+      putlog "%str(ERR)OR: Invalid expression for FORMAT";
+      stop;
+    end;
+    drop &tempcol;
+  end;
+  if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
+  else &outcol=0;
+%end;
 
 %mend mp_validatecol;
 /**

base/mf_existvar.sas

@@ -7,8 +7,12 @@
 
         %put %mf_existvar(work.someds, somevar)
 
-  @param libds (positional) - 2 part dataset or view reference
-  @param var (positional) - variable name
+  @param [in] libds 2 part dataset or view reference
+  @param [in] var variable name
+
+  <h4> Related Macros </h4>
+  @li mf_existvar.test.sas
+
   @version 9.2
   @author Allan Bowe
 **/

base/mf_getengine.sas

@@ -12,9 +12,10 @@
   contributors of Chris Hemedingers blog [post](
   http://blogs.sas.com/content/sasdummy/2013/06/04/find-a-sas-library-engine/)
 
-  @param libref Library reference (also accepts a 2 level libds ref).
+  @param [in] libref Library reference (also accepts a 2 level libds ref).
 
-  @return output returns the library engine for the FIRST library encountered.
+  @return output returns the library engine (uppercase) for the FIRST library
+    encountered.
 
   @warning will only return the FIRST library engine - for concatenated
     libraries, with different engines, inconsistent results may be encountered.
@@ -46,7 +47,7 @@
     %let rc= %sysfunc(close(&dsid));
   %end;
 
-  &engine
+  %upcase(&engine)
 
 %mend mf_getengine;
 

base/mf_writefile.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Creates a text file using pure macro
+  @details Creates a text file of up to 10 lines.  If further lines are
+    desired, feel free to [create an issue](
+    https://github.com/sasjs/core/issues/new), or make a pull request!
+
+    The use of PARMBUFF was considered for this macro, but it would have made
+    things problematic for writing lines containing commas.
+
+    Usage:
+
+        %mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=more content)
+        data _null_;
+          infile "&sasjswork/myfile.txt";
+          input;
+          list;
+        run;
+
+  @param [in] fpath Full path to file to be created or appended to
+  @param [in] mode= (O) Available options are A or O as follows:
+    @li A APPEND mode, writes new records after the current end of the file.
+    @li O OUTPUT mode, writes new records from the beginning of the file.
+  @param [in] l1= First line
+  @param [in] l2= Second line (etc through to l10)
+
+  <h4> Related Macros </h4>
+  @li mf_writefile.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mf_writefile(fpath,mode=O,l1=,l2=,l3=,l4=,l5=,l6=,l7=,l8=,l9=,l10=
+)/*/STORE SOURCE*/;
+%local fref rc fid i total_lines;
+
+/* find number of lines by reference to first non-blank param */
+%do i=10 %to 1 %by -1;
+  %if %str(&&l&i) ne %str() %then %goto continue;
+%end;
+%continue:
+%let total_lines=&i;
+
+%if %sysfunc(filename(fref,&fpath)) ne 0 %then %do;
+  %put &=fref &=fpath;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let fid=%sysfunc(fopen(&fref,&mode));
+
+%if &fid=0 %then %do;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%do i=1 %to &total_lines;
+  %let rc=%sysfunc(fput(&fid, &&l&i));
+  %let rc=%sysfunc(fwrite(&fid));
+%end;
+%let rc=%sysfunc(fclose(&fid));
+%let rc=%sysfunc(filename(&fref));
+
+%mend mf_writefile;
+/** @endcond */

base/mp_applyformats.sas

@@ -0,0 +1,181 @@
+/**
+  @file
+  @brief Apply a set of formats to a table
+  @details Applies a set of formats to the metadata of one or more SAS datasets.
+    Can be used to migrate formats from one table to another.  The input table
+    must contain the following columns:
+
+    @li lib - the libref of the table to be updated
+    @li ds - the dataset to be updated
+    @li var - the variable to be updated
+    @li fmt - the format to apply.  Missing or default ($CHAR, 8.) formats are
+      ignored.
+
+  The macro will abort in the following scenarios:
+
+    @li Libref not assigned
+    @li Dataset does not exist
+    @li Input table contains null or invalid values
+
+  Example usage:
+
+      data work.example;
+        set sashelp.prdsale;
+        format _all_ clear;
+      run;
+
+      %mp_getcols(sashelp.prdsale,outds=work.cols)
+
+      data work.cols2;
+        set work.cols;
+        lib='WORK';
+        ds='EXAMPLE';
+        var=name;
+        fmt=format;
+        keep lib ds var fmt;
+      run;
+
+      %mp_applyformats(work.cols2)
+
+  @param [in] inds The input dataset containing the formats to apply (and where
+    to apply them).  Example structure:
+  |LIB:$8.|DS:$32.|VAR:$32.|FMT:$49.|
+  |---|---|---|---|
+  |`WORK `|`EXAMPLE `|`ACTUAL `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`COUNTRY `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`DIVISION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`MONTH `|`MONNAME3. `|
+  |`WORK `|`EXAMPLE `|`PREDICT `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`PRODTYPE `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`PRODUCT `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`QUARTER `|`8. `|
+  |`WORK `|`EXAMPLE `|`REGION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`YEAR `|`8. `|
+
+  @param [out] errds= (0) Provide a libds reference here to export the
+    error messages to a table.  In this case, they will not be printed to the
+    log.
+
+  <h4> SAS Macros </h4>
+  @li mf_getengine.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_validatecol.sas
+
+
+  <h4> Related Macros </h4>
+  @li mp_getformats.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_applyformats(inds,errds=0
+)/*/STORE SOURCE*/;
+%local outds liblist i engine lib msg ;
+
+/**
+  * Validations
+  */
+proc sort data=&inds;
+  by lib ds var fmt;
+run;
+
+%if &errds=0 %then %let outds=%mf_getuniquename(prefix=mp_applyformats);
+%else %let outds=&errds;
+
+data &outds;
+  set &inds;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  length msg $128;
+  by lib ds var fmt;
+  if libref(lib) ne 0 then do;
+    msg=catx(' ','libref',lib,'is not assigned!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  if exist(cats(lib,'.',ds)) ne 1 then do;
+    msg=catx(' ','libds',lib,'.',ds,'does not exist!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  %mp_validatecol(fmt,FORMAT,is_fmt)
+  if is_fmt=0 then do;
+    msg=catx(' ','format',fmt,'on libds',lib,'.',ds,'.',var,'is not valid!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+
+  if first.ds then do;
+    retain dsid;
+    dsid=open(cats(lib,'.',ds));
+    if dsid=0 then do;
+      msg=catx(' ','libds',lib,'.',ds,' could not be opened!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+      return;
+    end;
+    if varnum(dsid,var)<1 then do;
+      msg=catx(' ','Variable',lib,'.',ds,'.',var,' was not found!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+    end;
+  end;
+  if last.ds then rc=close(dsid);
+run;
+
+proc sql noprint;
+select distinct lib into: liblist separated by ' ' from &inds;
+%put &=liblist;
+%do i=1 %to %sysfunc(countw(&liblist));
+  %let lib=%scan(&liblist,1);
+  %let engine=%mf_getengine(&lib);
+  %if &engine ne V9 and &engine ne BASE %then %do;
+    %let msg=&lib has &engine engine - formats cannot be applied;
+    proc sql;
+    insert into &outds set lib="&lib",ds="_all_",var="_all", msg="&msg" ;
+    %if &errds=0 %then %put %str(ERR)OR: &msg;
+  %end;
+%end;
+
+%if %mf_nobs(&outds)>0 %then %return;
+
+/**
+  * Validations complete - now apply the actual formats!
+  */
+%let fref=%mf_getuniquefileref();
+data _null_;
+  set &inds;
+  by lib ds var fmt;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  file &fref;
+  if first.lib then put 'proc datasets nolist lib=' lib ';';
+  if first.ds then put '  modify ' ds ';';
+  put '    format ' var fmt ';';
+  if last.ds then put '  run;';
+  if last.lib then put 'quit;';
+run;
+
+%inc &fref/source2;
+
+%if &errds=0 %then %do;
+  proc sql;
+  drop table &outds;
+%end;
+
+%mend mp_applyformats;

base/mp_coretable.sas

@@ -0,0 +1,92 @@
+/**
+  @file
+  @brief Create the permanent Core tables
+  @details Several macros in the [core](https://github.com/sasjs/core) library
+    make use of permanent tables.  To avoid duplication in definitions, this
+    macro provides a central location for managing the corresponding DDL.
+
+  Example usage:
+
+      %mp_coretable(LOCKTABLE,libds=work.locktable)
+
+  @param [in] table_ref The type of table to create.  Example values:
+    @li FILTER_DETAIL - For storing detailed filter values.  Used by
+      mp_filterstore.sas.
+    @li FILTER_SUMMARY - For storing summary filter values.  Used by
+      mp_filterstore.sas.
+    @li LOCKANYTABLE - For "locking" tables prior to multipass loads. Used by
+      mp_lockanytable.sas
+    @li MAXKEYTABLE - For storing the maximum retained key information.  Used
+      by mp_retainedkey.sas
+  @param [in] libds= (0) The library.dataset reference used to create the table.
+    If not provided, then the DDL is simply printed to the log.
+
+  <h4> Related Macros </h4>
+  @li mp_filterstore.sas
+  @li mp_lockanytable.sas
+  @li mp_retainedkey.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_coretable(table_ref,libds=0
+)/*/STORE SOURCE*/;
+%local outds ;
+%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
+proc sql;
+%if &table_ref=LOCKTABLE %then %do;
+  create table &outds(
+      lock_lib char(8),
+      lock_ds char(32),
+      lock_status_cd char(10) not null,
+      lock_user_nm char(100) not null ,
+      lock_ref char(200),
+      lock_pid char(10),
+      lock_start_dttm num format=E8601DT26.6,
+      lock_end_dttm num format=E8601DT26.6,
+    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+%end;
+%else %if &table_ref=FILTER_SUMMARY %then %do;
+  create table &outds(
+      filter_rk num not null,
+      filter_hash char(32) not null,
+      filter_table char(41) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_rk));
+%end;
+%else %if &table_ref=FILTER_DETAIL %then %do;
+  create table &outds(
+      filter_hash char(32) not null,
+      filter_line num not null,
+      group_logic char(3) not null,
+      subgroup_logic char(3) not null,
+      subgroup_id num not null,
+      variable_nm varchar(32) not null,
+      operator_nm varchar(12) not null,
+      raw_value varchar(4000) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_hash,filter_line));
+%end;
+%else %if &table_ref=MAXKEYTABLE %then %do;
+  create table &outds(
+      keytable varchar(41) label='Base table in libref.dataset format',
+      keycolumn char(32) format=$32.
+        label='The Retained key field containing the key values.',
+      max_key num label=
+        'Integer representing current max RK or SK value in the KEYTABLE',
+      processed_dttm num format=E8601DT26.6
+        label='Datetime this value was last updated',
+    constraint pk_mpe_maxkeyvalues
+        primary key(keytable));
+%end;
+
+
+%if &libds=0 %then %do;
+  describe table &syslast;
+  drop table &syslast;
+%end;
+%mend mp_coretable;

base/mp_filterstore.sas

@@ -0,0 +1,213 @@
+/**
+  @file
+  @brief Checks & Stores an input filter table and returns the Filter Key
+  @details Used to generate a FILTER_RK from an input query dataset.  This
+  process requires several permanent tables (names are configurable).  The
+  benefit of storing query values at backend is to enable stored 'views' of
+  filtered tables at frontend (ie, when building [SAS-Powered Apps](
+  https://sasapps.io)).  This macro is also used in [Data Controller for SAS](
+  https://datacontroller.io).
+
+
+  @param [in] libds= The target dataset to be filtered (lib should be assigned)
+  @param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to
+    be validated.  Has the following format:
+|GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
+|---|---|---|---|---|---|
+|AND|AND|1|SOME_BESTNUM|>|1|
+|AND|AND|1|SOME_TIME|=|77333|
+  @param [in] filter_summary= (PERM.FILTER_SUMMARY) Permanent table containing
+    summary filter values.  The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(FILTER_SUMMARY)`. Example
+    values:
+|FILTER_RK:best.|FILTER_HASH:$32.|FILTER_TABLE:$41.|PROCESSED_DTTM:datetime19.|
+|---|---|---|---|
+|`1 `|`540E96F566D194AB58DD4C413C99C9DB `|`VIYA6014.MPE_TABLES `|`1956084246 `|
+|`2 `|`87737DB9EEE2650F5C89956CEAD0A14F `|`VIYA6014.MPE_X_TEST `|`1956084452.1`|
+|`3 `|`8048BD908DBBD83D013560734E90D394 `|`VIYA6014.MPE_TABLES `|`1956093620.6`|
+  @param [in] filter_detail= (PERM.FILTER_DETAIL) Permanent table containing
+    detailed (raw) filter values. The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(FILTER_DETAIL)`. Example
+    values:
+|FILTER_HASH:$32.|FILTER_LINE:best.|GROUP_LOGIC:$3.|SUBGROUP_LOGIC:$3.|SUBGROUP_ID:best.|VARIABLE_NM:$32.|OPERATOR_NM:$12.|RAW_VALUE:$4000.|PROCESSED_DTTM:datetime19.|
+|---|---|---|---|---|---|---|---|---|
+|`540E96F566D194AB58DD4C413C99C9DB `|`1 `|`AND `|`AND `|`1 `|`LIBREF `|`CONTAINS `|`DC`|`1956084245.8 `|
+|`540E96F566D194AB58DD4C413C99C9DB `|`2 `|`AND `|`OR `|`2 `|`DSN `|`= `|` MPE_LOCK_ANYTABLE `|`1956084245.8 `|
+|`87737DB9EEE2650F5C89956CEAD0A14F `|`1 `|`AND `|`AND `|`1 `|`PRIMARY_KEY_FIELD `|`IN `|`(1,2,3) `|`1956084451.9 `|
+  @param [in] lock_table= (PERM.LOCK_TABLE) Permanent locking table.  Used to
+    manage concurrent access.  The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(LOCKTABLE)`.
+  @param [in] maxkeytable= (0) Optional permanent reference table used for
+    retained key tracking.  Described in mp_retainedkey.sas.
+  @param [in] mdebug= set to 1 to enable DEBUG messages
+  @param [out] outresult= The result table with the FILTER_RK
+  @param [out] outquery= The original query, taken as extract after table load
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_getvalue.sas
+  @li mf_islibds.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_filtercheck.sas
+  @li mp_hashdataset.sas
+  @li mp_retainedkey.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filtercheck.sas
+  @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
+  @li mp_filterstore.test.sas
+
+  @version 9.2
+  @author [Allan Bowe](https://www.linkedin.com/in/allanbowe)
+
+**/
+
+%macro mp_filterstore(libds=,
+  queryds=work.filterquery,
+  filter_summary=PERM.FILTER_SUMMARY,
+  filter_detail=PERM.FILTER_DETAIL,
+  lock_table=PERM.LOCK_TABLE,
+  maxkeytable=PERM.MAXKEYTABLE,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+);
+%put &sysmacroname entry vars:;
+%put _local_;
+
+%local ds1 ds2 ds3 ds4 filter_hash;
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+%mp_abort(iftrue= (%mf_islibds(&filter_summary)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid filter_summary value: &filter_summary)
+)
+%mp_abort(iftrue= (%mf_islibds(&filter_detail)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid filter_detail value: &filter_detail)
+)
+%mp_abort(iftrue= (%mf_islibds(&lock_table)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid lock_table value: &lock_table)
+)
+
+/* validate query */
+%mp_filtercheck(&queryds,targetds=&libds,abort=YES)
+
+/* hash the result */
+%let ds1=%mf_getuniquename(prefix=hashds);
+%mp_hashdataset(&queryds,outds=&ds1,salt=&libds)
+%let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
+%if &mdebug=1 %then %do;
+  data _null_;
+    putlog "filter_hash=&filter_hash";
+    set &ds1;
+    putlog (_all_)(=);
+  run;
+%end;
+
+/* check if data already exists for this hash */
+data &outresult;
+  set &filter_summary;
+  where filter_hash="&filter_hash";
+run;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc after hash check)
+)
+%mp_abort(iftrue= ("&filter_hash"=" ")
+  ,mac=mp_filterstore
+  ,msg=%str(problem with filter_hash generation)
+)
+
+%if %mf_nobs(&outresult)=0 %then %do;
+
+  /* update detail table first */
+  %let ds2=%mf_getuniquename(prefix=filterdetail);
+  data &ds2;
+    if 0 then set &filter_detail;
+    set &queryds;
+    format filter_hash $hex32. filter_line 8. processed_dttm E8601DT26.6;
+    filter_hash="&filter_hash";
+    filter_line=_n_;
+    PROCESSED_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt;
+  run;
+  %mp_lockanytable(LOCK,
+    lib=%scan(&filter_detail,1,.)
+    ,ds=%scan(&filter_detail,2,.)
+    ,ref=MP_FILTERSTORE update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+  proc append base=&filter_detail data=&ds2;
+  run;
+
+  %mp_lockanytable(UNLOCK,
+    lib=%scan(&filter_detail,1,.)
+    ,ds=%scan(&filter_detail,2,.)
+    ,ref=MP_FILTERSTORE update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  /* now update summary table */
+  %let ds3=%mf_getuniquename(prefix=filtersum);
+  data &ds3;
+    if 0 then set &filter_summary;
+    filter_table=symget('libds');
+    filter_hash="&filter_hash";
+    PROCESSED_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt;
+    output;
+    stop;
+  run;
+
+  %mp_lockanytable(LOCK,
+    lib=%scan(&filter_summary,1,.)
+    ,ds=%scan(&filter_summary,2,.)
+    ,ref=MP_FILTERSTORE update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  %let ds4=%mf_getuniquename(prefix=filtersumappend);
+  %mp_retainedkey(
+    base_lib=%scan(&filter_summary,1,.)
+    ,base_dsn=%scan(&filter_summary,2,.)
+    ,append_lib=work
+    ,append_dsn=&ds3
+    ,retained_key=filter_rk
+    ,business_key=filter_hash
+    ,maxkeytable=&maxkeytable
+    ,locktable=&lock_table
+    ,outds=work.&ds4
+  )
+  proc append base=&filter_summary data=&ds4;
+  run;
+
+  %mp_lockanytable(UNLOCK,
+    lib=%scan(&filter_summary,1,.)
+    ,ds=%scan(&filter_summary,2,.)
+    ,ref=MP_FILTERSTORE update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  data &outresult;
+    set &filter_summary;
+    where filter_hash="&filter_hash";
+  run;
+
+%end;
+
+proc sort data=&filter_detail(where=(filter_hash="&filter_hash")) out=&outquery;
+  by filter_line;
+run;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc on macro exit)
+)
+
+%mend mp_filterstore;

base/mp_getformats.sas

@@ -48,6 +48,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 
 
   <h4> Related Macros </h4>
+  @li mp_applyformats.sas
   @li mp_getformats.test.sas
 
   @version 9.2

base/mp_lockanytable.sas

@@ -5,19 +5,18 @@
   Only useful if every update uses the macro!   Used heavily within
   [Data Controller for SAS](https://datacontroller.io).
 
-  The underlying table is structured as per the MAKETABLE action.
-
   @param [in] action The action to be performed.  Valid values:
     @li LOCK - Sets the lock flag, also confirms if a SAS lock is available
     @li UNLOCK - Unlocks the table
-    @li MAKETABLE - creates the control table (ctl_ds)
   @param [in] lib= (WORK) The libref of the table to lock.  Should already be
     assigned.
   @param [in] ds= The dataset to lock
   @param [in] ref= A meaningful reference to enable the lock to be traced. Max
     length is 200 characters.
   @param [out] ctl_ds= (0) The control table which controls the actual locking.
-    Should already be assigned and available.
+    Should already be assigned and available.  The definition is available by
+    running mp_coretable.sas as follows:  `mp_coretable(LOCKTABLE)`.
+
   @param [in] loops= (25) Number of times to check for a lock.
   @param [in] loop_secs= (1) Seconds to wait between each lock attempt
 
@@ -221,19 +220,6 @@ run;
     %let abortme=1;
   %end;
 %end;
-%else %if &action=MAKETABLE %then %do;
-  proc sql;
-  create table &ctl_ds(
-      lock_lib char(8),
-      lock_ds char(32),
-      lock_status_cd char(10) not null,
-      lock_user_nm char(100) not null ,
-      lock_ref char(200),
-      lock_pid char(10),
-      lock_start_dttm num format=E8601DT26.6,
-      lock_end_dttm num format=E8601DT26.6,
-    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
-%end;
 %else %do;
   %let msg=lock_anytable given unsupported action (&action);
   %let abortme=1;

base/mp_makedata.sas

@@ -92,6 +92,7 @@ data &ds2;
     %end;
     output;
   end;
+  stop;
 run;
 proc sort data=&ds2 nodupkey;
   by &pk_fields;

base/mp_retainedkey.sas

@@ -0,0 +1,246 @@
+/**
+  @file
+  @brief Generate and apply retained key values to a staging table
+  @details This macro will populate a staging table with a Retained Key based on
+  a business key and a base (target) table.
+
+  Definition of retained key ([source](
+    http://bukhantsov.org/2012/04/what-is-data-vault/)):
+
+  > The retained key is a key which is mapped to business key one-to-one. In
+  > comparison,  the surrogate key includes time and there can be many surrogate
+  > keys corresponding to one business key. This explains the name of the key,
+  > it is retained with insertion of a new version of a row while surrogate key
+  > is increasing.
+
+  This macro is designed to be used as part of a wider load / ETL process (such
+  as the one in [Data Controller for SAS](https://datacontroller.io)).
+
+  Specifically, the macro assumes that the base table has already been 'locked'
+  (eg with the mp_lockanytable.sas macro) prior to invocation.  Also, several
+  tables are assumed to exist (names are configurable):
+
+  @li work.staging_table - the staged data, minus the retained key element
+  @li permlib.base_table - the target table to be loaded (**not** loaded by this
+    macro)
+  @li permlib.maxkeytable - optional, used to store load metaadata.
+    The definition is available by running mp_coretable.sas as follows:
+    `mp_coretable(MAXKEYTABLE)`.
+  @li permlib.locktable - Necessary if maxkeytable is being populated. The
+    definition is available by running mp_coretable.sas as follows:
+    `mp_coretable(LOCKTABLE)`.
+
+
+  @param [in] base_lib= (WORK) Libref of the base (target) table.
+  @param [in] base_dsn= (BASETABLE) Name of the base (target) table.
+  @param [in] append_lib= (WORK) Libref of the staging table
+  @param [in] append_dsn= (APPENDTABLE) Name of the staging table
+  @param [in] retained_key= (DEFAULT_RK) Name of RK to generate (should exist on
+    base table)
+  @param [in] business_key= (PK1 PK2) Business key against which to generate
+    RK values.  Should be unique and not null on the staging table.
+  @param [in] check_uniqueness=(NO) Set to yes to perform a uniqueness check.
+    Recommended if there is a chance that the staging data is not unique on the
+    business key.
+  @param [in] maxkeytable= (0) Provide a maxkeytable libds reference here, to
+    store load metadata (maxkey val, load time).  Set to zero if metadata is not
+    required, eg, when preparing a 'dummy' load. Structure is described above.
+    See below for sample data.
+    |KEYTABLE:$32.|KEYCOLUMN:$32.|MAX_KEY:best.|PROCESSED_DTTM:E8601DT26.6|
+    |---|---|---|---|
+    |`DC487173.MPE_SELECTBOX `|`SELECTBOX_RK `|`55 `|`1950427787.8 `|
+    |`DC487173.MPE_FILTERANYTABLE `|`filter_rk `|`14 `|`1951053886.8 `|
+  @param [in] locktable= (0) If updating the maxkeytable, provide the libds
+    reference to the lock table (per mp_lockanytable.sas macro)
+  @param [in] filter_str= Apply a filter - useful for SCD2 or BITEMPORAL loads.
+    Example: `filter_str=%str( (where=( &now < &tech_to)) )`
+  @param [out] outds= (WORK.APPEND) Output table (staging table + retained key)
+
+  <h4> SAS Macros </h4>
+  @li mf_existvar.sas
+  @li mf_getquotedstr.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_lockanytable.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filterstore.sas
+  @li mp_retainedkey.test.sas
+
+  @version 9.2
+
+**/
+
+%macro mp_retainedkey(
+  base_lib=WORK
+  ,base_dsn=BASETABLE
+  ,append_lib=WORK
+  ,append_dsn=APPENDTABLE
+  ,retained_key=DEFAULT_RK
+  ,business_key= PK1 PK2
+  ,check_uniqueness=NO
+  ,maxkeytable=0
+  ,locktable=0
+  ,outds=WORK.APPEND
+  ,filter_str=
+);
+%put &sysmacroname entry vars:;
+%put _local_;
+
+%local base_libds app_libds key_field check maxkey idx_pk newkey_cnt iserr
+  msg x tempds1 tempds2 comma_pk appnobs checknobs dropvar tempvar idx_val;
+%let base_libds=%upcase(&base_lib..&base_dsn);
+%let app_libds=%upcase(&append_lib..&append_dsn);
+%let tempds1=%mf_getuniquename();
+%let tempds2=%mf_getuniquename();
+%let comma_pk=%mf_getquotedstr(in_str=%str(&business_key),dlm=%str(,),quote=);
+%let outds=%sysfunc(ifc(%index(&outds,.)=0,work.&outds,&outds));
+/* validation checks */
+%let iserr=0;
+%if &syscc>0 %then %do;
+  %let iserr=1;
+  %let msg=%str(SYSCC=&syscc on macro entry);
+%end;
+%else %if %sysfunc(exist(&base_libds))=0 %then %do;
+  %let iserr=1;
+  %let msg=%str(Base LIBDS (&base_libds) expected but NOT FOUND);
+%end;
+%else %if %sysfunc(exist(&app_libds))=0 %then %do;
+  %let iserr=1;
+  %let msg=%str(Append LIBDS (&app_libds) expected but NOT FOUND);
+%end;
+%else %if &maxkeytable ne 0 and %sysfunc(exist(&maxkeytable))=0  %then %do;
+  %let iserr=1;
+  %let msg=%str(Maxkeytable (&maxkeytable) expected but NOT FOUND);
+%end;
+%else %if &maxkeytable ne 0 and %sysfunc(exist(&locktable))=0  %then %do;
+  %let iserr=1;
+  %let msg=%str(Locktable (&locktable) expected but NOT FOUND);
+%end;
+%else %if %length(&business_key)=0 %then %do;
+  %let iserr=1;
+  %let msg=%str(Business key (&business_key) expected but NOT FOUND);
+%end;
+
+%do x=1 %to %sysfunc(countw(&business_key));
+  /* check business key values exist */
+  %let key_field=%scan(&business_key,&x,%str( ));
+  %if not %mf_existvar(&app_libds,&key_field) %then %do;
+    %let iserr=1;
+    %let msg=Business key (&key_field) not found on &app_libds!;
+    %goto err;
+  %end;
+  %else %if not %mf_existvar(&base_libds,&key_field) %then %do;
+    %let iserr=1;
+    %let msg=Business key (&key_field) not found on &base_libds!;
+    %goto err;
+  %end;
+%end;
+%err:
+%if &iserr=1 %then %do;
+  /* err case so first perform an unlock of the base table before exiting */
+  %mp_lockanytable(
+    UNLOCK,lib=&base_lib,ds=&base_dsn,ref=%superq(msg),ctl_ds=&locktable
+  )
+%end;
+%mp_abort(iftrue=(&iserr=1),mac=mp_retainedkey,msg=%superq(msg))
+
+proc sql noprint;
+select sum(max(&retained_key),0) into: maxkey from &base_libds;
+
+/**
+  * get base table RK and bus field values for lookup
+  */
+proc sql noprint;
+create table &tempds1 as
+  select distinct &comma_pk,&retained_key
+  from &base_libds &filter_str
+  order by &comma_pk,&retained_key;
+
+%if &check_uniqueness=YES %then %do;
+  select count(*) into:checknobs
+    from (select distinct &comma_pk from &app_libds);
+  select count(*) into: appnobs from &app_libds; /* might be view */
+  %if &checknobs ne &appnobs %then %do;
+    %let msg=Source table &app_libds is not unique on (&business_key);
+    %let iserr=1;
+  %end;
+%end;
+%if &iserr=1 %then %do;
+  /* err case so first perform an unlock of the base table before exiting */
+  %mp_lockanytable(
+    UNLOCK,lib=&base_lib,ds=&base_dsn,ref=%superq(msg),ctl_ds=&locktable
+  )
+%end;
+%mp_abort(iftrue= (&iserr=1),mac=mp_retainedkey,msg=%superq(msg))
+
+%if %mf_existvar(&app_libds,&retained_key)
+%then %let dropvar=(drop=&retained_key);
+
+/* prepare interim table with retained key populated for matching keys */
+proc sql noprint;
+create table &tempds2 as
+  select b.&retained_key, a.*
+  from &app_libds &dropvar a
+  left join &tempds1 b
+  on 1
+  %do idx_pk=1 %to %sysfunc(countw(&business_key));
+    %let idx_val=%scan(&business_key,&idx_pk);
+    and a.&idx_val=b.&idx_val
+  %end;
+  order by &retained_key;
+
+/* identify the number of entries without retained keys (new records) */
+select count(*) into: newkey_cnt
+  from &tempds2
+  where missing(&retained_key);
+quit;
+
+/**
+  * Update maxkey table if link provided
+  */
+%if &maxkeytable ne 0 %then %do;
+  proc sql noprint;
+  select count(*) into: check from &maxkeytable
+    where upcase(keytable)="&base_libds";
+
+  %mp_lockanytable(LOCK
+    ,lib=%scan(&maxkeytable,1,.)
+    ,ds=%scan(&maxkeytable,2,.)
+    ,ref=Updating maxkeyvalues with mp_retainedkey
+    ,ctl_ds=&locktable
+  )
+  proc sql;
+  %if &check=0 %then %do;
+  insert into &maxkeytable
+    set keytable="&base_libds"
+      ,keycolumn="&retained_key"
+      ,max_key=%eval(&maxkey+&newkey_cnt)
+      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt;
+  %end;
+  %else %do;
+  update &maxkeytable
+    set max_key=%eval(&maxkey+&newkey_cnt)
+      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt
+    where keytable="&base_libds";
+  %end;
+  %mp_lockanytable(UNLOCK
+    ,lib=%scan(&maxkeytable,1,.)
+    ,ds=%scan(&maxkeytable,2,.)
+    ,ref=Updating maxkeyvalues with maxkey=%eval(&maxkey+&newkey_cnt)
+    ,ctl_ds=&locktable
+  )
+%end;
+
+/* fill in the missing retained key values */
+%let tempvar=%mf_getuniquename();
+data &outds(drop=&tempvar);
+  retain &tempvar %eval(&maxkey+1);
+  set &tempds2;
+  if &retained_key =. then &retained_key=&tempvar;
+  &tempvar=&tempvar+1;
+run;
+
+%mend mp_retainedkey;
+

base/mp_validatecol.sas

@@ -20,15 +20,24 @@
       ;;;;
       run;
 
+  For more examples, see mp_validatecol.test.sas
+
+  Tip - when contributing, use https://regex101.com to test the regex validity!
+
   @param [in] incol The column to be validated
   @param [in] rule The rule to apply.  Current rules:
+    @li ISINT - checks if the variable is an integer
     @li ISNUM - checks if the variable is numeric
     @li LIBDS - matches LIBREF.DATASET format
+    @li FORMAT - checks if the provided format is syntactically valid
   @param [out] outcol The variable to create, with the results of the match
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas
 
+  <h4> Related Macros </h4>
+  @li mp_validatecol.test.sas
+
   @version 9.3
 **/
 
@@ -38,7 +47,13 @@
 %local tempcol;
 %let tempcol=%mf_getuniquename();
 
-%if &rule=ISNUM %then %do;
+%if &rule=ISINT %then %do;
+  &tempcol=input(&incol,?? best32.);
+  &outcol=0;
+  if not missing(&tempcol) then if mod(&incol,1)=0 then &outcol=1;
+  drop &tempcol;
+%end;
+%else %if &rule=ISNUM %then %do;
   /*
     credit SØREN LASSEN
     https://sasmacro.blogspot.com/2009/06/welcome-isnum-macro.html
@@ -62,5 +77,19 @@
   if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
   else &outcol=0;
 %end;
+%else %if &rule=FORMAT %then %do;
+  /* match valid format - regex could probably be improved */
+  if _n_=1 then do;
+    retain &tempcol;
+    &tempcol=prxparse('/^[_a-z\$]\w{0,31}\.[0-9]*$/i');
+    if missing(&tempcol) then do;
+      putlog "%str(ERR)OR: Invalid expression for FORMAT";
+      stop;
+    end;
+    drop &tempcol;
+  end;
+  if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
+  else &outcol=0;
+%end;
 
 %mend mp_validatecol;

tests/crossplatform/mf_existvar.test.sas

@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Testing mf_existvar macro
+
+  <h4> SAS Macros </h4>
+  @li mf_existvar.sas
+  @li mp_assert.sas
+
+**/
+
+
+%mp_assert(
+  iftrue=(%mf_existvar(sashelp.class,age)=1),
+  desc=Checking existing var exists,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_existvar(sashelp.class,isjustanumber)=0),
+  desc=Checking non existing var does not exist,
+  outds=work.test_results
+)

tests/crossplatform/mf_writefile.test.sas

@@ -0,0 +1,74 @@
+/**
+  @file
+  @brief Testing mf_writefile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_writefile.sas
+  @li mp_assert.sas
+
+**/
+
+%mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=more content)
+data _null_;
+  infile "&sasjswork/myfile.txt";
+  input;
+  if _n_=2 then call symputx('test1',_infile_);
+run;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check code ran without errors,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&test1=more content),
+  desc=Checking line was created,
+  outds=work.test_results
+)
+
+%mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=different content)
+data _null_;
+  infile "&sasjswork/myfile.txt";
+  input;
+  if _n_=2 then call symputx('test2',_infile_);
+run;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check code ran without errors for test2,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&test2=different content),
+  desc=Checking second line was overwritten,
+  outds=work.test_results
+)
+
+%global test3 test4;
+%mf_writefile(&sasjswork/myfile.txt
+  ,mode=a
+  ,l1=%str(aah, content)
+  ,l2=append content
+)
+data _null_;
+  infile "&sasjswork/myfile.txt";
+  input;
+  if _n_=2 then call symputx('test3',_infile_);
+  if _n_=4 then call symputx('test4',_infile_);
+run;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check code ran without errors for test2,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&test3=different content),
+  desc=Checking second line was not overwritten,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&test4=append content),
+  desc=Checking fourth line was appended,
+  outds=work.test_results
+)

tests/crossplatform/mp_applyformats.test.sas

@@ -0,0 +1,45 @@
+/**
+  @file
+  @brief Testing mp_applyformats.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getvarformat.sas
+  @li mp_applyformats.sas
+  @li mp_assert.sas
+  @li mp_getcols.sas
+
+**/
+
+/**
+  * Test 1 Base case
+  */
+
+data work.example;
+  set sashelp.prdsale;
+  format _all_;
+run;
+%let origfmt=%mf_getvarformat(work.example,month);
+
+%mp_getcols(sashelp.prdsale,outds=work.cols)
+
+data work.cols2;
+  set work.cols;
+  lib='WORK';
+  ds='EXAMPLE';
+  var=name;
+  fmt=format;
+  keep lib ds var fmt;
+run;
+
+%mp_applyformats(work.cols2)
+
+%mp_assert(
+  iftrue=("&orig_fmt"=""),
+  desc=Check that formats were cleared,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("%mf_getvarformat(work.example,month)"="MONNAME3."),
+  desc=Check that formats were applied,
+  outds=work.test_results
+)

tests/crossplatform/mp_coretable.test.sas

@@ -0,0 +1,31 @@
+/**
+  @file
+  @brief Testing mp_coretable.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mp_coretable.sas
+  @li mp_assert.sas
+
+**/
+
+
+%mp_coretable(LOCKTABLE,libds=work.lock)
+%mp_assert(
+  iftrue=(%mf_existds(work.lock)=1),
+  desc=Lock table created,
+  outds=work.test_results
+)
+%mp_coretable(LOCKTABLE)
+%mp_assert(
+  iftrue=("&syscc"="0"),
+  desc=DDL export ran without errors,
+  outds=work.test_results
+)
+
+%mp_coretable(FILTER_SUMMARY,libds=work.sum)
+%mp_assert(
+  iftrue=(%mf_existds(work.sum)=1),
+  desc=Filter summary table created,
+  outds=work.test_results
+)

tests/crossplatform/mp_filterstore.test.sas

@@ -0,0 +1,85 @@
+/**
+  @file
+  @brief Testing mp_filterstore macro
+
+  <h4> SAS Macros </h4>
+  @li mp_coretable.sas
+  @li mp_filterstore.sas
+  @li mp_assertdsobs.sas
+  @li mp_assert.sas
+
+**/
+
+libname permlib (work);
+
+%mp_coretable(LOCKTABLE,libds=permlib.locktable)
+%mp_coretable(FILTER_SUMMARY,libds=permlib.filtsum)
+%mp_coretable(FILTER_DETAIL,libds=permlib.filtdet)
+%mp_coretable(MAXKEYTABLE,libds=permlib.maxkey)
+
+/* valid filter */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$12. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,AGE,=,12
+AND,AND,1,SEX,<=,"'M'"
+AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
+AND,OR,2,Weight,>=,77.7
+AND,OR,2,Weight,NE,77.7
+;;;;
+run;
+
+%mp_filterstore(libds=sashelp.class,
+  queryds=work.inds,
+  filter_summary=permlib.filtsum,
+  filter_detail=permlib.filtdet,
+  lock_table=permlib.locktable,
+  maxkeytable=permlib.maxkey,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+)
+%mp_assert(iftrue=(&syscc>0),
+  desc=Ensure macro runs without errors,
+  outds=work.test_results
+)
+/* ensure only one record created */
+%mp_assertdsobs(permlib.filtsum,
+  desc=Initial query,
+  test=ATMOST 1,
+  outds=work.test_results
+)
+/* check RK is correct */
+proc sql noprint;
+select max(filter_rk) into: test1 from work.result;
+%mp_assert(iftrue=(&test1=1),
+  desc=Ensure filter rk is correct,
+  outds=work.test_results
+)
+
+/* Test 2 - load same table again and ensure we get the same RK */
+%mp_filterstore(libds=sashelp.class,
+  queryds=work.inds,
+  filter_summary=permlib.filtsum,
+  filter_detail=permlib.filtdet,
+  lock_table=permlib.locktable,
+  maxkeytable=permlib.maxkey,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+)
+/* ensure only one record created */
+%mp_assertdsobs(permlib.filtsum,
+  desc=Initial query - same obs,
+  test=ATMOST 1,
+  outds=work.test_results
+)
+/* check RK is correct */
+proc sql noprint;
+select max(filter_rk) into: test2 from work.result;
+%mp_assert(iftrue=(&test2=1),
+  desc=Ensure filter rk is correct for second run,
+  outds=work.test_results
+)

tests/crossplatform/mp_retainedkey.test.sas

@@ -0,0 +1,116 @@
+/**
+  @file
+  @brief Testing mp_retainedkey macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertcolvals.sas
+  @li mp_retainedkey.sas
+
+**/
+
+/**
+  * Setup base tables
+  */
+proc sql;
+create table work.maxkeytable(
+    keytable varchar(41) label='Base table in libref.dataset format',
+    keycolumn char(32) format=$32.
+      label='The Retained key field containing the key values.',
+    max_key num label=
+      'Integer representing current max RK or SK value in the KEYTABLE',
+    processed_dttm num format=E8601DT26.6
+      label='Datetime this value was last updated',
+  constraint pk_mpe_maxkeyvalues
+      primary key(keytable));
+
+create table work.locktable(
+    lock_lib char(8),
+    lock_ds char(32),
+    lock_status_cd char(10) not null,
+    lock_user_nm char(100) not null ,
+    lock_ref char(200),
+    lock_pid char(10),
+    lock_start_dttm num format=E8601DT26.6,
+    lock_end_dttm num format=E8601DT26.6,
+  constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+
+data work.targetds;
+  rk_col=_n_;
+  set sashelp.class;
+run;
+
+data work.appendtable;
+  set sashelp.class;
+  if mod(_n_,2)=0 then name=cats('New',_n_);
+  if _n_<7;
+run;
+
+libname x (work);
+
+/** Test 1 - base case **/
+%mp_retainedkey(
+  base_lib=X
+  ,base_dsn=targetds
+  ,append_lib=X
+  ,append_dsn=APPENDTABLE
+  ,retained_key=rk_col
+  ,business_key= name
+  ,check_uniqueness=NO
+  ,maxkeytable=0
+  ,locktable=0
+  ,outds=work.APPEND
+  ,filter_str=
+)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking errors in test 1,
+  outds=work.test_results
+)
+
+data work.check;
+  do val=1,3,5,20,21,22;
+    output;
+  end;
+run;
+%mp_assertcolvals(work.append.rk_col,
+  checkvals=work.check.val,
+  desc=All values have a match,
+  test=ALLVALS
+)
+
+/** Test 2 - all new records, with metadata logging and unique check **/
+data work.targetds2;
+  rk_col=_n_;
+  set sashelp.class;
+run;
+
+data work.appendtable2;
+  set sashelp.class;
+  do x=1 to 21;
+    name=cats('New',x);
+    output;
+  end;
+  stop;
+run;
+
+%mp_retainedkey(base_dsn=targetds2
+  ,append_dsn=APPENDTABLE2
+  ,retained_key=rk_col
+  ,business_key= name
+  ,check_uniqueness=YES
+  ,maxkeytable=x.maxkeytable
+  ,locktable=work.locktable
+  ,outds=WORK.APPEND2
+  ,filter_str=
+)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking errors in test 2,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(work.append2)=21),
+  desc=Checking append records created,
+  outds=work.test_results
+)

tests/crossplatform/mp_validatecol.test.sas

@@ -59,4 +59,71 @@ run;
   desc=Test2 - ISNUM,
   test=EQUALS 4,
   outds=work.test_results
+)
+
+/**
+  * Test 3 - FORMAT
+  */
+data test3;
+  infile datalines4 dsd;
+  input;
+  infile=_infile_;
+  %mp_validatecol(infile,FORMAT,is_format)
+  if is_format=1;
+datalines4;
+$.
+$format.
+$format12.2
+somenum.
+somenum12.4
+above are good
+the rest are bad
+%abort
+1&somethingverybad.
+&
++-1
+.
+a.A
+$format12.1b
+$format12.1b1
+;;;;
+run;
+%mp_assertdsobs(work.test3,
+  desc=Test3 - ISFORMAT,
+  test=EQUALS 5,
+  outds=work.test_results
+)
+
+/**
+  * Test 4 - ISINT
+  */
+data test4;
+  infile datalines4 dsd;
+  input;
+  infile=_infile_;
+  %mp_validatecol(infile,ISINT,is_integer)
+  if is_integer=1;
+datalines4;
+1
+1234
+-134
+-1.0
+1.0
+0
+above are good
+the rest are bad
+%abort
+1&somethingverybad.
+&
++-1
+.
+a.A
+$format12.1b
+$format12.1b1
+;;;;
+run;
+%mp_assertdsobs(work.test4,
+  desc=Test4 - ISFORMAT,
+  test=EQUALS 6,
+  outds=work.test_results
 )