Merge pull request #133 from sasjs/fmtname

fix: adding fmtname to mp_getcols() macro

all.sas

@@ -3351,6 +3351,97 @@ run;
   drop table work.&tempds;
 
 %mend mp_copyfolder;/**
+  @file
+  @brief Create the permanent Core tables
+  @details Several macros in the [core](https://github.com/sasjs/core) library
+    make use of permanent tables.  To avoid duplication in definitions, this
+    macro provides a central location for managing the corresponding DDL.
+
+  Example usage:
+
+      %mp_coretable(LOCKTABLE,libds=work.locktable)
+
+  @param [in] table_ref The type of table to create.  Example values:
+    @li FILTER_DETAIL - For storing detailed filter values.  Used by
+      mp_filterstore.sas.
+    @li FILTER_SUMMARY - For storing summary filter values.  Used by
+      mp_filterstore.sas.
+    @li LOCKANYTABLE - For "locking" tables prior to multipass loads. Used by
+      mp_lockanytable.sas
+    @li MAXKEYTABLE - For storing the maximum retained key information.  Used
+      by mp_retainedkey.sas
+  @param [in] libds= (0) The library.dataset reference used to create the table.
+    If not provided, then the DDL is simply printed to the log.
+
+  <h4> Related Macros </h4>
+  @li mp_filterstore.sas
+  @li mp_lockanytable.sas
+  @li mp_retainedkey.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_coretable(table_ref,libds=0
+)/*/STORE SOURCE*/;
+%local outds ;
+%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
+proc sql;
+%if &table_ref=LOCKTABLE %then %do;
+  create table &outds(
+      lock_lib char(8),
+      lock_ds char(32),
+      lock_status_cd char(10) not null,
+      lock_user_nm char(100) not null ,
+      lock_ref char(200),
+      lock_pid char(10),
+      lock_start_dttm num format=E8601DT26.6,
+      lock_end_dttm num format=E8601DT26.6,
+    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+%end;
+%else %if &table_ref=FILTER_SUMMARY %then %do;
+  create table &outds(
+      filter_rk num not null,
+      filter_hash char(32) not null,
+      filter_table char(41) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_rk));
+%end;
+%else %if &table_ref=FILTER_DETAIL %then %do;
+  create table &outds(
+      filter_hash char(32) not null,
+      filter_line num not null,
+      group_logic char(3) not null,
+      subgroup_logic char(3) not null,
+      subgroup_id num not null,
+      variable_nm varchar(32) not null,
+      operator_nm varchar(12) not null,
+      raw_value varchar(4000) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_hash,filter_line));
+%end;
+%else %if &table_ref=MAXKEYTABLE %then %do;
+  create table &outds(
+      keytable varchar(41) label='Base table in libref.dataset format',
+      keycolumn char(32) format=$32.
+        label='The Retained key field containing the key values.',
+      max_key num label=
+        'Integer representing current max RK or SK value in the KEYTABLE',
+      processed_dttm num format=E8601DT26.6
+        label='Datetime this value was last updated',
+    constraint pk_mpe_maxkeyvalues
+        primary key(keytable));
+%end;
+
+
+%if &libds=0 %then %do;
+  describe table &syslast;
+  drop table &syslast;
+%end;
+%mend mp_coretable;/**
   @file mp_createconstraints.sas
   @brief Creates constraints
   @details Takes the output from mp_getconstraints.sas as input
@@ -5098,6 +5189,235 @@ filename &outref temp;
 
 %mend mp_filtergenerate;
 /**
+  @file
+  @brief Checks & Stores an input filter table and returns the Filter Key
+  @details Used to generate a FILTER_RK from an input query dataset.  This
+  process requires several permanent tables (names are configurable).  The
+  benefit of storing query values at backend is to enable stored 'views' of
+  filtered tables at frontend (ie, when building [SAS-Powered Apps](
+  https://sasapps.io)).  This macro is also used in [Data Controller for SAS](
+  https://datacontroller.io).
+
+
+  @param [in] libds= The target dataset to be filtered (lib should be assigned)
+  @param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to
+    be validated.  Has the following format:
+|GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
+|---|---|---|---|---|---|
+|AND|AND|1|SOME_BESTNUM|>|1|
+|AND|AND|1|SOME_TIME|=|77333|
+  @param [in] filter_summary= (PERM.FILTER_SUMMARY) Permanent table containing
+    summary filter values.  The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(FILTER_SUMMARY)`. Example
+    values:
+|FILTER_RK:best.|FILTER_HASH:$32.|FILTER_TABLE:$41.|PROCESSED_DTTM:datetime19.|
+|---|---|---|---|
+|`1 `|`540E96F566D194AB58DD4C413C99C9DB `|`VIYA6014.MPE_TABLES `|`1956084246 `|
+|`2 `|`87737DB9EEE2650F5C89956CEAD0A14F `|`VIYA6014.MPE_X_TEST `|`1956084452.1`|
+|`3 `|`8048BD908DBBD83D013560734E90D394 `|`VIYA6014.MPE_TABLES `|`1956093620.6`|
+  @param [in] filter_detail= (PERM.FILTER_DETAIL) Permanent table containing
+    detailed (raw) filter values. The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(FILTER_DETAIL)`. Example
+    values:
+|FILTER_HASH:$32.|FILTER_LINE:best.|GROUP_LOGIC:$3.|SUBGROUP_LOGIC:$3.|SUBGROUP_ID:best.|VARIABLE_NM:$32.|OPERATOR_NM:$12.|RAW_VALUE:$4000.|PROCESSED_DTTM:datetime19.|
+|---|---|---|---|---|---|---|---|---|
+|`540E96F566D194AB58DD4C413C99C9DB `|`1 `|`AND `|`AND `|`1 `|`LIBREF `|`CONTAINS `|`DC`|`1956084245.8 `|
+|`540E96F566D194AB58DD4C413C99C9DB `|`2 `|`AND `|`OR `|`2 `|`DSN `|`= `|` MPE_LOCK_ANYTABLE `|`1956084245.8 `|
+|`87737DB9EEE2650F5C89956CEAD0A14F `|`1 `|`AND `|`AND `|`1 `|`PRIMARY_KEY_FIELD `|`IN `|`(1,2,3) `|`1956084451.9 `|
+  @param [in] lock_table= (PERM.LOCK_TABLE) Permanent locking table.  Used to
+    manage concurrent access.  The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(LOCKTABLE)`.
+  @param [in] maxkeytable= (0) Optional permanent reference table used for
+    retained key tracking.  Described in mp_retainedkey.sas.
+  @param [in] mdebug= set to 1 to enable DEBUG messages
+  @param [out] outresult= The result table with the FILTER_RK
+  @param [out] outquery= The original query, taken as extract after table load
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_getvalue.sas
+  @li mf_islibds.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_filtercheck.sas
+  @li mp_hashdataset.sas
+  @li mp_retainedkey.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filtercheck.sas
+  @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
+  @li mp_filterstore.test.sas
+
+  @version 9.2
+  @author [Allan Bowe](https://www.linkedin.com/in/allanbowe)
+
+**/
+
+%macro mp_filterstore(libds=,
+  queryds=work.filterquery,
+  filter_summary=PERM.FILTER_SUMMARY,
+  filter_detail=PERM.FILTER_DETAIL,
+  lock_table=PERM.LOCK_TABLE,
+  maxkeytable=PERM.MAXKEYTABLE,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+);
+%put &sysmacroname entry vars:;
+%put _local_;
+
+%local ds1 ds2 ds3 ds4 filter_hash;
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+%mp_abort(iftrue= (%mf_islibds(&filter_summary)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid filter_summary value: &filter_summary)
+)
+%mp_abort(iftrue= (%mf_islibds(&filter_detail)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid filter_detail value: &filter_detail)
+)
+%mp_abort(iftrue= (%mf_islibds(&lock_table)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid lock_table value: &lock_table)
+)
+
+/* validate query */
+%mp_filtercheck(&queryds,targetds=&libds,abort=YES)
+
+/* hash the result */
+%let ds1=%mf_getuniquename(prefix=hashds);
+%mp_hashdataset(&queryds,outds=&ds1,salt=&libds)
+%let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
+%if &mdebug=1 %then %do;
+  data _null_;
+    putlog "filter_hash=&filter_hash";
+    set &ds1;
+    putlog (_all_)(=);
+  run;
+%end;
+
+/* check if data already exists for this hash */
+data &outresult;
+  set &filter_summary;
+  where filter_hash="&filter_hash";
+run;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc after hash check)
+)
+%mp_abort(iftrue= ("&filter_hash "=" ")
+  ,mac=mp_filterstore
+  ,msg=%str(problem with filter_hash generation)
+)
+
+%if %mf_nobs(&outresult)=0 %then %do;
+
+  /* first update summary table */
+  %let ds3=%mf_getuniquename(prefix=filtersum);
+  data work.&ds3;
+    if 0 then set &filter_summary;
+    filter_table=symget('libds');
+    filter_hash="&filter_hash";
+    PROCESSED_DTTM=%sysfunc(datetime());
+    output;
+    stop;
+  run;
+
+  %mp_lockanytable(LOCK,
+    lib=%scan(&filter_summary,1,.)
+    ,ds=%scan(&filter_summary,2,.)
+    ,ref=MP_FILTERSTORE summary update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  %let ds4=%mf_getuniquename(prefix=filtersumappend);
+  %mp_retainedkey(
+    base_lib=%scan(&filter_summary,1,.)
+    ,base_dsn=%scan(&filter_summary,2,.)
+    ,append_lib=work
+    ,append_dsn=&ds3
+    ,retained_key=filter_rk
+    ,business_key=filter_hash
+    ,maxkeytable=&maxkeytable
+    ,locktable=&lock_table
+    ,outds=work.&ds4
+  )
+  proc append base=&filter_summary data=&ds4;
+  run;
+
+  %mp_lockanytable(UNLOCK,
+    lib=%scan(&filter_summary,1,.)
+    ,ds=%scan(&filter_summary,2,.)
+    ,ref=MP_FILTERSTORE summary update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  %if &syscc ne 0 %then %do;
+    data _null_;
+      set &ds4;
+      putlog (_all_)(=);
+    run;
+    %goto err;
+  %end;
+
+  data &outresult;
+    set &filter_summary;
+    where filter_hash="&filter_hash";
+  run;
+
+  /* Next, update detail table */
+  %let ds2=%mf_getuniquename(prefix=filterdetail);
+  data &ds2;
+    if 0 then set &filter_detail;
+    set &queryds;
+    format filter_hash $hex32. filter_line 8.;
+    filter_hash="&filter_hash";
+    filter_line=_n_;
+    PROCESSED_DTTM=%sysfunc(datetime());
+  run;
+  %mp_lockanytable(LOCK,
+    lib=%scan(&filter_detail,1,.)
+    ,ds=%scan(&filter_detail,2,.)
+    ,ref=MP_FILTERSTORE update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+  proc append base=&filter_detail data=&ds2;
+  run;
+
+  %mp_lockanytable(UNLOCK,
+    lib=%scan(&filter_detail,1,.)
+    ,ds=%scan(&filter_detail,2,.)
+    ,ref=MP_FILTERSTORE detail update &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  %if &syscc ne 0 %then %do;
+    data _null_;
+      set &ds2;
+      putlog (_all_)(=);
+    run;
+    %goto err;
+  %end;
+
+%end;
+
+proc sort data=&filter_detail(where=(filter_hash="&filter_hash")) out=&outquery;
+  by filter_line;
+run;
+
+%err:
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc on macro exit)
+)
+
+%mend mp_filterstore;/**
   @file
   @brief Checks a generated filter query for validity
   @details Runs a generated filter in proc sql with the validate option.
@@ -5219,11 +5539,11 @@ filename &fref1 clear;
 
   @param ds The dataset from which to obtain column metadata
   @param outds= (work.cols) The output dataset to create. Sample data:
-  |NAME $|LENGTH 8|VARNUM 8|LABEL $|FORMAT $49|TYPE $1 |DDTYPE $|
-  |---|---|---|---|---|---|---|
-  |AIR|8|2|international airline travel (thousands)|8.|N|NUMERIC|
-  |DATE|8|1|DATE|MONYY.|N|DATE|
-  |REGION|3|3|REGION|$3.|C|CHARACTER|
+|NAME:$32.|LENGTH:best.|VARNUM:best.|LABEL:$256.|FMTNAME:$32.|FORMAT:$49.|TYPE:$1.|DDTYPE:$9.|
+|---|---|---|---|---|---|---|---|
+|`AIR `|`8 `|`2 `|`international airline travel (thousands) `|` `|`8. `|`N `|`NUMERIC `|
+|`DATE `|`8 `|`1 `|`DATE `|`MONYY `|`MONYY. `|`N `|`DATE `|
+|`REGION `|`3 `|`3 `|`REGION `|` `|`$3. `|`C `|`CHARACTER `|
 
   <h4> Related Macros </h4>
   @li mf_getvarlist.sas
@@ -5235,26 +5555,27 @@ filename &fref1 clear;
 **/
 
 %macro mp_getcols(ds, outds=work.cols);
-
+%local dropds;
 proc contents noprint data=&ds
   out=_data_ (keep=name type length label varnum format:);
 run;
-data &outds(keep=name type length varnum format label ddtype);
-  set &syslast(rename=(format=format2 type=type2));
+%let dropds=&syslast;
+data &outds(keep=name type length varnum format label ddtype fmtname);
+  set &dropds(rename=(format=fmtname type=type2));
   name=upcase(name);
   if type2=2 then do;
     length format $49.;
-    if format2='' then format=cats('$',length,'.');
-    else if formatl=0 then format=cats(format2,'.');
-    else format=cats(format2,formatl,'.');
+    if fmtname='' then format=cats('$',length,'.');
+    else if formatl=0 then format=cats(fmtname,'.');
+    else format=cats(fmtname,formatl,'.');
     type='C';
     ddtype='CHARACTER';
   end;
   else do;
-    if format2='' then format=cats(length,'.');
-    else if formatl=0 then format=cats(format2,'.');
-    else if formatd=0 then format=cats(format2,formatl,'.');
-    else format=cats(format2,formatl,'.',formatd);
+    if fmtname='' then format=cats(length,'.');
+    else if formatl=0 then format=cats(fmtname,'.');
+    else if formatd=0 then format=cats(fmtname,formatl,'.');
+    else format=cats(fmtname,formatl,'.',formatd);
     type='N';
     if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
     else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
@@ -5266,7 +5587,8 @@ data &outds(keep=name type length varnum format label ddtype);
   end;
   if label='' then label=name;
 run;
-
+proc sql;
+drop table &dropds;
 %mend mp_getcols;/**
   @file mp_getconstraints.sas
   @brief Get constraint details at column level
@@ -7584,19 +7906,8 @@ select distinct lowcase(memname)
   @param [in] ref= A meaningful reference to enable the lock to be traced. Max
     length is 200 characters.
   @param [out] ctl_ds= (0) The control table which controls the actual locking.
-    Should already be assigned and available.  Definition as follows:
-
-        proc sql;
-        create table &ctl_ds(
-            lock_lib char(8),
-            lock_ds char(32),
-            lock_status_cd char(10) not null,
-            lock_user_nm char(100) not null ,
-            lock_ref char(200),
-            lock_pid char(10),
-            lock_start_dttm num format=E8601DT26.6,
-            lock_end_dttm num format=E8601DT26.6,
-          constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+    Should already be assigned and available.  The definition is available by
+    running mp_coretable.sas as follows:  `mp_coretable(LOCKTABLE)`.
 
   @param [in] loops= (25) Number of times to check for a lock.
   @param [in] loop_secs= (1) Seconds to wait between each lock attempt
@@ -8326,19 +8637,12 @@ run;
   @li permlib.base_table - the target table to be loaded (**not** loaded by this
     macro)
   @li permlib.maxkeytable - optional, used to store load metaadata.
-  The structure is as follows:
+    The definition is available by running mp_coretable.sas as follows:
+    `mp_coretable(MAXKEYTABLE)`.
+  @li permlib.locktable - Necessary if maxkeytable is being populated. The
+    definition is available by running mp_coretable.sas as follows:
+    `mp_coretable(LOCKTABLE)`.
 
-      proc sql;
-      create table yourlib.maxkeytable(
-          keytable varchar(41) label='Base table in libref.dataset format',
-          keycolumn char(32) format=$32.
-            label='The Retained key field containing the key values.',
-          max_key num label=
-            'Integer representing current max RK or SK value in the KEYTABLE',
-          processed_dttm num format=E8601DT26.6
-            label='Datetime this value was last updated',
-        constraint pk_mpe_maxkeyvalues
-            primary key(keytable));
 
   @param [in] base_lib= (WORK) Libref of the base (target) table.
   @param [in] base_dsn= (BASETABLE) Name of the base (target) table.
@@ -8374,6 +8678,7 @@ run;
   @li mp_lockanytable.sas
 
   <h4> Related Macros </h4>
+  @li mp_filterstore.sas
   @li mp_retainedkey.test.sas
 
   @version 9.2
@@ -8403,7 +8708,7 @@ run;
 %let tempds1=%mf_getuniquename();
 %let tempds2=%mf_getuniquename();
 %let comma_pk=%mf_getquotedstr(in_str=%str(&business_key),dlm=%str(,),quote=);
-
+%let outds=%sysfunc(ifc(%index(&outds,.)=0,work.&outds,&outds));
 /* validation checks */
 %let iserr=0;
 %if &syscc>0 %then %do;
@@ -8434,14 +8739,18 @@ run;
 %do x=1 %to %sysfunc(countw(&business_key));
   /* check business key values exist */
   %let key_field=%scan(&business_key,&x,%str( ));
-  %if (not %mf_existvar(&app_libds,&key_field))
-    or (not %mf_existvar(&base_libds,&key_field))
-  %then %do;
+  %if not %mf_existvar(&app_libds,&key_field) %then %do;
     %let iserr=1;
-    %let msg=Business key (&key_field) not found!;
+    %let msg=Business key (&key_field) not found on &app_libds!;
+    %goto err;
+  %end;
+  %else %if not %mf_existvar(&base_libds,&key_field) %then %do;
+    %let iserr=1;
+    %let msg=Business key (&key_field) not found on &base_libds!;
+    %goto err;
   %end;
 %end;
-
+%err:
 %if &iserr=1 %then %do;
   /* err case so first perform an unlock of the base table before exiting */
   %mp_lockanytable(
@@ -8505,7 +8814,7 @@ quit;
   * Update maxkey table if link provided
   */
 %if &maxkeytable ne 0 %then %do;
-  proc sql;
+  proc sql noprint;
   select count(*) into: check from &maxkeytable
     where upcase(keytable)="&base_libds";
 

base/mp_coretable.sas

@@ -0,0 +1,92 @@
+/**
+  @file
+  @brief Create the permanent Core tables
+  @details Several macros in the [core](https://github.com/sasjs/core) library
+    make use of permanent tables.  To avoid duplication in definitions, this
+    macro provides a central location for managing the corresponding DDL.
+
+  Example usage:
+
+      %mp_coretable(LOCKTABLE,libds=work.locktable)
+
+  @param [in] table_ref The type of table to create.  Example values:
+    @li FILTER_DETAIL - For storing detailed filter values.  Used by
+      mp_filterstore.sas.
+    @li FILTER_SUMMARY - For storing summary filter values.  Used by
+      mp_filterstore.sas.
+    @li LOCKANYTABLE - For "locking" tables prior to multipass loads. Used by
+      mp_lockanytable.sas
+    @li MAXKEYTABLE - For storing the maximum retained key information.  Used
+      by mp_retainedkey.sas
+  @param [in] libds= (0) The library.dataset reference used to create the table.
+    If not provided, then the DDL is simply printed to the log.
+
+  <h4> Related Macros </h4>
+  @li mp_filterstore.sas
+  @li mp_lockanytable.sas
+  @li mp_retainedkey.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_coretable(table_ref,libds=0
+)/*/STORE SOURCE*/;
+%local outds ;
+%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
+proc sql;
+%if &table_ref=LOCKTABLE %then %do;
+  create table &outds(
+      lock_lib char(8),
+      lock_ds char(32),
+      lock_status_cd char(10) not null,
+      lock_user_nm char(100) not null ,
+      lock_ref char(200),
+      lock_pid char(10),
+      lock_start_dttm num format=E8601DT26.6,
+      lock_end_dttm num format=E8601DT26.6,
+    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+%end;
+%else %if &table_ref=FILTER_SUMMARY %then %do;
+  create table &outds(
+      filter_rk num not null,
+      filter_hash char(32) not null,
+      filter_table char(41) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_rk));
+%end;
+%else %if &table_ref=FILTER_DETAIL %then %do;
+  create table &outds(
+      filter_hash char(32) not null,
+      filter_line num not null,
+      group_logic char(3) not null,
+      subgroup_logic char(3) not null,
+      subgroup_id num not null,
+      variable_nm varchar(32) not null,
+      operator_nm varchar(12) not null,
+      raw_value varchar(4000) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_hash,filter_line));
+%end;
+%else %if &table_ref=MAXKEYTABLE %then %do;
+  create table &outds(
+      keytable varchar(41) label='Base table in libref.dataset format',
+      keycolumn char(32) format=$32.
+        label='The Retained key field containing the key values.',
+      max_key num label=
+        'Integer representing current max RK or SK value in the KEYTABLE',
+      processed_dttm num format=E8601DT26.6
+        label='Datetime this value was last updated',
+    constraint pk_mpe_maxkeyvalues
+        primary key(keytable));
+%end;
+
+
+%if &libds=0 %then %do;
+  describe table &syslast;
+  drop table &syslast;
+%end;
+%mend mp_coretable;

base/mp_filterstore.sas

@@ -0,0 +1,230 @@
+/**
+  @file
+  @brief Checks & Stores an input filter table and returns the Filter Key
+  @details Used to generate a FILTER_RK from an input query dataset.  This
+  process requires several permanent tables (names are configurable).  The
+  benefit of storing query values at backend is to enable stored 'views' of
+  filtered tables at frontend (ie, when building [SAS-Powered Apps](
+  https://sasapps.io)).  This macro is also used in [Data Controller for SAS](
+  https://datacontroller.io).
+
+
+  @param [in] libds= The target dataset to be filtered (lib should be assigned)
+  @param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to
+    be validated.  Has the following format:
+|GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
+|---|---|---|---|---|---|
+|AND|AND|1|SOME_BESTNUM|>|1|
+|AND|AND|1|SOME_TIME|=|77333|
+  @param [in] filter_summary= (PERM.FILTER_SUMMARY) Permanent table containing
+    summary filter values.  The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(FILTER_SUMMARY)`. Example
+    values:
+|FILTER_RK:best.|FILTER_HASH:$32.|FILTER_TABLE:$41.|PROCESSED_DTTM:datetime19.|
+|---|---|---|---|
+|`1 `|`540E96F566D194AB58DD4C413C99C9DB `|`VIYA6014.MPE_TABLES `|`1956084246 `|
+|`2 `|`87737DB9EEE2650F5C89956CEAD0A14F `|`VIYA6014.MPE_X_TEST `|`1956084452.1`|
+|`3 `|`8048BD908DBBD83D013560734E90D394 `|`VIYA6014.MPE_TABLES `|`1956093620.6`|
+  @param [in] filter_detail= (PERM.FILTER_DETAIL) Permanent table containing
+    detailed (raw) filter values. The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(FILTER_DETAIL)`. Example
+    values:
+|FILTER_HASH:$32.|FILTER_LINE:best.|GROUP_LOGIC:$3.|SUBGROUP_LOGIC:$3.|SUBGROUP_ID:best.|VARIABLE_NM:$32.|OPERATOR_NM:$12.|RAW_VALUE:$4000.|PROCESSED_DTTM:datetime19.|
+|---|---|---|---|---|---|---|---|---|
+|`540E96F566D194AB58DD4C413C99C9DB `|`1 `|`AND `|`AND `|`1 `|`LIBREF `|`CONTAINS `|`DC`|`1956084245.8 `|
+|`540E96F566D194AB58DD4C413C99C9DB `|`2 `|`AND `|`OR `|`2 `|`DSN `|`= `|` MPE_LOCK_ANYTABLE `|`1956084245.8 `|
+|`87737DB9EEE2650F5C89956CEAD0A14F `|`1 `|`AND `|`AND `|`1 `|`PRIMARY_KEY_FIELD `|`IN `|`(1,2,3) `|`1956084451.9 `|
+  @param [in] lock_table= (PERM.LOCK_TABLE) Permanent locking table.  Used to
+    manage concurrent access.  The definition is available by running
+    mp_coretable.sas as follows:  `mp_coretable(LOCKTABLE)`.
+  @param [in] maxkeytable= (0) Optional permanent reference table used for
+    retained key tracking.  Described in mp_retainedkey.sas.
+  @param [in] mdebug= set to 1 to enable DEBUG messages
+  @param [out] outresult= The result table with the FILTER_RK
+  @param [out] outquery= The original query, taken as extract after table load
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_getvalue.sas
+  @li mf_islibds.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_filtercheck.sas
+  @li mp_hashdataset.sas
+  @li mp_retainedkey.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filtercheck.sas
+  @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
+  @li mp_filterstore.test.sas
+
+  @version 9.2
+  @author [Allan Bowe](https://www.linkedin.com/in/allanbowe)
+
+**/
+
+%macro mp_filterstore(libds=,
+  queryds=work.filterquery,
+  filter_summary=PERM.FILTER_SUMMARY,
+  filter_detail=PERM.FILTER_DETAIL,
+  lock_table=PERM.LOCK_TABLE,
+  maxkeytable=PERM.MAXKEYTABLE,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+);
+%put &sysmacroname entry vars:;
+%put _local_;
+
+%local ds1 ds2 ds3 ds4 filter_hash;
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+%mp_abort(iftrue= (%mf_islibds(&filter_summary)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid filter_summary value: &filter_summary)
+)
+%mp_abort(iftrue= (%mf_islibds(&filter_detail)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid filter_detail value: &filter_detail)
+)
+%mp_abort(iftrue= (%mf_islibds(&lock_table)=0)
+  ,mac=mp_filterstore
+  ,msg=%str(Invalid lock_table value: &lock_table)
+)
+
+/* validate query */
+%mp_filtercheck(&queryds,targetds=&libds,abort=YES)
+
+/* hash the result */
+%let ds1=%mf_getuniquename(prefix=hashds);
+%mp_hashdataset(&queryds,outds=&ds1,salt=&libds)
+%let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
+%if &mdebug=1 %then %do;
+  data _null_;
+    putlog "filter_hash=&filter_hash";
+    set &ds1;
+    putlog (_all_)(=);
+  run;
+%end;
+
+/* check if data already exists for this hash */
+data &outresult;
+  set &filter_summary;
+  where filter_hash="&filter_hash";
+run;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc after hash check)
+)
+%mp_abort(iftrue= ("&filter_hash "=" ")
+  ,mac=mp_filterstore
+  ,msg=%str(problem with filter_hash generation)
+)
+
+%if %mf_nobs(&outresult)=0 %then %do;
+
+  /* first update summary table */
+  %let ds3=%mf_getuniquename(prefix=filtersum);
+  data work.&ds3;
+    if 0 then set &filter_summary;
+    filter_table=symget('libds');
+    filter_hash="&filter_hash";
+    PROCESSED_DTTM=%sysfunc(datetime());
+    output;
+    stop;
+  run;
+
+  %mp_lockanytable(LOCK,
+    lib=%scan(&filter_summary,1,.)
+    ,ds=%scan(&filter_summary,2,.)
+    ,ref=MP_FILTERSTORE summary update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  %let ds4=%mf_getuniquename(prefix=filtersumappend);
+  %mp_retainedkey(
+    base_lib=%scan(&filter_summary,1,.)
+    ,base_dsn=%scan(&filter_summary,2,.)
+    ,append_lib=work
+    ,append_dsn=&ds3
+    ,retained_key=filter_rk
+    ,business_key=filter_hash
+    ,maxkeytable=&maxkeytable
+    ,locktable=&lock_table
+    ,outds=work.&ds4
+  )
+  proc append base=&filter_summary data=&ds4;
+  run;
+
+  %mp_lockanytable(UNLOCK,
+    lib=%scan(&filter_summary,1,.)
+    ,ds=%scan(&filter_summary,2,.)
+    ,ref=MP_FILTERSTORE summary update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  %if &syscc ne 0 %then %do;
+    data _null_;
+      set &ds4;
+      putlog (_all_)(=);
+    run;
+    %goto err;
+  %end;
+
+  data &outresult;
+    set &filter_summary;
+    where filter_hash="&filter_hash";
+  run;
+
+  /* Next, update detail table */
+  %let ds2=%mf_getuniquename(prefix=filterdetail);
+  data &ds2;
+    if 0 then set &filter_detail;
+    set &queryds;
+    format filter_hash $hex32. filter_line 8.;
+    filter_hash="&filter_hash";
+    filter_line=_n_;
+    PROCESSED_DTTM=%sysfunc(datetime());
+  run;
+  %mp_lockanytable(LOCK,
+    lib=%scan(&filter_detail,1,.)
+    ,ds=%scan(&filter_detail,2,.)
+    ,ref=MP_FILTERSTORE update - &filter_hash
+    ,ctl_ds=&lock_table
+  )
+  proc append base=&filter_detail data=&ds2;
+  run;
+
+  %mp_lockanytable(UNLOCK,
+    lib=%scan(&filter_detail,1,.)
+    ,ds=%scan(&filter_detail,2,.)
+    ,ref=MP_FILTERSTORE detail update &filter_hash
+    ,ctl_ds=&lock_table
+  )
+
+  %if &syscc ne 0 %then %do;
+    data _null_;
+      set &ds2;
+      putlog (_all_)(=);
+    run;
+    %goto err;
+  %end;
+
+%end;
+
+proc sort data=&filter_detail(where=(filter_hash="&filter_hash")) out=&outquery;
+  by filter_line;
+run;
+
+%err:
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=mp_filterstore
+  ,msg=%str(syscc=&syscc on macro exit)
+)
+
+%mend mp_filterstore;

base/mp_getcols.sas

@@ -14,11 +14,11 @@
 
   @param ds The dataset from which to obtain column metadata
   @param outds= (work.cols) The output dataset to create. Sample data:
-  |NAME $|LENGTH 8|VARNUM 8|LABEL $|FORMAT $49|TYPE $1 |DDTYPE $|
-  |---|---|---|---|---|---|---|
-  |AIR|8|2|international airline travel (thousands)|8.|N|NUMERIC|
-  |DATE|8|1|DATE|MONYY.|N|DATE|
-  |REGION|3|3|REGION|$3.|C|CHARACTER|
+|NAME:$32.|LENGTH:best.|VARNUM:best.|LABEL:$256.|FMTNAME:$32.|FORMAT:$49.|TYPE:$1.|DDTYPE:$9.|
+|---|---|---|---|---|---|---|---|
+|`AIR `|`8 `|`2 `|`international airline travel (thousands) `|` `|`8. `|`N `|`NUMERIC `|
+|`DATE `|`8 `|`1 `|`DATE `|`MONYY `|`MONYY. `|`N `|`DATE `|
+|`REGION `|`3 `|`3 `|`REGION `|` `|`$3. `|`C `|`CHARACTER `|
 
   <h4> Related Macros </h4>
   @li mf_getvarlist.sas
@@ -30,26 +30,27 @@
 **/
 
 %macro mp_getcols(ds, outds=work.cols);
-
+%local dropds;
 proc contents noprint data=&ds
   out=_data_ (keep=name type length label varnum format:);
 run;
-data &outds(keep=name type length varnum format label ddtype);
-  set &syslast(rename=(format=format2 type=type2));
+%let dropds=&syslast;
+data &outds(keep=name type length varnum format label ddtype fmtname);
+  set &dropds(rename=(format=fmtname type=type2));
   name=upcase(name);
   if type2=2 then do;
     length format $49.;
-    if format2='' then format=cats('$',length,'.');
-    else if formatl=0 then format=cats(format2,'.');
-    else format=cats(format2,formatl,'.');
+    if fmtname='' then format=cats('$',length,'.');
+    else if formatl=0 then format=cats(fmtname,'.');
+    else format=cats(fmtname,formatl,'.');
     type='C';
     ddtype='CHARACTER';
   end;
   else do;
-    if format2='' then format=cats(length,'.');
-    else if formatl=0 then format=cats(format2,'.');
-    else if formatd=0 then format=cats(format2,formatl,'.');
-    else format=cats(format2,formatl,'.',formatd);
+    if fmtname='' then format=cats(length,'.');
+    else if formatl=0 then format=cats(fmtname,'.');
+    else if formatd=0 then format=cats(fmtname,formatl,'.');
+    else format=cats(fmtname,formatl,'.',formatd);
     type='N';
     if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
     else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
@@ -61,5 +62,6 @@ data &outds(keep=name type length varnum format label ddtype);
   end;
   if label='' then label=name;
 run;
-
+proc sql;
+drop table &dropds;
 %mend mp_getcols;

base/mp_lockanytable.sas

@@ -14,19 +14,8 @@
   @param [in] ref= A meaningful reference to enable the lock to be traced. Max
     length is 200 characters.
   @param [out] ctl_ds= (0) The control table which controls the actual locking.
-    Should already be assigned and available.  Definition as follows:
-
-        proc sql;
-        create table &ctl_ds(
-            lock_lib char(8),
-            lock_ds char(32),
-            lock_status_cd char(10) not null,
-            lock_user_nm char(100) not null ,
-            lock_ref char(200),
-            lock_pid char(10),
-            lock_start_dttm num format=E8601DT26.6,
-            lock_end_dttm num format=E8601DT26.6,
-          constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+    Should already be assigned and available.  The definition is available by
+    running mp_coretable.sas as follows:  `mp_coretable(LOCKTABLE)`.
 
   @param [in] loops= (25) Number of times to check for a lock.
   @param [in] loop_secs= (1) Seconds to wait between each lock attempt

base/mp_retainedkey.sas

@@ -24,19 +24,12 @@
   @li permlib.base_table - the target table to be loaded (**not** loaded by this
     macro)
   @li permlib.maxkeytable - optional, used to store load metaadata.
-  The structure is as follows:
+    The definition is available by running mp_coretable.sas as follows:
+    `mp_coretable(MAXKEYTABLE)`.
+  @li permlib.locktable - Necessary if maxkeytable is being populated. The
+    definition is available by running mp_coretable.sas as follows:
+    `mp_coretable(LOCKTABLE)`.
 
-      proc sql;
-      create table yourlib.maxkeytable(
-          keytable varchar(41) label='Base table in libref.dataset format',
-          keycolumn char(32) format=$32.
-            label='The Retained key field containing the key values.',
-          max_key num label=
-            'Integer representing current max RK or SK value in the KEYTABLE',
-          processed_dttm num format=E8601DT26.6
-            label='Datetime this value was last updated',
-        constraint pk_mpe_maxkeyvalues
-            primary key(keytable));
 
   @param [in] base_lib= (WORK) Libref of the base (target) table.
   @param [in] base_dsn= (BASETABLE) Name of the base (target) table.
@@ -72,6 +65,7 @@
   @li mp_lockanytable.sas
 
   <h4> Related Macros </h4>
+  @li mp_filterstore.sas
   @li mp_retainedkey.test.sas
 
   @version 9.2
@@ -101,7 +95,7 @@
 %let tempds1=%mf_getuniquename();
 %let tempds2=%mf_getuniquename();
 %let comma_pk=%mf_getquotedstr(in_str=%str(&business_key),dlm=%str(,),quote=);
-
+%let outds=%sysfunc(ifc(%index(&outds,.)=0,work.&outds,&outds));
 /* validation checks */
 %let iserr=0;
 %if &syscc>0 %then %do;
@@ -132,14 +126,18 @@
 %do x=1 %to %sysfunc(countw(&business_key));
   /* check business key values exist */
   %let key_field=%scan(&business_key,&x,%str( ));
-  %if (not %mf_existvar(&app_libds,&key_field))
-    or (not %mf_existvar(&base_libds,&key_field))
-  %then %do;
+  %if not %mf_existvar(&app_libds,&key_field) %then %do;
     %let iserr=1;
-    %let msg=Business key (&key_field) not found!;
+    %let msg=Business key (&key_field) not found on &app_libds!;
+    %goto err;
+  %end;
+  %else %if not %mf_existvar(&base_libds,&key_field) %then %do;
+    %let iserr=1;
+    %let msg=Business key (&key_field) not found on &base_libds!;
+    %goto err;
   %end;
 %end;
-
+%err:
 %if &iserr=1 %then %do;
   /* err case so first perform an unlock of the base table before exiting */
   %mp_lockanytable(
@@ -203,7 +201,7 @@ quit;
   * Update maxkey table if link provided
   */
 %if &maxkeytable ne 0 %then %do;
-  proc sql;
+  proc sql noprint;
   select count(*) into: check from &maxkeytable
     where upcase(keytable)="&base_libds";
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sasjs/core",
-  "description": "Production Ready Macros for SAS Application Developers",
+  "description": "Macros for SAS Application Developers",
   "license": "MIT",
   "keywords": [
     "SAS",
@@ -38,4 +38,4 @@
   "dependencies": {
     "ts-loader": "^9.2.6"
   }
-}
+}

sasjs/doxy/new_header.html

@@ -1,67 +1,73 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!DOCTYPE html
+  PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 <!-- HTML header for doxygen 1.8.17-->
 <html xmlns="https://www.w3.org/1999/xhtml" lang="en">
-  <head>
-    <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8" />
-    <meta http-equiv="X-UA-Compatible" content="IE=9" />
-    <meta property="og:type" content="website">
-    <meta name="author" content="Allan Bowe">
-    <meta name="generator" content="Doxygen $doxygenversion" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <!--BEGIN PROJECT_NAME-->
-    <meta name="description" content="$projectbrief" />
-    <!--END PROJECT_NAME-->
-    <!--BEGIN !PROJECT_NAME-->
-    <title>$title</title>
-    <!--END !PROJECT_NAME-->
-    <link href="$relpath^tabs.css" rel="stylesheet" type="text/css" />
-    <script type="text/javascript" src="$relpath^jquery.js"></script>
-    <script type="text/javascript" src="$relpath^dynsections.js"></script>
-    $treeview $search $mathjax
-    <link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
-    <link rel="shortcut icon" href="$relpath^favicon.ico" type="image/x-icon" />
-    $extrastylesheet
-  </head>
-  <body>
-    <div id="top">
-      <!-- do not remove this div, it is closed by doxygen! -->
 
-      <!--BEGIN TITLEAREA-->
-      <div id="titlearea">
-        <table cellspacing="0" cellpadding="0">
-          <tbody>
-            <tr style="height: 56px">
-              <!--BEGIN PROJECT_LOGO-->
-              <td id="projectlogo">
-                <a href="$relpath^"
-                  ><img alt="Logo" src="$relpath^$projectlogo"
-                /></a>
-              </td>
-              <!--END PROJECT_LOGO-->
-              <td id="projectalign" style="padding-left: 0.5em">
-                <div id="projectbrief">
-                  Production Ready Macros for SAS Application Developers<br />
-                  <a href="https://github.com/sasjs/core">
-                    https://github.com/sasjs/core
-                  </a>
-                </div>
-              </td>
-                  </tr>
-                </table>
-              </td>
+<head>
+  <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8" />
+  <meta http-equiv="X-UA-Compatible" content="IE=9" />
+  <meta property="og:type" content="website">
+  <meta property="og:title" content="MacroCore" />
+  <meta property="og:url" content="https://core.sasjs.io" />
+  <meta property="og:image" content="https://core.sasjs.io/Macro_core_website_1.png" />
+  <meta name="author" content="Allan Bowe">
+  <meta name="generator" content="Doxygen $doxygenversion" />
+  <meta name="viewport" content="width=device-width, initial-scale=1" />
+  <!--BEGIN PROJECT_NAME-->
+  <meta name="description" content="$projectbrief" />
+  <meta name="og:description" content="$projectbrief" />
+  <!--END PROJECT_NAME-->
+  <!--BEGIN !PROJECT_NAME-->
+  <title>$title</title>
+  <!--END !PROJECT_NAME-->
+  <link href="$relpath^tabs.css" rel="stylesheet" type="text/css" />
+  <script type="text/javascript" src="$relpath^jquery.js"></script>
+  <script type="text/javascript" src="$relpath^dynsections.js"></script>
+  $treeview $search $mathjax
+  <link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
+  <link rel="shortcut icon" href="$relpath^favicon.ico" type="image/x-icon" />
+  $extrastylesheet
+</head>
+
+<body>
+  <div id="top">
+    <!-- do not remove this div, it is closed by doxygen! -->
+
+    <!--BEGIN TITLEAREA-->
+    <div id="titlearea">
+      <table cellspacing="0" cellpadding="0">
+        <tbody>
+          <tr style="height: 56px">
+            <!--BEGIN PROJECT_LOGO-->
+            <td id="projectlogo">
+              <a href="$relpath^"><img alt="Logo" src="$relpath^$projectlogo" /></a>
+            </td>
+            <!--END PROJECT_LOGO-->
+            <td id="projectalign" style="padding-left: 0.5em">
+              <div id="projectbrief">
+                Macros for SAS Application Developers<br />
+                <a href="https://github.com/sasjs/core">
+                  https://github.com/sasjs/core
+                </a>
+              </div>
+            </td>
+          </tr>
+      </table>
+      </td>
 
 
-              <!--BEGIN DISABLE_INDEX-->
-              <!--BEGIN SEARCHENGINE-->
-              <td>$searchbox</td>
-              <!--END SEARCHENGINE-->
-              <!--END DISABLE_INDEX-->
-            </tr>
-          </tbody>
-        </table>
-      </div>
-      <!--END TITLEAREA-->
-      <!-- end header part -->
+      <!--BEGIN DISABLE_INDEX-->
+      <!--BEGIN SEARCHENGINE-->
+      <td>$searchbox</td>
+      <!--END SEARCHENGINE-->
+      <!--END DISABLE_INDEX-->
+      </tr>
+      </tbody>
+      </table>
     </div>
-  </body>
-</html>
+    <!--END TITLEAREA-->
+    <!-- end header part -->
+  </div>
+</body>
+
+</html>

sasjs/sasjsconfig.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://cli.sasjs.io/sasjsconfig-schema.json",
+  "$schema": "https://raw.githubusercontent.com/sasjs/utils/main/src/types/sasjsconfig-schema.json",
   "macroFolders": [
     "base",
     "fcmp",

tests/crossplatform/mp_coretable.test.sas

@@ -0,0 +1,31 @@
+/**
+  @file
+  @brief Testing mp_coretable.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mp_coretable.sas
+  @li mp_assert.sas
+
+**/
+
+
+%mp_coretable(LOCKTABLE,libds=work.lock)
+%mp_assert(
+  iftrue=(%mf_existds(work.lock)=1),
+  desc=Lock table created,
+  outds=work.test_results
+)
+%mp_coretable(LOCKTABLE)
+%mp_assert(
+  iftrue=("&syscc"="0"),
+  desc=DDL export ran without errors,
+  outds=work.test_results
+)
+
+%mp_coretable(FILTER_SUMMARY,libds=work.sum)
+%mp_assert(
+  iftrue=(%mf_existds(work.sum)=1),
+  desc=Filter summary table created,
+  outds=work.test_results
+)

tests/crossplatform/mp_filterstore.test.sas

@@ -0,0 +1,85 @@
+/**
+  @file
+  @brief Testing mp_filterstore macro
+
+  <h4> SAS Macros </h4>
+  @li mp_coretable.sas
+  @li mp_filterstore.sas
+  @li mp_assertdsobs.sas
+  @li mp_assert.sas
+
+**/
+
+libname permlib (work);
+
+%mp_coretable(LOCKTABLE,libds=permlib.locktable)
+%mp_coretable(FILTER_SUMMARY,libds=permlib.filtsum)
+%mp_coretable(FILTER_DETAIL,libds=permlib.filtdet)
+%mp_coretable(MAXKEYTABLE,libds=permlib.maxkey)
+
+/* valid filter */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$12. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,AGE,=,12
+AND,AND,1,SEX,<=,"'M'"
+AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
+AND,OR,2,Weight,>=,77.7
+AND,OR,2,Weight,NE,77.7
+;;;;
+run;
+
+%mp_filterstore(libds=sashelp.class,
+  queryds=work.inds,
+  filter_summary=permlib.filtsum,
+  filter_detail=permlib.filtdet,
+  lock_table=permlib.locktable,
+  maxkeytable=permlib.maxkey,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+)
+%mp_assert(iftrue=(&syscc>0),
+  desc=Ensure macro runs without errors,
+  outds=work.test_results
+)
+/* ensure only one record created */
+%mp_assertdsobs(permlib.filtsum,
+  desc=Initial query,
+  test=ATMOST 1,
+  outds=work.test_results
+)
+/* check RK is correct */
+proc sql noprint;
+select max(filter_rk) into: test1 from work.result;
+%mp_assert(iftrue=(&test1=1),
+  desc=Ensure filter rk is correct,
+  outds=work.test_results
+)
+
+/* Test 2 - load same table again and ensure we get the same RK */
+%mp_filterstore(libds=sashelp.class,
+  queryds=work.inds,
+  filter_summary=permlib.filtsum,
+  filter_detail=permlib.filtdet,
+  lock_table=permlib.locktable,
+  maxkeytable=permlib.maxkey,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+)
+/* ensure only one record created */
+%mp_assertdsobs(permlib.filtsum,
+  desc=Initial query - same obs,
+  test=ATMOST 1,
+  outds=work.test_results
+)
+/* check RK is correct */
+proc sql noprint;
+select max(filter_rk) into: test2 from work.result;
+%mp_assert(iftrue=(&test2=1),
+  desc=Ensure filter rk is correct for second run,
+  outds=work.test_results
+)

tests/crossplatform/mp_getcols.test.sas

@@ -4,6 +4,7 @@
 
   <h4> SAS Macros </h4>
   @li mp_getcols.sas
+  @li mp_assertcols.sas
   @li mp_assertcolvals.sas
   @li mp_assertdsobs.sas
 
@@ -30,4 +31,10 @@ run;
   checkvals=work.check.val,
   desc=All values have a match,
   test=ALLVALS
+)
+
+%mp_assertcols(work.info,
+  cols=name type length varnum format label ddtype fmtname,
+  test=ALL,
+  desc=check all columns exist
 )