diff --git a/README.md b/README.md
index 42d1e30..b10007f 100644
--- a/README.md
+++ b/README.md
@@ -31,19 +31,28 @@ Documentation: https://core.sasjs.io
## Components
-### BASE library (SAS9/Viya)
+### BASE library (All Platforms)
- OS independent
-- Not metadata aware
+- Works on all SAS Platforms
- No X command
- Prefixes: _mf_, _mp_
-#### FCMP library (SAS9/Viya)
+### DDL library (All Platforms)
+
+- OS independent
+- Works on all SAS Platforms
+- No X command
+- Prefixes: _mddl_(lib)_ -> where lib can be "SAS" (in relation to a SAS component) or "DC" (in relation to a Data Controller component)
+
+This library will not be used for storing data entries (such as formats or datalines). Where this becomes necessary in the future, a new repo will be created, in order to keep the NPM bundle size down (for the benefit of those looking to embed purely macros in their applications).
+
+#### FCMP library (All Platforms)
+
- Function and macro names are identical, except for special cases
- Prefixes: _mcf_
-The fcmp macros are used to generate fcmp functions, and can be used with or
-without the `proc fcmp` wrapper.
+The fcmp macros are used to generate fcmp functions, and can be used with or without the `proc fcmp` wrapper.
### META library (SAS9 only)
@@ -125,6 +134,7 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
- one macro per file
- prefixes:
- _mcf_ for macro compiled functions (proc fcmp)
+ - _mddl_ for macros containing DDL (Data Definition Language)
- _mf_ for macro functions (can be used in open code).
- _ml_ for macros that are used to compile LUA modules
- _mm_ for metadata macros (interface with the metadata server).
@@ -198,6 +208,7 @@ We are currently on major release v4. Breaking changes should be marked with th
* `insert_cmplib` option of mcf_xxx macros will be deprecated (the option is now checked automatically with value inserted only if needed)
* mcf_xxx macros to have `wrap=` option defaulted to YES for convenience. Set this option explicitly to avoid issues.
* mp_getddl.sas to be renamed to mp_ds2ddl.sas (consistent with other ds2xxx macros). A wrapper macro is already in place, and you are able to use this immediately. The default for SHOWLOG will also be YES instead of NO.
+* mp_coretable.sas will be replaced by the standalone macros in the `ddl` folder (which are already available)
## Star Gazing
diff --git a/all.sas b/all.sas
index f2925f2..5a709f4 100644
--- a/all.sas
+++ b/all.sas
@@ -3544,19 +3544,21 @@ run;
%mp_coretable(LOCKTABLE,libds=work.locktable)
@param [in] table_ref The type of table to create. Example values:
- @li DIFFTABLE - Used to store changes to tables. Used by mp_storediffs.sas
- and mp_stackdiffs.sas
- @li FILTER_DETAIL - For storing detailed filter values. Used by
- mp_filterstore.sas.
- @li FILTER_SUMMARY - For storing summary filter values. Used by
- mp_filterstore.sas.
- @li LOCKANYTABLE - For "locking" tables prior to multipass loads. Used by
- mp_lockanytable.sas
- @li MAXKEYTABLE - For storing the maximum retained key information. Used
- by mp_retainedkey.sas
+ @li DIFFTABLE
+ @li FILTER_DETAIL
+ @li FILTER_SUMMARY
+ @li LOCKANYTABLE
+ @li MAXKEYTABLE
@param [in] libds= (0) The library.dataset reference used to create the table.
If not provided, then the DDL is simply printed to the log.
+
SAS Macros
+ @li mddl_dc_difftable.sas
+ @li mddl_dc_filterdetail.sas
+ @li mddl_dc_filtersummary.sas
+ @li mddl_dc_locktable.sas
+ @li mddl_dc_maxkeytable.sas
+
Related Macros
@li mp_filterstore.sas
@li mp_lockanytable.sas
@@ -3575,76 +3577,21 @@ run;
%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
proc sql;
%if &table_ref=DIFFTABLE %then %do;
- create table &outds(
- load_ref char(36) label='unique load reference',
- processed_dttm num format=E8601DT26.6 label='Processed at timestamp',
- libref char(8) label='Library Reference (8 chars)',
- dsn char(32) label='Dataset Name (32 chars)',
- key_hash char(32) label=
- 'MD5 Hash of primary key values (pipe seperated)',
- move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified',
- is_pk num label='Is Primary Key Field? (1/0)',
- is_diff num label=
- 'Did value change? (1/0/-1). Always -1 for appends and deletes.',
- tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
- tgtvar_nm char(32) label='Target variable name (32 chars)',
- oldval_num num format=best32. label='Old (numeric) value',
- newval_num num format=best32. label='New (numeric) value',
- oldval_char char(32765) label='Old (character) value',
- newval_char char(32765) label='New (character) value',
- constraint pk_mpe_audit
- primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
- );
+ %mddl_dc_difftable(libds=&outds)
%end;
%else %if &table_ref=LOCKTABLE %then %do;
- create table &outds(
- lock_lib char(8),
- lock_ds char(32),
- lock_status_cd char(10) not null,
- lock_user_nm char(100) not null ,
- lock_ref char(200),
- lock_pid char(10),
- lock_start_dttm num format=E8601DT26.6,
- lock_end_dttm num format=E8601DT26.6,
- constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+ %mddl_dc_locktable(libds=&outds)
%end;
%else %if &table_ref=FILTER_SUMMARY %then %do;
- create table &outds(
- filter_rk num not null,
- filter_hash char(32) not null,
- filter_table char(41) not null,
- processed_dttm num not null format=E8601DT26.6,
- constraint pk_mpe_filteranytable
- primary key(filter_rk));
+ %mddl_dc_filtersummary(libds=&outds)
%end;
%else %if &table_ref=FILTER_DETAIL %then %do;
- create table &outds(
- filter_hash char(32) not null,
- filter_line num not null,
- group_logic char(3) not null,
- subgroup_logic char(3) not null,
- subgroup_id num not null,
- variable_nm varchar(32) not null,
- operator_nm varchar(12) not null,
- raw_value varchar(4000) not null,
- processed_dttm num not null format=E8601DT26.6,
- constraint pk_mpe_filteranytable
- primary key(filter_hash,filter_line));
+ %mddl_dc_filterdetail(libds=&outds)
%end;
%else %if &table_ref=MAXKEYTABLE %then %do;
- create table &outds(
- keytable varchar(41) label='Base table in libref.dataset format',
- keycolumn char(32) format=$32.
- label='The Retained key field containing the key values.',
- max_key num label=
- 'Integer representing current max RK or SK value in the KEYTABLE',
- processed_dttm num format=E8601DT26.6
- label='Datetime this value was last updated',
- constraint pk_mpe_maxkeyvalues
- primary key(keytable));
+ %mddl_dc_maxkeytable(libds=&outds)
%end;
-
%if &libds=0 %then %do;
describe table &syslast;
drop table &syslast;
@@ -5779,6 +5726,7 @@ filename &outref temp;
SAS Macros
+ @li mddl_sas_cntlout.sas
@li mf_getuniquename.sas
@li mf_getvalue.sas
@li mf_islibds.sas
@@ -5840,36 +5788,14 @@ filename &outref temp;
%if "%substr(&libds,%length(&libds)-2,3)"="-FC" %then %do;
%let libds=%scan(&libds,1,-); /* chop off -FC extension */
%let ds0=%mf_getuniquename(prefix=fmtds_);
+ %let libds=&ds0;
/*
There is no need to export the entire format catalog here - the validations
are done against the data model, not the data values. So we can simply
hardcode the structure based on the cntlout dataset.
*/
- proc sql;
- create table &ds0(
- FMTNAME char(32)
- ,START char(16)
- ,END char(16)
- ,LABEL char(256)
- ,MIN num length=3
- ,MAX num length=3
- ,DEFAULT num length=3
- ,LENGTH num length=3
- ,FUZZ num
- ,PREFIX char(2)
- ,MULT num
- ,FILL char(1)
- ,NOEDIT num length=3
- ,TYPE char(1)
- ,SEXCL char(1)
- ,EEXCL char(1)
- ,HLO char(13)
- ,DECSEP char(1)
- ,DIG3SEP char(1)
- ,DATATYPE char(8)
- ,LANGUAGE char(8)
- );
- %let libds=&ds0;
+ %mddl_sas_cntlout(libds=&ds0)
+
%end;
%mp_filtercheck(&queryds,targetds=&libds,abort=YES)
@@ -7042,6 +6968,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
|`WHICHPATH `|`**OTHER** `|`**OTHER** `|`big fat problem if not path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|`O `|` `|` `|` `|` `|
SAS Macros
+ @li mddl_sas_cntlout.sas
@li mf_dedup.sas
@li mf_getfmtlist.sas
@li mf_getfmtname.sas
@@ -7096,30 +7023,7 @@ create table &outsummary as
%if "&outdetail" ne "0" %then %do;
/* ensure base table always exists */
- proc sql;
- create table &outdetail(
- FMTNAME char(32) label='Format name'
- ,START char(16) label='Starting value for format'
- ,END char(16) label='Ending value for format'
- ,LABEL char(256) label='Format value label'
- ,MIN num length=3 label='Minimum length'
- ,MAX num length=3 label='Maximum length'
- ,DEFAULT num length=3 label='Default length'
- ,LENGTH num length=3 label='Format length'
- ,FUZZ num label='Fuzz value'
- ,PREFIX char(2) label='Prefix characters'
- ,MULT num label='Multiplier'
- ,FILL char(1) label='Fill character'
- ,NOEDIT num length=3 label='Is picture string noedit?'
- ,TYPE char(1) label='Type of format'
- ,SEXCL char(1) label='Start exclusion'
- ,EEXCL char(1) label='End exclusion'
- ,HLO char(13) label='Additional information'
- ,DECSEP char(1) label='Decimal separator'
- ,DIG3SEP char(1) label='Three-digit separator'
- ,DATATYPE char(8) label='Date/time/datetime?'
- ,LANGUAGE char(8) label='Language for date strings'
- );
+ %mddl_sas_cntlout(libds=&outdetail)
/* grab the location of each format */
%let fmtcnt=0;
data _null_;
@@ -12002,6 +11906,170 @@ ods package publish archive properties
ods package close;
%mend mp_zip;/**
+ @file
+ @brief Difftable DDL
+ @details Used to store changes to tables. Used by mp_storediffs.sas
+ and mp_stackdiffs.sas
+
+**/
+
+
+%macro mddl_dc_difftable(libds=WORK.DIFFTABLE);
+
+ proc sql;
+ create table &libds(
+ load_ref char(36) label='unique load reference',
+ processed_dttm num format=E8601DT26.6 label='Processed at timestamp',
+ libref char(8) label='Library Reference (8 chars)',
+ dsn char(32) label='Dataset Name (32 chars)',
+ key_hash char(32) label=
+ 'MD5 Hash of primary key values (pipe seperated)',
+ move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified',
+ is_pk num label='Is Primary Key Field? (1/0)',
+ is_diff num label=
+ 'Did value change? (1/0/-1). Always -1 for appends and deletes.',
+ tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
+ tgtvar_nm char(32) label='Target variable name (32 chars)',
+ oldval_num num format=best32. label='Old (numeric) value',
+ newval_num num format=best32. label='New (numeric) value',
+ oldval_char char(32765) label='Old (character) value',
+ newval_char char(32765) label='New (character) value',
+ constraint pk_mpe_audit
+ primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
+ );
+
+%mend mddl_dc_difftable;/**
+ @file
+ @brief Filtertable DDL
+ @details For storing detailed filter values. Used by
+ mp_filterstore.sas.
+
+**/
+
+
+%macro mddl_dc_filterdetail(libds=WORK.FILTER_DETAIL);
+
+ proc sql;
+ create table &libds(
+ filter_hash char(32) not null,
+ filter_line num not null,
+ group_logic char(3) not null,
+ subgroup_logic char(3) not null,
+ subgroup_id num not null,
+ variable_nm varchar(32) not null,
+ operator_nm varchar(12) not null,
+ raw_value varchar(4000) not null,
+ processed_dttm num not null format=E8601DT26.6,
+ constraint pk_mpe_filteranytable
+ primary key(filter_hash,filter_line)
+ );
+
+%mend mddl_dc_filterdetail;/**
+ @file
+ @brief Filtersummary DDL
+ @details For storing summary filter values. Used by
+ mp_filterstore.sas.
+
+**/
+
+
+%macro mddl_dc_filtersummary(libds=WORK.FILTER_SUMMARY);
+
+ proc sql;
+ create table &libds(
+ filter_rk num not null,
+ filter_hash char(32) not null,
+ filter_table char(41) not null,
+ processed_dttm num not null format=E8601DT26.6,
+ constraint pk_mpe_filteranytable
+ primary key(filter_rk)
+ );
+
+%mend mddl_dc_filtersummary;/**
+ @file
+ @brief Locktable DDL
+ @details For "locking" tables prior to multipass loads. Used by
+ mp_lockanytable.sas
+
+**/
+
+
+%macro mddl_dc_locktable(libds=WORK.LOCKTABLE);
+
+ proc sql;
+ create table &libds(
+ lock_lib char(8),
+ lock_ds char(32),
+ lock_status_cd char(10) not null,
+ lock_user_nm char(100) not null ,
+ lock_ref char(200),
+ lock_pid char(10),
+ lock_start_dttm num format=E8601DT26.6,
+ lock_end_dttm num format=E8601DT26.6,
+ constraint pk_mp_lockanytable primary key(lock_lib,lock_ds)
+ );
+
+%mend mddl_dc_locktable;/**
+ @file
+ @brief Maxkeytable DDL
+ @details For storing the maximum retained key information. Used
+ by mp_retainedkey.sas
+
+**/
+
+
+%macro mddl_dc_maxkeytable(libds=WORK.MAXKEYTABLE);
+
+ proc sql;
+ create table &libds(
+ keytable varchar(41) label='Base table in libref.dataset format',
+ keycolumn char(32) format=$32.
+ label='The Retained key field containing the key values.',
+ max_key num label=
+ 'Integer representing current max RK or SK value in the KEYTABLE',
+ processed_dttm num format=E8601DT26.6
+ label='Datetime this value was last updated',
+ constraint pk_mpe_maxkeyvalues
+ primary key(keytable));
+
+%mend mddl_dc_maxkeytable;/**
+ @file
+ @brief The CNTLOUT table generated by proc format
+ @details This table will actually change format depending on the data values,
+ therefore the max possible lengths are described here to enable consistency
+ when dealing with format data.
+
+**/
+
+
+%macro mddl_sas_cntlout(libds=WORK.CNTLOUT);
+
+proc sql;
+create table &libds(
+ FMTNAME char(32) label='Format name'
+ ,START char(16) label='Starting value for format'
+ ,END char(16) label='Ending value for format'
+ ,LABEL char(256) label='Format value label'
+ ,MIN num length=3 label='Minimum length'
+ ,MAX num length=3 label='Maximum length'
+ ,DEFAULT num length=3 label='Default length'
+ ,LENGTH num length=3 label='Format length'
+ ,FUZZ num label='Fuzz value'
+ ,PREFIX char(2) label='Prefix characters'
+ ,MULT num label='Multiplier'
+ ,FILL char(1) label='Fill character'
+ ,NOEDIT num length=3 label='Is picture string noedit?'
+ ,TYPE char(1) label='Type of format'
+ ,SEXCL char(1) label='Start exclusion'
+ ,EEXCL char(1) label='End exclusion'
+ ,HLO char(13) label='Additional information'
+ ,DECSEP char(1) label='Decimal separator'
+ ,DIG3SEP char(1) label='Three-digit separator'
+ ,DATATYPE char(8) label='Date/time/datetime?'
+ ,LANGUAGE char(8) label='Language for date strings'
+);
+
+%mend mddl_sas_cntlout;/**
@file mm_adduser2group.sas
@brief Adds a user to a group
@details Adds a user to a metadata group. The macro first checks whether the
diff --git a/base/mp_coretable.sas b/base/mp_coretable.sas
index 52a980e..76e786e 100644
--- a/base/mp_coretable.sas
+++ b/base/mp_coretable.sas
@@ -16,19 +16,21 @@
%mp_coretable(LOCKTABLE,libds=work.locktable)
@param [in] table_ref The type of table to create. Example values:
- @li DIFFTABLE - Used to store changes to tables. Used by mp_storediffs.sas
- and mp_stackdiffs.sas
- @li FILTER_DETAIL - For storing detailed filter values. Used by
- mp_filterstore.sas.
- @li FILTER_SUMMARY - For storing summary filter values. Used by
- mp_filterstore.sas.
- @li LOCKANYTABLE - For "locking" tables prior to multipass loads. Used by
- mp_lockanytable.sas
- @li MAXKEYTABLE - For storing the maximum retained key information. Used
- by mp_retainedkey.sas
+ @li DIFFTABLE
+ @li FILTER_DETAIL
+ @li FILTER_SUMMARY
+ @li LOCKANYTABLE
+ @li MAXKEYTABLE
@param [in] libds= (0) The library.dataset reference used to create the table.
If not provided, then the DDL is simply printed to the log.
+ SAS Macros
+ @li mddl_dc_difftable.sas
+ @li mddl_dc_filterdetail.sas
+ @li mddl_dc_filtersummary.sas
+ @li mddl_dc_locktable.sas
+ @li mddl_dc_maxkeytable.sas
+
Related Macros
@li mp_filterstore.sas
@li mp_lockanytable.sas
@@ -47,76 +49,21 @@
%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
proc sql;
%if &table_ref=DIFFTABLE %then %do;
- create table &outds(
- load_ref char(36) label='unique load reference',
- processed_dttm num format=E8601DT26.6 label='Processed at timestamp',
- libref char(8) label='Library Reference (8 chars)',
- dsn char(32) label='Dataset Name (32 chars)',
- key_hash char(32) label=
- 'MD5 Hash of primary key values (pipe seperated)',
- move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified',
- is_pk num label='Is Primary Key Field? (1/0)',
- is_diff num label=
- 'Did value change? (1/0/-1). Always -1 for appends and deletes.',
- tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
- tgtvar_nm char(32) label='Target variable name (32 chars)',
- oldval_num num format=best32. label='Old (numeric) value',
- newval_num num format=best32. label='New (numeric) value',
- oldval_char char(32765) label='Old (character) value',
- newval_char char(32765) label='New (character) value',
- constraint pk_mpe_audit
- primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
- );
+ %mddl_dc_difftable(libds=&outds)
%end;
%else %if &table_ref=LOCKTABLE %then %do;
- create table &outds(
- lock_lib char(8),
- lock_ds char(32),
- lock_status_cd char(10) not null,
- lock_user_nm char(100) not null ,
- lock_ref char(200),
- lock_pid char(10),
- lock_start_dttm num format=E8601DT26.6,
- lock_end_dttm num format=E8601DT26.6,
- constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+ %mddl_dc_locktable(libds=&outds)
%end;
%else %if &table_ref=FILTER_SUMMARY %then %do;
- create table &outds(
- filter_rk num not null,
- filter_hash char(32) not null,
- filter_table char(41) not null,
- processed_dttm num not null format=E8601DT26.6,
- constraint pk_mpe_filteranytable
- primary key(filter_rk));
+ %mddl_dc_filtersummary(libds=&outds)
%end;
%else %if &table_ref=FILTER_DETAIL %then %do;
- create table &outds(
- filter_hash char(32) not null,
- filter_line num not null,
- group_logic char(3) not null,
- subgroup_logic char(3) not null,
- subgroup_id num not null,
- variable_nm varchar(32) not null,
- operator_nm varchar(12) not null,
- raw_value varchar(4000) not null,
- processed_dttm num not null format=E8601DT26.6,
- constraint pk_mpe_filteranytable
- primary key(filter_hash,filter_line));
+ %mddl_dc_filterdetail(libds=&outds)
%end;
%else %if &table_ref=MAXKEYTABLE %then %do;
- create table &outds(
- keytable varchar(41) label='Base table in libref.dataset format',
- keycolumn char(32) format=$32.
- label='The Retained key field containing the key values.',
- max_key num label=
- 'Integer representing current max RK or SK value in the KEYTABLE',
- processed_dttm num format=E8601DT26.6
- label='Datetime this value was last updated',
- constraint pk_mpe_maxkeyvalues
- primary key(keytable));
+ %mddl_dc_maxkeytable(libds=&outds)
%end;
-
%if &libds=0 %then %do;
describe table &syslast;
drop table &syslast;
diff --git a/base/mp_filterstore.sas b/base/mp_filterstore.sas
index 8a78d48..a2a79f5 100644
--- a/base/mp_filterstore.sas
+++ b/base/mp_filterstore.sas
@@ -50,6 +50,7 @@
SAS Macros
+ @li mddl_sas_cntlout.sas
@li mf_getuniquename.sas
@li mf_getvalue.sas
@li mf_islibds.sas
@@ -111,36 +112,14 @@
%if "%substr(&libds,%length(&libds)-2,3)"="-FC" %then %do;
%let libds=%scan(&libds,1,-); /* chop off -FC extension */
%let ds0=%mf_getuniquename(prefix=fmtds_);
+ %let libds=&ds0;
/*
There is no need to export the entire format catalog here - the validations
are done against the data model, not the data values. So we can simply
hardcode the structure based on the cntlout dataset.
*/
- proc sql;
- create table &ds0(
- FMTNAME char(32)
- ,START char(16)
- ,END char(16)
- ,LABEL char(256)
- ,MIN num length=3
- ,MAX num length=3
- ,DEFAULT num length=3
- ,LENGTH num length=3
- ,FUZZ num
- ,PREFIX char(2)
- ,MULT num
- ,FILL char(1)
- ,NOEDIT num length=3
- ,TYPE char(1)
- ,SEXCL char(1)
- ,EEXCL char(1)
- ,HLO char(13)
- ,DECSEP char(1)
- ,DIG3SEP char(1)
- ,DATATYPE char(8)
- ,LANGUAGE char(8)
- );
- %let libds=&ds0;
+ %mddl_sas_cntlout(libds=&ds0)
+
%end;
%mp_filtercheck(&queryds,targetds=&libds,abort=YES)
diff --git a/base/mp_getformats.sas b/base/mp_getformats.sas
index 82a0c63..73f413b 100644
--- a/base/mp_getformats.sas
+++ b/base/mp_getformats.sas
@@ -40,6 +40,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
|`WHICHPATH `|`**OTHER** `|`**OTHER** `|`big fat problem if not path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|`O `|` `|` `|` `|` `|
SAS Macros
+ @li mddl_sas_cntlout.sas
@li mf_dedup.sas
@li mf_getfmtlist.sas
@li mf_getfmtname.sas
@@ -94,30 +95,7 @@ create table &outsummary as
%if "&outdetail" ne "0" %then %do;
/* ensure base table always exists */
- proc sql;
- create table &outdetail(
- FMTNAME char(32) label='Format name'
- ,START char(16) label='Starting value for format'
- ,END char(16) label='Ending value for format'
- ,LABEL char(256) label='Format value label'
- ,MIN num length=3 label='Minimum length'
- ,MAX num length=3 label='Maximum length'
- ,DEFAULT num length=3 label='Default length'
- ,LENGTH num length=3 label='Format length'
- ,FUZZ num label='Fuzz value'
- ,PREFIX char(2) label='Prefix characters'
- ,MULT num label='Multiplier'
- ,FILL char(1) label='Fill character'
- ,NOEDIT num length=3 label='Is picture string noedit?'
- ,TYPE char(1) label='Type of format'
- ,SEXCL char(1) label='Start exclusion'
- ,EEXCL char(1) label='End exclusion'
- ,HLO char(13) label='Additional information'
- ,DECSEP char(1) label='Decimal separator'
- ,DIG3SEP char(1) label='Three-digit separator'
- ,DATATYPE char(8) label='Date/time/datetime?'
- ,LANGUAGE char(8) label='Language for date strings'
- );
+ %mddl_sas_cntlout(libds=&outdetail)
/* grab the location of each format */
%let fmtcnt=0;
data _null_;
diff --git a/build.py b/build.py
index 6080aac..7df8e39 100755
--- a/build.py
+++ b/build.py
@@ -84,7 +84,7 @@ options noquotelenmax;
"""
f = open('all.sas', "w") # r / r+ / rb / rb+ / w / wb
f.write(header)
-folders=['base','meta','metax','server','viya','lua','fcmp']
+folders=['base','ddl','meta','metax','server','viya','lua','fcmp']
for folder in folders:
filenames = [fn for fn in Path('./' + folder).iterdir() if fn.match("*.sas")]
filenames.sort()
diff --git a/ddl/mddl_dc_difftable.sas b/ddl/mddl_dc_difftable.sas
new file mode 100644
index 0000000..78265e0
--- /dev/null
+++ b/ddl/mddl_dc_difftable.sas
@@ -0,0 +1,34 @@
+/**
+ @file
+ @brief Difftable DDL
+ @details Used to store changes to tables. Used by mp_storediffs.sas
+ and mp_stackdiffs.sas
+
+**/
+
+
+%macro mddl_dc_difftable(libds=WORK.DIFFTABLE);
+
+ proc sql;
+ create table &libds(
+ load_ref char(36) label='unique load reference',
+ processed_dttm num format=E8601DT26.6 label='Processed at timestamp',
+ libref char(8) label='Library Reference (8 chars)',
+ dsn char(32) label='Dataset Name (32 chars)',
+ key_hash char(32) label=
+ 'MD5 Hash of primary key values (pipe seperated)',
+ move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified',
+ is_pk num label='Is Primary Key Field? (1/0)',
+ is_diff num label=
+ 'Did value change? (1/0/-1). Always -1 for appends and deletes.',
+ tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
+ tgtvar_nm char(32) label='Target variable name (32 chars)',
+ oldval_num num format=best32. label='Old (numeric) value',
+ newval_num num format=best32. label='New (numeric) value',
+ oldval_char char(32765) label='Old (character) value',
+ newval_char char(32765) label='New (character) value',
+ constraint pk_mpe_audit
+ primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
+ );
+
+%mend mddl_dc_difftable;
\ No newline at end of file
diff --git a/ddl/mddl_dc_filterdetail.sas b/ddl/mddl_dc_filterdetail.sas
new file mode 100644
index 0000000..a49ac9d
--- /dev/null
+++ b/ddl/mddl_dc_filterdetail.sas
@@ -0,0 +1,27 @@
+/**
+ @file
+ @brief Filtertable DDL
+ @details For storing detailed filter values. Used by
+ mp_filterstore.sas.
+
+**/
+
+
+%macro mddl_dc_filterdetail(libds=WORK.FILTER_DETAIL);
+
+ proc sql;
+ create table &libds(
+ filter_hash char(32) not null,
+ filter_line num not null,
+ group_logic char(3) not null,
+ subgroup_logic char(3) not null,
+ subgroup_id num not null,
+ variable_nm varchar(32) not null,
+ operator_nm varchar(12) not null,
+ raw_value varchar(4000) not null,
+ processed_dttm num not null format=E8601DT26.6,
+ constraint pk_mpe_filteranytable
+ primary key(filter_hash,filter_line)
+ );
+
+%mend mddl_dc_filterdetail;
\ No newline at end of file
diff --git a/ddl/mddl_dc_filtersummary.sas b/ddl/mddl_dc_filtersummary.sas
new file mode 100644
index 0000000..55b04fb
--- /dev/null
+++ b/ddl/mddl_dc_filtersummary.sas
@@ -0,0 +1,22 @@
+/**
+ @file
+ @brief Filtersummary DDL
+ @details For storing summary filter values. Used by
+ mp_filterstore.sas.
+
+**/
+
+
+%macro mddl_dc_filtersummary(libds=WORK.FILTER_SUMMARY);
+
+ proc sql;
+ create table &libds(
+ filter_rk num not null,
+ filter_hash char(32) not null,
+ filter_table char(41) not null,
+ processed_dttm num not null format=E8601DT26.6,
+ constraint pk_mpe_filteranytable
+ primary key(filter_rk)
+ );
+
+%mend mddl_dc_filtersummary;
\ No newline at end of file
diff --git a/ddl/mddl_dc_locktable.sas b/ddl/mddl_dc_locktable.sas
new file mode 100644
index 0000000..bd3ecd7
--- /dev/null
+++ b/ddl/mddl_dc_locktable.sas
@@ -0,0 +1,25 @@
+/**
+ @file
+ @brief Locktable DDL
+ @details For "locking" tables prior to multipass loads. Used by
+ mp_lockanytable.sas
+
+**/
+
+
+%macro mddl_dc_locktable(libds=WORK.LOCKTABLE);
+
+ proc sql;
+ create table &libds(
+ lock_lib char(8),
+ lock_ds char(32),
+ lock_status_cd char(10) not null,
+ lock_user_nm char(100) not null ,
+ lock_ref char(200),
+ lock_pid char(10),
+ lock_start_dttm num format=E8601DT26.6,
+ lock_end_dttm num format=E8601DT26.6,
+ constraint pk_mp_lockanytable primary key(lock_lib,lock_ds)
+ );
+
+%mend mddl_dc_locktable;
\ No newline at end of file
diff --git a/ddl/mddl_dc_maxkeytable.sas b/ddl/mddl_dc_maxkeytable.sas
new file mode 100644
index 0000000..163b8ff
--- /dev/null
+++ b/ddl/mddl_dc_maxkeytable.sas
@@ -0,0 +1,24 @@
+/**
+ @file
+ @brief Maxkeytable DDL
+ @details For storing the maximum retained key information. Used
+ by mp_retainedkey.sas
+
+**/
+
+
+%macro mddl_dc_maxkeytable(libds=WORK.MAXKEYTABLE);
+
+ proc sql;
+ create table &libds(
+ keytable varchar(41) label='Base table in libref.dataset format',
+ keycolumn char(32) format=$32.
+ label='The Retained key field containing the key values.',
+ max_key num label=
+ 'Integer representing current max RK or SK value in the KEYTABLE',
+ processed_dttm num format=E8601DT26.6
+ label='Datetime this value was last updated',
+ constraint pk_mpe_maxkeyvalues
+ primary key(keytable));
+
+%mend mddl_dc_maxkeytable;
\ No newline at end of file
diff --git a/ddl/mddl_sas_cntlout.sas b/ddl/mddl_sas_cntlout.sas
new file mode 100644
index 0000000..aca4ab9
--- /dev/null
+++ b/ddl/mddl_sas_cntlout.sas
@@ -0,0 +1,38 @@
+/**
+ @file
+ @brief The CNTLOUT table generated by proc format
+ @details This table will actually change format depending on the data values,
+ therefore the max possible lengths are described here to enable consistency
+ when dealing with format data.
+
+**/
+
+
+%macro mddl_sas_cntlout(libds=WORK.CNTLOUT);
+
+proc sql;
+create table &libds(
+ FMTNAME char(32) label='Format name'
+ ,START char(16) label='Starting value for format'
+ ,END char(16) label='Ending value for format'
+ ,LABEL char(256) label='Format value label'
+ ,MIN num length=3 label='Minimum length'
+ ,MAX num length=3 label='Maximum length'
+ ,DEFAULT num length=3 label='Default length'
+ ,LENGTH num length=3 label='Format length'
+ ,FUZZ num label='Fuzz value'
+ ,PREFIX char(2) label='Prefix characters'
+ ,MULT num label='Multiplier'
+ ,FILL char(1) label='Fill character'
+ ,NOEDIT num length=3 label='Is picture string noedit?'
+ ,TYPE char(1) label='Type of format'
+ ,SEXCL char(1) label='Start exclusion'
+ ,EEXCL char(1) label='End exclusion'
+ ,HLO char(13) label='Additional information'
+ ,DECSEP char(1) label='Decimal separator'
+ ,DIG3SEP char(1) label='Three-digit separator'
+ ,DATATYPE char(8) label='Date/time/datetime?'
+ ,LANGUAGE char(8) label='Language for date strings'
+);
+
+%mend mddl_sas_cntlout;
\ No newline at end of file
diff --git a/sasjs/sasjsconfig.json b/sasjs/sasjsconfig.json
index 15e60e6..39402a2 100644
--- a/sasjs/sasjsconfig.json
+++ b/sasjs/sasjsconfig.json
@@ -2,13 +2,15 @@
"$schema": "https://raw.githubusercontent.com/sasjs/utils/main/src/types/sasjsconfig-schema.json",
"macroFolders": [
"base",
+ "ddl",
"fcmp",
"meta",
"metax",
"server",
"viya",
"lua",
- "tests/crossplatform"
+ "tests/crossplatform",
+ "tests/ddl"
],
"docConfig": {
"displayMacroCore": false,
diff --git a/tests/ddl/mddl_dc_difftable.test.sas b/tests/ddl/mddl_dc_difftable.test.sas
new file mode 100644
index 0000000..b0ba18c
--- /dev/null
+++ b/tests/ddl/mddl_dc_difftable.test.sas
@@ -0,0 +1,19 @@
+/**
+ @file
+ @brief Difftable DDL test
+
+ SAS Macros
+ @li mddl_dc_difftable.sas
+ @li mf_existds.sas
+ @li mp_assert.sas
+
+**/
+
+
+%mddl_dc_difftable(libds=WORK.DIFFTABLE)
+
+%mp_assert(
+ iftrue=(%mf_existds(WORK.DIFFTABLE)=1),
+ desc=Checking table was created,
+ outds=work.test_results
+)
\ No newline at end of file
diff --git a/tests/ddl/mddl_dc_filterdetail.test.sas b/tests/ddl/mddl_dc_filterdetail.test.sas
new file mode 100644
index 0000000..dddf8eb
--- /dev/null
+++ b/tests/ddl/mddl_dc_filterdetail.test.sas
@@ -0,0 +1,18 @@
+/**
+ @file
+ @brief Filtertable DDL test
+
+ SAS Macros
+ @li mddl_dc_filterdetail.sas
+ @li mf_existds.sas
+ @li mp_assert.sas
+
+**/
+
+%mddl_dc_filterdetail(libds=WORK.TEST)
+
+%mp_assert(
+ iftrue=(%mf_existds(WORK.TEST)=1),
+ desc=Checking table was created,
+ outds=work.test_results
+)
\ No newline at end of file
diff --git a/tests/ddl/mddl_dc_filtersummary.test.sas b/tests/ddl/mddl_dc_filtersummary.test.sas
new file mode 100644
index 0000000..22ef87e
--- /dev/null
+++ b/tests/ddl/mddl_dc_filtersummary.test.sas
@@ -0,0 +1,18 @@
+/**
+ @file
+ @brief Filtersummary DDL test
+
+ SAS Macros
+ @li mddl_dc_filtersummary.sas
+ @li mf_existds.sas
+ @li mp_assert.sas
+
+**/
+
+%mddl_dc_filtersummary(libds=WORK.TEST)
+
+%mp_assert(
+ iftrue=(%mf_existds(WORK.TEST)=1),
+ desc=Checking table was created,
+ outds=work.test_results
+)
\ No newline at end of file
diff --git a/tests/ddl/mddl_dc_locktable.test.sas b/tests/ddl/mddl_dc_locktable.test.sas
new file mode 100644
index 0000000..cebe92c
--- /dev/null
+++ b/tests/ddl/mddl_dc_locktable.test.sas
@@ -0,0 +1,18 @@
+/**
+ @file
+ @brief Locktable DDL test
+
+ SAS Macros
+ @li mddl_dc_locktable.sas
+ @li mf_existds.sas
+ @li mp_assert.sas
+
+**/
+
+%mddl_dc_locktable(libds=WORK.TEST)
+
+%mp_assert(
+ iftrue=(%mf_existds(WORK.TEST)=1),
+ desc=Checking table was created,
+ outds=work.test_results
+)
\ No newline at end of file
diff --git a/tests/ddl/mddl_dc_maxkeytable.test.sas b/tests/ddl/mddl_dc_maxkeytable.test.sas
new file mode 100644
index 0000000..7ba444b
--- /dev/null
+++ b/tests/ddl/mddl_dc_maxkeytable.test.sas
@@ -0,0 +1,18 @@
+/**
+ @file
+ @brief Maxkeytable DDL test
+
+ SAS Macros
+ @li mddl_dc_maxkeytable.sas
+ @li mf_existds.sas
+ @li mp_assert.sas
+
+**/
+
+%mddl_dc_maxkeytable(libds=WORK.TEST)
+
+%mp_assert(
+ iftrue=(%mf_existds(WORK.TEST)=1),
+ desc=Checking table was created,
+ outds=work.test_results
+)
\ No newline at end of file
diff --git a/tests/ddl/mddl_sas_cntlout.test.sas b/tests/ddl/mddl_sas_cntlout.test.sas
new file mode 100644
index 0000000..ea45cf0
--- /dev/null
+++ b/tests/ddl/mddl_sas_cntlout.test.sas
@@ -0,0 +1,18 @@
+/**
+ @file
+ @brief mddl_sas_cntlout ddl test
+
+ SAS Macros
+ @li mddl_sas_cntlout.sas
+ @li mf_existds.sas
+ @li mp_assert.sas
+
+**/
+
+%mddl_sas_cntlout(libds=WORK.TEST)
+
+%mp_assert(
+ iftrue=(%mf_existds(WORK.TEST)=1),
+ desc=Checking table was created,
+ outds=work.test_results
+)
\ No newline at end of file