diff --git a/all.sas b/all.sas
index aeeb0dc..815f3b1 100644
--- a/all.sas
+++ b/all.sas
@@ -6011,10 +6011,10 @@ data _null_;
* there is not much point importing a short length numeric like this,
* eg with best4., as the resulting variable will still be stored as
* length 8. We need a length or format statement to ensure variable
- * is creatd with the smaller length...
+ * is created with the smaller length...
**/
else if vlen<8 then header=cats(varnm,':best',vlen,'.');
- else header=cats(varnm,':best.');
+ else header=cats(varnm,':best32.');
end;
%end;
%else %do;
@@ -9721,7 +9721,7 @@ run;
@li mf_getattrn.sas
@li mf_getuniquename.sas
@li mf_getvarlist.sas
- @li mp_md5.sas
+ @li mp_rowhash.sas
Related Files
@li mp_hashdataset.test.sas
@@ -9750,8 +9750,7 @@ run;
%local keyvar /* roll up the md5 */
prevkeyvar /* retain prev record md5 */
- lastvar /* last var in input ds */
- cvars nvars;
+ lastvar /* last var in input ds */;
%if not(%eval(%unquote(&iftrue))) %then %return;
@@ -9783,10 +9782,11 @@ run;
retain &prevkeyvar;
if _n_=1 then &prevkeyvar=put(md5("&salt"),$hex32.);
set &libds end=&lastvar;
- /* hash should include previous row */
- &keyvar=%mp_md5(
- cvars=%mf_getvarlist(&libds,typefilter=C) &prevkeyvar,
- nvars=%mf_getvarlist(&libds,typefilter=N)
+ /* hash should include previous row - listed first so it is hashed first */
+ %mp_rowhash(
+ md5_col=&keyvar
+ ,cvars=&prevkeyvar %mf_getvarlist(&libds,typefilter=C)
+ ,nvars=%mf_getvarlist(&libds,typefilter=N)
);
&prevkeyvar=&keyvar;
if &lastvar then output;
@@ -10747,7 +10747,7 @@ select distinct lowcase(memname)
@li mp_aligndecimal.sas
@li mp_cntlout.sas
@li mp_lockanytable.sas
- @li mp_md5.sas
+ @li mp_rowhash.sas
@li mp_storediffs.sas
Related Macros
@@ -10876,14 +10876,19 @@ select distinct
%let nvars=FMTROW MIN MAX DEFAULT LENGTH FUZZ MULT NOEDIT;
data &base_fmts/note2err;
set &base_fmts;
- fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
+ length fmthash $32;
+ %mp_rowhash(
+ md5_col=fmthash
+ ,cvars=&cvars
+ ,nvars=&nvars
+ )
run;
/**
* Ensure input table and base_formats have consistent lengths and types
*/
data &inlibds/nonote2err;
- length &delete_col $3 FMTROW 8 start end label $32767;
+ length &delete_col $3 FMTROW 8 start end label $32767 fmthash $32;
if 0 then set &base_fmts;
set &libds;
by type fmtname notsorted;
@@ -10905,7 +10910,11 @@ data &inlibds/nonote2err;
%mp_aligndecimal(end,width=16)
end;
- fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
+ %mp_rowhash(
+ md5_col=fmthash
+ ,cvars=&cvars
+ ,nvars=&nvars
+ )
run;
/**
@@ -11571,11 +11580,16 @@ drop table &ds1, &ds2;
@li Global option: `options dsoptions=nonote2err;`
@li Data step option: `data YOURLIB.YOURDATASET /nonote2err;`
+ For very wide tables (hundreds or thousands of columns) consider using
+ `mp_rowhash.sas`, which builds the same type of row hash iteratively and
+ avoids creating a single, very long concatenated SAS expression.
+
@param [in] cvars= () Space seperated list of character variables
@param [in] nvars= () Space seperated list of numeric variables
Related Programs
@li mp_init.sas
+ @li mp_rowhash.sas
@version 9.2
@author Allan Bowe
@@ -12283,6 +12297,98 @@ run;
%mend mp_retainedkey;
+/**
+ @file
+ @brief Iterative row-level MD5 hash generator
+ @details Generates DATA step statements that compute a deterministic,
+ row-level MD5 hash using a Merkle-style construction. Each variable is
+ hashed independently and the resulting raw 16-byte digests are combined
+ iteratively. This avoids concatenating the per-variable hex digests into a
+ single SAS character expression, which overflows when datasets contain many
+ variables.
+
+ Temporary variables use five leading underscores.
+
+ This macro is called from inside a DATA step. The caller is responsible
+ for declaring the output hash column if it does not already exist on the
+ input dataset.
+
+ Variables are hashed in the order supplied. If a particular variable
+ needs to influence the hash first (for example the previous row hash in
+ `mp_hashdataset`, or business dates in Data Controller's bitemporal
+ loader) simply list it first in `cvars` or `nvars`.
+
+ @param [in] md5_col= Name of the output hash column.
+ @param [in] cvars= Space separated list of character variables to hash.
+ @param [in] nvars= Space separated list of numeric variables to hash.
+
+ @version 9.3M5
+ @author Allan Bowe
+**/
+
+%macro mp_rowhash(
+ md5_col=
+ ,cvars=
+ ,nvars=
+ );
+
+ /* DATA step temp variables use five leading underscores. The names
+ are generated at macro invocation to avoid clashing with data columns. */
+ %local state digest pair numtext normal i chars nums;
+ %let state=%mf_getuniquename(prefix=_____state_);
+ %let digest=%mf_getuniquename(prefix=_____digest_);
+ %let pair=%mf_getuniquename(prefix=_____pair_);
+ %let numtext=%mf_getuniquename(prefix=_____numtext_);
+ %let normal=%mf_getuniquename(prefix=_____normal_);
+ %let i=%mf_getuniquename(prefix=_____i_);
+ %let chars=%mf_getuniquename(prefix=_____chars_);
+ %let nums=%mf_getuniquename(prefix=_____nums_);
+
+ length &state $16
+ &digest $16
+ &pair $32
+ &numtext $64
+ &normal &i 8;
+ if _n_=1 then call missing(&state,&digest,&pair,&numtext,&normal,&i);
+ drop &state &digest &pair &numtext &normal &i;
+
+ /* Versioned seed prevents confusion with other hashing schemes. */
+ &state = md5('DC HASH v2');
+
+ %if %length(&cvars)>0 %then %do;
+ array &chars{*} $ &cvars;
+ do &i = 1 to dim(&chars);
+ /* Leading blanks are retained. */
+ &digest = md5(trimn(&chars[&i]));
+ substr(&pair, 1, 16) = &state;
+ substr(&pair, 17, 16) = &digest;
+ &state = md5(&pair);
+ end;
+ %end;
+
+ %if %length(&nvars)>0 %then %do;
+ array &nums{*} &nvars;
+ do &i = 1 to dim(&nums);
+ /*
+ multiply-by-one for consistent cross-system precision.
+ Ignore null to protect SAS special missing values.
+ */
+ &normal = ifn(
+ missing(&nums[&i]),
+ &nums[&i],
+ &nums[&i] * 1
+ );
+ &numtext = put(&normal, binary64.);
+ &digest = md5(trim(&numtext));
+ substr(&pair, 1, 16) = &state;
+ substr(&pair, 17, 16) = &digest;
+ &state = md5(&pair);
+ end;
+ %end;
+
+ &md5_col = put(&state, $hex32.);
+
+%mend mp_rowhash;
/**
@file mp_runddl.sas
@brief An opinionated way to execute DDL files in SAS.