Merge pull request #433 from sasjs/mp_rowhash

refactor mf_wordsinstr1andstr2 + mf_wordsinstr1butnotstr2

AGENTS.md

@@ -0,0 +1,31 @@
+# Agent Instructions for @sasjs/core
+
+Follow these rules when editing or generating code for the @sasjs/core SAS macro library.
+
+## Project Context
+This repo is the SASjs Macro Core library — a collection of MIT-licensed, production-quality SAS macros for SAS application development.
+
+## Versioning
+- NEVER bump or modify the version in `package.json`.
+- Versioning is handled entirely by the CI/CD pipeline using semantic-release.
+
+## SAS Style & Standards
+- Read and follow the standards documented in `README.md` (Sections: Components, Standards, File Properties, Header Properties, Coding Standards).
+- Read and follow `.sasjslint`:
+  - No trailing spaces.
+  - Requires a Doxygen header on every macro (`@file`, `@brief`, etc.).
+  - Lowercase file names without spaces.
+  - Lowercase macro names.
+  - Macro definitions must use parentheses, e.g. `%macro x();` not `%macro x;`.
+  - Indentation = 2 spaces (or multiple thereof); no tabs.
+  - Max line length 300.
+  - No gremlins / invisible characters.
+- One macro per file; filename must match macro name.
+- Macro *calls* should NOT be terminated with a semicolon. Use `%my_macro()` not `%my_macro();`.
+- Macro variables must always be local, to prevent scope leakage.
+
+
+## Build / Generated Files
+- Do not run the build script locally; it is executed in the CI/CD pipeline.
+- Generated files, including the consolidated `all.sas`, the per-folder `mc_*.sas` files, and the LUA macro wrappers in the `lua` folder, can generally be ignored unless the pipeline requires an update. Do not edit generated files by hand.
+- run sasjs lint after each change

all.sas

@@ -2621,9 +2621,12 @@ Usage:
   @brief Returns words that are in both string 1 and string 2
   @details  Compares two space separated strings and returns the words that are
   in both.
+
+  If either string is empty, nothing is returned.
+
   Usage:
 
-      %put %mf_wordsInStr1andStr2(
+      %put %mf_wordsinstr1andstr2(
         Str1=blah sss blaaah brah bram boo
         ,Str2=   blah blaaah brah ssss
       );
@@ -2641,34 +2644,26 @@ Usage:
 
 **/
 
-%macro mf_wordsInStr1andStr2(
+%macro mf_wordsinstr1andstr2(
   Str1= /* string containing words to extract */
   ,Str2= /* used to compare with the extract string */
 )/*/STORE SOURCE*/;
 
-%local count_base count_extr i i2 extr_word base_word match outvar;
+%local count_extr i extr_word outvar;
 %if %length(&str1)=0 or %length(&str2)=0 %then %do;
-  %put base string (str1)= &str1;
-  %put compare string (str2) = &str2;
+  %put &sysmacroname: empty input string, nothing to compare;
   %return;
 %end;
-%let count_base=%sysfunc(countw(&Str2));
 %let count_extr=%sysfunc(countw(&Str1));
 
 %do i=1 %to &count_extr;
   %let extr_word=%scan(&Str1,&i,%str( ));
-  %let match=0;
-  %do i2=1 %to &count_base;
-    %let base_word=%scan(&Str2,&i2,%str( ));
-    %if &extr_word=&base_word %then %let match=1;
-  %end;
-  %if &match=1 %then %let outvar=&outvar &extr_word;
+  %if %sysfunc(indexw(%superq(str2),%superq(extr_word)))>0 %then
+    %let outvar=&outvar &extr_word;
 %end;
-
-  &outvar
-
-%mend mf_wordsInStr1andStr2;
-
+/* send out the result without any surrounding whitespace */
+%do;&outvar%end;
+%mend mf_wordsinstr1andstr2;
 /**
   @file
   @brief Returns words that are in string 1 but not in string 2
@@ -2677,9 +2672,12 @@ Usage:
 
   Note - case sensitive!
 
+  If str1 is empty, nothing is returned.  If str2 is empty, all the words in
+  str1 are returned.
+
   Usage:
 
-      %let x= %mf_wordsInStr1ButNotStr2(
+      %let x= %mf_wordsinstr1butnotstr2(
         Str1=blah sss blaaah brah bram boo
         ,Str2=   blah blaaah brah ssss
       );
@@ -2695,34 +2693,26 @@ Usage:
 
 **/
 
-%macro mf_wordsInStr1ButNotStr2(
+%macro mf_wordsinstr1butnotstr2(
   Str1= /* string containing words to extract */
   ,Str2= /* used to compare with the extract string */
 )/*/STORE SOURCE*/;
 
-%local count_base count_extr i i2 extr_word base_word match outvar;
-%if %length(&str1)=0 or %length(&str2)=0 %then %do;
-  %put base string (str1)= &str1;
-  %put compare string (str2) = &str2;
+%local count_extr i extr_word outvar;
+%if %length(&str1)=0 %then %do;
+  %put &sysmacroname: str1 is empty, nothing to compare;
   %return;
 %end;
-%let count_base=%sysfunc(countw(&Str2));
 %let count_extr=%sysfunc(countw(&Str1));
 
 %do i=1 %to &count_extr;
   %let extr_word=%scan(&Str1,&i,%str( ));
-  %let match=0;
-  %do i2=1 %to &count_base;
-    %let base_word=%scan(&Str2,&i2,%str( ));
-    %if &extr_word=&base_word %then %let match=1;
-  %end;
-  %if &match=0 %then %let outvar=&outvar &extr_word;
+  %if %sysfunc(indexw(%superq(str2),%superq(extr_word)))=0 %then
+    %let outvar=&outvar &extr_word;
 %end;
-
-  &outvar
-
-%mend mf_wordsInStr1ButNotStr2;
-
+/* send out the result without any surrounding whitespace */
+%do;&outvar%end;
+%mend mf_wordsinstr1butnotstr2;
 /**
   @file
   @brief Creates a text file using pure macro
@@ -3495,7 +3485,7 @@ run;
 )/*/STORE SOURCE*/;
 
   data ;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     test_comments="&sysmacroname: Test result of "!!symget('iftrue');
   %if %eval(%unquote(&iftrue)) %then %do;
@@ -3624,7 +3614,7 @@ run;
   %end;
 
   data;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     if test_description='0'
     then test_description="Testing &inds for existence of &test of: &cols";
@@ -3803,7 +3793,7 @@ run;
   )
 
   data;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
     test_comments="&sysmacroname: &lib..&ds..&col has &result/&orig values "
@@ -3918,7 +3908,7 @@ run;
   %end;
 
   data &ds;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
     test_comments="&sysmacroname: Dataset &inds has &nobs observations.";
@@ -4082,7 +4072,7 @@ run;
   %end;
 
   data ;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     test_comments=symget('test_comments');
     test_result=symget('test_result');
@@ -6011,10 +6001,10 @@ data _null_;
         * there is not much point importing a short length numeric like this,
         * eg with best4., as the resulting variable will still be stored as
         * length 8.  We need a length or format statement to ensure variable
-        * is creatd with the smaller length...
+        * is created with the smaller length...
         **/
       else if vlen<8 then header=cats(varnm,':best',vlen,'.');
-      else header=cats(varnm,':best.');
+      else header=cats(varnm,':best32.');
     end;
   %end;
   %else %do;
@@ -9721,7 +9711,7 @@ run;
   @li mf_getattrn.sas
   @li mf_getuniquename.sas
   @li mf_getvarlist.sas
-  @li mp_md5.sas
+  @li mp_rowhash.sas
 
   <h4> Related Files </h4>
   @li mp_hashdataset.test.sas
@@ -9750,8 +9740,7 @@ run;
 
 %local keyvar /* roll up the md5 */
   prevkeyvar /* retain prev record md5 */
-  lastvar /* last var in input ds */
-  cvars nvars;
+  lastvar /* last var in input ds */;
 
 %if not(%eval(%unquote(&iftrue))) %then %return;
 
@@ -9783,10 +9772,11 @@ run;
     retain &prevkeyvar;
     if _n_=1 then &prevkeyvar=put(md5("&salt"),$hex32.);
     set &libds end=&lastvar;
-    /* hash should include previous row */
-    &keyvar=%mp_md5(
-      cvars=%mf_getvarlist(&libds,typefilter=C) &prevkeyvar,
-      nvars=%mf_getvarlist(&libds,typefilter=N)
+    /* hash should include previous row - listed first so it is hashed first */
+    %mp_rowhash(
+      md5_col=&keyvar
+      ,cvars=&prevkeyvar %mf_getvarlist(&libds,typefilter=C)
+      ,nvars=%mf_getvarlist(&libds,typefilter=N)
     );
     &prevkeyvar=&keyvar;
     if &lastvar then output;
@@ -10101,6 +10091,7 @@ filename &tempref clear;
 
 %global
   SASJS_PREFIX       /* the ONLY hard-coded global macro variable in SASjs    */
+  &prefix.PROCESSMODE &prefix._STPSRV_HEADER_LOC /* SASjs server specific vars*/
   &prefix._FUNCTIONS /* used in mcf_init() to track core function compilation */
   &prefix._INIT_NUM  /* initialisation time as numeric                        */
   &prefix._INIT_DTTM /* initialisation time in E8601DT26.6 format             */
@@ -10746,7 +10737,7 @@ select distinct lowcase(memname)
   @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
-  @li mp_md5.sas
+  @li mp_rowhash.sas
   @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
@@ -10875,14 +10866,19 @@ select distinct
 %let nvars=FMTROW MIN MAX DEFAULT LENGTH FUZZ MULT NOEDIT;
 data &base_fmts/note2err;
   set &base_fmts;
-  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
+  length fmthash $32;
+  %mp_rowhash(
+    md5_col=fmthash
+    ,cvars=&cvars
+    ,nvars=&nvars
+  )
 run;
 
 /**
   * Ensure input table and base_formats have consistent lengths and types
   */
 data &inlibds/nonote2err;
-  length &delete_col $3 FMTROW 8 start end label $32767;
+  length &delete_col $3 FMTROW 8 start end label $32767 fmthash $32;
   if 0 then set &base_fmts;
   set &libds;
   by type fmtname notsorted;
@@ -10904,7 +10900,11 @@ data &inlibds/nonote2err;
     %mp_aligndecimal(end,width=16)
   end;
 
-  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
+  %mp_rowhash(
+    md5_col=fmthash
+    ,cvars=&cvars
+    ,nvars=&nvars
+  )
 run;
 
 /**
@@ -11570,11 +11570,16 @@ drop table &ds1, &ds2;
   @li Global option:  `options dsoptions=nonote2err;`
   @li Data step option: `data YOURLIB.YOURDATASET /nonote2err;`
 
+  For very wide tables (hundreds or thousands of columns) consider using
+  `mp_rowhash.sas`, which builds the same type of row hash iteratively and
+  avoids creating a single, very long concatenated SAS expression.
+
   @param [in] cvars= () Space seperated list of character variables
   @param [in] nvars= () Space seperated list of numeric variables
 
   <h4> Related Programs </h4>
   @li mp_init.sas
+  @li mp_rowhash.sas
 
   @version 9.2
   @author Allan Bowe
@@ -12282,6 +12287,96 @@ run;
 
 %mend mp_retainedkey;
 
+/**
+  @file
+  @brief Iterative row-level MD5 hash generator
+  @details Generates DATA step statements that compute a deterministic,
+    row-level MD5 hash using a Merkle-style construction.  Each variable is
+    hashed independently and the resulting raw 16-byte digests are combined
+    iteratively.  This avoids concatenating the per-variable hex digests into a
+    single SAS character expression, which overflows when datasets contain many
+    variables.
+
+    Temporary variables use five leading underscores.
+
+    This macro is called from inside a DATA step.  The caller is responsible
+    for declaring the output hash column if it does not already exist on the
+    input dataset.
+
+    Variables are hashed in the order supplied.  If a particular variable
+    needs to influence the hash first (for example the previous row hash in
+    `mp_hashdataset`, or business dates in Data Controller's bitemporal
+    loader) simply list it first in `cvars` or `nvars`.
+
+  @param [in] md5_col= Name of the output hash column.
+  @param [in] cvars= Space separated list of character variables to hash.
+  @param [in] nvars= Space separated list of numeric variables to hash.
+
+  @version 9.3M5
+  @author Allan Bowe
+**/
+
+%macro mp_rowhash(
+    md5_col=
+    ,cvars=
+    ,nvars=
+  );
+
+  /* DATA step temp variables use five leading underscores.  The names
+      are generated at macro invocation to avoid clashing with data columns. */
+  %local state digest pair numtext normal i chars nums;
+  %let state=%mf_getuniquename(prefix=_____state_);
+  %let digest=%mf_getuniquename(prefix=_____digest_);
+  %let pair=%mf_getuniquename(prefix=_____pair_);
+  %let numtext=%mf_getuniquename(prefix=_____numtext_);
+  %let normal=%mf_getuniquename(prefix=_____normal_);
+  %let i=%mf_getuniquename(prefix=_____i_);
+  %let chars=%mf_getuniquename(prefix=_____chars_);
+  %let nums=%mf_getuniquename(prefix=_____nums_);
+
+  length &state $16
+        &digest $16
+        &pair $32
+        &numtext $64
+        &normal &i 8;
+  if _n_=1 then call missing(&state,&digest,&pair,&numtext,&normal,&i);
+  drop &state &digest &pair &numtext &normal &i;
+
+  /* Versioned seed prevents confusion with other hashing schemes. */
+  &state = md5('DC HASH v2');
+
+  %if %length(&cvars)>0 %then %do;
+    array &chars{*} $ &cvars;
+    do &i = 1 to dim(&chars);
+      /* Leading blanks are retained. */
+      &digest = md5(trimn(&chars[&i]));
+      substr(&pair,  1, 16) = &state;
+      substr(&pair, 17, 16) = &digest;
+      &state = md5(&pair);
+    end;
+  %end;
+
+  %if %length(&nvars)>0 %then %do;
+    array &nums{*} &nvars;
+    do &i = 1 to dim(&nums);
+      /*
+        multiply-by-one for consistent cross-system precision.
+        Ignore null to protect SAS special missing values.
+        Cannot use IFN() as it evaluates both sides
+      */
+      if missing(&nums[&i]) then &normal = &nums[&i];
+      else &normal = &nums[&i] * 1;
+      &numtext = put(&normal, binary64.);
+      &digest = md5(trim(&numtext));
+      substr(&pair,  1, 16) = &state;
+      substr(&pair, 17, 16) = &digest;
+      &state = md5(&pair);
+    end;
+  %end;
+
+  &md5_col = put(&state, $hex32.);
+
+%mend mp_rowhash;
 /**
   @file mp_runddl.sas
   @brief An opinionated way to execute DDL files in SAS.

base/mf_wordsinstr1andstr2.sas

@@ -3,9 +3,12 @@
   @brief Returns words that are in both string 1 and string 2
   @details  Compares two space separated strings and returns the words that are
   in both.
+
+  If either string is empty, nothing is returned.
+
   Usage:
 
-      %put %mf_wordsInStr1andStr2(
+      %put %mf_wordsinstr1andstr2(
         Str1=blah sss blaaah brah bram boo
         ,Str2=   blah blaaah brah ssss
       );
@@ -23,31 +26,23 @@
 
 **/
 
-%macro mf_wordsInStr1andStr2(
+%macro mf_wordsinstr1andstr2(
   Str1= /* string containing words to extract */
   ,Str2= /* used to compare with the extract string */
 )/*/STORE SOURCE*/;
 
-%local count_base count_extr i i2 extr_word base_word match outvar;
+%local count_extr i extr_word outvar;
 %if %length(&str1)=0 or %length(&str2)=0 %then %do;
-  %put base string (str1)= &str1;
-  %put compare string (str2) = &str2;
+  %put &sysmacroname: empty input string, nothing to compare;
   %return;
 %end;
-%let count_base=%sysfunc(countw(&Str2));
 %let count_extr=%sysfunc(countw(&Str1));
 
 %do i=1 %to &count_extr;
   %let extr_word=%scan(&Str1,&i,%str( ));
-  %let match=0;
-  %do i2=1 %to &count_base;
-    %let base_word=%scan(&Str2,&i2,%str( ));
-    %if &extr_word=&base_word %then %let match=1;
-  %end;
-  %if &match=1 %then %let outvar=&outvar &extr_word;
+  %if %sysfunc(indexw(%superq(str2),%superq(extr_word)))>0 %then
+    %let outvar=&outvar &extr_word;
 %end;
-
-  &outvar
-
-%mend mf_wordsInStr1andStr2;
-
+/* send out the result without any surrounding whitespace */
+%do;&outvar%end;
+%mend mf_wordsinstr1andstr2;

base/mf_wordsinstr1butnotstr2.sas

@@ -6,9 +6,12 @@
 
   Note - case sensitive!
 
+  If str1 is empty, nothing is returned.  If str2 is empty, all the words in
+  str1 are returned.
+
   Usage:
 
-      %let x= %mf_wordsInStr1ButNotStr2(
+      %let x= %mf_wordsinstr1butnotstr2(
         Str1=blah sss blaaah brah bram boo
         ,Str2=   blah blaaah brah ssss
       );
@@ -24,31 +27,23 @@
 
 **/
 
-%macro mf_wordsInStr1ButNotStr2(
+%macro mf_wordsinstr1butnotstr2(
   Str1= /* string containing words to extract */
   ,Str2= /* used to compare with the extract string */
 )/*/STORE SOURCE*/;
 
-%local count_base count_extr i i2 extr_word base_word match outvar;
-%if %length(&str1)=0 or %length(&str2)=0 %then %do;
-  %put base string (str1)= &str1;
-  %put compare string (str2) = &str2;
+%local count_extr i extr_word outvar;
+%if %length(&str1)=0 %then %do;
+  %put &sysmacroname: str1 is empty, nothing to compare;
   %return;
 %end;
-%let count_base=%sysfunc(countw(&Str2));
 %let count_extr=%sysfunc(countw(&Str1));
 
 %do i=1 %to &count_extr;
   %let extr_word=%scan(&Str1,&i,%str( ));
-  %let match=0;
-  %do i2=1 %to &count_base;
-    %let base_word=%scan(&Str2,&i2,%str( ));
-    %if &extr_word=&base_word %then %let match=1;
-  %end;
-  %if &match=0 %then %let outvar=&outvar &extr_word;
+  %if %sysfunc(indexw(%superq(str2),%superq(extr_word)))=0 %then
+    %let outvar=&outvar &extr_word;
 %end;
-
-  &outvar
-
-%mend mf_wordsInStr1ButNotStr2;
-
+/* send out the result without any surrounding whitespace */
+%do;&outvar%end;
+%mend mf_wordsinstr1butnotstr2;

base/mp_assert.sas

@@ -35,7 +35,7 @@
 )/*/STORE SOURCE*/;
 
   data ;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     test_comments="&sysmacroname: Test result of "!!symget('iftrue');
   %if %eval(%unquote(&iftrue)) %then %do;

base/mp_assertcols.sas

@@ -108,7 +108,7 @@
   %end;
 
   data;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     if test_description='0'
     then test_description="Testing &inds for existence of &test of: &cols";

base/mp_assertcolvals.sas

@@ -143,7 +143,7 @@
   )
 
   data;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
     test_comments="&sysmacroname: &lib..&ds..&col has &result/&orig values "

base/mp_assertdsobs.sas

@@ -87,7 +87,7 @@
   %end;
 
   data &ds;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
     test_comments="&sysmacroname: Dataset &inds has &nobs observations.";

base/mp_assertscope.sas

@@ -131,7 +131,7 @@
   %end;
 
   data ;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     test_comments=symget('test_comments');
     test_result=symget('test_result');

base/mp_ds2csv.sas

@@ -130,10 +130,10 @@ data _null_;
         * there is not much point importing a short length numeric like this,
         * eg with best4., as the resulting variable will still be stored as
         * length 8.  We need a length or format statement to ensure variable
-        * is creatd with the smaller length...
+        * is created with the smaller length...
         **/
       else if vlen<8 then header=cats(varnm,':best',vlen,'.');
-      else header=cats(varnm,':best.');
+      else header=cats(varnm,':best32.');
     end;
   %end;
   %else %do;

base/mp_hashdataset.sas

@@ -17,7 +17,7 @@
   @li mf_getattrn.sas
   @li mf_getuniquename.sas
   @li mf_getvarlist.sas
-  @li mp_md5.sas
+  @li mp_rowhash.sas
 
   <h4> Related Files </h4>
   @li mp_hashdataset.test.sas
@@ -46,8 +46,7 @@
 
 %local keyvar /* roll up the md5 */
   prevkeyvar /* retain prev record md5 */
-  lastvar /* last var in input ds */
-  cvars nvars;
+  lastvar /* last var in input ds */;
 
 %if not(%eval(%unquote(&iftrue))) %then %return;
 
@@ -79,10 +78,11 @@
     retain &prevkeyvar;
     if _n_=1 then &prevkeyvar=put(md5("&salt"),$hex32.);
     set &libds end=&lastvar;
-    /* hash should include previous row */
-    &keyvar=%mp_md5(
-      cvars=%mf_getvarlist(&libds,typefilter=C) &prevkeyvar,
-      nvars=%mf_getvarlist(&libds,typefilter=N)
+    /* hash should include previous row - listed first so it is hashed first */
+    %mp_rowhash(
+      md5_col=&keyvar
+      ,cvars=&prevkeyvar %mf_getvarlist(&libds,typefilter=C)
+      ,nvars=%mf_getvarlist(&libds,typefilter=N)
     );
     &prevkeyvar=&keyvar;
     if &lastvar then output;

base/mp_init.sas

@@ -37,6 +37,7 @@
 
 %global
   SASJS_PREFIX       /* the ONLY hard-coded global macro variable in SASjs    */
+  &prefix.PROCESSMODE &prefix._STPSRV_HEADER_LOC /* SASjs server specific vars*/
   &prefix._FUNCTIONS /* used in mcf_init() to track core function compilation */
   &prefix._INIT_NUM  /* initialisation time as numeric                        */
   &prefix._INIT_DTTM /* initialisation time in E8601DT26.6 format             */

base/mp_loadformat.sas

@@ -45,7 +45,7 @@
   @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
-  @li mp_md5.sas
+  @li mp_rowhash.sas
   @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
@@ -174,14 +174,19 @@ select distinct
 %let nvars=FMTROW MIN MAX DEFAULT LENGTH FUZZ MULT NOEDIT;
 data &base_fmts/note2err;
   set &base_fmts;
-  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
+  length fmthash $32;
+  %mp_rowhash(
+    md5_col=fmthash
+    ,cvars=&cvars
+    ,nvars=&nvars
+  )
 run;
 
 /**
   * Ensure input table and base_formats have consistent lengths and types
   */
 data &inlibds/nonote2err;
-  length &delete_col $3 FMTROW 8 start end label $32767;
+  length &delete_col $3 FMTROW 8 start end label $32767 fmthash $32;
   if 0 then set &base_fmts;
   set &libds;
   by type fmtname notsorted;
@@ -203,7 +208,11 @@ data &inlibds/nonote2err;
     %mp_aligndecimal(end,width=16)
   end;
 
-  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
+  %mp_rowhash(
+    md5_col=fmthash
+    ,cvars=&cvars
+    ,nvars=&nvars
+  )
 run;
 
 /**

base/mp_md5.sas

@@ -28,11 +28,16 @@
   @li Global option:  `options dsoptions=nonote2err;`
   @li Data step option: `data YOURLIB.YOURDATASET /nonote2err;`
 
+  For very wide tables (hundreds or thousands of columns) consider using
+  `mp_rowhash.sas`, which builds the same type of row hash iteratively and
+  avoids creating a single, very long concatenated SAS expression.
+
   @param [in] cvars= () Space seperated list of character variables
   @param [in] nvars= () Space seperated list of numeric variables
 
   <h4> Related Programs </h4>
   @li mp_init.sas
+  @li mp_rowhash.sas
 
   @version 9.2
   @author Allan Bowe

base/mp_rowhash.sas

@@ -0,0 +1,90 @@
+/**
+  @file
+  @brief Iterative row-level MD5 hash generator
+  @details Generates DATA step statements that compute a deterministic,
+    row-level MD5 hash using a Merkle-style construction.  Each variable is
+    hashed independently and the resulting raw 16-byte digests are combined
+    iteratively.  This avoids concatenating the per-variable hex digests into a
+    single SAS character expression, which overflows when datasets contain many
+    variables.
+
+    Temporary variables use five leading underscores.
+
+    This macro is called from inside a DATA step.  The caller is responsible
+    for declaring the output hash column if it does not already exist on the
+    input dataset.
+
+    Variables are hashed in the order supplied.  If a particular variable
+    needs to influence the hash first (for example the previous row hash in
+    `mp_hashdataset`, or business dates in Data Controller's bitemporal
+    loader) simply list it first in `cvars` or `nvars`.
+
+  @param [in] md5_col= Name of the output hash column.
+  @param [in] cvars= Space separated list of character variables to hash.
+  @param [in] nvars= Space separated list of numeric variables to hash.
+
+  @version 9.3M5
+  @author Allan Bowe
+**/
+
+%macro mp_rowhash(
+    md5_col=
+    ,cvars=
+    ,nvars=
+  );
+
+  /* DATA step temp variables use five leading underscores.  The names
+      are generated at macro invocation to avoid clashing with data columns. */
+  %local state digest pair numtext normal i chars nums;
+  %let state=%mf_getuniquename(prefix=_____state_);
+  %let digest=%mf_getuniquename(prefix=_____digest_);
+  %let pair=%mf_getuniquename(prefix=_____pair_);
+  %let numtext=%mf_getuniquename(prefix=_____numtext_);
+  %let normal=%mf_getuniquename(prefix=_____normal_);
+  %let i=%mf_getuniquename(prefix=_____i_);
+  %let chars=%mf_getuniquename(prefix=_____chars_);
+  %let nums=%mf_getuniquename(prefix=_____nums_);
+
+  length &state $16
+        &digest $16
+        &pair $32
+        &numtext $64
+        &normal &i 8;
+  if _n_=1 then call missing(&state,&digest,&pair,&numtext,&normal,&i);
+  drop &state &digest &pair &numtext &normal &i;
+
+  /* Versioned seed prevents confusion with other hashing schemes. */
+  &state = md5('DC HASH v2');
+
+  %if %length(&cvars)>0 %then %do;
+    array &chars{*} $ &cvars;
+    do &i = 1 to dim(&chars);
+      /* Leading blanks are retained. */
+      &digest = md5(trimn(&chars[&i]));
+      substr(&pair,  1, 16) = &state;
+      substr(&pair, 17, 16) = &digest;
+      &state = md5(&pair);
+    end;
+  %end;
+
+  %if %length(&nvars)>0 %then %do;
+    array &nums{*} &nvars;
+    do &i = 1 to dim(&nums);
+      /*
+        multiply-by-one for consistent cross-system precision.
+        Ignore null to protect SAS special missing values.
+        Cannot use IFN() as it evaluates both sides
+      */
+      if missing(&nums[&i]) then &normal = &nums[&i];
+      else &normal = &nums[&i] * 1;
+      &numtext = put(&normal, binary64.);
+      &digest = md5(trim(&numtext));
+      substr(&pair,  1, 16) = &state;
+      substr(&pair, 17, 16) = &digest;
+      &state = md5(&pair);
+    end;
+  %end;
+
+  &md5_col = put(&state, $hex32.);
+
+%mend mp_rowhash;

tests/base/mf_getfilesize.test.sas

@@ -14,7 +14,7 @@ data test;
 run;
 
 %mp_assertscope(SNAPSHOT)
-%put %mf_getfilesize(libds=work.test)
+%put %mf_getfilesize(libds=work.test);
 %mp_assertscope(COMPARE)
 
 %mp_assert(

tests/base/mf_getfmtlist.test.sas

@@ -5,9 +5,14 @@
   <h4> SAS Macros </h4>
   @li mf_getfmtlist.sas
   @li mp_assert.sas
+  @li mp_assertscope.sas
 
 **/
 
+%mp_assertscope(SNAPSHOT)
+%put %mf_getfmtlist(sashelp.prdsale);
+%mp_assertscope(COMPARE)
+
 %mp_assert(
   iftrue=(
     "%mf_getfmtlist(sashelp.prdsale)"="DOLLAR $CHAR W MONNAME"
@@ -23,11 +28,3 @@
   desc=Checking basic char,
   outds=work.test_results
 )
-
-%mp_assert(
-  iftrue=(
-    "%mf_getfmtlist(sashelp.demographics)"="BEST Z $CHAR COMMA PERCENTN"
-  ),
-  desc=Checking longer numeric,
-  outds=work.test_results
-)

tests/base/mf_getvalue.test.sas

@@ -74,18 +74,5 @@ run;
   desc=Test fetching value from 1st row of empty (filtered) data,
   outds=work.test_results
 )
+%let syscc=0;
 
-%let syscc = 0; /* Reset w@rning To ensure confidence in next test */
-
-/* - Test 6 -
-  Get value from default observation.
-  Dataset does not exist.
-*/
-%let test_value=%mf_getvalue(work.test_data_x,i);
-%mp_assert(
-  iftrue=(&test_value=%str() and &syscc gt 0),
-  desc=Test fetching value from 1st row of non-existent data,
-  outds=work.test_results
-)
-
-%let syscc = 0; /* To reset expected error and allow test job to exit clean. */

tests/base/mf_getvarlist.test.sas

@@ -15,7 +15,7 @@
 %let test5=%mf_getvarlist(sashelp.class,typefilter=C);
 
 data work.test_results;
-  length test_description $256 test_result $4 test_comments base result  $256;
+  length test_result $4 test_description $256 test_comments base result  $256;
   test_description="Basic test";
   base=symget('test1');
   result='Name Sex Age Height Weight';

tests/base/mf_mimetype.test.sas

@@ -8,9 +8,12 @@
 
 **/
 
+%mp_assertscope(SNAPSHOT)
+%let test_value=%mf_mimetype(CSV);
+%mp_assertscope(COMPARE,ignorelist=test_value)
 
 %mp_assert(
-  iftrue=("%mf_mimetype(XLS)"="application/vnd.ms-excel",
+  iftrue=("%mf_mimetype(XLS)"="application/vnd.ms-excel"),
   desc=Checking correct value
 )
 

tests/base/mf_wordsinstr1andstr2.test.sas

@@ -5,12 +5,17 @@
   <h4> SAS Macros </h4>
   @li mf_wordsinstr1andstr2.sas
   @li mp_assert.sas
+  @li mp_assertscope.sas
 
 **/
 
+/* basic test, with scope check */
+%mp_assertscope(SNAPSHOT)
 %let x=%mf_wordsinstr1andstr2(str1=xx DOLLAR x $CHAR xxx W MONNAME
   ,str2=DOLLAR $CHAR W MONNAME xxxxxx
 );
+%mp_assertscope(COMPARE,ignorelist=x)
+
 %mp_assert(
   iftrue=(
     "&x"="DOLLAR $CHAR W MONNAME"
@@ -18,3 +23,92 @@
   desc=Checking basic string,
   outds=work.test_results
 )
+
+/* word boundary - var should not match var1 or var10 */
+%mp_assert(
+  iftrue=(
+    "%mf_wordsinstr1andstr2(str1=var1 var10,str2=var var1)"="var1"
+  ),
+  desc=Checking word boundaries,
+  outds=work.test_results
+)
+
+/* case sensitivity - dollar does not match DOLLAR */
+%mp_assert(
+  iftrue=(
+    "%mf_wordsinstr1andstr2(str1=DOLLAR dollar,str2=DOLLAR)"="DOLLAR"
+  ),
+  desc=Checking case sensitivity,
+  outds=work.test_results
+)
+
+/* duplicate words in str1 are preserved */
+%mp_assert(
+  iftrue=(
+    "%mf_wordsinstr1andstr2(str1=a a b,str2=a)"="a a"
+  ),
+  desc=Checking duplicate words,
+  outds=work.test_results
+)
+
+/* when no words match, nothing is returned */
+%mp_assert(
+  iftrue=(
+    "%mf_wordsinstr1andstr2(str1=a b,str2=c d)"=""
+  ),
+  desc=Checking empty result,
+  outds=work.test_results
+)
+
+/* when str1 is empty, nothing is returned */
+%mp_assert(
+  iftrue=(
+    "%mf_wordsinstr1andstr2(str1=,str2=a b)"=""
+  ),
+  desc=Checking empty str1,
+  outds=work.test_results
+)
+
+/* when str2 is empty, nothing is returned */
+%mp_assert(
+  iftrue=(
+    "%mf_wordsinstr1andstr2(str1=a b,str2=)"=""
+  ),
+  desc=Checking empty str2,
+  outds=work.test_results
+)
+
+/* build strings containing 1000 variables */
+/* str2 is kept to 100 words to avoid excessive macro iterations */
+data _null_;
+  length str1 str2 $32767;
+  do i=1 to 1000;
+    word=cats('var',i);
+    str1=catx(' ',str1,word);
+    if mod(i,10)=0 then str2=catx(' ',str2,word);
+  end;
+  call symputx('str1',str1);
+  call symputx('str2',str2);
+run;
+
+%mp_assertscope(SNAPSHOT)
+%let result=%mf_wordsinstr1andstr2(str1=&str1,str2=&str2);
+%mp_assertscope(COMPARE,ignorelist=result)
+
+%let count=%sysfunc(countw(&result));
+
+%mp_assert(
+  iftrue=(
+    "&count"="100"
+  ),
+  desc=Checking 1000 variable string returns 100 words,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "&result"="&str2"
+  ),
+  desc=Checking 1000 variable string content,
+  outds=work.test_results
+)

tests/base/mf_wordsinstr1butnotstr2.test.sas

@@ -5,12 +5,17 @@
   <h4> SAS Macros </h4>
   @li mf_wordsinstr1butnotstr2.sas
   @li mp_assert.sas
+  @li mp_assertscope.sas
 
 **/
 
+/* basic test, with scope check */
+%mp_assertscope(SNAPSHOT)
 %let x=%mf_wordsinstr1butnotstr2(str1=xx DOLLAR x $CHAR xxx W MONNAME
   ,str2=ff xx x xxx xxxxxx
 );
+%mp_assertscope(COMPARE,ignorelist=x)
+
 %mp_assert(
   iftrue=(
     "&x"="DOLLAR $CHAR W MONNAME"
@@ -18,3 +23,95 @@
   desc=Checking basic string,
   outds=work.test_results
 )
+
+/* word boundary - var1 should not match var10 or var100 */
+%mp_assert(
+  iftrue=(
+    "%mf_wordsinstr1butnotstr2(str1=var1 var10 var100,str2=var1)"
+      ="var10 var100"
+  ),
+  desc=Checking word boundaries,
+  outds=work.test_results
+)
+
+/* case sensitivity - dollar does not match DOLLAR */
+%mp_assert(
+  iftrue=(
+    "%mf_wordsinstr1butnotstr2(str1=DOLLAR dollar,str2=DOLLAR)"="dollar"
+  ),
+  desc=Checking case sensitivity,
+  outds=work.test_results
+)
+
+/* duplicate words in str1 are preserved */
+%mp_assert(
+  iftrue=(
+    "%mf_wordsinstr1butnotstr2(str1=a a b a,str2=b)"="a a a"
+  ),
+  desc=Checking duplicate words,
+  outds=work.test_results
+)
+
+/* when all words match, nothing is returned */
+%mp_assert(
+  iftrue=(
+    "%mf_wordsinstr1butnotstr2(str1=a b,str2=b a)"=""
+  ),
+  desc=Checking empty result,
+  outds=work.test_results
+)
+
+/* when str1 is empty, nothing is returned */
+%mp_assert(
+  iftrue=(
+    "%mf_wordsinstr1butnotstr2(str1=,str2=a b)"=""
+  ),
+  desc=Checking empty str1,
+  outds=work.test_results
+)
+
+/* when str2 is empty, all of str1 is returned */
+%mp_assert(
+  iftrue=(
+    "%mf_wordsinstr1butnotstr2(str1=a b c,str2=)"="a b c"
+  ),
+  desc=Checking empty str2 returns all words,
+  outds=work.test_results
+)
+
+/* build strings containing 1000 variables */
+/* str2 is kept to 100 words to avoid excessive macro iterations */
+data _null_;
+  length str1 str2 expected $32767;
+  do i=1 to 1000;
+    word=cats('var',i);
+    str1=catx(' ',str1,word);
+    if mod(i,10)=0 then str2=catx(' ',str2,word);
+    else expected=catx(' ',expected,word);
+  end;
+  call symputx('str1',str1);
+  call symputx('str2',str2);
+  call symputx('expected',expected);
+run;
+
+%mp_assertscope(SNAPSHOT)
+%let result=%mf_wordsinstr1butnotstr2(str1=&str1,str2=&str2);
+%mp_assertscope(COMPARE,ignorelist=result)
+
+%let count=%sysfunc(countw(&result));
+
+%mp_assert(
+  iftrue=(
+    "&count"="900"
+  ),
+  desc=Checking 1000 variable string returns 900 words,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "&result"="&expected"
+  ),
+  desc=Checking 1000 variable string content,
+  outds=work.test_results
+)

tests/base/mp_ds2csv.test.1.sas

@@ -85,7 +85,7 @@ data _null_;
 run;
 
 %mp_assert(
-  iftrue=("&test3a"="X:best. Y:$char7. Z:best."),
+  iftrue=("&test3a"="X:best32. Y:$char7. Z:best32."),
   desc=Checking header row Test 3,
   outds=work.test_results
 )
@@ -96,8 +96,87 @@ run;
 )
 
 /* test 4 - sasjs with compare */
-filename example temp;
-%mp_ds2csv(sashelp.air,outref=example,headerformat=SASJS)
+data work.baseball ;
+attrib
+Name                             length= $18 label="Player's Name"
+Team                             length= $14 label="Team at the End of 1986"
+nAtBat                           length= 8 label="Times at Bat in 1986"
+nHits                            length= 8 label="Hits in 1986"
+nHome                            length= 8 label="Home Runs in 1986"
+nRuns                            length= 8 label="Runs in 1986"
+nRBI                             length= 8 label="RBIs in 1986"
+nBB                              length= 8 label="Walks in 1986"
+YrMajor                          length= 8 label="Years in the Major Leagues"
+CrAtBat                          length= 8 label="Career Times at Bat"
+CrHits                           length= 8 label="Career Hits"
+CrHome                           length= 8 label="Career Home Runs"
+CrRuns                           length= 8 label="Career Runs"
+CrRbi                            length= 8 label="Career RBIs"
+CrBB                             length= 8 label="Career Walks"
+League                           length= $8 label="League at the End of 1986"
+Division                         length= $8 label="Division at the End of 1986"
+Position                         length= $8 label="Position(s) in 1986"
+nOuts                            length= 8 label="Put Outs in 1986"
+nAssts                           length= 8 label="Assists in 1986"
+nError                           length= 8 label="Errors in 1986"
+Salary                           length= 8 label="1987 Salary in $ Thousands"
+Div                              length= $16 label="League and Division"
+logSalary                        length= 8 label="Log Salary"
+;
+infile cards dsd;
+input
+  Name                             :$char.
+  Team                             :$char.
+  nAtBat
+  nHits
+  nHome
+  nRuns
+  nRBI
+  nBB
+  YrMajor
+  CrAtBat
+  CrHits
+  CrHome
+  CrRuns
+  CrRbi
+  CrBB
+  League                           :$char.
+  Division                         :$char.
+  Position                         :$char.
+  nOuts
+  nAssts
+  nError
+  Salary
+  Div                              :$char.
+  logSalary
+;
+missing a b c d e f g h i j k l m n o p q r s t u v w x y z _;
+/* fix precision issues */
+if not missing(logSalary) then logsalary=logSalary*1;
+datalines4;
+"Allanson, Andy",Cleveland,293,66,1,30,29,14,1,293,66,1,30,29,14,American,East,C,446,33,20,.,AE,.
+"Ashby, Alan",Houston,315,81,7,24,38,39,14,3449,835,69,321,414,375,National,West,C,632,43,10,475,NW,6.16331480403464
+"Davis, Alan",Seattle,479,130,18,66,72,76,3,1624,457,63,224,266,263,American,West,1B,880,82,14,480,AW,6.17378610390193
+"Dawson, Andre",Montreal,496,141,20,65,78,37,11,5628,1575,225,828,838,354,National,East,RF,200,11,3,500,NE,6.21460809842219
+"Galarraga, Andres",Montreal,321,87,10,39,42,30,2,396,101,12,48,46,33,National,East,1B,805,40,4,91.5,NE,4.51633897228147
+"Griffin, Alfredo",Oakland,594,169,4,74,51,35,11,4408,1133,19,501,336,194,American,West,SS,282,421,25,750,AW,6.62007320653035
+"Newman, Al",Montreal,185,37,1,23,8,21,2,214,42,1,30,9,24,National,East,2B,76,127,7,70,NE,4.24849524204936
+"Salazar, Argenis",Kansas City,298,73,0,24,24,7,3,509,108,0,41,37,12,American,West,SS,121,283,9,100,AW,4.60517018598809
+"Thomas, Andres",Atlanta,323,81,6,26,32,8,2,341,86,6,32,34,8,National,West,SS,143,290,19,75,NW,4.31748811353631
+"Thornton, Andre",Cleveland,401,92,17,49,66,65,13,5206,1332,253,784,890,866,American,East,DH,0,0,0,1100,AE,7.00306545878646
+"Trammell, Alan",Detroit,574,159,21,107,75,59,10,4631,1300,90,702,504,488,American,East,SS,238,445,22,517.143,AE,6.24831943200756
+"Trevino, Alex",Los Angeles,202,53,4,31,26,27,9,1876,467,15,192,186,161,National,West,C,304,45,11,512.5,NW,6.23930071101256
+"Van Slyke, Andy",St Louis,418,113,13,48,61,47,4,1512,392,41,205,204,203,National,East,RF,211,11,7,550,NE,6.30991827822651
+"Wiggins, Alan",Baltimore,239,60,0,30,11,22,6,1941,510,4,309,103,207,American,East,2B,121,151,6,700,AE,6.5510803350434
+"Almon, Bill",Pittsburgh,196,43,7,29,27,30,13,3231,825,36,376,290,238,National,East,UT,80,45,8,240,NE,5.48063892334199
+"Beane, Billy",Minneapolis,183,39,3,20,15,11,3,201,42,3,20,16,11,American,West,OF,118,0,0,.,AW,.
+"Bell, Buddy",Cincinnati,568,158,20,89,75,73,15,8068,2273,177,1045,993,732,National,West,3B,105,290,10,775,NW,6.65286302935334
+"Biancalana, Buddy",Kansas City,190,46,2,24,8,15,5,479,102,5,65,23,39,American,West,SS,102,177,16,175,AW,5.16478597392351
+"Bochte, Bruce",Oakland,407,104,6,57,43,65,12,5233,1478,100,643,658,653,American,West,1B,912,88,9,.,AW,.
+;;;;
+run;
+filename example temp lrecl=5000;
+%mp_ds2csv(work.baseball,outref=example,headerformat=SASJS)
 data _null_; infile example; input;put _infile_; if _n_>5 then stop;run;
 
 data _null_;
@@ -113,15 +192,16 @@ run;
 
 %mp_assert(
   iftrue=(&syscc =0),
-  desc=Checking syscc prior to compare of sashelp.air,
+  desc=Checking syscc prior to compare of work.baseball,
   outds=work.test_results
 )
 
-proc compare base=want compare=sashelp.air;
+proc compare base=want compare=work.baseball
+  method = absolute criterion = 0.0000000001   ;
 run;
 %mp_assert(
   iftrue=(&sysinfo le 41),
-  desc=Checking compare of sashelp.air,
+  desc=Checking compare of work.baseball,
   outds=work.test_results
 )
 

tests/base/mp_ds2csv.test.2.sas

@@ -32,7 +32,7 @@ data _null_;
 run;
 
 %mp_assert(
-  iftrue=("&test1a"="A:best3. B:best4. C:best."),
+  iftrue=("&test1a"="A:best3. B:best4. C:best32."),
   desc=Checking header row Test 1,
   outds=work.test_results
 )

tests/base/mp_gitreleaseinfo.test.sas

@@ -24,7 +24,7 @@ data _null_;
 run;
 
 %mp_assert(
-  iftrue=(&author=sasjsbot),
+  iftrue=("&author"="github-actions[bot]"),
   desc=release info extracted successfully,
   outds=work.test_results
 )

tests/base/mp_rowhash.test.sas

@@ -0,0 +1,195 @@
+/**
+  @file
+  @brief Testing mp_rowhash.sas macro
+  @details The mp_rowhash macro generates DATA step statements that compute a
+    deterministic row-level MD5 hash.  These tests exercise the versioned seed,
+    character and numeric hashing rules, variable ordering, sensitivity to
+    whitespace, and reproducibility.
+
+  <h4> SAS Macros </h4>
+  @li mp_rowhash.sas
+  @li mp_assert.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
+
+**/
+
+%mp_assertscope(SNAPSHOT)
+
+/* test 1 - versioned seed only (no input variables) */
+data _null_;
+  length actual expected $32;
+  actual='';
+  %mp_rowhash(md5_col=actual)
+  expected=put(md5('DC HASH v2'),$hex32.);
+  call symputx('t1_actual',actual);
+  call symputx('t1_expected',expected);
+run;
+%mp_assertscope(COMPARE,ignorelist=T1_ACTUAL T1_EXPECTED)
+%mp_assert(
+  iftrue=("&t1_actual"="&t1_expected"),
+  desc=Versioned seed is returned when no variables are supplied,
+  outds=work.test_results
+)
+
+/* test 2 - single character variable reference */
+data _null_;
+  length c1 $1 actual expected $32;
+  length state digest $16 pair $32;
+  c1='A';
+  actual='';
+  %mp_rowhash(md5_col=actual, cvars=c1)
+  state=md5('DC HASH v2');
+  digest=md5(trimn(c1));
+  pair=state||digest;
+  expected=put(md5(pair),$hex32.);
+  call symputx('t2_actual',actual);
+  call symputx('t2_expected',expected);
+run;
+%mp_assert(
+  iftrue=("&t2_actual"="&t2_expected"),
+  desc=Single character variable hash matches reference construction,
+  outds=work.test_results
+)
+
+/* test 3 - single numeric variable reference */
+data _null_;
+  length actual expected $32;
+  length state digest $16 numtext $64 pair $32;
+  n1=42;
+  actual='';
+  %mp_rowhash(md5_col=actual, nvars=n1)
+  state=md5('DC HASH v2');
+  /* multiply-by-one matches the internal normalisation */
+  normal=n1*1;
+  numtext=put(normal,binary64.);
+  digest=md5(trim(numtext));
+  pair=state||digest;
+  expected=put(md5(pair),$hex32.);
+  call symputx('t3_actual',actual);
+  call symputx('t3_expected',expected);
+run;
+%mp_assert(
+  iftrue=("&t3_actual"="&t3_expected"),
+  desc=Single numeric variable hash matches reference construction,
+  outds=work.test_results
+)
+
+/* test 4 - order of cvars affects the resulting hash */
+data _null_;
+  length c1 c2 $1 h1 h2 $32;
+  c1='A';
+  c2='B';
+  h1='';
+  %mp_rowhash(md5_col=h1, cvars=c1 c2)
+  h2='';
+  %mp_rowhash(md5_col=h2, cvars=c2 c1)
+  call symputx('t4_h1',h1);
+  call symputx('t4_h2',h2);
+run;
+%mp_assert(
+  iftrue=("&t4_h1" ne "&t4_h2"),
+  desc=Order of cvars changes the resulting hash,
+  outds=work.test_results
+)
+
+/* test 5 - order of nvars affects the resulting hash */
+data _null_;
+  length h1 h2 $32;
+  n1=1;
+  n2=2;
+  h1='';
+  %mp_rowhash(md5_col=h1, nvars=n1 n2)
+  h2='';
+  %mp_rowhash(md5_col=h2, nvars=n2 n1)
+  call symputx('t5_h1',h1);
+  call symputx('t5_h2',h2);
+run;
+%mp_assert(
+  iftrue=("&t5_h1" ne "&t5_h2"),
+  desc=Order of nvars changes the resulting hash,
+  outds=work.test_results
+)
+
+/* test 6 - identical rows produce identical hashes */
+data work.test6;
+  length c $5 n 8;
+  c='ABC';
+  n=42;
+  output;
+  output;
+run;
+
+data work.test6_hashed;
+  set work.test6;
+  length hash $32;
+  %mp_rowhash(md5_col=hash, cvars=c, nvars=n)
+run;
+
+proc sql noprint;
+  select count(distinct hash) into: t6_distinct trimmed
+    from work.test6_hashed;
+quit;
+
+%mp_assert(
+  iftrue=(&t6_distinct=1),
+  desc=Identical rows produce identical hashes,
+  outds=work.test_results
+)
+
+/* test 7 - different rows produce different hashes */
+data work.test7;
+  length c $5 n 8;
+  c='ABC';
+  n=42;
+  output;
+  c='ABD';
+  n=42;
+  output;
+run;
+
+data work.test7_hashed;
+  set work.test7;
+  length hash $32;
+  %mp_rowhash(md5_col=hash, cvars=c, nvars=n)
+run;
+
+proc sql noprint;
+  select count(distinct hash) into: t7_distinct trimmed
+    from work.test7_hashed;
+quit;
+
+%mp_assert(
+  iftrue=(&t7_distinct=2),
+  desc=Different rows produce different hashes,
+  outds=work.test_results
+)
+
+/* test 8 - leading blanks in character variables are retained */
+data work.test8;
+  length c $5;
+  c=' f';
+  output;
+  c='f';
+  output;
+run;
+
+data work.test8_hashed;
+  set work.test8;
+  length hash $32;
+  %mp_rowhash(md5_col=hash, cvars=c)
+run;
+
+proc sql noprint;
+  select count(distinct hash) into: t8_distinct trimmed
+    from work.test8_hashed;
+quit;
+
+%mp_assert(
+  iftrue=(&t8_distinct=2),
+  desc=Leading blanks are retained in character hashes,
+  outds=work.test_results
+)
+
+
+

tests/serveronly/ms_adduser2group.test.sas

@@ -32,6 +32,7 @@ run;
 %ms_adduser2group(uid=1,gid=&groupid,mdebug=&sasjs_mdebug,outds=test1)
 %mp_assertscope(COMPARE
   ,ignorelist=MCLIB2_JADP1LEN MCLIB2_JADP2LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
+    MC2_JADP1LEN MC2_JADP2LEN MC2_JADPNUM MC2_JADVLEN
 )
 
 /* check the user is in the output list */

tests/serveronly/ms_creategroup.test.sas

@@ -16,7 +16,8 @@
 %mp_assertscope(SNAPSHOT)
 %ms_creategroup(&group, desc=The description,mdebug=&sasjs_mdebug,outds=test1)
 %mp_assertscope(COMPARE
-  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN
+  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN MC0_JADP1LEN
+    MC0_JADPNUM MC0_JADVLEN
 )
 
 %let id=0;

tests/serveronly/ms_createuser.test.sas

@@ -16,7 +16,8 @@
 %mp_assertscope(SNAPSHOT)
 %ms_createuser(&user,passwrd,outds=test1,mdebug=&sasjs_mdebug)
 %mp_assertscope(COMPARE
-  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN
+  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN MC0_JADP1LEN
+    MC0_JADPNUM MC0_JADVLEN
 )
 
 %let id=0;

tests/serveronly/ms_getgroups.test.sas

@@ -21,7 +21,8 @@
 %mp_assertscope(SNAPSHOT)
 %ms_getgroups(outds=work.test1,mdebug=&sasjs_mdebug)
 %mp_assertscope(COMPARE
-  ,ignorelist=MCLIB2_JADP1LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
+  ,ignorelist=MCLIB2_JADP1LEN MCLIB2_JADPNUM MCLIB2_JADVLEN MC2_JADP1LEN
+    MC2_JADPNUM MC2_JADVLEN
 )
 
 /* check the group was created */

tests/serveronly/ms_getusers.test.sas

@@ -15,7 +15,8 @@
 %mp_assertscope(SNAPSHOT)
 %ms_getusers(outds=work.test1,mdebug=&sasjs_mdebug)
 %mp_assertscope(COMPARE
-  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN
+  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN MC0_JADP1LEN
+    MC0_JADPNUM MC0_JADVLEN
 )
 
 %mp_assertdsobs(work.test1,test=ATLEAST 1)

tests/serveronly/ms_triggerstp.test.sas

@@ -51,12 +51,13 @@ options mprint;
   %ms_triggerstp(/sasjs/tests/&fname2
     ,outds=work.mySessions
   )
-%mp_assertscope(COMPARE
-                ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN)
+%mp_assertscope(COMPARE,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM
+  MCLIB0_JADVLEN MC0_JADP1LEN MC0_JADPNUM MC0_JADVLEN
+)
 
 %mp_assert(iftrue=%str(%mf_existds(work.mySessions)=1)
-          ,desc=Testing output exists
-          ,outds=work.test_results
+  ,desc=Testing output exists
+  ,outds=work.test_results
 )
 
 %mp_assertdsobs(work.mySessions,

tests/viyaonly/mv_createwebservice.test.sas

@@ -32,7 +32,7 @@ run;
 )
 %put TEST1: checking web service code;
 data work.test_results;
-  length test_description $256 test_result $4 test_comments $256;
+  length test_result $4 test_description $256 test_comments $256;
   if _n_=1 then call missing (of _all_);
   infile compare end=eof;
   input;

tests/x-platform/mx_createjob.test.sas

@@ -26,7 +26,10 @@ parmcards4;
 ;;;;
 %mp_assertscope(SNAPSHOT)
 %mx_createjob(path=&mcTestAppLoc/jobs,name=testjob1,replace=YES)
-%mp_assertscope(COMPARE)
+%mp_assertscope(COMPARE,ignorelist=MC0_JADP1LEN MC0_JADP2LEN MC0_JADPNUM
+  MC0_JADVLEN MC2_JADP1LEN MC2_JADP2LEN MC2_JADPNUM MC2_JADVLEN MC4_JADP1LEN
+  MC4_JADP2LEN MC4_JADPNUM MC4_JADVLEN
+)
 
 %mp_assert(
   iftrue=(&syscc=0),