fix: more dedup fixes on mp_stripdiffs

all.sas

@@ -13393,7 +13393,7 @@ run;
 
     @li deleted rows - these are re-inserted
     @li changed rows - differences are reverted
-    @li added rows - these are marked with `_____DELETE_THIS_RECORD_____="YES"`
+    @li added rows - marked with `_____DELETE__THIS__RECORD_____="YES"`
 
     These changes are NOT applied to the base table - a staging dataset is
     simply prepared for an ETL process to action.  In Data Controller, this
@@ -13409,7 +13409,7 @@ run;
     in mddl_dc_difftable.sas
   @param [out] outds= (work.mp_stripdiffs) Output table containing the diffs.
     Has the same format as the base datset, plus a
-    `_____DELETE_THIS_RECORD_____` variable.
+    `_____DELETE__THIS__RECORD_____` variable.
   @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
@@ -13483,7 +13483,7 @@ create table &ds1 (drop=libref dsn) as
 /* extract key values only */
 %let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_pks));
 create table &ds2 as
-  select key_hash,
+  select distinct key_hash,
     tgtvar_nm,
     tgtvar_type,
     coalescec(oldval_char,newval_char) as charval,
@@ -13497,15 +13497,15 @@ create table &ds2 as
 %local pk;
 data _null_;
   set &ds2;
-  by key_hash;
+  by key_hash processed_dttm;
   call symputx('pk',catx(' ',symget('pk'),tgtvar_nm),'l');
-  if last.key_hash then stop;
+  if last.processed_dttm then stop;
 run;
 
 %let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_keychar));
 proc transpose data=&ds2(where=(tgtvar_type='C'))
     out=&ds3(drop=_name_);
-  by KEY_HASH;
+  by KEY_HASH PROCESSED_DTTM;
   id TGTVAR_NM;
   var charval;
 run;
@@ -13513,7 +13513,7 @@ run;
 %let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_keynum));
 proc transpose data=&ds2(where=(tgtvar_type='N'))
     out=&ds4(drop=_name_);
-  by KEY_HASH;
+  by KEY_HASH PROCESSED_DTTM;
   id TGTVAR_NM;
   var numval;
 run;
@@ -13521,17 +13521,21 @@ run;
 %mp_ds2squeeze(&ds3,outds=&ds3)
 %mp_ds2squeeze(&ds4,outds=&ds4)
 
+/* now merge to get all key values and de-dup */
 %let ds5=%upcase(work.%mf_getuniquename(prefix=mpsd_merged));
 data &ds5;
-  length key_hash $32;
+  length key_hash $32 processed_dttm 8;
   merge &ds3 &ds4;
   by key_hash;
   if not missing(key_hash);
 run;
+proc sort data=&ds5 nodupkey;
+  by &pk;
+run;
 
 /* join to base table for preliminary stage DS */
 proc sql;
-create table &outds as select "No " as _____DELETE_THIS_RECORD_____,
+create table &outds as select "No " as _____DELETE__THIS__RECORD_____,
     b.*
   from &ds5 a
   inner join &libds b
@@ -13544,7 +13548,8 @@ create table &outds as select "No " as _____DELETE_THIS_RECORD_____,
 /* create SAS code to apply to stage_ds */
 data _null_;
   set &ds1;
-  file &fref1;
+  file &fref1 lrecl=33000;
+  length charval $32767;
   if _n_=1 then put 'proc sql noprint;';
   by descending processed_dttm key_hash is_pk;
   if move_type='M' then do;
@@ -13553,7 +13558,8 @@ data _null_;
     end;
     if IS_PK=0 then do;
       put "  " tgtvar_nm '=' @@;
-      charval=quote(cats(oldval_char));
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
       if tgtvar_type='C' then put charval @@;
       else put oldval_num @@;
       if not last.is_pk then put ',';
@@ -13561,28 +13567,31 @@ data _null_;
     else do;
       if first.is_pk then put "  where 1=1 " @@;
       put "  and " tgtvar_nm '=' @@;
-      charval=quote(cats(oldval_char));
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
       if tgtvar_type='C' then put charval @@;
       else put oldval_num @@;
     end;
   end;
   else if move_type='A' then do;
     if first.key_hash then do;
-      put "update &outds set _____DELETE_THIS_RECORD_____='Yes'  where 1=1 " @@;
+      put "update &outds set _____DELETE__THIS__RECORD_____='Yes' where 1=1 "@@;
     end;
     /* gating if - as only need PK now */
     if is_pk=1;
     put '  AND ' tgtvar_nm '=' @@;
-    charval=quote(cats(newval_char));
+    cnt=count(newval_char,'"');
+    charval=quote(trim(substr(newval_char,1,32765-cnt)));
     if tgtvar_type='C' then put charval @@;
     else put newval_num @@;
   end;
   else if move_type='D' then do;
     if first.key_hash then do;
-      put "insert into &outds set _____DELETE_THIS_RECORD_____='No' " @@;
+      put "insert into &outds set _____DELETE__THIS__RECORD_____='No' " @@;
     end;
     put "  ," tgtvar_nm '=' @@;
-    charval=quote(cats(oldval_char));
+    cnt=count(oldval_char,'"');
+    charval=quote(trim(substr(oldval_char,1,32765-cnt)));
     if tgtvar_type='C' then put charval @@;
     else put oldval_num @@;
   end;
@@ -13590,7 +13599,7 @@ data _null_;
 run;
 
 /* apply the modification statements */
-%inc &fref1/source2;
+%inc &fref1/source2 lrecl=33000;
 
 %if &mdebug=0 %then %do;
   proc sql;

base/mp_stripdiffs.sas

@@ -8,7 +8,7 @@
 
     @li deleted rows - these are re-inserted
     @li changed rows - differences are reverted
-    @li added rows - these are marked with `_____DELETE_THIS_RECORD_____="YES"`
+    @li added rows - marked with `_____DELETE__THIS__RECORD_____="YES"`
 
     These changes are NOT applied to the base table - a staging dataset is
     simply prepared for an ETL process to action.  In Data Controller, this
@@ -24,7 +24,7 @@
     in mddl_dc_difftable.sas
   @param [out] outds= (work.mp_stripdiffs) Output table containing the diffs.
     Has the same format as the base datset, plus a
-    `_____DELETE_THIS_RECORD_____` variable.
+    `_____DELETE__THIS__RECORD_____` variable.
   @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
@@ -98,7 +98,7 @@ create table &ds1 (drop=libref dsn) as
 /* extract key values only */
 %let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_pks));
 create table &ds2 as
-  select key_hash,
+  select distinct key_hash,
     tgtvar_nm,
     tgtvar_type,
     coalescec(oldval_char,newval_char) as charval,
@@ -112,15 +112,15 @@ create table &ds2 as
 %local pk;
 data _null_;
   set &ds2;
-  by key_hash;
+  by key_hash processed_dttm;
   call symputx('pk',catx(' ',symget('pk'),tgtvar_nm),'l');
-  if last.key_hash then stop;
+  if last.processed_dttm then stop;
 run;
 
 %let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_keychar));
 proc transpose data=&ds2(where=(tgtvar_type='C'))
     out=&ds3(drop=_name_);
-  by KEY_HASH;
+  by KEY_HASH PROCESSED_DTTM;
   id TGTVAR_NM;
   var charval;
 run;
@@ -128,7 +128,7 @@ run;
 %let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_keynum));
 proc transpose data=&ds2(where=(tgtvar_type='N'))
     out=&ds4(drop=_name_);
-  by KEY_HASH;
+  by KEY_HASH PROCESSED_DTTM;
   id TGTVAR_NM;
   var numval;
 run;
@@ -136,17 +136,21 @@ run;
 %mp_ds2squeeze(&ds3,outds=&ds3)
 %mp_ds2squeeze(&ds4,outds=&ds4)
 
+/* now merge to get all key values and de-dup */
 %let ds5=%upcase(work.%mf_getuniquename(prefix=mpsd_merged));
 data &ds5;
-  length key_hash $32;
+  length key_hash $32 processed_dttm 8;
   merge &ds3 &ds4;
   by key_hash;
   if not missing(key_hash);
 run;
+proc sort data=&ds5 nodupkey;
+  by &pk;
+run;
 
 /* join to base table for preliminary stage DS */
 proc sql;
-create table &outds as select "No " as _____DELETE_THIS_RECORD_____,
+create table &outds as select "No " as _____DELETE__THIS__RECORD_____,
     b.*
   from &ds5 a
   inner join &libds b
@@ -159,7 +163,8 @@ create table &outds as select "No " as _____DELETE_THIS_RECORD_____,
 /* create SAS code to apply to stage_ds */
 data _null_;
   set &ds1;
-  file &fref1;
+  file &fref1 lrecl=33000;
+  length charval $32767;
   if _n_=1 then put 'proc sql noprint;';
   by descending processed_dttm key_hash is_pk;
   if move_type='M' then do;
@@ -168,7 +173,8 @@ data _null_;
     end;
     if IS_PK=0 then do;
       put "  " tgtvar_nm '=' @@;
-      charval=quote(cats(oldval_char));
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
       if tgtvar_type='C' then put charval @@;
       else put oldval_num @@;
       if not last.is_pk then put ',';
@@ -176,28 +182,31 @@ data _null_;
     else do;
       if first.is_pk then put "  where 1=1 " @@;
       put "  and " tgtvar_nm '=' @@;
-      charval=quote(cats(oldval_char));
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
       if tgtvar_type='C' then put charval @@;
       else put oldval_num @@;
     end;
   end;
   else if move_type='A' then do;
     if first.key_hash then do;
-      put "update &outds set _____DELETE_THIS_RECORD_____='Yes'  where 1=1 " @@;
+      put "update &outds set _____DELETE__THIS__RECORD_____='Yes' where 1=1 "@@;
     end;
     /* gating if - as only need PK now */
     if is_pk=1;
     put '  AND ' tgtvar_nm '=' @@;
-    charval=quote(cats(newval_char));
+    cnt=count(newval_char,'"');
+    charval=quote(trim(substr(newval_char,1,32765-cnt)));
     if tgtvar_type='C' then put charval @@;
     else put newval_num @@;
   end;
   else if move_type='D' then do;
     if first.key_hash then do;
-      put "insert into &outds set _____DELETE_THIS_RECORD_____='No' " @@;
+      put "insert into &outds set _____DELETE__THIS__RECORD_____='No' " @@;
     end;
     put "  ," tgtvar_nm '=' @@;
-    charval=quote(cats(oldval_char));
+    cnt=count(oldval_char,'"');
+    charval=quote(trim(substr(oldval_char,1,32765-cnt)));
     if tgtvar_type='C' then put charval @@;
     else put oldval_num @@;
   end;
@@ -205,7 +214,7 @@ data _null_;
 run;
 
 /* apply the modification statements */
-%inc &fref1/source2;
+%inc &fref1/source2 lrecl=33000;
 
 %if &mdebug=0 %then %do;
   proc sql;

tests/base/mp_stripdiffs.test.sas

@@ -82,10 +82,10 @@ run;
 %let addpass=0;
 data _null_;
   set work.mp_stripdiffs;
-  if upcase(_____DELETE_THIS_RECORD_____)='NO' and name="&delname"
+  if upcase(_____DELETE__THIS__RECORD_____)='NO' and name="&delname"
   then call symputx('delpass',1);
   if name="&modname" and age=&modval then call symputx('modpass',1);
-  if upcase(_____DELETE_THIS_RECORD_____)='YES' and name="Newbie"
+  if upcase(_____DELETE__THIS__RECORD_____)='YES' and name="Newbie"
   then call symputx('addpass',1);
 run;