fix: when all the entries in a format are deleted, then delete the format completely

includes 3 tests (regular delete, delete all but one, delete all and add one) Closes #341
2025-12-11 06:24:35 +00:00 · 2023-07-10 19:50:17 +01:00
parent 724de80d0f
commit da5244cda9
6 changed files with 238 additions and 38 deletions
--- a/all.sas
+++ b/all.sas
@@ -695,10 +695,9 @@ or %index(&pgm,/tests/testteardown)

  returns:

-  > DOLLAR $CHAR W MONNAME
-  > $CHAR BEST DOLLAR
-  > BEST Z $CHAR COMMA PERCENTN
-
+      DOLLAR $CHAR W MONNAME
+      $CHAR BEST DOLLAR
+      BEST Z $CHAR COMMA PERCENTN

  @param [in] libds Two part library.dataset reference.

@@ -4152,9 +4151,9 @@ run;
 %end;

 proc format lib=&libcat cntlout=&cntlds;
-%if "&fmtlist" ne "0" %then %do;
+%if "&fmtlist" ne "0" and "&fmtlist" ne "" %then %do;
  select
-  %do i=1 %to %sysfunc(countw(&fmtlist));
+  %do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
    %scan(&fmtlist,&i,%str( ))
  %end;
  ;
@@ -7894,10 +7893,15 @@ run;
  Formats are taken from the library / dataset reference and / or a static
  format list.

+  Note - the source for this information is the dictionary.formats table. This
+  does NOT show formats that do not have
+
  Example usage:

      %mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable)

+      %mp_getformats(fmtlist=FORMAT1 $FORMAT2 @INFMT3,outsummary=work.table2)
+
  @param [in] lib= (0) The libref for which to return formats.
  @todo Enable exporting of formats for an entire library
  @param [in] ds= (0) The dataset from which to obtain format definitions
@@ -7936,7 +7940,9 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#


  <h4> Related Macros </h4>
+  @li mf_getfmtlist.sas
  @li mp_applyformats.sas
+  @li mp_cntlout.sas
  @li mp_getformats.test.sas

  @version 9.2
@@ -7953,7 +7959,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#

 %local i fmt allfmts tempds fmtcnt;

-%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( )));
+%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
  /* ensure format list contains format _name_ only */
  %let fmt=%scan(&fmtlist,&i,%str( ));
  %let fmt=%mf_getfmtname(&fmt);
@@ -7977,8 +7983,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 proc sql;
 create table &outsummary as
  select * from dictionary.formats
-  where fmtname in (%mf_getquotedstr(&allfmts,quote=D))
-    and fmttype='F';
+  where fmtname in (%mf_getquotedstr(&allfmts,quote=D));

 %if "&outdetail" ne "0" %then %do;
  /* ensure base table always exists */
@@ -8002,6 +8007,10 @@ create table &outsummary as
    data &tempds;
      if 0 then set &outdetail;
      set &tempds;
+      /* set fmtrow (position of record within the format) */
+      by type fmtname notsorted;
+      if first.fmtname then fmtrow=1;
+      else fmtrow+1;
    run;
    proc append base=&outdetail data=&tempds ;
    run;
@@ -10162,7 +10171,7 @@ select distinct lowcase(memname)
 );
 /* set up local macro variables and temporary tables (with a prefix) */
 %local err msg prefix dslist i var fmtlist ibufsize;
-%let dslist=base_fmts template inlibds ds1 stagedata storediffs;
+%let dslist=base_fmts template inlibds ds1 stagedata storediffs del1 del2;
 %if &outds_add=0 %then %let dslist=&dslist outds_add;
 %if &outds_del=0 %then %let dslist=&dslist outds_del;
 %if &outds_mod=0 %then %let dslist=&dslist outds_mod;
@@ -10293,6 +10302,18 @@ create table &outds_add(drop=&delete_col) as
    and upcase(a.&delete_col) ne "YES"
  order by type, fmtname, fmtrow;

+/**
+  * Identify modified records
+  */
+create table &outds_mod (drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
+  where upcase(a.&delete_col) ne "YES"
+    and a.fmthash ne b.fmthash
+  order by type, fmtname, fmtrow;
+
 /**
  * Identify deleted records
  */
@@ -10305,16 +10326,23 @@ create table &outds_del(drop=&delete_col) as
  order by type, fmtname, fmtrow;

 /**
-  * Identify modified records
+  * Identify fully deleted formats (where every record is removed)
+  * These require to be explicitly deleted in proc format
+  * del1 - identify _partial_ deletes
+  * del2 - exclude these, and also formats that come with _additions_
  */
-create table &outds_mod (drop=&delete_col) as
+create table &del1 as
  select a.*
-  from &inlibds a
-  inner join &base_fmts b
+  from &base_fmts a
+  left join &outds_del b
  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
-  where upcase(a.&delete_col) ne "YES"
-    and a.fmthash ne b.fmthash
-  order by type, fmtname, fmtrow;
+  where b.fmtrow is null;
+
+create table &del2 as
+  select * from &outds_del
+  where cats(type,fmtname) not in (select cats(type,fmtname) from &outds_add)
+    and cats(type,fmtname) not in (select cats(type,fmtname) from &del1);
+

 %mp_abort(
  iftrue=(&syscc ne 0)
@@ -10347,7 +10375,7 @@ create table &outds_mod (drop=&delete_col) as
  ,msg=%str(SYSCC=&syscc prior to actual load)
 )
 %if &loadtarget=YES %then %do;
-  %if %mf_nobs(&stagedata)=0 %then %do;
+  %if %mf_nobs(&stagedata)=0 and %mf_nobs(&del2)=0 %then %do;
    %put There are no changes to load in &libcat!;
    %return;
  %end;
@@ -10363,6 +10391,22 @@ create table &outds_mod (drop=&delete_col) as
  /* do the actual load */
  proc format lib=&libcat cntlin=&stagedata;
  run;
+  /* apply any full deletes */
+  %if %mf_nobs(&del2)>0 %then %do;
+    %local delfmtlist;
+    proc sql noprint;
+    select distinct case when type='N' then cats(fmtname,'.FORMAT')
+        when type='C' then cats(fmtname,'.FORMATC')
+        when type='J' then cats(fmtname,'.INFMTC')
+        when type='I' then cats(fmtname,'.INFMT')
+        else cats(fmtname,'.BADENTRY!!!') end
+      into: delfmtlist
+      separated by ' '
+      from &del2;
+    proc catalog catalog=&libcat;
+      delete &delfmtlist;
+    quit;
+  %end;
  %if &locklibds ne 0 %then %do;
    /* unlock the table */
    %mp_lockanytable(UNLOCK
@@ -12831,7 +12875,7 @@ data &ds4;
  if upcase(&inds_auto)="&ds2" then tgtvar_type='N';
  else if upcase(&inds_auto)="&ds3" then tgtvar_type='C';
  else do;
-    putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
+    putlog 'ERR' +(-1) "OR: unidentified vartype input!" &inds_auto;
    call symputx('syscc',98);
  end;

@@ -12840,7 +12884,7 @@ data &ds4;
  else if &inds_keep="&modds" then move_type='M';
  else if &inds_keep="&origds" then move_type='O';
  else do;
-    putlog "%str(ERR)OR: unidentified movetype input!" &inds_keep;
+    putlog 'ERR' +(-1) "OR: unidentified movetype input!" &inds_keep;
    call symputx('syscc',99);
  end;
  tgtvar_nm=upcase(tgtvar_nm);
--- a/base/mp_cntlout.sas
+++ b/base/mp_cntlout.sas
@@ -58,9 +58,9 @@
 %end;

 proc format lib=&libcat cntlout=&cntlds;
-%if "&fmtlist" ne "0" %then %do;
+%if "&fmtlist" ne "0" and "&fmtlist" ne "" %then %do;
  select
-  %do i=1 %to %sysfunc(countw(&fmtlist));
+  %do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
    %scan(&fmtlist,&i,%str( ))
  %end;
  ;
--- a/base/mp_getformats.sas
+++ b/base/mp_getformats.sas
@@ -7,10 +7,15 @@
  Formats are taken from the library / dataset reference and / or a static
  format list.

+  Note - the source for this information is the dictionary.formats table. This
+  does NOT show formats that do not have
+
  Example usage:

      %mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable)

+      %mp_getformats(fmtlist=FORMAT1 $FORMAT2 @INFMT3,outsummary=work.table2)
+
  @param [in] lib= (0) The libref for which to return formats.
  @todo Enable exporting of formats for an entire library
  @param [in] ds= (0) The dataset from which to obtain format definitions
@@ -49,7 +54,9 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#


  <h4> Related Macros </h4>
+  @li mf_getfmtlist.sas
  @li mp_applyformats.sas
+  @li mp_cntlout.sas
  @li mp_getformats.test.sas

  @version 9.2
@@ -66,7 +73,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#

 %local i fmt allfmts tempds fmtcnt;

-%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( )));
+%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
  /* ensure format list contains format _name_ only */
  %let fmt=%scan(&fmtlist,&i,%str( ));
  %let fmt=%mf_getfmtname(&fmt);
@@ -90,8 +97,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 proc sql;
 create table &outsummary as
  select * from dictionary.formats
-  where fmtname in (%mf_getquotedstr(&allfmts,quote=D))
-    and fmttype='F';
+  where fmtname in (%mf_getquotedstr(&allfmts,quote=D));

 %if "&outdetail" ne "0" %then %do;
  /* ensure base table always exists */
@@ -115,6 +121,10 @@ create table &outsummary as
    data &tempds;
      if 0 then set &outdetail;
      set &tempds;
+      /* set fmtrow (position of record within the format) */
+      by type fmtname notsorted;
+      if first.fmtname then fmtrow=1;
+      else fmtrow+1;
    run;
    proc append base=&outdetail data=&tempds ;
    run;
--- a/base/mp_loadformat.sas
+++ b/base/mp_loadformat.sas
@@ -68,7 +68,7 @@
 );
 /* set up local macro variables and temporary tables (with a prefix) */
 %local err msg prefix dslist i var fmtlist ibufsize;
-%let dslist=base_fmts template inlibds ds1 stagedata storediffs;
+%let dslist=base_fmts template inlibds ds1 stagedata storediffs del1 del2;
 %if &outds_add=0 %then %let dslist=&dslist outds_add;
 %if &outds_del=0 %then %let dslist=&dslist outds_del;
 %if &outds_mod=0 %then %let dslist=&dslist outds_mod;
@@ -199,6 +199,18 @@ create table &outds_add(drop=&delete_col) as
    and upcase(a.&delete_col) ne "YES"
  order by type, fmtname, fmtrow;

+/**
+  * Identify modified records
+  */
+create table &outds_mod (drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
+  where upcase(a.&delete_col) ne "YES"
+    and a.fmthash ne b.fmthash
+  order by type, fmtname, fmtrow;
+
 /**
  * Identify deleted records
  */
@@ -211,16 +223,23 @@ create table &outds_del(drop=&delete_col) as
  order by type, fmtname, fmtrow;

 /**
-  * Identify modified records
+  * Identify fully deleted formats (where every record is removed)
+  * These require to be explicitly deleted in proc format
+  * del1 - identify _partial_ deletes
+  * del2 - exclude these, and also formats that come with _additions_
  */
-create table &outds_mod (drop=&delete_col) as
+create table &del1 as
  select a.*
-  from &inlibds a
-  inner join &base_fmts b
+  from &base_fmts a
+  left join &outds_del b
  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
-  where upcase(a.&delete_col) ne "YES"
-    and a.fmthash ne b.fmthash
-  order by type, fmtname, fmtrow;
+  where b.fmtrow is null;
+
+create table &del2 as
+  select * from &outds_del
+  where cats(type,fmtname) not in (select cats(type,fmtname) from &outds_add)
+    and cats(type,fmtname) not in (select cats(type,fmtname) from &del1);
+

 %mp_abort(
  iftrue=(&syscc ne 0)
@@ -253,7 +272,7 @@ create table &outds_mod (drop=&delete_col) as
  ,msg=%str(SYSCC=&syscc prior to actual load)
 )
 %if &loadtarget=YES %then %do;
-  %if %mf_nobs(&stagedata)=0 %then %do;
+  %if %mf_nobs(&stagedata)=0 and %mf_nobs(&del2)=0 %then %do;
    %put There are no changes to load in &libcat!;
    %return;
  %end;
@@ -269,6 +288,22 @@ create table &outds_mod (drop=&delete_col) as
  /* do the actual load */
  proc format lib=&libcat cntlin=&stagedata;
  run;
+  /* apply any full deletes */
+  %if %mf_nobs(&del2)>0 %then %do;
+    %local delfmtlist;
+    proc sql noprint;
+    select distinct case when type='N' then cats(fmtname,'.FORMAT')
+        when type='C' then cats(fmtname,'.FORMATC')
+        when type='J' then cats(fmtname,'.INFMTC')
+        when type='I' then cats(fmtname,'.INFMT')
+        else cats(fmtname,'.BADENTRY!!!') end
+      into: delfmtlist
+      separated by ' '
+      from &del2;
+    proc catalog catalog=&libcat;
+      delete &delfmtlist;
+    quit;
+  %end;
  %if &locklibds ne 0 %then %do;
    /* unlock the table */
    %mp_lockanytable(UNLOCK
--- a/base/mp_storediffs.sas
+++ b/base/mp_storediffs.sas
@@ -165,7 +165,7 @@ data &ds4;
  if upcase(&inds_auto)="&ds2" then tgtvar_type='N';
  else if upcase(&inds_auto)="&ds3" then tgtvar_type='C';
  else do;
-    putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
+    putlog 'ERR' +(-1) "OR: unidentified vartype input!" &inds_auto;
    call symputx('syscc',98);
  end;

@@ -174,7 +174,7 @@ data &ds4;
  else if &inds_keep="&modds" then move_type='M';
  else if &inds_keep="&origds" then move_type='O';
  else do;
-    putlog "%str(ERR)OR: unidentified movetype input!" &inds_keep;
+    putlog 'ERR' +(-1) "OR: unidentified movetype input!" &inds_keep;
    call symputx('syscc',99);
  end;
  tgtvar_nm=upcase(tgtvar_nm);
--- a/tests/base/mp_loadformat.test.2.sas
+++ b/tests/base/mp_loadformat.test.2.sas
@@ -33,13 +33,16 @@
  @li mp_loadformat.sas
  @li mp_assert.sas
  @li mp_assertdsobs.sas
+  @li mp_getformats.sas
+  @li mp_ds2md.sas
+

 **/

 /* prep format catalog */
 libname perm (work);

-/* create some multilable formats */
+/* create some multilabel formats */
 %let cat1=perm.test1;
 proc format library=&cat1;
  value $genderml (multilabel notsorted)
@@ -120,4 +123,112 @@ run;
  iftrue=(&check1=1 and &check2=1),
  desc=Ensuring Farmale values retain their order,
  outds=work.test_results
-)
+)
+
+/**
+  * completely delete a format and make sure it is removed
+  */
+
+/* first, make sure these three formats exist */
+options insert=(fmtsearch=(&cat1));
+%mp_getformats(fmtlist=AGEMLA AGEMLB AGEMLC $GENDERML,outsummary=work.fmtdels)
+
+%let fmtlist=NONE;
+proc sql;
+select distinct cats(fmtname) into: fmtlist separated by ' ' from work.fmtdels;
+
+%mp_assert(
+  iftrue=(%mf_nobs(fmtdels)=4),
+  desc=Deletion test 1 - ensure formats exist for deletion (&fmtlist found),
+  outds=work.test_results
+)
+
+/* deltest1 - deleting every record */
+%mp_cntlout(libcat=&cat1,cntlout=work.cntloutdel1)
+data work.stagedatadel1;
+  set work.cntloutdel1;
+  if fmtname='AGEMLA';
+  deleteme='Yes';
+run;
+%mp_loadformat(&cat1
+  ,work.stagedatadel1
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_testdel1
+  ,outds_del=del_testdel1
+  ,outds_mod=mod_testdel1
+  ,mdebug=1
+)
+%mp_getformats(fmtlist=AGEMLA,outsummary=work.fmtdel1)
+%mp_assert(
+  iftrue=(%mf_nobs(fmtdel1)=0),
+  desc=Deletion test 1 - ensure AGEMLA format was fully deleted,
+  outds=work.test_results
+)
+
+/* deltest2 - deleting every record except 1 */
+data work.stagedatadel2;
+  set work.cntloutdel1;
+  if fmtname='AGEMLB';
+  x+1;
+  if x>1 then deleteme='Yes';
+run;
+%mp_loadformat(&cat1
+  ,work.stagedatadel2
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_testdel2
+  ,outds_del=del_testdel2
+  ,outds_mod=mod_testdel2
+  ,mdebug=1
+)
+%mp_getformats(fmtlist=AGEMLB,outsummary=work.fmtdel2)
+%mp_assert(
+  iftrue=(%mf_nobs(fmtdel2)=1),
+  desc=Deletion test 2 - ensure AGEMLB format was not fully deleted,
+  outds=work.test_results
+)
+
+
+/* deltest3 - deleting every record, and adding a new one */
+data work.stagedatadel3;
+  set work.cntloutdel1;
+  if fmtname='GENDERML';
+  deleteme='Yes';
+run;
+data work.stagedatadel3;
+  set work.stagedatadel3 end=last;
+  output;
+  if last then do;
+    deleteme='No';
+    /* must be a new fmtrow (key value) if adding new row in same load! */
+    fmtrow=1000;
+    start='Mail';
+    end='Mail';
+    output;
+  end;
+run;
+
+%mp_loadformat(&cat1
+  ,work.stagedatadel3
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_testdel2
+  ,outds_del=del_testdel2
+  ,outds_mod=mod_testdel2
+  ,mdebug=1
+)
+%mp_getformats(fmtlist=$GENDERML,outsummary=work.fmtdel3)
+%mp_assert(
+  iftrue=(%mf_nobs(fmtdel3)=1),
+  desc=Deletion test 3 - ensure GENDERML format was not fully deleted,
+  outds=work.test_results
+)
+
+%mp_ds2md(work.fmtdel3)