From da5244cda942030d8afe570eb9e3cec0c5f63f46 Mon Sep 17 00:00:00 2001 From: Allan Date: Mon, 10 Jul 2023 19:50:17 +0100 Subject: [PATCH] fix: when all the entries in a format are deleted, then delete the format completely includes 3 tests (regular delete, delete all but one, delete all and add one) Closes #341 --- all.sas | 84 +++++++++++++++----- base/mp_cntlout.sas | 4 +- base/mp_getformats.sas | 16 +++- base/mp_loadformat.sas | 53 ++++++++++--- base/mp_storediffs.sas | 4 +- tests/base/mp_loadformat.test.2.sas | 115 +++++++++++++++++++++++++++- 6 files changed, 238 insertions(+), 38 deletions(-) diff --git a/all.sas b/all.sas index 4c762c7..b820ece 100644 --- a/all.sas +++ b/all.sas @@ -695,10 +695,9 @@ or %index(&pgm,/tests/testteardown) returns: - > DOLLAR $CHAR W MONNAME - > $CHAR BEST DOLLAR - > BEST Z $CHAR COMMA PERCENTN - + DOLLAR $CHAR W MONNAME + $CHAR BEST DOLLAR + BEST Z $CHAR COMMA PERCENTN @param [in] libds Two part library.dataset reference. @@ -4152,9 +4151,9 @@ run; %end; proc format lib=&libcat cntlout=&cntlds; -%if "&fmtlist" ne "0" %then %do; +%if "&fmtlist" ne "0" and "&fmtlist" ne "" %then %do; select - %do i=1 %to %sysfunc(countw(&fmtlist)); + %do i=1 %to %sysfunc(countw(&fmtlist,%str( ))); %scan(&fmtlist,&i,%str( )) %end; ; @@ -7894,10 +7893,15 @@ run; Formats are taken from the library / dataset reference and / or a static format list. + Note - the source for this information is the dictionary.formats table. This + does NOT show formats that do not have + Example usage: %mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable) + %mp_getformats(fmtlist=FORMAT1 $FORMAT2 @INFMT3,outsummary=work.table2) + @param [in] lib= (0) The libref for which to return formats. @todo Enable exporting of formats for an entire library @param [in] ds= (0) The dataset from which to obtain format definitions @@ -7936,7 +7940,9 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#

Related Macros

+ @li mf_getfmtlist.sas @li mp_applyformats.sas + @li mp_cntlout.sas @li mp_getformats.test.sas @version 9.2 @@ -7953,7 +7959,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm# %local i fmt allfmts tempds fmtcnt; -%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( ))); +%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,%str( ))); /* ensure format list contains format _name_ only */ %let fmt=%scan(&fmtlist,&i,%str( )); %let fmt=%mf_getfmtname(&fmt); @@ -7977,8 +7983,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm# proc sql; create table &outsummary as select * from dictionary.formats - where fmtname in (%mf_getquotedstr(&allfmts,quote=D)) - and fmttype='F'; + where fmtname in (%mf_getquotedstr(&allfmts,quote=D)); %if "&outdetail" ne "0" %then %do; /* ensure base table always exists */ @@ -8002,6 +8007,10 @@ create table &outsummary as data &tempds; if 0 then set &outdetail; set &tempds; + /* set fmtrow (position of record within the format) */ + by type fmtname notsorted; + if first.fmtname then fmtrow=1; + else fmtrow+1; run; proc append base=&outdetail data=&tempds ; run; @@ -10162,7 +10171,7 @@ select distinct lowcase(memname) ); /* set up local macro variables and temporary tables (with a prefix) */ %local err msg prefix dslist i var fmtlist ibufsize; -%let dslist=base_fmts template inlibds ds1 stagedata storediffs; +%let dslist=base_fmts template inlibds ds1 stagedata storediffs del1 del2; %if &outds_add=0 %then %let dslist=&dslist outds_add; %if &outds_del=0 %then %let dslist=&dslist outds_del; %if &outds_mod=0 %then %let dslist=&dslist outds_mod; @@ -10293,6 +10302,18 @@ create table &outds_add(drop=&delete_col) as and upcase(a.&delete_col) ne "YES" order by type, fmtname, fmtrow; +/** + * Identify modified records + */ +create table &outds_mod (drop=&delete_col) as + select a.* + from &inlibds a + inner join &base_fmts b + on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow + where upcase(a.&delete_col) ne "YES" + and a.fmthash ne b.fmthash + order by type, fmtname, fmtrow; + /** * Identify deleted records */ @@ -10305,16 +10326,23 @@ create table &outds_del(drop=&delete_col) as order by type, fmtname, fmtrow; /** - * Identify modified records + * Identify fully deleted formats (where every record is removed) + * These require to be explicitly deleted in proc format + * del1 - identify _partial_ deletes + * del2 - exclude these, and also formats that come with _additions_ */ -create table &outds_mod (drop=&delete_col) as +create table &del1 as select a.* - from &inlibds a - inner join &base_fmts b + from &base_fmts a + left join &outds_del b on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow - where upcase(a.&delete_col) ne "YES" - and a.fmthash ne b.fmthash - order by type, fmtname, fmtrow; + where b.fmtrow is null; + +create table &del2 as + select * from &outds_del + where cats(type,fmtname) not in (select cats(type,fmtname) from &outds_add) + and cats(type,fmtname) not in (select cats(type,fmtname) from &del1); + %mp_abort( iftrue=(&syscc ne 0) @@ -10347,7 +10375,7 @@ create table &outds_mod (drop=&delete_col) as ,msg=%str(SYSCC=&syscc prior to actual load) ) %if &loadtarget=YES %then %do; - %if %mf_nobs(&stagedata)=0 %then %do; + %if %mf_nobs(&stagedata)=0 and %mf_nobs(&del2)=0 %then %do; %put There are no changes to load in &libcat!; %return; %end; @@ -10363,6 +10391,22 @@ create table &outds_mod (drop=&delete_col) as /* do the actual load */ proc format lib=&libcat cntlin=&stagedata; run; + /* apply any full deletes */ + %if %mf_nobs(&del2)>0 %then %do; + %local delfmtlist; + proc sql noprint; + select distinct case when type='N' then cats(fmtname,'.FORMAT') + when type='C' then cats(fmtname,'.FORMATC') + when type='J' then cats(fmtname,'.INFMTC') + when type='I' then cats(fmtname,'.INFMT') + else cats(fmtname,'.BADENTRY!!!') end + into: delfmtlist + separated by ' ' + from &del2; + proc catalog catalog=&libcat; + delete &delfmtlist; + quit; + %end; %if &locklibds ne 0 %then %do; /* unlock the table */ %mp_lockanytable(UNLOCK @@ -12831,7 +12875,7 @@ data &ds4; if upcase(&inds_auto)="&ds2" then tgtvar_type='N'; else if upcase(&inds_auto)="&ds3" then tgtvar_type='C'; else do; - putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto; + putlog 'ERR' +(-1) "OR: unidentified vartype input!" &inds_auto; call symputx('syscc',98); end; @@ -12840,7 +12884,7 @@ data &ds4; else if &inds_keep="&modds" then move_type='M'; else if &inds_keep="&origds" then move_type='O'; else do; - putlog "%str(ERR)OR: unidentified movetype input!" &inds_keep; + putlog 'ERR' +(-1) "OR: unidentified movetype input!" &inds_keep; call symputx('syscc',99); end; tgtvar_nm=upcase(tgtvar_nm); diff --git a/base/mp_cntlout.sas b/base/mp_cntlout.sas index 0329360..7e15bf0 100644 --- a/base/mp_cntlout.sas +++ b/base/mp_cntlout.sas @@ -58,9 +58,9 @@ %end; proc format lib=&libcat cntlout=&cntlds; -%if "&fmtlist" ne "0" %then %do; +%if "&fmtlist" ne "0" and "&fmtlist" ne "" %then %do; select - %do i=1 %to %sysfunc(countw(&fmtlist)); + %do i=1 %to %sysfunc(countw(&fmtlist,%str( ))); %scan(&fmtlist,&i,%str( )) %end; ; diff --git a/base/mp_getformats.sas b/base/mp_getformats.sas index 7244064..3e69636 100644 --- a/base/mp_getformats.sas +++ b/base/mp_getformats.sas @@ -7,10 +7,15 @@ Formats are taken from the library / dataset reference and / or a static format list. + Note - the source for this information is the dictionary.formats table. This + does NOT show formats that do not have + Example usage: %mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable) + %mp_getformats(fmtlist=FORMAT1 $FORMAT2 @INFMT3,outsummary=work.table2) + @param [in] lib= (0) The libref for which to return formats. @todo Enable exporting of formats for an entire library @param [in] ds= (0) The dataset from which to obtain format definitions @@ -49,7 +54,9 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#

Related Macros

+ @li mf_getfmtlist.sas @li mp_applyformats.sas + @li mp_cntlout.sas @li mp_getformats.test.sas @version 9.2 @@ -66,7 +73,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm# %local i fmt allfmts tempds fmtcnt; -%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( ))); +%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,%str( ))); /* ensure format list contains format _name_ only */ %let fmt=%scan(&fmtlist,&i,%str( )); %let fmt=%mf_getfmtname(&fmt); @@ -90,8 +97,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm# proc sql; create table &outsummary as select * from dictionary.formats - where fmtname in (%mf_getquotedstr(&allfmts,quote=D)) - and fmttype='F'; + where fmtname in (%mf_getquotedstr(&allfmts,quote=D)); %if "&outdetail" ne "0" %then %do; /* ensure base table always exists */ @@ -115,6 +121,10 @@ create table &outsummary as data &tempds; if 0 then set &outdetail; set &tempds; + /* set fmtrow (position of record within the format) */ + by type fmtname notsorted; + if first.fmtname then fmtrow=1; + else fmtrow+1; run; proc append base=&outdetail data=&tempds ; run; diff --git a/base/mp_loadformat.sas b/base/mp_loadformat.sas index 3dadee3..aa66b83 100644 --- a/base/mp_loadformat.sas +++ b/base/mp_loadformat.sas @@ -68,7 +68,7 @@ ); /* set up local macro variables and temporary tables (with a prefix) */ %local err msg prefix dslist i var fmtlist ibufsize; -%let dslist=base_fmts template inlibds ds1 stagedata storediffs; +%let dslist=base_fmts template inlibds ds1 stagedata storediffs del1 del2; %if &outds_add=0 %then %let dslist=&dslist outds_add; %if &outds_del=0 %then %let dslist=&dslist outds_del; %if &outds_mod=0 %then %let dslist=&dslist outds_mod; @@ -199,6 +199,18 @@ create table &outds_add(drop=&delete_col) as and upcase(a.&delete_col) ne "YES" order by type, fmtname, fmtrow; +/** + * Identify modified records + */ +create table &outds_mod (drop=&delete_col) as + select a.* + from &inlibds a + inner join &base_fmts b + on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow + where upcase(a.&delete_col) ne "YES" + and a.fmthash ne b.fmthash + order by type, fmtname, fmtrow; + /** * Identify deleted records */ @@ -211,16 +223,23 @@ create table &outds_del(drop=&delete_col) as order by type, fmtname, fmtrow; /** - * Identify modified records + * Identify fully deleted formats (where every record is removed) + * These require to be explicitly deleted in proc format + * del1 - identify _partial_ deletes + * del2 - exclude these, and also formats that come with _additions_ */ -create table &outds_mod (drop=&delete_col) as +create table &del1 as select a.* - from &inlibds a - inner join &base_fmts b + from &base_fmts a + left join &outds_del b on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow - where upcase(a.&delete_col) ne "YES" - and a.fmthash ne b.fmthash - order by type, fmtname, fmtrow; + where b.fmtrow is null; + +create table &del2 as + select * from &outds_del + where cats(type,fmtname) not in (select cats(type,fmtname) from &outds_add) + and cats(type,fmtname) not in (select cats(type,fmtname) from &del1); + %mp_abort( iftrue=(&syscc ne 0) @@ -253,7 +272,7 @@ create table &outds_mod (drop=&delete_col) as ,msg=%str(SYSCC=&syscc prior to actual load) ) %if &loadtarget=YES %then %do; - %if %mf_nobs(&stagedata)=0 %then %do; + %if %mf_nobs(&stagedata)=0 and %mf_nobs(&del2)=0 %then %do; %put There are no changes to load in &libcat!; %return; %end; @@ -269,6 +288,22 @@ create table &outds_mod (drop=&delete_col) as /* do the actual load */ proc format lib=&libcat cntlin=&stagedata; run; + /* apply any full deletes */ + %if %mf_nobs(&del2)>0 %then %do; + %local delfmtlist; + proc sql noprint; + select distinct case when type='N' then cats(fmtname,'.FORMAT') + when type='C' then cats(fmtname,'.FORMATC') + when type='J' then cats(fmtname,'.INFMTC') + when type='I' then cats(fmtname,'.INFMT') + else cats(fmtname,'.BADENTRY!!!') end + into: delfmtlist + separated by ' ' + from &del2; + proc catalog catalog=&libcat; + delete &delfmtlist; + quit; + %end; %if &locklibds ne 0 %then %do; /* unlock the table */ %mp_lockanytable(UNLOCK diff --git a/base/mp_storediffs.sas b/base/mp_storediffs.sas index 631453c..b226d16 100644 --- a/base/mp_storediffs.sas +++ b/base/mp_storediffs.sas @@ -165,7 +165,7 @@ data &ds4; if upcase(&inds_auto)="&ds2" then tgtvar_type='N'; else if upcase(&inds_auto)="&ds3" then tgtvar_type='C'; else do; - putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto; + putlog 'ERR' +(-1) "OR: unidentified vartype input!" &inds_auto; call symputx('syscc',98); end; @@ -174,7 +174,7 @@ data &ds4; else if &inds_keep="&modds" then move_type='M'; else if &inds_keep="&origds" then move_type='O'; else do; - putlog "%str(ERR)OR: unidentified movetype input!" &inds_keep; + putlog 'ERR' +(-1) "OR: unidentified movetype input!" &inds_keep; call symputx('syscc',99); end; tgtvar_nm=upcase(tgtvar_nm); diff --git a/tests/base/mp_loadformat.test.2.sas b/tests/base/mp_loadformat.test.2.sas index 918f534..0354da1 100644 --- a/tests/base/mp_loadformat.test.2.sas +++ b/tests/base/mp_loadformat.test.2.sas @@ -33,13 +33,16 @@ @li mp_loadformat.sas @li mp_assert.sas @li mp_assertdsobs.sas + @li mp_getformats.sas + @li mp_ds2md.sas + **/ /* prep format catalog */ libname perm (work); -/* create some multilable formats */ +/* create some multilabel formats */ %let cat1=perm.test1; proc format library=&cat1; value $genderml (multilabel notsorted) @@ -120,4 +123,112 @@ run; iftrue=(&check1=1 and &check2=1), desc=Ensuring Farmale values retain their order, outds=work.test_results -) \ No newline at end of file +) + +/** + * completely delete a format and make sure it is removed + */ + +/* first, make sure these three formats exist */ +options insert=(fmtsearch=(&cat1)); +%mp_getformats(fmtlist=AGEMLA AGEMLB AGEMLC $GENDERML,outsummary=work.fmtdels) + +%let fmtlist=NONE; +proc sql; +select distinct cats(fmtname) into: fmtlist separated by ' ' from work.fmtdels; + +%mp_assert( + iftrue=(%mf_nobs(fmtdels)=4), + desc=Deletion test 1 - ensure formats exist for deletion (&fmtlist found), + outds=work.test_results +) + +/* deltest1 - deleting every record */ +%mp_cntlout(libcat=&cat1,cntlout=work.cntloutdel1) +data work.stagedatadel1; + set work.cntloutdel1; + if fmtname='AGEMLA'; + deleteme='Yes'; +run; +%mp_loadformat(&cat1 + ,work.stagedatadel1 + ,loadtarget=YES + ,auditlibds=perm.audit + ,locklibds=0 + ,delete_col=deleteme + ,outds_add=add_testdel1 + ,outds_del=del_testdel1 + ,outds_mod=mod_testdel1 + ,mdebug=1 +) +%mp_getformats(fmtlist=AGEMLA,outsummary=work.fmtdel1) +%mp_assert( + iftrue=(%mf_nobs(fmtdel1)=0), + desc=Deletion test 1 - ensure AGEMLA format was fully deleted, + outds=work.test_results +) + +/* deltest2 - deleting every record except 1 */ +data work.stagedatadel2; + set work.cntloutdel1; + if fmtname='AGEMLB'; + x+1; + if x>1 then deleteme='Yes'; +run; +%mp_loadformat(&cat1 + ,work.stagedatadel2 + ,loadtarget=YES + ,auditlibds=perm.audit + ,locklibds=0 + ,delete_col=deleteme + ,outds_add=add_testdel2 + ,outds_del=del_testdel2 + ,outds_mod=mod_testdel2 + ,mdebug=1 +) +%mp_getformats(fmtlist=AGEMLB,outsummary=work.fmtdel2) +%mp_assert( + iftrue=(%mf_nobs(fmtdel2)=1), + desc=Deletion test 2 - ensure AGEMLB format was not fully deleted, + outds=work.test_results +) + + +/* deltest3 - deleting every record, and adding a new one */ +data work.stagedatadel3; + set work.cntloutdel1; + if fmtname='GENDERML'; + deleteme='Yes'; +run; +data work.stagedatadel3; + set work.stagedatadel3 end=last; + output; + if last then do; + deleteme='No'; + /* must be a new fmtrow (key value) if adding new row in same load! */ + fmtrow=1000; + start='Mail'; + end='Mail'; + output; + end; +run; + +%mp_loadformat(&cat1 + ,work.stagedatadel3 + ,loadtarget=YES + ,auditlibds=perm.audit + ,locklibds=0 + ,delete_col=deleteme + ,outds_add=add_testdel2 + ,outds_del=del_testdel2 + ,outds_mod=mod_testdel2 + ,mdebug=1 +) +%mp_getformats(fmtlist=$GENDERML,outsummary=work.fmtdel3) +%mp_assert( + iftrue=(%mf_nobs(fmtdel3)=1), + desc=Deletion test 3 - ensure GENDERML format was not fully deleted, + outds=work.test_results +) + +%mp_ds2md(work.fmtdel3) \ No newline at end of file