From dcb7958950346b2ec159d8be2546bc56e45afc73 Mon Sep 17 00:00:00 2001 From: Allan Date: Tue, 20 Jun 2023 00:21:19 +0100 Subject: [PATCH] feat: enabling informats to be ingested with the mp_loadformat macro --- base/mp_cntlout.sas | 9 ++- base/mp_loadformat.sas | 45 ++++++++--- ddl/mddl_sas_cntlout.sas | 11 ++- tests/base/mp_loadformat.test.sas | 127 +++++++++++++++++++++++++++++- 4 files changed, 171 insertions(+), 21 deletions(-) diff --git a/base/mp_cntlout.sas b/base/mp_cntlout.sas index 357b630..5a8e69a 100644 --- a/base/mp_cntlout.sas +++ b/base/mp_cntlout.sas @@ -28,6 +28,7 @@

Related Macros

@li mf_getvarformat.sas + @li mp_aligndecimal.sas @li mp_getformats.sas @li mp_loadformat.sas @li mp_ds2fmtds.sas @@ -69,13 +70,13 @@ run; data &cntlout; if 0 then set &ddlds; set &cntlds; - if type="N" then do; - start=cats(start); - end=cats(end); + if type in ("I","N") then do; /* numeric (in)format */ + %mp_aligndecimal(start,width=16) + %mp_aligndecimal(end,width=16) end; run; proc sort; - by fmtname start; + by type fmtname start; run; proc sql; diff --git a/base/mp_loadformat.sas b/base/mp_loadformat.sas index 44f4371..cb33ff4 100644 --- a/base/mp_loadformat.sas +++ b/base/mp_loadformat.sas @@ -45,6 +45,7 @@

Related Macros

@li mddl_dc_difftable.sas @li mddl_dc_locktable.sas + @li mp_aligndecimal.sas @li mp_loadformat.test.sas @li mp_lockanytable.sas @li mp_stackdiffs.sas @@ -134,7 +135,16 @@ run; * First, extract only relevant formats from the catalog */ proc sql noprint; -select distinct upcase(fmtname) into: fmtlist separated by ' ' from &libds; +select distinct + case + when type='N' then upcase(fmtname) + when type='C' then cats('$',upcase(fmtname)) + when type='I' then cats('@',upcase(fmtname)) + when type='J' then cats('@$',upcase(fmtname)) + else "&sysmacroname:UNHANDLED" + end + into: fmtlist separated by ' ' + from &libds; %mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts) @@ -146,16 +156,24 @@ select distinct upcase(fmtname) into: fmtlist separated by ' ' from &libds; data &inlibds; length &delete_col $3; if 0 then set &template; + length start end $10000; set &libds; if &delete_col='' then &delete_col='No'; fmtname=upcase(fmtname); + type=upcase(type); if missing(type) then do; - if substr(fmtname,1,1)='$' then type='C'; - else type='N'; + if substr(fmtname,1,1)='@' then do; + if substr(fmtname,2,1)='$' then type='J'; + else type='I'; + end; + else do; + if substr(fmtname,1,1)='$' then type='C'; + else type='N'; + end; end; - if type='N' then do; - start=cats(start); - end=cats(end); + if type in ('N','I') then do; + %mp_aligndecimal(start,width=16) + %mp_aligndecimal(end,width=16) end; run; @@ -169,9 +187,10 @@ create table &outds_add(drop=&delete_col) as left join &base_fmts b on a.fmtname=b.fmtname and a.start=b.start + and a.type=b.type where b.fmtname is null and upcase(a.&delete_col) ne "YES" - order by fmtname, start;; + order by type, fmtname, start; /** * Identify deleted records @@ -182,8 +201,9 @@ create table &outds_del(drop=&delete_col) as inner join &base_fmts b on a.fmtname=b.fmtname and a.start=b.start + and a.type=b.type where upcase(a.&delete_col)="YES" - order by fmtname, start; + order by type, fmtname, start; /** * Identify modified records @@ -194,8 +214,9 @@ create table &outds_mod (drop=&delete_col) as inner join &base_fmts b on a.fmtname=b.fmtname and a.start=b.start + and a.type=b.type where upcase(a.&delete_col) ne "YES" - order by fmtname, start; + order by type, fmtname, start; options ibufsize=&ibufsize; @@ -212,13 +233,13 @@ options ibufsize=&ibufsize; &outds_add(in=add) &outds_del(in=del); if not del and not mod; - by fmtname start; + by type fmtname start; run; data &stagedata; set &ds1 &outds_mod; run; proc sort; - by fmtname start; + by type fmtname start; run; %end; /* mp abort needs to run outside of conditional blocks */ @@ -266,7 +287,7 @@ options ibufsize=&ibufsize; %mp_storediffs(&libcat-FC ,&base_fmts - ,FMTNAME START + ,TYPE FMTNAME START ,delds=&outds_del ,modds=&outds_mod ,appds=&outds_add diff --git a/ddl/mddl_sas_cntlout.sas b/ddl/mddl_sas_cntlout.sas index 3b4ffc4..18ce174 100644 --- a/ddl/mddl_sas_cntlout.sas +++ b/ddl/mddl_sas_cntlout.sas @@ -12,14 +12,20 @@ proc sql; create table &libds( - FMTNAME char(32) label='Format name' + TYPE char(1) label='Type of format' + ,FMTNAME char(32) label='Format name' /* to accommodate larger START values, mp_loadformat.sas will need the SQL dependency removed (proc sql needs to accommodate 3 index values in a 32767 ibufsize limit) */ ,START char(10000) label='Starting value for format' - ,END char(32767) label='Ending value for format' + /* + Keep lengths of START and END the same to avoid this err: + "Start is greater than end: -<." + Similar usage note: https://support.sas.com/kb/69/330.html + */ + ,END char(10000) label='Ending value for format' ,LABEL char(32767) label='Format value label' ,MIN num length=3 label='Minimum length' ,MAX num length=3 label='Maximum length' @@ -30,7 +36,6 @@ create table &libds( ,MULT num label='Multiplier' ,FILL char(1) label='Fill character' ,NOEDIT num length=3 label='Is picture string noedit?' - ,TYPE char(1) label='Type of format' ,SEXCL char(1) label='Start exclusion' ,EEXCL char(1) label='End exclusion' ,HLO char(13) label='Additional information' diff --git a/tests/base/mp_loadformat.test.sas b/tests/base/mp_loadformat.test.sas index 9c9be6d..9925784 100644 --- a/tests/base/mp_loadformat.test.sas +++ b/tests/base/mp_loadformat.test.sas @@ -1,9 +1,11 @@ /** @file @brief Testing mp_loadformat.sas macro + @details first test regular formats, then informats

SAS Macros

@li mddl_dc_difftable.sas + @li mp_aligndecimal.sas @li mp_loadformat.sas @li mp_assert.sas @li mp_assertscope.sas @@ -15,8 +17,10 @@ libname perm (work); %mddl_dc_difftable(libds=perm.audit) +/* set up regular formats */ data work.loadfmts; - length fmtname $32; + /* matching start / end lengths (to baseds) are important */ + length fmtname $32 start end $10000; eexcl='Y'; type='N'; do i=1 to 100; @@ -24,7 +28,9 @@ data work.loadfmts; do j=1 to 100; start=cats(j); end=cats(j+1); - label= cats('Dummy ',start); + %mp_aligndecimal(start,width=16) + %mp_aligndecimal(end,width=16) + label= cats('Numeric Format ',start); output; end; end; @@ -42,6 +48,8 @@ data work.stagedata; else if _n_<350 then do; start=cats(_n_); end=cats(_n_+1); + %mp_aligndecimal(start,width=16) + %mp_aligndecimal(end,width=16) label='newval'!!cats(_N_); end; else stop; @@ -91,3 +99,118 @@ run; desc=Test 1 - diffs were found, outds=work.test_results ) + +/* set up a mix of formats */ +data work.loadfmts2; + length fmtname $32 start end $10000; + eexcl='Y'; + type='J'; + do i=1 to 3; + fmtname=cats('SASJS_CI_',i,'X'); + do j=1 to 4; + start=cats(j); + end=start; + label= cats('Char INFORMAT ',start); + output; + end; + end; + type='I'; + do i=1 to 3; + fmtname=cats('SASJS_NI_',i,'X'); + do j=1 to 4; + start=cats(j); + end=cats(j+1); + %mp_aligndecimal(start,width=16) + %mp_aligndecimal(end,width=16) + label= cats(ranuni(0)); + output; + end; + end; + type='N'; + do i=1 to 3; + fmtname=cats('SASJS_NF_',i,'X'); + do j=1 to 4; + start=cats(j); + end=cats(j+1); + %mp_aligndecimal(start,width=16) + %mp_aligndecimal(end,width=16) + label= cats('Numeric Format ',start); + output; + end; + end; + type='C'; + do i=1 to 3; + fmtname=cats('SASJS_CF_',i,'X'); + do j=1 to 4; + start=cats(j); + end=start; + label= cats('Char Format ',start); + output; + end; + end; + drop i j; +run; +proc format cntlin=work.loadfmts2 library=perm.testcat2; +run; + +/* make some test data */ +data work.stagedata2; + set work.loadfmts2; + where type in ('I','J'); + eexcl='Y'; + if type='I' then do; + i+1; + if i<3 then deleteme='Yes'; + else if i<7 then label= cats(ranuni(0)*100); + else if i<12 then do; + /* new values */ + z=ranuni(0)*1000000; + start=cats(z); + end=cats(z+1); + %mp_aligndecimal(start,width=16) + %mp_aligndecimal(end,width=16) + label= cats(ranuni(0)*100); + end; + if i<12 then output; + end; + else do; + j+1; + if j<3 then deleteme='Yes'; + else if j<7 then label= cats(ranuni(0)*100); + else if j<12 then do; + start= cats("NEWVAL",start); + end=start; + label= "NEWVAL "||cats(ranuni(0)*100); + end; + if j<12 then output; + end; + +run; + +%mp_loadformat(perm.testcat2 + ,work.stagedata2 + ,loadtarget=YES + ,auditlibds=perm.audit + ,locklibds=0 + ,delete_col=deleteme + ,outds_add=add_test2 + ,outds_del=del_test2 + ,outds_mod=mod_test2 + ,mdebug=1 +) + +%mp_assert( + iftrue=(%mf_nobs(del_test2)=4), + desc=Test 2 - delete obs, + outds=work.test_results +) +%mp_assert( + iftrue=(%mf_nobs(mod_test2)=8), + desc=Test 2 - mod obs, + outds=work.test_results +) +%mp_assert( + iftrue=(%mf_nobs(add_test2)=10), + desc=Test 2 - add obs, + outds=work.test_results +)