From 6ba3588eff4e309f2942597236d9b75ff67a78f0 Mon Sep 17 00:00:00 2001 From: munja Date: Fri, 11 Feb 2022 14:22:38 +0100 Subject: [PATCH] feat: new mp_cntlout.sas macro --- base/mp_cntlout.sas | 85 ++++++++++++++++++++++ base/mp_loadformat.sas | 63 ++++++++-------- base/mp_lockanytable.sas | 2 +- base/mp_lockfilecheck.sas | 8 +- base/mp_storediffs.sas | 13 +++- tests/crossplatform/mp_cntlout.test.sas | 38 ++++++++++ tests/crossplatform/mp_stackdiffs.test.sas | 1 + 7 files changed, 172 insertions(+), 38 deletions(-) create mode 100644 base/mp_cntlout.sas create mode 100644 tests/crossplatform/mp_cntlout.test.sas diff --git a/base/mp_cntlout.sas b/base/mp_cntlout.sas new file mode 100644 index 0000000..357b630 --- /dev/null +++ b/base/mp_cntlout.sas @@ -0,0 +1,85 @@ +/** + @file mp_cntlout.sas + @brief Creates a cntlout dataset in a consistent format + @details The dataset produced by proc format in the cntlout option will vary + according to its contents. + + When dealing with formats from an ETL perspective (eg in [Data Controller for + SAS](https://datacontroller.io)), it is important that the output dataset + has a consistent model (and compariable values). + + This macro makes use of mddl_sas_cntlout.sas to provide the consistent model, + and will left-align the start and end values when dealing with numeric ranges + to enable consistency when checking for differences. + + usage: + + %mp_cntlout(libcat=yourlib.cat,cntlout=work.formatexport) + + @param [in] libcat The library.catalog reference + @param [in] fmtlist= (0) provide a space separated list of specific formats to + extract + @param [in] iftrue= (1=1) A condition under which the macro should be executed + @param [out] cntlout= (work.fmtextract) Libds reference for the output dataset + +

SAS Macros

+ @li mddl_sas_cntlout.sas + @li mf_getuniquename.sas + +

Related Macros

+ @li mf_getvarformat.sas + @li mp_getformats.sas + @li mp_loadformat.sas + @li mp_ds2fmtds.sas + + @version 9.2 + @author Allan Bowe + @cond +**/ + +%macro mp_cntlout( + iftrue=(1=1) + ,libcat= + ,cntlout=work.fmtextract + ,fmtlist=0 +)/*/STORE SOURCE*/; +%local ddlds cntlds i; + +%if not(%eval(%unquote(&iftrue))) %then %return; + +%let ddlds=%mf_getuniquename(); +%let cntlds=%mf_getuniquename(); + +%mddl_sas_cntlout(libds=&ddlds) + +%if %index(&libcat,-)>0 and %scan(&libcat,2,-)=FC %then %do; + %let libcat=%scan(&libcat,1,-); +%end; + +proc format lib=&libcat cntlout=&cntlds; +%if "&fmtlist" ne "0" %then %do; + select + %do i=1 %to %sysfunc(countw(&fmtlist)); + %scan(&fmtlist,&i,%str( )) + %end; + ; +%end; +run; + +data &cntlout; + if 0 then set &ddlds; + set &cntlds; + if type="N" then do; + start=cats(start); + end=cats(end); + end; +run; +proc sort; + by fmtname start; +run; + +proc sql; +drop table &ddlds,&cntlds; + +%mend mp_cntlout; +/** @endcond */ \ No newline at end of file diff --git a/base/mp_loadformat.sas b/base/mp_loadformat.sas index 0d6dc3f..6d6e308 100644 --- a/base/mp_loadformat.sas +++ b/base/mp_loadformat.sas @@ -38,6 +38,7 @@ @li mf_getuniquename.sas @li mf_nobs.sas @li mp_abort.sas + @li mp_cntlout.sas @li mp_lockanytable.sas

Related Macros

@@ -66,17 +67,26 @@ ); /* set up local macro variables and temporary tables (with a prefix) */ %local err msg prefix dslist i var fmtlist ibufsize; -%let dslist=base base_fmts template inlibds ds1 stagedata storediffs; +%let dslist=base_fmts template inlibds ds1 stagedata storediffs; %if &outds_add=0 %then %let dslist=&dslist outds_add; %if &outds_del=0 %then %let dslist=&dslist outds_del; %if &outds_mod=0 %then %let dslist=&dslist outds_mod; -%let prefix=%substr(%mf_getuniquename(),1,22); +%let prefix=%substr(%mf_getuniquename(),1,21); %do i=1 %to %sysfunc(countw(&dslist)); %let var=%scan(&dslist,&i); %local &var; %let &var=%upcase(&prefix._&var); %end; +/* +format values can be up to 32767 wide. SQL joins on such a wide column can +cause buffer issues. Update ibufsize and reset at the end. +*/ +%let ibufsize=%sysfunc(getoption(ibufsize)); +options ibufsize=32767 ; + +/* in DC, format catalogs maybe specified in the libds with a -FC extension */ +%let libcat=%scan(&libcat,1,-); /* perform input validations */ %let err=0; @@ -125,17 +135,9 @@ run; */ proc sql noprint; select distinct fmtname into: fmtlist separated by ' ' from &libds; -proc format lib=&libcat cntlout=&base; - select - /* send formats individually to avoid line truncation in the input stack */ - %do i=1 %to %sysfunc(countw(&fmtlist)); - %scan(&fmtlist,&i,%str( )) - %end; - ; -run; -proc sort data=&base; - by fmtname start; -run; + +%mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts) + /** * Ensure input table and base_formats have consistent lengths and types @@ -148,19 +150,12 @@ data &inlibds; if substr(fmtname,1,1)='$' then type='C'; else type='N'; end; - if type='N' then start=put(input(start,best.),best16.); -run; -data &base_fmts; - if 0 then set &template; - set &base; + if type='N' then do; + start=cats(start); + end=cats(end); + end; run; -/* -format values can be up to 32767 wide. SQL joins on such a wide column can -cause buffer issues. Update ibufsize and reset at the end. -*/ -%let ibufsize=%sysfunc(getoption(ibufsize)); -options ibufsize=32767 ; /** * Identify new records @@ -237,9 +232,9 @@ options ibufsize=&ibufsize; %end; %if &locklibds ne 0 %then %do; /* prevent parallel updates */ - %mp_lockanytable(LOCK, - lib=%scan(&libcat,1,.) - ,ds=%scan(&libcat,2,.) + %mp_lockanytable(LOCK + ,lib=%scan(&libcat,1,.) + ,ds=%scan(&libcat,2,.)-FC ,ref=MP_LOADFORMAT commencing format load ,ctl_ds=&locklibds ) @@ -250,8 +245,8 @@ options ibufsize=&ibufsize; %if &locklibds ne 0 %then %do; /* unlock the table */ %mp_lockanytable(UNLOCK - lib=%scan(&libcat,1,.) - ,ds=%scan(&libcat,2,.) + ,lib=%scan(&libcat,1,.) + ,ds=%scan(&libcat,2,.)-FC ,ref=MP_LOADFORMAT completed format load ,ctl_ds=&locklibds ) @@ -259,16 +254,16 @@ options ibufsize=&ibufsize; /* track the changes */ %if &auditlibds ne 0 %then %do; %if &locklibds ne 0 %then %do; - %mp_lockanytable(LOCK, - lib=%scan(&auditlibds,1,.) + %mp_lockanytable(LOCK + ,lib=%scan(&auditlibds,1,.) ,ds=%scan(&auditlibds,2,.) ,ref=MP_LOADFORMAT commencing audit table load ,ctl_ds=&locklibds ) %end; - %mp_storediffs(&libcat - ,&stageds + %mp_storediffs(&libcat-FC + ,&inlibds ,FMTNAME START ,delds=&outds_del ,modds=&outds_mod @@ -279,7 +274,7 @@ options ibufsize=&ibufsize; %if &locklibds ne 0 %then %do; %mp_lockanytable(UNLOCK - lib=%scan(&auditlibds,1,.) + ,lib=%scan(&auditlibds,1,.) ,ds=%scan(&auditlibds,2,.) ,ref=MP_LOADFORMAT commencing audit table load ,ctl_ds=&locklibds diff --git a/base/mp_lockanytable.sas b/base/mp_lockanytable.sas index abae924..a47e09b 100644 --- a/base/mp_lockanytable.sas +++ b/base/mp_lockanytable.sas @@ -48,7 +48,7 @@ data _null_; put name '=' value; run; -%mp_abort(iftrue= (&ds=0 and &action ne MAKETABLE) +%mp_abort(iftrue= ("&ds"="0" and &action ne MAKETABLE) ,mac=&sysmacroname ,msg=%str(dataset was not provided) ) diff --git a/base/mp_lockfilecheck.sas b/base/mp_lockfilecheck.sas index 0a2ae82..ee0997d 100644 --- a/base/mp_lockfilecheck.sas +++ b/base/mp_lockfilecheck.sas @@ -37,7 +37,7 @@ run; ,mac=checklock.sas ,msg=Aborting with syscc=&syscc on entry. ) -%mp_abort(iftrue= (&libds=0) +%mp_abort(iftrue= ("&libds"="0") ,mac=&sysmacroname ,msg=%str(libds not provided) ) @@ -46,6 +46,12 @@ run; %let lib=%upcase(%scan(&libds,1,.)); %let ds=%upcase(%scan(&libds,2,.)); +/* in DC, format catalogs are passed with a -FC suffix. No saslock here! */ +%if %scan(&libds,2,-)=FC %then %do; + %put &sysmacroname: Format Catalog detected, no lockfile applied to &libds; + %return; +%end; + /* do not proceed if no observations can be processed */ %let msg=options obs = 0. syserrortext=%superq(syserrortext); %mp_abort(iftrue= (%sysfunc(getoption(OBS))=0) diff --git a/base/mp_storediffs.sas b/base/mp_storediffs.sas index f5ddd4d..878cb55 100644 --- a/base/mp_storediffs.sas +++ b/base/mp_storediffs.sas @@ -90,7 +90,7 @@ %else %let dbg=*; /* set up unique and temporary vars */ -%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist; +%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist vlist; %let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_ds1)); %let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_ds2)); %let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_ds3)); @@ -144,12 +144,21 @@ proc transpose data=&ds1 by &inds_keep &hashkey; var _character_; run; + +%if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do; + /* this is a format catalog - cannot query cols directly */ + %let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH" + ,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO" + ,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE"; +%end; +%else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE); + data &ds4; length &inds_keep $41 tgtvar_nm $32; set &ds2 &ds3 indsname=&inds_auto; tgtvar_nm=upcase(tgtvar_nm); - if tgtvar_nm in (%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE))); + if tgtvar_nm in (%upcase(&vlist)); if &inds_auto="&ds2" then tgtvar_type='N'; else if &inds_auto="&ds3" then tgtvar_type='C'; diff --git a/tests/crossplatform/mp_cntlout.test.sas b/tests/crossplatform/mp_cntlout.test.sas new file mode 100644 index 0000000..bf91aaa --- /dev/null +++ b/tests/crossplatform/mp_cntlout.test.sas @@ -0,0 +1,38 @@ +/** + @file + @brief Testing mp_cntlout.sas macro + +

SAS Macros

+ @li mp_cntlout.sas + @li mp_assert.sas + @li mp_assertscope.sas + +**/ + +libname perm (work); +data work.loadfmts; + length fmtname $32; + eexcl='Y'; + type='N'; + do i=1 to 100; + fmtname=cats('SASJS_',i,'X'); + do j=1 to 100; + start=cats(j); + end=cats(j+1); + label= cats('Dummy ',start); + output; + end; + end; +run; +proc format cntlin=work.loadfmts library=perm.testcat; +run; + +%mp_assertscope(SNAPSHOT) +%mp_cntlout(libcat=perm.testcat,cntlout=work.cntlout) +%mp_assertscope(COMPARE) + +%mp_assert( + iftrue=(%mf_nobs(work.cntlout)=10000), + desc=Checking first hash diff, + outds=work.test_results +) diff --git a/tests/crossplatform/mp_stackdiffs.test.sas b/tests/crossplatform/mp_stackdiffs.test.sas index fd9a32e..82989e5 100644 --- a/tests/crossplatform/mp_stackdiffs.test.sas +++ b/tests/crossplatform/mp_stackdiffs.test.sas @@ -42,6 +42,7 @@ run; ,mdebug=1 ) + %mp_assertscope(SNAPSHOT) /**