sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2025-12-10 14:04:36 +00:00
Code Issues Projects Releases Wiki Activity

feat: new mp_cntlout.sas macro

Browse Source
This commit is contained in:
munja
2022-02-11 14:22:38 +01:00
parent 53aa403630
commit 6ba3588eff
7 changed files with 172 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
base/mp_cntlout.sas Normal file
View File

@@ -0,0 +1,85 @@
/**
@file mp_cntlout.sas
@brief Creates a cntlout dataset in a consistent format
@details The dataset produced by proc format in the cntlout option will vary
according to its contents.
When dealing with formats from an ETL perspective (eg in [Data Controller for
SAS](https://datacontroller.io)), it is important that the output dataset
has a consistent model (and compariable values).
This macro makes use of mddl_sas_cntlout.sas to provide the consistent model,
and will left-align the start and end values when dealing with numeric ranges
to enable consistency when checking for differences.
usage:
%mp_cntlout(libcat=yourlib.cat,cntlout=work.formatexport)
@param [in] libcat The library.catalog reference
@param [in] fmtlist= (0) provide a space separated list of specific formats to
extract
@param [in] iftrue= (1=1) A condition under which the macro should be executed
@param [out] cntlout= (work.fmtextract) Libds reference for the output dataset
<h4> SAS Macros </h4>
@li mddl_sas_cntlout.sas
@li mf_getuniquename.sas
<h4> Related Macros </h4>
@li mf_getvarformat.sas
@li mp_getformats.sas
@li mp_loadformat.sas
@li mp_ds2fmtds.sas
@version 9.2
@author Allan Bowe
@cond
**/
%macro mp_cntlout(
iftrue=(1=1)
,libcat=
,cntlout=work.fmtextract
,fmtlist=0
)/*/STORE SOURCE*/;
%local ddlds cntlds i;
%if not(%eval(%unquote(&iftrue))) %then %return;
%let ddlds=%mf_getuniquename();
%let cntlds=%mf_getuniquename();
%mddl_sas_cntlout(libds=&ddlds)
%if %index(&libcat,-)>0 and %scan(&libcat,2,-)=FC %then %do;
%let libcat=%scan(&libcat,1,-);
%end;
proc format lib=&libcat cntlout=&cntlds;
%if "&fmtlist" ne "0" %then %do;
select
%do i=1 %to %sysfunc(countw(&fmtlist));
%scan(&fmtlist,&i,%str( ))
%end;
;
%end;
run;
data &cntlout;
if 0 then set &ddlds;
set &cntlds;
if type="N" then do;
start=cats(start);
end=cats(end);
end;
run;
proc sort;
by fmtname start;
run;
proc sql;
drop table &ddlds,&cntlds;
%mend mp_cntlout;
/** @endcond */

63
base/mp_loadformat.sas
View File

@@ -38,6 +38,7 @@
@li mf_getuniquename.sas
@li mf_nobs.sas
@li mp_abort.sas
@li mp_cntlout.sas
@li mp_lockanytable.sas
<h4> Related Macros </h4>
@@ -66,17 +67,26 @@
);
/* set up local macro variables and temporary tables (with a prefix) */
%local err msg prefix dslist i var fmtlist ibufsize;
%let dslist=base base_fmts template inlibds ds1 stagedata storediffs;
%let dslist=base_fmts template inlibds ds1 stagedata storediffs;
%if &outds_add=0 %then %let dslist=&dslist outds_add;
%if &outds_del=0 %then %let dslist=&dslist outds_del;
%if &outds_mod=0 %then %let dslist=&dslist outds_mod;
%let prefix=%substr(%mf_getuniquename(),1,22);
%let prefix=%substr(%mf_getuniquename(),1,21);
%do i=1 %to %sysfunc(countw(&dslist));
%let var=%scan(&dslist,&i);
%local &var;
%let &var=%upcase(&prefix._&var);
%end;
/*
format values can be up to 32767 wide. SQL joins on such a wide column can
cause buffer issues. Update ibufsize and reset at the end.
*/
%let ibufsize=%sysfunc(getoption(ibufsize));
options ibufsize=32767 ;
/* in DC, format catalogs maybe specified in the libds with a -FC extension */
%let libcat=%scan(&libcat,1,-);
/* perform input validations */
%let err=0;
@@ -125,17 +135,9 @@ run;
*/
proc sql noprint;
select distinct fmtname into: fmtlist separated by ' ' from &libds;
proc format lib=&libcat cntlout=&base;
select
/* send formats individually to avoid line truncation in the input stack */
%do i=1 %to %sysfunc(countw(&fmtlist));
%scan(&fmtlist,&i,%str( ))
%end;
;
run;
proc sort data=&base;
by fmtname start;
run;
%mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
/**
* Ensure input table and base_formats have consistent lengths and types
@@ -148,19 +150,12 @@ data &inlibds;
if substr(fmtname,1,1)='$' then type='C';
else type='N';
end;
if type='N' then start=put(input(start,best.),best16.);
run;
data &base_fmts;
if 0 then set &template;
set &base;
if type='N' then do;
start=cats(start);
end=cats(end);
end;
run;
/*
format values can be up to 32767 wide. SQL joins on such a wide column can
cause buffer issues. Update ibufsize and reset at the end.
*/
%let ibufsize=%sysfunc(getoption(ibufsize));
options ibufsize=32767 ;
/**
* Identify new records
@@ -237,9 +232,9 @@ options ibufsize=&ibufsize;
%end;
%if &locklibds ne 0 %then %do;
/* prevent parallel updates */
%mp_lockanytable(LOCK,
lib=%scan(&libcat,1,.)
,ds=%scan(&libcat,2,.)
%mp_lockanytable(LOCK
,lib=%scan(&libcat,1,.)
,ds=%scan(&libcat,2,.)-FC
,ref=MP_LOADFORMAT commencing format load
,ctl_ds=&locklibds
)
@@ -250,8 +245,8 @@ options ibufsize=&ibufsize;
%if &locklibds ne 0 %then %do;
/* unlock the table */
%mp_lockanytable(UNLOCK
lib=%scan(&libcat,1,.)
,ds=%scan(&libcat,2,.)
,lib=%scan(&libcat,1,.)
,ds=%scan(&libcat,2,.)-FC
,ref=MP_LOADFORMAT completed format load
,ctl_ds=&locklibds
)
@@ -259,16 +254,16 @@ options ibufsize=&ibufsize;
/* track the changes */
%if &auditlibds ne 0 %then %do;
%if &locklibds ne 0 %then %do;
%mp_lockanytable(LOCK,
lib=%scan(&auditlibds,1,.)
%mp_lockanytable(LOCK
,lib=%scan(&auditlibds,1,.)
,ds=%scan(&auditlibds,2,.)
,ref=MP_LOADFORMAT commencing audit table load
,ctl_ds=&locklibds
)
%end;
%mp_storediffs(&libcat
,&stageds
%mp_storediffs(&libcat-FC
,&inlibds
,FMTNAME START
,delds=&outds_del
,modds=&outds_mod
@@ -279,7 +274,7 @@ options ibufsize=&ibufsize;
%if &locklibds ne 0 %then %do;
%mp_lockanytable(UNLOCK
lib=%scan(&auditlibds,1,.)
,lib=%scan(&auditlibds,1,.)
,ds=%scan(&auditlibds,2,.)
,ref=MP_LOADFORMAT commencing audit table load
,ctl_ds=&locklibds

2
base/mp_lockanytable.sas
View File

@@ -48,7 +48,7 @@ data _null_;
put name '=' value;
run;
%mp_abort(iftrue= (&ds=0 and &action ne MAKETABLE)
%mp_abort(iftrue= ("&ds"="0" and &action ne MAKETABLE)
,mac=&sysmacroname
,msg=%str(dataset was not provided)
)

8
base/mp_lockfilecheck.sas
View File

@@ -37,7 +37,7 @@ run;
,mac=checklock.sas
,msg=Aborting with syscc=&syscc on entry.
)
%mp_abort(iftrue= (&libds=0)
%mp_abort(iftrue= ("&libds"="0")
,mac=&sysmacroname
,msg=%str(libds not provided)
)
@@ -46,6 +46,12 @@ run;
%let lib=%upcase(%scan(&libds,1,.));
%let ds=%upcase(%scan(&libds,2,.));
/* in DC, format catalogs are passed with a -FC suffix. No saslock here! */
%if %scan(&libds,2,-)=FC %then %do;
%put &sysmacroname: Format Catalog detected, no lockfile applied to &libds;
%return;
%end;
/* do not proceed if no observations can be processed */
%let msg=options obs = 0. syserrortext=%superq(syserrortext);
%mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)

13
base/mp_storediffs.sas
View File

@@ -90,7 +90,7 @@
%else %let dbg=*;
/* set up unique and temporary vars */
%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist;
%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist vlist;
%let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_ds1));
%let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_ds2));
%let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_ds3));
@@ -144,12 +144,21 @@ proc transpose data=&ds1
by &inds_keep &hashkey;
var _character_;
run;
%if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do;
/* this is a format catalog - cannot query cols directly */
%let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH"
,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO"
,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
%end;
%else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
data &ds4;
length &inds_keep $41 tgtvar_nm $32;
set &ds2 &ds3 indsname=&inds_auto;
tgtvar_nm=upcase(tgtvar_nm);
if tgtvar_nm in (%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE)));
if tgtvar_nm in (%upcase(&vlist));
if &inds_auto="&ds2" then tgtvar_type='N';
else if &inds_auto="&ds3" then tgtvar_type='C';

38
tests/crossplatform/mp_cntlout.test.sas Normal file
View File

@@ -0,0 +1,38 @@
/**
@file
@brief Testing mp_cntlout.sas macro
<h4> SAS Macros </h4>
@li mp_cntlout.sas
@li mp_assert.sas
@li mp_assertscope.sas
**/
libname perm (work);
data work.loadfmts;
length fmtname $32;
eexcl='Y';
type='N';
do i=1 to 100;
fmtname=cats('SASJS_',i,'X');
do j=1 to 100;
start=cats(j);
end=cats(j+1);
label= cats('Dummy ',start);
output;
end;
end;
run;
proc format cntlin=work.loadfmts library=perm.testcat;
run;
%mp_assertscope(SNAPSHOT)
%mp_cntlout(libcat=perm.testcat,cntlout=work.cntlout)
%mp_assertscope(COMPARE)
%mp_assert(
iftrue=(%mf_nobs(work.cntlout)=10000),
desc=Checking first hash diff,
outds=work.test_results
)

1
tests/crossplatform/mp_stackdiffs.test.sas
View File

@@ -42,6 +42,7 @@ run;
,mdebug=1
)
%mp_assertscope(SNAPSHOT)
/**