sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2025-12-10 22:14:35 +00:00
Code Issues Projects Releases Wiki Activity

fix: adding support for multilabel and notsorted formats

Browse Source 
included additional test job covering multiple scenarios.  Closes #337
This commit is contained in:
Allan
2023-06-21 16:41:46 +01:00
parent b8cec22a88
commit 4f481ec8b4
8 changed files with 328 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
all.sas
View File

@@ -4161,16 +4161,24 @@ proc format lib=&libcat cntlout=&cntlds;
%end; %end;
run; run;
data &cntlout; data &cntlout/nonote2err;
if 0 then set &ddlds; if 0 then set &ddlds;
set &cntlds; set &cntlds;
if type in ("I","N") then do; /* numeric (in)format */ by type fmtname notsorted;
/* align the numeric values to avoid overlapping ranges */
if type in ("I","N") then do;
%mp_aligndecimal(start,width=16) %mp_aligndecimal(start,width=16)
%mp_aligndecimal(end,width=16) %mp_aligndecimal(end,width=16)
end; end;
/* create row marker. Data cannot be sorted without it! */
if first.fmtname then fmtrow=0;
fmtrow+1;
run; run;
proc sort; proc sort;
by type fmtname start; by type fmtname fmtrow;
run; run;
proc sql; proc sql;
@@ -6014,6 +6022,7 @@ run;
options ps=max lrecl=max; options ps=max lrecl=max;
data _null_; data _null_;
infile &outref; infile &outref;
if _n_=1 then putlog "# &libds" /;
input; input;
putlog _infile_; putlog _infile_;
run; run;
@@ -10119,19 +10128,20 @@ select distinct lowcase(memname)
@param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
<h4> SAS Macros </h4> <h4> SAS Macros </h4>
@li mddl_sas_cntlout.sas
@li mf_getuniquename.sas @li mf_getuniquename.sas
@li mf_nobs.sas @li mf_nobs.sas
@li mp_abort.sas @li mp_abort.sas
@li mp_aligndecimal.sas @li mp_aligndecimal.sas
@li mp_cntlout.sas @li mp_cntlout.sas
@li mp_lockanytable.sas @li mp_lockanytable.sas
@li mp_md5.sas
@li mp_storediffs.sas @li mp_storediffs.sas
<h4> Related Macros </h4> <h4> Related Macros </h4>
@li mddl_dc_difftable.sas @li mddl_dc_difftable.sas
@li mddl_dc_locktable.sas @li mddl_dc_locktable.sas
@li mp_loadformat.test.sas @li mp_loadformat.test.1.sas
@li mp_loadformat.test.2.sas
@li mp_lockanytable.sas @li mp_lockanytable.sas
@li mp_stackdiffs.sas @li mp_stackdiffs.sas
@@ -10164,13 +10174,6 @@ select distinct lowcase(memname)
%let &var=%upcase(&prefix._&var); %let &var=%upcase(&prefix._&var);
%end; %end;
/*
format values can be up to 32767 wide. SQL joins on such a wide column can
cause buffer issues. Update ibufsize and reset at the end.
*/
%let ibufsize=%sysfunc(getoption(ibufsize));
options ibufsize=32767 ;
/* in DC, format catalogs maybe specified in the libds with a -FC extension */ /* in DC, format catalogs maybe specified in the libds with a -FC extension */
%let libcat=%scan(&libcat,1,-); %let libcat=%scan(&libcat,1,-);
@@ -10233,16 +10236,24 @@ select distinct
%mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts) %mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
/* get a hash of the row */
%local cvars nvars;
%let cvars=TYPE FMTNAME START END LABEL PREFIX FILL SEXCL EEXCL HLO DECSEP
DIG3SEP DATATYPE LANGUAGE;
%let nvars=FMTROW MIN MAX DEFAULT LENGTH FUZZ MULT NOEDIT;
data &base_fmts/note2err;
set &base_fmts;
fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
run;
/** /**
* Ensure input table and base_formats have consistent lengths and types * Ensure input table and base_formats have consistent lengths and types
*/ */
%mddl_sas_cntlout(libds=&template) data &inlibds/nonote2err;
data &inlibds; length &delete_col $3 FMTROW 8 start end label $32767;
length &delete_col $3; if 0 then set &base_fmts;
if 0 then set &template;
length start end $10000;
set &libds; set &libds;
by type fmtname notsorted;
if &delete_col='' then &delete_col='No'; if &delete_col='' then &delete_col='No';
fmtname=upcase(fmtname); fmtname=upcase(fmtname);
type=upcase(type); type=upcase(type);
@@ -10260,6 +10271,14 @@ data &inlibds;
%mp_aligndecimal(start,width=16) %mp_aligndecimal(start,width=16)
%mp_aligndecimal(end,width=16) %mp_aligndecimal(end,width=16)
end; end;
/* update row marker - retain new var as fmtrow may already be in libds */
if first.fmtname then row=1;
else row+1;
drop row;
fmtrow=row;
fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
run; run;
/** /**
@@ -10270,12 +10289,10 @@ create table &outds_add(drop=&delete_col) as
select a.* select a.*
from &inlibds a from &inlibds a
left join &base_fmts b left join &base_fmts b
on a.fmtname=b.fmtname on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
and a.start=b.start
and a.type=b.type
where b.fmtname is null where b.fmtname is null
and upcase(a.&delete_col) ne "YES" and upcase(a.&delete_col) ne "YES"
order by type, fmtname, start; order by type, fmtname, fmtrow;
/** /**
* Identify deleted records * Identify deleted records
@@ -10284,11 +10301,9 @@ create table &outds_del(drop=&delete_col) as
select a.* select a.*
from &inlibds a from &inlibds a
inner join &base_fmts b inner join &base_fmts b
on a.fmtname=b.fmtname on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
and a.start=b.start
and a.type=b.type
where upcase(a.&delete_col)="YES" where upcase(a.&delete_col)="YES"
order by type, fmtname, start; order by type, fmtname, fmtrow;
/** /**
* Identify modified records * Identify modified records
@@ -10297,13 +10312,10 @@ create table &outds_mod (drop=&delete_col) as
select a.* select a.*
from &inlibds a from &inlibds a
inner join &base_fmts b inner join &base_fmts b
on a.fmtname=b.fmtname on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
and a.start=b.start
and a.type=b.type
where upcase(a.&delete_col) ne "YES" where upcase(a.&delete_col) ne "YES"
order by type, fmtname, start; and a.fmthash ne b.fmthash
order by type, fmtname, fmtrow;
options ibufsize=&ibufsize;
%mp_abort( %mp_abort(
iftrue=(&syscc ne 0) iftrue=(&syscc ne 0)
@@ -10312,19 +10324,21 @@ options ibufsize=&ibufsize;
) )
%if &loadtarget=YES %then %do; %if &loadtarget=YES %then %do;
/* new records plus base records that are not deleted or modified */
data &ds1; data &ds1;
merge &base_fmts(in=base) merge &base_fmts(in=base)
&outds_mod(in=mod) &outds_mod(in=mod)
&outds_add(in=add) &outds_add(in=add)
&outds_del(in=del); &outds_del(in=del);
if not del and not mod; if not del and not mod;
by type fmtname start; by type fmtname fmtrow;
run; run;
/* add back the modified records */
data &stagedata; data &stagedata;
set &ds1 &outds_mod; set &ds1 &outds_mod;
run; run;
proc sort; proc sort;
by type fmtname start; by type fmtname fmtrow;
run; run;
%end; %end;
/* mp abort needs to run outside of conditional blocks */ /* mp abort needs to run outside of conditional blocks */
@@ -10372,7 +10386,7 @@ options ibufsize=&ibufsize;
%mp_storediffs(&libcat-FC %mp_storediffs(&libcat-FC
,&base_fmts ,&base_fmts
,TYPE FMTNAME START ,TYPE FMTNAME FMTROW
,delds=&outds_del ,delds=&outds_del
,modds=&outds_mod ,modds=&outds_mod
,appds=&outds_add ,appds=&outds_add
@@ -12796,9 +12810,9 @@ run;
%if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do; %if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do;
/* this is a format catalog - cannot query cols directly */ /* this is a format catalog - cannot query cols directly */
%let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH" %let vlist="TYPE","FMTNAME","FMTROW","START","END","LABEL","MIN","MAX"
,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO" ,"DEFAULT","LENGTH","FUZZ","PREFIX","MULT","FILL","NOEDIT","SEXCL"
,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE"; ,"EEXCL","HLO","DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
%end; %end;
%else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE); %else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
@@ -14076,22 +14090,18 @@ ods package close;
%macro mddl_sas_cntlout(libds=WORK.CNTLOUT); %macro mddl_sas_cntlout(libds=WORK.CNTLOUT);
proc sql; proc sql;
create table &libds( create table &libds(
TYPE char(1) label='Type of format' TYPE char(1) label='Type of format - either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
,FMTNAME char(32) label='Format name' ,FMTNAME char(32) label='Format name'
/* ,FMTROW num label='CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
to accommodate larger START values, mp_loadformat.sas will need the ,START char(32767) label='Starting value for format'
SQL dependency removed (proc sql needs to accommodate 3 index values in
a 32767 ibufsize limit)
*/
,START char(10000) label='Starting value for format'
/* /*
Keep lengths of START and END the same to avoid this err: Keep lengths of START and END the same to avoid this err:
"Start is greater than end: -<." "Start is greater than end: -<."
Similar usage note: https://support.sas.com/kb/69/330.html Similar usage note: https://support.sas.com/kb/69/330.html
*/ */
,END char(10000) label='Ending value for format' ,END char(32767) label='Ending value for format'
,LABEL char(32767) label='Format value label' ,LABEL char(32767) label='Format value label'
,MIN num length=3 label='Minimum length' ,MIN num length=3 label='Minimum length'
,MAX num length=3 label='Maximum length' ,MAX num length=3 label='Maximum length'
@@ -14104,12 +14114,24 @@ create table &libds(
,NOEDIT num length=3 label='Is picture string noedit?' ,NOEDIT num length=3 label='Is picture string noedit?'
,SEXCL char(1) label='Start exclusion' ,SEXCL char(1) label='Start exclusion'
,EEXCL char(1) label='End exclusion' ,EEXCL char(1) label='End exclusion'
,HLO char(13) label='Additional information' ,HLO char(13) label='Additional information. M=MultiLabel'
,DECSEP char(1) label='Decimal separator' ,DECSEP char(1) label='Decimal separator'
,DIG3SEP char(1) label='Three-digit separator' ,DIG3SEP char(1) label='Three-digit separator'
,DATATYPE char(8) label='Date/time/datetime?' ,DATATYPE char(8) label='Date/time/datetime?'
,LANGUAGE char(8) label='Language for date strings' ,LANGUAGE char(8) label='Language for date strings'
); );
%local lib;
%let libds=%upcase(&libds);
%if %index(&libds,.)=0 %then %let lib=WORK;
%else %let lib=%scan(&libds,1,.);
proc datasets lib=&lib noprint;
modify %scan(&libds,-1,.);
index create
pk_cntlout=(type fmtname fmtrow)
/nomiss unique;
quit;
%mend mddl_sas_cntlout; %mend mddl_sas_cntlout;
/** /**

14
base/mp_cntlout.sas
View File

@@ -67,16 +67,24 @@ proc format lib=&libcat cntlout=&cntlds;
%end; %end;
run; run;
data &cntlout; data &cntlout/nonote2err;
if 0 then set &ddlds; if 0 then set &ddlds;
set &cntlds; set &cntlds;
if type in ("I","N") then do; /* numeric (in)format */ by type fmtname notsorted;
/* align the numeric values to avoid overlapping ranges */
if type in ("I","N") then do;
%mp_aligndecimal(start,width=16) %mp_aligndecimal(start,width=16)
%mp_aligndecimal(end,width=16) %mp_aligndecimal(end,width=16)
end; end;
/* create row marker. Data cannot be sorted without it! */
if first.fmtname then fmtrow=0;
fmtrow+1;
run; run;
proc sort; proc sort;
by type fmtname start; by type fmtname fmtrow;
run; run;
proc sql; proc sql;

1
base/mp_ds2md.sas
View File

@@ -95,6 +95,7 @@ run;
options ps=max lrecl=max; options ps=max lrecl=max;
data _null_; data _null_;
infile &outref; infile &outref;
if _n_=1 then putlog "# &libds" /;
input; input;
putlog _infile_; putlog _infile_;
run; run;

67
base/mp_loadformat.sas
View File

@@ -34,19 +34,20 @@
@param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
<h4> SAS Macros </h4> <h4> SAS Macros </h4>
@li mddl_sas_cntlout.sas
@li mf_getuniquename.sas @li mf_getuniquename.sas
@li mf_nobs.sas @li mf_nobs.sas
@li mp_abort.sas @li mp_abort.sas
@li mp_aligndecimal.sas @li mp_aligndecimal.sas
@li mp_cntlout.sas @li mp_cntlout.sas
@li mp_lockanytable.sas @li mp_lockanytable.sas
@li mp_md5.sas
@li mp_storediffs.sas @li mp_storediffs.sas
<h4> Related Macros </h4> <h4> Related Macros </h4>
@li mddl_dc_difftable.sas @li mddl_dc_difftable.sas
@li mddl_dc_locktable.sas @li mddl_dc_locktable.sas
@li mp_loadformat.test.sas @li mp_loadformat.test.1.sas
@li mp_loadformat.test.2.sas
@li mp_lockanytable.sas @li mp_lockanytable.sas
@li mp_stackdiffs.sas @li mp_stackdiffs.sas
@@ -79,13 +80,6 @@
%let &var=%upcase(&prefix._&var); %let &var=%upcase(&prefix._&var);
%end; %end;
/*
format values can be up to 32767 wide. SQL joins on such a wide column can
cause buffer issues. Update ibufsize and reset at the end.
*/
%let ibufsize=%sysfunc(getoption(ibufsize));
options ibufsize=32767 ;
/* in DC, format catalogs maybe specified in the libds with a -FC extension */ /* in DC, format catalogs maybe specified in the libds with a -FC extension */
%let libcat=%scan(&libcat,1,-); %let libcat=%scan(&libcat,1,-);
@@ -148,16 +142,24 @@ select distinct
%mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts) %mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
/* get a hash of the row */
%local cvars nvars;
%let cvars=TYPE FMTNAME START END LABEL PREFIX FILL SEXCL EEXCL HLO DECSEP
DIG3SEP DATATYPE LANGUAGE;
%let nvars=FMTROW MIN MAX DEFAULT LENGTH FUZZ MULT NOEDIT;
data &base_fmts/note2err;
set &base_fmts;
fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
run;
/** /**
* Ensure input table and base_formats have consistent lengths and types * Ensure input table and base_formats have consistent lengths and types
*/ */
%mddl_sas_cntlout(libds=&template) data &inlibds/nonote2err;
data &inlibds; length &delete_col $3 FMTROW 8 start end label $32767;
length &delete_col $3; if 0 then set &base_fmts;
if 0 then set &template;
length start end $10000;
set &libds; set &libds;
by type fmtname notsorted;
if &delete_col='' then &delete_col='No'; if &delete_col='' then &delete_col='No';
fmtname=upcase(fmtname); fmtname=upcase(fmtname);
type=upcase(type); type=upcase(type);
@@ -175,6 +177,14 @@ data &inlibds;
%mp_aligndecimal(start,width=16) %mp_aligndecimal(start,width=16)
%mp_aligndecimal(end,width=16) %mp_aligndecimal(end,width=16)
end; end;
/* update row marker - retain new var as fmtrow may already be in libds */
if first.fmtname then row=1;
else row+1;
drop row;
fmtrow=row;
fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
run; run;
/** /**
@@ -185,12 +195,10 @@ create table &outds_add(drop=&delete_col) as
select a.* select a.*
from &inlibds a from &inlibds a
left join &base_fmts b left join &base_fmts b
on a.fmtname=b.fmtname on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
and a.start=b.start
and a.type=b.type
where b.fmtname is null where b.fmtname is null
and upcase(a.&delete_col) ne "YES" and upcase(a.&delete_col) ne "YES"
order by type, fmtname, start; order by type, fmtname, fmtrow;
/** /**
* Identify deleted records * Identify deleted records
@@ -199,11 +207,9 @@ create table &outds_del(drop=&delete_col) as
select a.* select a.*
from &inlibds a from &inlibds a
inner join &base_fmts b inner join &base_fmts b
on a.fmtname=b.fmtname on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
and a.start=b.start
and a.type=b.type
where upcase(a.&delete_col)="YES" where upcase(a.&delete_col)="YES"
order by type, fmtname, start; order by type, fmtname, fmtrow;
/** /**
* Identify modified records * Identify modified records
@@ -212,13 +218,10 @@ create table &outds_mod (drop=&delete_col) as
select a.* select a.*
from &inlibds a from &inlibds a
inner join &base_fmts b inner join &base_fmts b
on a.fmtname=b.fmtname on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
and a.start=b.start
and a.type=b.type
where upcase(a.&delete_col) ne "YES" where upcase(a.&delete_col) ne "YES"
order by type, fmtname, start; and a.fmthash ne b.fmthash
order by type, fmtname, fmtrow;
options ibufsize=&ibufsize;
%mp_abort( %mp_abort(
iftrue=(&syscc ne 0) iftrue=(&syscc ne 0)
@@ -227,19 +230,21 @@ options ibufsize=&ibufsize;
) )
%if &loadtarget=YES %then %do; %if &loadtarget=YES %then %do;
/* new records plus base records that are not deleted or modified */
data &ds1; data &ds1;
merge &base_fmts(in=base) merge &base_fmts(in=base)
&outds_mod(in=mod) &outds_mod(in=mod)
&outds_add(in=add) &outds_add(in=add)
&outds_del(in=del); &outds_del(in=del);
if not del and not mod; if not del and not mod;
by type fmtname start; by type fmtname fmtrow;
run; run;
/* add back the modified records */
data &stagedata; data &stagedata;
set &ds1 &outds_mod; set &ds1 &outds_mod;
run; run;
proc sort; proc sort;
by type fmtname start; by type fmtname fmtrow;
run; run;
%end; %end;
/* mp abort needs to run outside of conditional blocks */ /* mp abort needs to run outside of conditional blocks */
@@ -287,7 +292,7 @@ options ibufsize=&ibufsize;
%mp_storediffs(&libcat-FC %mp_storediffs(&libcat-FC
,&base_fmts ,&base_fmts
,TYPE FMTNAME START ,TYPE FMTNAME FMTROW
,delds=&outds_del ,delds=&outds_del
,modds=&outds_mod ,modds=&outds_mod
,appds=&outds_add ,appds=&outds_add

6
base/mp_storediffs.sas
View File

@@ -147,9 +147,9 @@ run;
%if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do; %if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do;
/* this is a format catalog - cannot query cols directly */ /* this is a format catalog - cannot query cols directly */
%let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH" %let vlist="TYPE","FMTNAME","FMTROW","START","END","LABEL","MIN","MAX"
,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO" ,"DEFAULT","LENGTH","FUZZ","PREFIX","MULT","FILL","NOEDIT","SEXCL"
,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE"; ,"EEXCL","HLO","DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
%end; %end;
%else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE); %else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);

34
ddl/mddl_sas_cntlout.sas
View File

@@ -11,22 +11,18 @@
%macro mddl_sas_cntlout(libds=WORK.CNTLOUT); %macro mddl_sas_cntlout(libds=WORK.CNTLOUT);
proc sql; proc sql;
create table &libds( create table &libds(
TYPE char(1) label='Type of format' TYPE char(1) label='Type of format - either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
,FMTNAME char(32) label='Format name' ,FMTNAME char(32) label='Format name'
/* ,FMTROW num label='CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
to accommodate larger START values, mp_loadformat.sas will need the ,START char(32767) label='Starting value for format'
SQL dependency removed (proc sql needs to accommodate 3 index values in
a 32767 ibufsize limit)
*/
,START char(10000) label='Starting value for format'
/* /*
Keep lengths of START and END the same to avoid this err: Keep lengths of START and END the same to avoid this err:
"Start is greater than end: -<." "Start is greater than end: -<."
Similar usage note: https://support.sas.com/kb/69/330.html Similar usage note: https://support.sas.com/kb/69/330.html
*/ */
,END char(10000) label='Ending value for format' ,END char(32767) label='Ending value for format'
,LABEL char(32767) label='Format value label' ,LABEL char(32767) label='Format value label'
,MIN num length=3 label='Minimum length' ,MIN num length=3 label='Minimum length'
,MAX num length=3 label='Maximum length' ,MAX num length=3 label='Maximum length'
@@ -39,11 +35,23 @@ create table &libds(
,NOEDIT num length=3 label='Is picture string noedit?' ,NOEDIT num length=3 label='Is picture string noedit?'
,SEXCL char(1) label='Start exclusion' ,SEXCL char(1) label='Start exclusion'
,EEXCL char(1) label='End exclusion' ,EEXCL char(1) label='End exclusion'
,HLO char(13) label='Additional information' ,HLO char(13) label='Additional information. M=MultiLabel'
,DECSEP char(1) label='Decimal separator' ,DECSEP char(1) label='Decimal separator'
,DIG3SEP char(1) label='Three-digit separator' ,DIG3SEP char(1) label='Three-digit separator'
,DATATYPE char(8) label='Date/time/datetime?' ,DATATYPE char(8) label='Date/time/datetime?'
,LANGUAGE char(8) label='Language for date strings' ,LANGUAGE char(8) label='Language for date strings'
); );
%local lib;
%let libds=%upcase(&libds);
%if %index(&libds,.)=0 %then %let lib=WORK;
%else %let lib=%scan(&libds,1,.);
proc datasets lib=&lib noprint;
modify %scan(&libds,-1,.);
index create
pk_cntlout=(type fmtname fmtrow)
/nomiss unique;
quit;
%mend mddl_sas_cntlout; %mend mddl_sas_cntlout;

97
tests/base/mp_loadformat.test.sas → tests/base/mp_loadformat.test.1.sas
View File

@@ -6,6 +6,7 @@
<h4> SAS Macros </h4> <h4> SAS Macros </h4>
@li mddl_dc_difftable.sas @li mddl_dc_difftable.sas
@li mp_aligndecimal.sas @li mp_aligndecimal.sas
@li mp_cntlout.sas
@li mp_loadformat.sas @li mp_loadformat.sas
@li mp_assert.sas @li mp_assert.sas
@li mp_assertscope.sas @li mp_assertscope.sas
@@ -23,9 +24,9 @@ data work.loadfmts;
length fmtname $32 start end $10000; length fmtname $32 start end $10000;
eexcl='Y'; eexcl='Y';
type='N'; type='N';
do i=1 to 100; do i=1 to 10;
fmtname=cats('SASJS_',i,'X'); fmtname=cats('SASJS_',put(i,z4.),'X');
do j=1 to 100; do j=1 to 20;
start=cats(j); start=cats(j);
end=cats(j+1); end=cats(j+1);
%mp_aligndecimal(start,width=16) %mp_aligndecimal(start,width=16)
@@ -38,21 +39,32 @@ run;
proc format cntlin=work.loadfmts library=perm.testcat; proc format cntlin=work.loadfmts library=perm.testcat;
run; run;
/*
use actual format data as test baseline, as proc format adds attributes eg
min/max etc
*/
%mp_cntlout(libcat=perm.testcat,cntlout=work.loadfmts2)
/* make some test data */ /* make some test data */
data work.stagedata; data work.stagedata;
set work.loadfmts; set work.loadfmts2 end=lastobs;
type='N'; by type fmtname;
eexcl='Y';
if _n_<150 then deleteme='Yes'; if lastobs then do;
else if _n_<250 then label='mod'!!cats(_n_); output;
else if _n_<350 then do; fmtname='NEWFMT'!!cats(_n_,'x'); /* 1 new record */
start=cats(_n_); start=cats(_n_);
end=cats(_n_+1); end=cats(_n_+1);
%mp_aligndecimal(start,width=16) %mp_aligndecimal(start,width=16)
%mp_aligndecimal(end,width=16) %mp_aligndecimal(end,width=16)
label='newval'!!cats(_N_); label='newval'!!cats(_N_,'X');
output;
stop;
end; end;
else stop; else if last.fmtname then deleteme='Yes'; /* 9 deletions */
else if first.fmtname then label='modified '!!cats(_n_); /* 10 changes */
output;
run; run;
/* load the above */ /* load the above */
@@ -71,22 +83,22 @@ run;
%mp_assertscope(COMPARE) %mp_assertscope(COMPARE)
%mp_assert( %mp_assert(
iftrue=(%mf_nobs(del_test1)=149), iftrue=(%mf_nobs(del_test1)=9),
desc=Test 1 - delete obs, desc=Test 1 - delete obs,
outds=work.test_results outds=work.test_results
) )
%mp_assert( %mp_assert(
iftrue=(%mf_nobs(add_test1)=100), iftrue=(%mf_nobs(add_test1)=1),
desc=Test 1 - add obs, desc=Test 1 - add obs,
outds=work.test_results outds=work.test_results
) )
%mp_assert( %mp_assert(
iftrue=(%mf_nobs(mod_test1)=100), iftrue=(%mf_nobs(mod_test1)=10),
desc=Test 1 - mod obs, desc=Test 1 - mod obs,
outds=work.test_results outds=work.test_results
) )
%mp_assert( %mp_assert(
iftrue=(%mf_nobs(perm.audit)=7329), iftrue=(%mf_nobs(perm.audit)=440),
desc=Test 1 - audit table updated, desc=Test 1 - audit table updated,
outds=work.test_results outds=work.test_results
) )
@@ -101,7 +113,7 @@ run;
) )
/* set up a mix of formats */ /* set up a mix of formats */
data work.loadfmts2; data work.loadfmts3;
length fmtname $32 start end $10000; length fmtname $32 start end $10000;
eexcl='Y'; eexcl='Y';
type='J'; type='J';
@@ -150,45 +162,47 @@ data work.loadfmts2;
end; end;
drop i j; drop i j;
run; run;
proc format cntlin=work.loadfmts2 library=perm.testcat2; proc format cntlin=work.loadfmts3 library=perm.testcat3;
run; run;
%mp_cntlout(libcat=perm.testcat3,cntlout=work.loadfmts4)
/* make some test data */ /* make some test data */
data work.stagedata2; data work.stagedata3;
set work.loadfmts2; set work.loadfmts4;
where type in ('I','J'); where type in ('I','J');
eexcl='Y'; by type fmtname notsorted;
if type='I' then do; if type='I' then do;
i+1; if last.fmtname then do;
if i<3 then deleteme='Yes'; deleteme='Yes'; /* 3 deletions */
else if i<7 then label= cats(ranuni(0)*100); output;
else if i<12 then do; end;
/* new values */ else if fmtrow le 3 then do; /* 9 changed values */
z=ranuni(0)*1000000; z=ranuni(0)*1000000;
start=cats(z); start=cats(z);
end=cats(z+1); end=cats(z+1);
%mp_aligndecimal(start,width=16) %mp_aligndecimal(start,width=16)
%mp_aligndecimal(end,width=16) %mp_aligndecimal(end,width=16)
label= cats(ranuni(0)*100); output;
end; end;
if i<12 then output;
end; end;
else do; else do;
j+1; if last.fmtname then do;
if j<3 then deleteme='Yes'; output; /* 6 new records */
else if j<7 then label= cats(ranuni(0)*100); x=_n_;
else if j<12 then do; x+1;start=cats("mod",x);end=start;label='newlabel1';output;
start= cats("NEWVAL",start); x+1;start=cats("mod",x);end=start;label='newlabel2';output;
end=start; end;
label= "NEWVAL "||cats(ranuni(0)*100); else if fmtrow le 3 then do; /* 9 more changed values */
start= cats("mod",_n_);
end=start;
label= "mod "||cats(ranuni(0)*100);
output;
end; end;
if j<12 then output;
end; end;
run; run;
%mp_loadformat(perm.testcat2 %mp_loadformat(perm.testcat3
,work.stagedata2 ,work.stagedata3
,loadtarget=YES ,loadtarget=YES
,auditlibds=perm.audit ,auditlibds=perm.audit
,locklibds=0 ,locklibds=0
@@ -200,17 +214,18 @@ run;
) )
%mp_assert( %mp_assert(
iftrue=(%mf_nobs(del_test2)=4), iftrue=(%mf_nobs(del_test2)=3),
desc=Test 2 - delete obs, desc=Test 2 - delete obs,
outds=work.test_results outds=work.test_results
) )
%mp_assert( %mp_assert(
iftrue=(%mf_nobs(mod_test2)=8), iftrue=(%mf_nobs(mod_test2)=18),
desc=Test 2 - mod obs, desc=Test 2 - mod obs,
outds=work.test_results outds=work.test_results
) )
%mp_assert( %mp_assert(
iftrue=(%mf_nobs(add_test2)=10), iftrue=(%mf_nobs(add_test2)=6),
desc=Test 2 - add obs, desc=Test 2 - add obs,
outds=work.test_results outds=work.test_results
) )

128
tests/base/mp_loadformat.test.2.sas Normal file
View File

@@ -0,0 +1,128 @@
/**
@file
@brief Testing mp_loadformat.sas macro for multilabel formats
@details Multilabel records can be complete duplicates!! Also, the order is
important.
The provided formats create a table as follows:
|TYPE:$1.|FMTNAME:$32.|START:$10000.|END:$10000.|LABEL:$32767.|MIN:best.|
MAX:best.|DEFAULT:best.|LENGTH:best.|FUZZ:best.|PREFIX:$2.|MULT:best.|FILL:$1.|
NOEDIT:best.|SEXCL:$1.|EEXCL:$1.|HLO:$13.|DECSEP:$1.|DIG3SEP:$1.|DATATYPE:$8.|
LANGUAGE:$8.|
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
---|---|
|`C `|`GENDERML `|` `|` `|`Total people `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
|`C `|`GENDERML `|`1 `|`1 `|`Male `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
|`C `|`GENDERML `|`1 `|`1 `|`Total people `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
|`C `|`GENDERML `|`2 `|`2 `|`Female `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
|`C `|`GENDERML `|`2 `|`2 `|`Female `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
|`C `|`GENDERML `|`2 `|`2 `|`Thormale `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
|`C `|`GENDERML `|`2 `|`2 `|`Total people `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
|`N `|`AGEMLA `|`1 `|`4 `|`Preschool `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
|`N `|`AGEMLA `|`1 `|`18 `|`Children `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
|`N `|`AGEMLA `|`19 `|`120 `|`Adults `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
|`N `|`AGEMLB `|`1 `|`4 `|`Preschool `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
|`N `|`AGEMLB `|`1 `|`18 `|`Children `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
|`N `|`AGEMLB `|`19 `|`120 `|`Adults `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
|`N `|`AGEMLC `|`1 `|`18 `|`Children `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
|`N `|`AGEMLC `|`1 `|`4 `|`Preschool `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
|`N `|`AGEMLC `|`19 `|`120 `|`Adults `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
<h4> SAS Macros </h4>
@li mf_nobs.sas
@li mp_cntlout.sas
@li mp_loadformat.sas
@li mp_assert.sas
@li mp_assertdsobs.sas
@li mp_ds2md.sas
**/
/* prep format catalog */
libname perm (work);
/* create some multilable formats */
%let cat1=perm.test1;
proc format library=&cat1;
value $genderml (multilabel notsorted)
'1'='Male'
'2'='Female'
'2'='Female'
'2'='Farmale'
'1','2',' '='Total people';
value agemla (multilabel)
1-4='Preschool'
1-18='Children'
19-120='Adults';
value agemlb (multilabel)
19-120='Adults'
1-18='Children'
1-4='Preschool';
value agemlc (multilabel notsorted)
19-120='Adults'
1-18='Children'
1-4='Preschool';
run;
%mp_cntlout(libcat=&cat1,cntlout=work.cntlout1)
%mp_assertdsobs(work.cntlout1,
desc=Has 16 records,
test=EQUALS 16
)
data work.stagedata3;
set work.cntlout1;
if fmtname='AGEMLA' and label ne 'Preschool' then deleteme='Yes';
if fmtname='AGEMLB' and label = 'Preschool' then label='Kids';
if fmtname='GENDERML' and label='Farmale' then output;
output;
run;
%mp_loadformat(&cat1
,work.stagedata3
,loadtarget=YES
,auditlibds=perm.audit
,locklibds=0
,delete_col=deleteme
,outds_add=add_test1
,outds_del=del_test1
,outds_mod=mod_test1
,mdebug=1
)
%mp_assert(
iftrue=(%mf_nobs(del_test1)=2),
desc=Test 1 - deleted obs,
outds=work.test_results
)
%mp_assert(
iftrue=(%mf_nobs(mod_test1)=4),
desc=Test 1 - mod obs,
outds=work.test_results
)
%mp_assert(
iftrue=(%mf_nobs(add_test1)=1),
desc=Test 1 - add obs,
outds=work.test_results
)
/* now check the order of the notsorted format */
%mp_cntlout(libcat=&cat1,cntlout=work.cntlout2)
%let check1=0;
%let check2=0;
data test;
set work.cntlout2;
where fmtname='GENDERML';
if _n_=4 and label='Farmale' then call symputx('check1',1);
if _n_=5 and label='Farmale' then call symputx('check2',1);
run;
%mp_assert(
iftrue=(&check1=1 and &check2=1),
desc=Ensuring Farmale values retain their order,
outds=work.test_results
)
%mp_ds2md(work.cntlout2)