sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2026-01-05 00:20:05 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #342 from sasjs/issue341

Browse Source 
Full Format Deletion
This commit is contained in:
Allan Bowe
2023-07-14 00:05:29 +01:00
committed by GitHub
parent 8de2dd4e7c 7b6d34028b
commit 461cda45ee
15 changed files with 516 additions and 2070 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.git-hooks/pre-commit
View File

@@ -2,7 +2,7 @@
# Ensure lint is passing
LINT=`sasjs lint`
if [[ "$LINT" != "✔ All matched files use @sasjs/lint code style!" ]]; then
if [[ "$LINT" != *"✔ All matched files use @sasjs/lint code style!" ]]; then
echo "$LINT"
echo "To commit in spite of these warnings, use the -n parameter."
exit 1

11
.github/workflows/run-tests.yml vendored
View File

@@ -12,7 +12,7 @@ jobs:
strategy:
matrix:
node-version: [lts/fermium]
node-version: [lts/hydrogen]
steps:
- uses: actions/checkout@v2
@@ -51,7 +51,7 @@ jobs:
run: npm ci
- name: Check code style (aborts if errors found)
run: npx sasjs lint
run: npx @sasjs/cli lint
- name: Add client
run: echo "CLIENT=${{secrets.SAS9_4GL_IO_CLIENT}}"> .env.server
@@ -59,11 +59,14 @@ jobs:
- name: Add access token
run: echo "ACCESS_TOKEN=${{secrets.SAS9_4GL_IO_ACCESS_TOKEN}}" >> .env.server
- name: Add refresh token
run: echo "REFRESH_TOKEN=${{secrets.SAS9_4GL_IO_REFRESH_TOKEN}}" >> .env.server
- name: Build & Deploy Project to SAS server
run: npx sasjs cbd -t server
run: npx @sasjs/cli cbd -t server
- name: Run all tests
run: npx sasjs test -t server
run: npx @sasjs/cli test -t server
env:
CI: true
CLIENT: ${{secrets.CLIENT}}

84
all.sas
View File

@@ -695,10 +695,9 @@ or %index(&pgm,/tests/testteardown)
returns:
> DOLLAR $CHAR W MONNAME
> $CHAR BEST DOLLAR
> BEST Z $CHAR COMMA PERCENTN
DOLLAR $CHAR W MONNAME
$CHAR BEST DOLLAR
BEST Z $CHAR COMMA PERCENTN
@param [in] libds Two part library.dataset reference.
@@ -4152,9 +4151,9 @@ run;
%end;
proc format lib=&libcat cntlout=&cntlds;
%if "&fmtlist" ne "0" %then %do;
%if "&fmtlist" ne "0" and "&fmtlist" ne "" %then %do;
select
%do i=1 %to %sysfunc(countw(&fmtlist));
%do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
%scan(&fmtlist,&i,%str( ))
%end;
;
@@ -7894,10 +7893,15 @@ run;
Formats are taken from the library / dataset reference and / or a static
format list.
Note - the source for this information is the dictionary.formats table. This
cannot show formats that are not already declared in the FMTSEARCH path.
Example usage:
%mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable)
%mp_getformats(fmtlist=FORMAT1 $FORMAT2 @INFMT3,outsummary=work.table2)
@param [in] lib= (0) The libref for which to return formats.
@todo Enable exporting of formats for an entire library
@param [in] ds= (0) The dataset from which to obtain format definitions
@@ -7936,7 +7940,9 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
<h4> Related Macros </h4>
@li mf_getfmtlist.sas
@li mp_applyformats.sas
@li mp_cntlout.sas
@li mp_getformats.test.sas
@version 9.2
@@ -7953,7 +7959,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
%local i fmt allfmts tempds fmtcnt;
%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( )));
%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
/* ensure format list contains format _name_ only */
%let fmt=%scan(&fmtlist,&i,%str( ));
%let fmt=%mf_getfmtname(&fmt);
@@ -7977,8 +7983,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
proc sql;
create table &outsummary as
select * from dictionary.formats
where fmtname in (%mf_getquotedstr(&allfmts,quote=D))
and fmttype='F';
where fmtname in (%mf_getquotedstr(&allfmts,quote=D));
%if "&outdetail" ne "0" %then %do;
/* ensure base table always exists */
@@ -8002,6 +8007,10 @@ create table &outsummary as
data &tempds;
if 0 then set &outdetail;
set &tempds;
/* set fmtrow (position of record within the format) */
by type fmtname notsorted;
if first.fmtname then fmtrow=1;
else fmtrow+1;
run;
proc append base=&outdetail data=&tempds ;
run;
@@ -10162,7 +10171,7 @@ select distinct lowcase(memname)
);
/* set up local macro variables and temporary tables (with a prefix) */
%local err msg prefix dslist i var fmtlist ibufsize;
%let dslist=base_fmts template inlibds ds1 stagedata storediffs;
%let dslist=base_fmts template inlibds ds1 stagedata storediffs del1 del2;
%if &outds_add=0 %then %let dslist=&dslist outds_add;
%if &outds_del=0 %then %let dslist=&dslist outds_del;
%if &outds_mod=0 %then %let dslist=&dslist outds_mod;
@@ -10293,6 +10302,18 @@ create table &outds_add(drop=&delete_col) as
and upcase(a.&delete_col) ne "YES"
order by type, fmtname, fmtrow;
/**
* Identify modified records
*/
create table &outds_mod (drop=&delete_col) as
select a.*
from &inlibds a
inner join &base_fmts b
on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
where upcase(a.&delete_col) ne "YES"
and a.fmthash ne b.fmthash
order by type, fmtname, fmtrow;
/**
* Identify deleted records
*/
@@ -10305,16 +10326,23 @@ create table &outds_del(drop=&delete_col) as
order by type, fmtname, fmtrow;
/**
* Identify modified records
* Identify fully deleted formats (where every record is removed)
* These require to be explicitly deleted in proc format
* del1 - identify _partial_ deletes
* del2 - exclude these, and also formats that come with _additions_
*/
create table &outds_mod (drop=&delete_col) as
create table &del1 as
select a.*
from &inlibds a
inner join &base_fmts b
from &base_fmts a
left join &outds_del b
on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
where upcase(a.&delete_col) ne "YES"
and a.fmthash ne b.fmthash
order by type, fmtname, fmtrow;
where b.fmtrow is null;
create table &del2 as
select * from &outds_del
where cats(type,fmtname) not in (select cats(type,fmtname) from &outds_add)
and cats(type,fmtname) not in (select cats(type,fmtname) from &del1);
%mp_abort(
iftrue=(&syscc ne 0)
@@ -10347,7 +10375,7 @@ create table &outds_mod (drop=&delete_col) as
,msg=%str(SYSCC=&syscc prior to actual load)
)
%if &loadtarget=YES %then %do;
%if %mf_nobs(&stagedata)=0 %then %do;
%if %mf_nobs(&stagedata)=0 and %mf_nobs(&del2)=0 %then %do;
%put There are no changes to load in &libcat!;
%return;
%end;
@@ -10363,6 +10391,22 @@ create table &outds_mod (drop=&delete_col) as
/* do the actual load */
proc format lib=&libcat cntlin=&stagedata;
run;
/* apply any full deletes */
%if %mf_nobs(&del2)>0 %then %do;
%local delfmtlist;
proc sql noprint;
select distinct case when type='N' then cats(fmtname,'.FORMAT')
when type='C' then cats(fmtname,'.FORMATC')
when type='J' then cats(fmtname,'.INFMTC')
when type='I' then cats(fmtname,'.INFMT')
else cats(fmtname,'.BADENTRY!!!') end
into: delfmtlist
separated by ' '
from &del2;
proc catalog catalog=&libcat;
delete &delfmtlist;
quit;
%end;
%if &locklibds ne 0 %then %do;
/* unlock the table */
%mp_lockanytable(UNLOCK
@@ -12831,7 +12875,7 @@ data &ds4;
if upcase(&inds_auto)="&ds2" then tgtvar_type='N';
else if upcase(&inds_auto)="&ds3" then tgtvar_type='C';
else do;
putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
putlog 'ERR' +(-1) "OR: unidentified vartype input!" &inds_auto;
call symputx('syscc',98);
end;
@@ -12840,7 +12884,7 @@ data &ds4;
else if &inds_keep="&modds" then move_type='M';
else if &inds_keep="&origds" then move_type='O';
else do;
putlog "%str(ERR)OR: unidentified movetype input!" &inds_keep;
putlog 'ERR' +(-1) "OR: unidentified movetype input!" &inds_keep;
call symputx('syscc',99);
end;
tgtvar_nm=upcase(tgtvar_nm);

7
base/mf_getfmtlist.sas
View File

@@ -10,10 +10,9 @@
returns:
> DOLLAR $CHAR W MONNAME
> $CHAR BEST DOLLAR
> BEST Z $CHAR COMMA PERCENTN
DOLLAR $CHAR W MONNAME
$CHAR BEST DOLLAR
BEST Z $CHAR COMMA PERCENTN
@param [in] libds Two part library.dataset reference.

4
base/mp_cntlout.sas
View File

@@ -58,9 +58,9 @@
%end;
proc format lib=&libcat cntlout=&cntlds;
%if "&fmtlist" ne "0" %then %do;
%if "&fmtlist" ne "0" and "&fmtlist" ne "" %then %do;
select
%do i=1 %to %sysfunc(countw(&fmtlist));
%do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
%scan(&fmtlist,&i,%str( ))
%end;
;

16
base/mp_getformats.sas
View File

@@ -7,10 +7,15 @@
Formats are taken from the library / dataset reference and / or a static
format list.
Note - the source for this information is the dictionary.formats table. This
cannot show formats that are not already declared in the FMTSEARCH path.
Example usage:
%mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable)
%mp_getformats(fmtlist=FORMAT1 $FORMAT2 @INFMT3,outsummary=work.table2)
@param [in] lib= (0) The libref for which to return formats.
@todo Enable exporting of formats for an entire library
@param [in] ds= (0) The dataset from which to obtain format definitions
@@ -49,7 +54,9 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
<h4> Related Macros </h4>
@li mf_getfmtlist.sas
@li mp_applyformats.sas
@li mp_cntlout.sas
@li mp_getformats.test.sas
@version 9.2
@@ -66,7 +73,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
%local i fmt allfmts tempds fmtcnt;
%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( )));
%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,%str( )));
/* ensure format list contains format _name_ only */
%let fmt=%scan(&fmtlist,&i,%str( ));
%let fmt=%mf_getfmtname(&fmt);
@@ -90,8 +97,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
proc sql;
create table &outsummary as
select * from dictionary.formats
where fmtname in (%mf_getquotedstr(&allfmts,quote=D))
and fmttype='F';
where fmtname in (%mf_getquotedstr(&allfmts,quote=D));
%if "&outdetail" ne "0" %then %do;
/* ensure base table always exists */
@@ -115,6 +121,10 @@ create table &outsummary as
data &tempds;
if 0 then set &outdetail;
set &tempds;
/* set fmtrow (position of record within the format) */
by type fmtname notsorted;
if first.fmtname then fmtrow=1;
else fmtrow+1;
run;
proc append base=&outdetail data=&tempds ;
run;

53
base/mp_loadformat.sas
View File

@@ -68,7 +68,7 @@
);
/* set up local macro variables and temporary tables (with a prefix) */
%local err msg prefix dslist i var fmtlist ibufsize;
%let dslist=base_fmts template inlibds ds1 stagedata storediffs;
%let dslist=base_fmts template inlibds ds1 stagedata storediffs del1 del2;
%if &outds_add=0 %then %let dslist=&dslist outds_add;
%if &outds_del=0 %then %let dslist=&dslist outds_del;
%if &outds_mod=0 %then %let dslist=&dslist outds_mod;
@@ -199,6 +199,18 @@ create table &outds_add(drop=&delete_col) as
and upcase(a.&delete_col) ne "YES"
order by type, fmtname, fmtrow;
/**
* Identify modified records
*/
create table &outds_mod (drop=&delete_col) as
select a.*
from &inlibds a
inner join &base_fmts b
on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
where upcase(a.&delete_col) ne "YES"
and a.fmthash ne b.fmthash
order by type, fmtname, fmtrow;
/**
* Identify deleted records
*/
@@ -211,16 +223,23 @@ create table &outds_del(drop=&delete_col) as
order by type, fmtname, fmtrow;
/**
* Identify modified records
* Identify fully deleted formats (where every record is removed)
* These require to be explicitly deleted in proc format
* del1 - identify _partial_ deletes
* del2 - exclude these, and also formats that come with _additions_
*/
create table &outds_mod (drop=&delete_col) as
create table &del1 as
select a.*
from &inlibds a
inner join &base_fmts b
from &base_fmts a
left join &outds_del b
on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
where upcase(a.&delete_col) ne "YES"
and a.fmthash ne b.fmthash
order by type, fmtname, fmtrow;
where b.fmtrow is null;
create table &del2 as
select * from &outds_del
where cats(type,fmtname) not in (select cats(type,fmtname) from &outds_add)
and cats(type,fmtname) not in (select cats(type,fmtname) from &del1);
%mp_abort(
iftrue=(&syscc ne 0)
@@ -253,7 +272,7 @@ create table &outds_mod (drop=&delete_col) as
,msg=%str(SYSCC=&syscc prior to actual load)
)
%if &loadtarget=YES %then %do;
%if %mf_nobs(&stagedata)=0 %then %do;
%if %mf_nobs(&stagedata)=0 and %mf_nobs(&del2)=0 %then %do;
%put There are no changes to load in &libcat!;
%return;
%end;
@@ -269,6 +288,22 @@ create table &outds_mod (drop=&delete_col) as
/* do the actual load */
proc format lib=&libcat cntlin=&stagedata;
run;
/* apply any full deletes */
%if %mf_nobs(&del2)>0 %then %do;
%local delfmtlist;
proc sql noprint;
select distinct case when type='N' then cats(fmtname,'.FORMAT')
when type='C' then cats(fmtname,'.FORMATC')
when type='J' then cats(fmtname,'.INFMTC')
when type='I' then cats(fmtname,'.INFMT')
else cats(fmtname,'.BADENTRY!!!') end
into: delfmtlist
separated by ' '
from &del2;
proc catalog catalog=&libcat;
delete &delfmtlist;
quit;
%end;
%if &locklibds ne 0 %then %do;
/* unlock the table */
%mp_lockanytable(UNLOCK

4
base/mp_storediffs.sas
View File

@@ -165,7 +165,7 @@ data &ds4;
if upcase(&inds_auto)="&ds2" then tgtvar_type='N';
else if upcase(&inds_auto)="&ds3" then tgtvar_type='C';
else do;
putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
putlog 'ERR' +(-1) "OR: unidentified vartype input!" &inds_auto;
call symputx('syscc',98);
end;
@@ -174,7 +174,7 @@ data &ds4;
else if &inds_keep="&modds" then move_type='M';
else if &inds_keep="&origds" then move_type='O';
else do;
putlog "%str(ERR)OR: unidentified movetype input!" &inds_keep;
putlog 'ERR' +(-1) "OR: unidentified movetype input!" &inds_keep;
call symputx('syscc',99);
end;
tgtvar_nm=upcase(tgtvar_nm);

2251
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

4
package.json
View File

@@ -33,6 +33,6 @@
"prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
},
"devDependencies": {
"@sasjs/cli": "3.24.0"
"@sasjs/cli": "^4.4.1"
}
}
}

115
tests/base/mp_loadformat.test.2.sas
View File

@@ -33,13 +33,16 @@
@li mp_loadformat.sas
@li mp_assert.sas
@li mp_assertdsobs.sas
@li mp_getformats.sas
@li mp_ds2md.sas
**/
/* prep format catalog */
libname perm (work);
/* create some multilable formats */
/* create some multilabel formats */
%let cat1=perm.test1;
proc format library=&cat1;
value $genderml (multilabel notsorted)
@@ -120,4 +123,112 @@ run;
iftrue=(&check1=1 and &check2=1),
desc=Ensuring Farmale values retain their order,
outds=work.test_results
)
)
/**
* completely delete a format and make sure it is removed
*/
/* first, make sure these three formats exist */
options insert=(fmtsearch=(&cat1));
%mp_getformats(fmtlist=AGEMLA AGEMLB AGEMLC $GENDERML,outsummary=work.fmtdels)
%let fmtlist=NONE;
proc sql;
select distinct cats(fmtname) into: fmtlist separated by ' ' from work.fmtdels;
%mp_assert(
iftrue=(%mf_nobs(fmtdels)=4),
desc=Deletion test 1 - ensure formats exist for deletion (&fmtlist found),
outds=work.test_results
)
/* deltest1 - deleting every record */
%mp_cntlout(libcat=&cat1,cntlout=work.cntloutdel1)
data work.stagedatadel1;
set work.cntloutdel1;
if fmtname='AGEMLA';
deleteme='Yes';
run;
%mp_loadformat(&cat1
,work.stagedatadel1
,loadtarget=YES
,auditlibds=perm.audit
,locklibds=0
,delete_col=deleteme
,outds_add=add_testdel1
,outds_del=del_testdel1
,outds_mod=mod_testdel1
,mdebug=1
)
%mp_getformats(fmtlist=AGEMLA,outsummary=work.fmtdel1)
%mp_assert(
iftrue=(%mf_nobs(fmtdel1)=0),
desc=Deletion test 1 - ensure AGEMLA format was fully deleted,
outds=work.test_results
)
/* deltest2 - deleting every record except 1 */
data work.stagedatadel2;
set work.cntloutdel1;
if fmtname='AGEMLB';
x+1;
if x>1 then deleteme='Yes';
run;
%mp_loadformat(&cat1
,work.stagedatadel2
,loadtarget=YES
,auditlibds=perm.audit
,locklibds=0
,delete_col=deleteme
,outds_add=add_testdel2
,outds_del=del_testdel2
,outds_mod=mod_testdel2
,mdebug=1
)
%mp_getformats(fmtlist=AGEMLB,outsummary=work.fmtdel2)
%mp_assert(
iftrue=(%mf_nobs(fmtdel2)=1),
desc=Deletion test 2 - ensure AGEMLB format was not fully deleted,
outds=work.test_results
)
/* deltest3 - deleting every record, and adding a new one */
data work.stagedatadel3;
set work.cntloutdel1;
if fmtname='GENDERML';
deleteme='Yes';
run;
data work.stagedatadel3;
set work.stagedatadel3 end=last;
output;
if last then do;
deleteme='No';
/* must be a new fmtrow (key value) if adding new row in same load! */
fmtrow=1000;
start='Mail';
end='Mail';
output;
end;
run;
%mp_loadformat(&cat1
,work.stagedatadel3
,loadtarget=YES
,auditlibds=perm.audit
,locklibds=0
,delete_col=deleteme
,outds_add=add_testdel2
,outds_del=del_testdel2
,outds_mod=mod_testdel2
,mdebug=1
)
%mp_getformats(fmtlist=$GENDERML,outsummary=work.fmtdel3)
%mp_assert(
iftrue=(%mf_nobs(fmtdel3)=1),
desc=Deletion test 3 - ensure GENDERML format was not fully deleted,
outds=work.test_results
)
%mp_ds2md(work.fmtdel3)

2
tests/serveronly/ms_adduser2group.test.sas
View File

@@ -31,7 +31,7 @@ run;
%mp_assertscope(SNAPSHOT)
%ms_adduser2group(uid=1,gid=&groupid,mdebug=&sasjs_mdebug,outds=test1)
%mp_assertscope(COMPARE
,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN
,ignorelist=MCLIB2_JADP1LEN MCLIB2_JADP2LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
)
/* check the user is in the output list */

2
tests/serveronly/ms_deletefile.test.sas
View File

@@ -53,7 +53,7 @@ data _null_;
run;
%mp_assert(
iftrue=("&test2"="%str(Err)or: File doesn't exist."),
iftrue=("&test2"="File doesn't exist."),
desc=Make sure the file was deleted,
outds=work.test_results
)

2
tests/serveronly/ms_getgroups.test.sas
View File

@@ -21,7 +21,7 @@
%mp_assertscope(SNAPSHOT)
%ms_getgroups(outds=work.test1,mdebug=&sasjs_mdebug)
%mp_assertscope(COMPARE
,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN
,ignorelist=MCLIB2_JADP1LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
)
/* check the group was created */

29
tests/serveronly/ms_runstp.test.sas
View File

@@ -17,6 +17,7 @@ data _null_;
file stpcode;
put '%put hello world;';
put '%put _all_;';
put 'data _null_; file _webout; put "runstptest";run;';
run;
options mprint;
@@ -34,25 +35,29 @@ options mprint;
)
%mp_assertscope(COMPARE)
libname webeen json fileref=weboot;
%let test1=0;
%let test2=0;
data _null_;
infile weboot;
input;
putlog _infile_;
if _n_=1 then call symputx('test1',_infile_);
if _n_=3 then do;
call symputx('test2',substr(_infile_,1,30));
putlog "SASJS_LOGS_SEPARATOR_xxx"; /* this marker affects the CLI parser */
end;
else putlog _infile_;
run;
%let test1=0;
data work.log;
set webeen.log;
put (_all_)(=);
if _n_>10 then call symputx('test1',1);
run;
%mp_assert(
iftrue=("&test1"="1"),
desc=Checking log was returned,
iftrue=("&test1"="runstptest"),
desc=Checking webout was created,
outds=work.test_results
)
%mp_assert(
iftrue=("&test2"="SASJS_LOGS_SEPARATOR_163ee17b6"),
desc=Checking debug was enabled,
outds=work.test_results
)