sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2026-06-08 20:10:20 +00:00
Code Issues Projects Releases Wiki Activity

fix: simplifying mv_castabload and improving tests

Browse Source
This commit is contained in:
4gl
2026-04-28 18:38:57 +01:00
parent 1c005586dc
commit 36452a2a02
2 changed files with 257 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/viyaonly/mv_castabload.test.sas
+98 -94
View File
@@ -7,26 +7,26 @@
@li mp_assert.sas @li mp_assert.sas
@li mp_assertscope.sas @li mp_assertscope.sas
@li mv_castabload.sas @li mv_castabload.sas
@li mv_createfile.sas
**/ **/
options mprint; options mprint;
/* ------------------------------------------------------------------------ */ /* -------------------------------------------------------------------- */
/* Setup: start a CAS session and stage a source file in the Public caslib */ /* Setup: start a CAS session and stage a source file in the caslib */
/* ------------------------------------------------------------------------ */ /* -------------------------------------------------------------------- */
cas mysess; cas mysess;
caslib _all_ assign; caslib _all_ assign;
%let testcaslib = Public; /* change this if Public isn't available */ %let testcaslib=Public;
proc cas; proc cas;
table.caslibInfo result=r / ; table.caslibInfo result=r / ;
found = 0; found=0;
do row over r.CASLibInfo; do row over r.CASLibInfo;
if upcase(row.Name) = upcase("&testcaslib") then found = 1; if upcase(row.Name)=upcase("&testcaslib") then found=1;
end; end;
if found = 0 then do; if found=0 then do;
print "ERROR: caslib &testcaslib not available"; print "ERROR: caslib &testcaslib not available";
exit; exit;
end; end;
@@ -34,113 +34,117 @@ quit;
%put NOTE: Using testcaslib=&testcaslib; %put NOTE: Using testcaslib=&testcaslib;
%let tab1=T%mf_uid(); %let tab1=T%mf_uid();
%let tab2=T%mf_uid();
%let tab3=T%mf_uid();
/* Create a SASHDAT source file in the Public caslib from SASHELP.BASEBALL /* Save a sashdat source file then drop the in-memory copy so the first
so that subsequent LOAD operations have something real to pick up. */ mv_castabload call has something to load */
proc casutil; proc casutil;
load data=sashelp.baseball outcaslib="&testcaslib" casout="&tab1" replace; load data=sashelp.baseball
outcaslib="&testcaslib" casout="&tab1" replace;
save casdata="&tab1" incaslib="&testcaslib" save casdata="&tab1" incaslib="&testcaslib"
casout="&tab1..sashdat" outcaslib="&testcaslib" replace; casout="&tab1..sashdat" outcaslib="&testcaslib" replace;
droptable casdata="&tab1" incaslib="&testcaslib" quiet; droptable casdata="&tab1" incaslib="&testcaslib" quiet;
quit; quit;
/* And a second hdat, with a name different from the table name, so that we libname mylib cas caslib="&testcaslib";
can exercise the explicit srcfile= path. */
proc casutil;
load data=sashelp.cars outcaslib="&testcaslib" casout="&tab2" replace; /* -------------------------------------------------------------------- */
save casdata="&tab2" incaslib="&testcaslib" %put TEST 1 - load a table that is not in memory;
casout="src_&tab2..sashdat" outcaslib="&testcaslib" replace; /* -------------------------------------------------------------------- */
droptable casdata="&tab2" incaslib="&testcaslib" quiet;
/* Confirm table is absent before the call */
%let _tabexists=0;
proc cas;
table.tableExists result=r /
caslib="&testcaslib" name="&tab1";
if r.exists > 0 then call symputx('_tabexists','1');
quit; quit;
%mp_assert(
iftrue=(&_tabexists=0),
desc=Check table is not in memory before mv_castabload
)
/* ------------------------------------------------------------------------ */ %mv_castabload(lib=mylib, table=&tab1, mdebug=1)
%put TEST 1 - missing required parameters returns without setting RC to 0/1;
/* ------------------------------------------------------------------------ */ %let _tabexists=0;
%let MV_CASTABLOAD_RC=; proc cas;
%mv_castabload(caslib=,table=,srcfile=) table.tableExists result=r /
caslib="&testcaslib" name="&tab1";
if r.exists > 0 then call symputx('_tabexists','1');
quit;
%mp_assert( %mp_assert(
iftrue=(&MV_CASTABLOAD_RC=3), iftrue=(&_tabexists=1),
desc=Check RC=3 (initial/failure value) when required params are missing desc=Check table is in memory after mv_castabload
) )
/* ------------------------------------------------------------------------ */ /* -------------------------------------------------------------------- */
%put TEST 2 - load a table that does not yet exist (default srcfile=table.sashdat); %put TEST 2 - reload fetches a fresh copy and discards in-memory changes;
/* ------------------------------------------------------------------------ */ /* -------------------------------------------------------------------- */
%mv_castabload(caslib=&testcaslib,table=&tab1,mdebug=1)
/* Append a sentinel row to the in-memory table */
data work.extra;
set mylib.&tab1;
name='TESTROW';
output;
stop;
run;
proc casutil;
load data=work.extra casout="&tab1"
outcaslib="&testcaslib" append;
quit;
%let _modified=0;
proc sql noprint;
select count(*) into :_modified
from mylib.&tab1
where name='TESTROW';
quit;
%mp_assert( %mp_assert(
iftrue=(&MV_CASTABLOAD_RC=1), iftrue=(&_modified=1),
desc=Check RC=1 when table is loaded and promoted for the first time desc=Check sentinel row is present in memory before reload
) )
/* Drop the table and reload - source file does not have the sentinel */
/* ------------------------------------------------------------------------ */
%put TEST 3 - calling again for the same table should be a no-op (RC=0);
/* also verify no scope leakage of macro variables */
/* ------------------------------------------------------------------------ */
%mp_assertscope(SNAPSHOT)
%mv_castabload(caslib=&testcaslib,table=&tab1,mdebug=1)
%mp_assertscope(COMPARE,
desc=Check mv_castabload does not leak macro variables into GLOBAL scope,
ignorelist=MV_CASTABLOAD_RC
)
%mp_assert(
iftrue=(&MV_CASTABLOAD_RC=0),
desc=Check RC=0 when table is already in-memory (skip load)
)
/* ------------------------------------------------------------------------ */
%put TEST 4 - explicit srcfile= where file name differs from table name;
/* ------------------------------------------------------------------------ */
%mv_castabload(
caslib=&testcaslib,
table=&tab2,
srcfile=src_&tab2..sashdat,
mdebug=1
)
%mp_assert(
iftrue=(&MV_CASTABLOAD_RC=1),
desc=Check RC=1 when loading with explicit srcfile= parameter
)
/* ------------------------------------------------------------------------ */
%put TEST 5 - load failure when srcfile does not exist in the caslib;
/* ------------------------------------------------------------------------ */
%mv_castabload(
caslib=&testcaslib,
table=&tab3,
srcfile=doesnotexist_%mf_uid..sashdat,
mdebug=1
)
%mp_assert(
iftrue=(&MV_CASTABLOAD_RC=3),
desc=Check RC=3 when source file cannot be found / load fails
)
/* reset so that a downstream failure RC does not break testterm */
%let syscc=0;
/* ------------------------------------------------------------------------ */
/* Teardown: drop promoted tables and remove source files */
/* ------------------------------------------------------------------------ */
proc casutil; proc casutil;
droptable casdata="&tab1" incaslib="&testcaslib" quiet; droptable casdata="&tab1" incaslib="&testcaslib" quiet;
droptable casdata="&tab2" incaslib="&testcaslib" quiet; droptable casdata="&tab1" incaslib="&testcaslib" quiet;
deletesource casdata="&tab1..sashdat" incaslib="&testcaslib" quiet; quit;
deletesource casdata="src_&tab2..sashdat" incaslib="&testcaslib" quiet;
%mp_assertscope(SNAPSHOT)
%mv_castabload(lib=mylib, table=&tab1, mdebug=1)
%mp_assertscope(COMPARE,
desc=Check mv_castabload does not leak macro variables into GLOBAL scope
)
%let _after=0;
proc sql noprint;
select count(*) into :_after
from mylib.&tab1
where name='TESTROW';
quit;
%mp_assert(
iftrue=(&_after=0),
desc=Check sentinel row is absent after reload from source
)
/* -------------------------------------------------------------------- */
/* Teardown */
/* -------------------------------------------------------------------- */
libname mylib clear;
proc casutil;
droptable casdata="&tab1" incaslib="&testcaslib" quiet;
droptable casdata="&tab1" incaslib="&testcaslib" quiet;
deletesource casdata="&tab1..sashdat"
incaslib="&testcaslib" quiet;
quit; quit;
cas mysess terminate; cas mysess terminate;
viya/mv_castabload.sas
+159 -65
View File
@@ -1,73 +1,78 @@
/** /**
@file mv_castabload.sas @file mv_castabload.sas
@brief Checks if a CAS table exists in a CASLIB; if not, loads & promotes it @brief Checks if a CAS table is loaded; if not, loads and promotes it
@details Runs in SPRE against an active CAS session. Uses @details Runs in SPRE against an active CAS session. Accepts a
`table.tableExists` to check whether the table is already in-memory, SAS libref, derives the CAS caslib and session UUID from
and PROC CASUTIL LOAD with the PROMOTE option to load it if not. sashelp.vlibnam, then checks whether the table is already
CASUTIL infers the file type from the source file extension. in-memory. If not, locates the owning CAS server via the
casManagement REST API, queries the table endpoint to discover
the source file and caslib, then loads and promotes the table.
A CAS session must already be established by the caller, eg: A CAS session must already be established by the caller, eg:
cas mysess; cas mysess;
%mv_castabload(caslib=Public, table=BASEBALL) libname mylib cas caslib=Public;
%mv_castabload(lib=mylib, table=BASEBALL)
or (if not a hdat source with the same name as the table): @param [in] lib= SAS libref for the CAS caslib
@param [in] table= Name of the CAS table to load
%mv_castabload(caslib=Public, table=BASEBALL, @param [in] mdebug= (0) Set to 1 to enable verbose logging:
srcfile=MYBASEBALL.parquet)
@param [in] caslib= CASLIB containing the source file
@param [in] table= Name to give the in-memory CAS table
@param [in] srcfile= (0) Source file name.ext in the caslib. If not provided,
the code assumes that srcfile=&table..sashdat
@param [in] mdebug= (0) Set to 1 to enable verbose logging:
- echoes resolved parameters - echoes resolved parameters
- prints tableExists result - prints tableExists result
- enables mprint/notes during PROC calls - enables mprint/notes during PROC calls
@returns Sets global macro variable `MV_CASTABLOAD_RC`:
0 = table already existed (no load performed)
1 = table was loaded & promoted successfully
3 = action failed (including source file missing)
<h4> SAS Macros </h4> <h4> SAS Macros </h4>
@li mfv_existsashdat.sas @li mf_getplatform.sas
@li mf_getuniquefileref.sas
@li mf_getuniquelibref.sas
@li mp_abort.sas
**/ **/
%macro mv_castabload( %macro mv_castabload(
caslib= lib=
,table= ,table=
,srcfile=0
,mdebug=0 ,mdebug=0
); );
%global MV_CASTABLOAD_RC; %local _sysopts base_uri caslib uuid server
%let MV_CASTABLOAD_RC=3; srcfile srccaslib fname1 libref1 ftmp i _svcount _exists;
%local _sysopts;
%let _sysopts=%sysfunc(getoption(mprint)) %sysfunc(getoption(notes)); %let _sysopts=%sysfunc(getoption(mprint)) %sysfunc(getoption(notes));
/* ---- input validation -------------------------------------------------- */ /* ---- input validation -------------------------------------------------- */
%if "&caslib"="" or "&table"="" or "&srcfile"="" %then %do; %mp_abort(
%put %str(ERR)OR: caslib=, table= and srcfile= are all required; iftrue=("&lib"="" or "&table"=""),
%return; msg=%str(lib= and table= are required)
%end; )
%if "&srcfile"="0" %then %let srcfile=&table..sashdat;
%if &mdebug=1 %then %do; %if &mdebug=1 %then %do;
%put &=caslib; %put &=lib;
%put &=table; %put &=table;
%put &=srcfile;
options mprint notes; options mprint notes;
%end; %end;
/* ---- check source file exists ------------------------------------------ */ /* ---- derive caslib and session UUID from sashelp.vlibnam --------------- */
%if not %mfv_existsashdat(&caslib..%scan(&srcfile,1,.)) %then %do; data _null_;
%put %str(ERR)OR: Source file "&srcfile" not found in caslib "&caslib"; set sashelp.vlibnam(
%let MV_CASTABLOAD_RC=3; where=(libname="%upcase(&lib)"
%return; and sysname in ("Caslib","Session UUID"))
%end; );
if sysname="Caslib" then call symputx('caslib',sysvalue,'L');
else call symputx('uuid',sysvalue,'L');
%if &mdebug=1 %then %do;
putlog sysname sysvalue;
%end;
run;
%mp_abort(
iftrue=("&caslib"=""),
msg=%str(&lib is not an assigned CAS libref)
)
%mp_abort(
iftrue=("&uuid"=""),
msg=%str(No session UUID found for libref &lib)
)
/* ---- existence check --------------------------------------------------- */ /* ---- existence check --------------------------------------------------- */
proc cas; proc cas;
@@ -77,37 +82,126 @@ proc cas;
%if &mdebug=1 %then %do; %if &mdebug=1 %then %do;
print r; print r;
%end; %end;
if r.exists = 0 then rc = 9; if r.exists > 0 then call symputx('_exists', '1', 'L');
else rc = 0; else call symputx('_exists', '0', 'L');
symputx('MV_CASTABLOAD_RC', rc, 'G');
quit; quit;
/* ---- already loaded: skip ---------------------------------------------- */
/* ---- load if absent ---------------------------------------------------- */ %if &_exists=1 %then %do;
%if &MV_CASTABLOAD_RC=9 %then %do; %put NOTE: Table &caslib..&table already loaded - skipping;
%return;
proc casutil;
load casdata="&srcfile"
incaslib="&caslib"
casout="&table"
outcaslib="&caslib"
promote;
quit;
%if &syserr=0 %then %let MV_CASTABLOAD_RC=1;
%else %let MV_CASTABLOAD_RC=3;
%end; %end;
%if &MV_CASTABLOAD_RC=0 %then /* ---- get list of CAS servers ----------------------------------------- */
%put NOTE: Table &caslib..&table already loaded - skipping; %let base_uri=%mf_getplatform(VIYARESTAPI);
%else %if &MV_CASTABLOAD_RC=1 %then %let fname1=%mf_getuniquefileref();
%put NOTE: Table &caslib..&table loaded and promoted; %let libref1=%mf_getuniquelibref();
%else %put %str(ERR)OR: load failed for &caslib..&table;
proc http method='GET' out=&fname1 oauth_bearer=sas_services
url="&base_uri/casManagement/servers";
run;
%mp_abort(
iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200),
msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
)
libname &libref1 JSON fileref=&fname1;
data _null_;
set &libref1..items;
call symputx(cats('_sv_', _n_), name, 'L');
call symputx('_svcount', _n_, 'L');
run;
libname &libref1 clear;
filename &fname1 clear;
/* ---- find which server owns this session ------------------------------ */
%do i=1 %to &_svcount;
%if "&server"="" %then %do;
%if &mdebug=1 %then %put checking server: &&_sv_&i;
%let ftmp=%mf_getuniquefileref();
proc http method='GET' out=&ftmp oauth_bearer=sas_services
url="&base_uri/casManagement/servers/&&_sv_&i/sessions/&uuid";
run;
%if &SYS_PROCHTTP_STATUS_CODE=200
%then %let server=&&_sv_&i;
filename &ftmp clear;
%end;
%end;
%mp_abort(
iftrue=("&server"=""),
msg=%str(Could not find owning server for CAS session &uuid)
)
%if &mdebug=1 %then %put &=server;
/* ---- discover source file from REST endpoint -------------------------- */
%let fname1=%mf_getuniquefileref();
%let libref1=%mf_getuniquelibref();
proc http method='GET' out=&fname1 oauth_bearer=sas_services
url="&base_uri/casManagement/servers/&server/caslibs/&caslib/tables/&table";
run;
%if &mdebug=1 %then %do;
%put &=SYS_PROCHTTP_STATUS_CODE &=SYS_PROCHTTP_STATUS_PHRASE;
data _null_;
infile &fname1;
input;
putlog _infile_;
run;
%end;
%mp_abort(
iftrue=(&SYS_PROCHTTP_STATUS_CODE=404),
msg=%str(&caslib..&table not found - is a source file registered?)
)
%mp_abort(
iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200),
msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
)
libname &libref1 JSON fileref=&fname1;
data _null_;
set &libref1..tablereference;
call symputx('srcfile', sourceTableName, 'L');
call symputx('srccaslib', sourceCaslibName, 'L');
stop;
run;
libname &libref1 clear;
filename &fname1 clear;
%mp_abort(
iftrue=("&srcfile"="" or "&srccaslib"=""),
msg=%str(No sourceTableName/sourceCaslibName for &caslib..&table)
)
%if &mdebug=1 %then %put &=srcfile &=srccaslib;
/* ---- load from discovered source -------------------------------------- */
proc casutil;
load casdata="&srcfile"
incaslib="&srccaslib"
casout="&table"
outcaslib="&caslib"
promote;
quit;
%mp_abort(
iftrue=(&syscc ne 0),
msg=%str(Load failed for &caslib..&table)
)
%put NOTE: Table &caslib..&table loaded and promoted from &srcfile;
/* ---- restore options --------------------------------------------------- */ /* ---- restore options --------------------------------------------------- */
%if &mdebug=1 %then %do; %if &mdebug=1 %then %do;
options &_sysopts; options &_sysopts;
%end; %end;
%mend mv_castabload; %mend mv_castabload;