From 36452a2a02b5ad0259f6d74b4710de7edd6a008d Mon Sep 17 00:00:00 2001 From: 4gl <@> Date: Tue, 28 Apr 2026 18:38:57 +0100 Subject: [PATCH] fix: simplifying mv_castabload and improving tests --- tests/viyaonly/mv_castabload.test.sas | 192 +++++++++++----------- viya/mv_castabload.sas | 224 ++++++++++++++++++-------- 2 files changed, 257 insertions(+), 159 deletions(-) diff --git a/tests/viyaonly/mv_castabload.test.sas b/tests/viyaonly/mv_castabload.test.sas index 8f5ca17..6c6d3f9 100644 --- a/tests/viyaonly/mv_castabload.test.sas +++ b/tests/viyaonly/mv_castabload.test.sas @@ -7,26 +7,26 @@ @li mp_assert.sas @li mp_assertscope.sas @li mv_castabload.sas - @li mv_createfile.sas **/ options mprint; -/* ------------------------------------------------------------------------ */ -/* Setup: start a CAS session and stage a source file in the Public caslib */ -/* ------------------------------------------------------------------------ */ +/* -------------------------------------------------------------------- */ +/* Setup: start a CAS session and stage a source file in the caslib */ +/* -------------------------------------------------------------------- */ cas mysess; caslib _all_ assign; -%let testcaslib = Public; /* change this if Public isn't available */ +%let testcaslib=Public; + proc cas; table.caslibInfo result=r / ; - found = 0; + found=0; do row over r.CASLibInfo; - if upcase(row.Name) = upcase("&testcaslib") then found = 1; + if upcase(row.Name)=upcase("&testcaslib") then found=1; end; - if found = 0 then do; + if found=0 then do; print "ERROR: caslib &testcaslib not available"; exit; end; @@ -34,113 +34,117 @@ quit; %put NOTE: Using testcaslib=&testcaslib; %let tab1=T%mf_uid(); -%let tab2=T%mf_uid(); -%let tab3=T%mf_uid(); -/* Create a SASHDAT source file in the Public caslib from SASHELP.BASEBALL - so that subsequent LOAD operations have something real to pick up. */ +/* Save a sashdat source file then drop the in-memory copy so the first + mv_castabload call has something to load */ proc casutil; - load data=sashelp.baseball outcaslib="&testcaslib" casout="&tab1" replace; + load data=sashelp.baseball + outcaslib="&testcaslib" casout="&tab1" replace; save casdata="&tab1" incaslib="&testcaslib" casout="&tab1..sashdat" outcaslib="&testcaslib" replace; droptable casdata="&tab1" incaslib="&testcaslib" quiet; quit; -/* And a second hdat, with a name different from the table name, so that we - can exercise the explicit srcfile= path. */ -proc casutil; - load data=sashelp.cars outcaslib="&testcaslib" casout="&tab2" replace; - save casdata="&tab2" incaslib="&testcaslib" - casout="src_&tab2..sashdat" outcaslib="&testcaslib" replace; - droptable casdata="&tab2" incaslib="&testcaslib" quiet; +libname mylib cas caslib="&testcaslib"; + + +/* -------------------------------------------------------------------- */ +%put TEST 1 - load a table that is not in memory; +/* -------------------------------------------------------------------- */ + +/* Confirm table is absent before the call */ +%let _tabexists=0; +proc cas; + table.tableExists result=r / + caslib="&testcaslib" name="&tab1"; + if r.exists > 0 then call symputx('_tabexists','1'); quit; +%mp_assert( + iftrue=(&_tabexists=0), + desc=Check table is not in memory before mv_castabload +) -/* ------------------------------------------------------------------------ */ -%put TEST 1 - missing required parameters returns without setting RC to 0/1; -/* ------------------------------------------------------------------------ */ -%let MV_CASTABLOAD_RC=; -%mv_castabload(caslib=,table=,srcfile=) +%mv_castabload(lib=mylib, table=&tab1, mdebug=1) + +%let _tabexists=0; +proc cas; + table.tableExists result=r / + caslib="&testcaslib" name="&tab1"; + if r.exists > 0 then call symputx('_tabexists','1'); +quit; %mp_assert( - iftrue=(&MV_CASTABLOAD_RC=3), - desc=Check RC=3 (initial/failure value) when required params are missing + iftrue=(&_tabexists=1), + desc=Check table is in memory after mv_castabload ) -/* ------------------------------------------------------------------------ */ -%put TEST 2 - load a table that does not yet exist (default srcfile=table.sashdat); -/* ------------------------------------------------------------------------ */ -%mv_castabload(caslib=&testcaslib,table=&tab1,mdebug=1) +/* -------------------------------------------------------------------- */ +%put TEST 2 - reload fetches a fresh copy and discards in-memory changes; +/* -------------------------------------------------------------------- */ + +/* Append a sentinel row to the in-memory table */ +data work.extra; + set mylib.&tab1; + name='TESTROW'; + output; + stop; +run; +proc casutil; + load data=work.extra casout="&tab1" + outcaslib="&testcaslib" append; +quit; + +%let _modified=0; +proc sql noprint; + select count(*) into :_modified + from mylib.&tab1 + where name='TESTROW'; +quit; %mp_assert( - iftrue=(&MV_CASTABLOAD_RC=1), - desc=Check RC=1 when table is loaded and promoted for the first time + iftrue=(&_modified=1), + desc=Check sentinel row is present in memory before reload ) - -/* ------------------------------------------------------------------------ */ -%put TEST 3 - calling again for the same table should be a no-op (RC=0); -/* also verify no scope leakage of macro variables */ -/* ------------------------------------------------------------------------ */ -%mp_assertscope(SNAPSHOT) - -%mv_castabload(caslib=&testcaslib,table=&tab1,mdebug=1) - -%mp_assertscope(COMPARE, - desc=Check mv_castabload does not leak macro variables into GLOBAL scope, - ignorelist=MV_CASTABLOAD_RC -) - -%mp_assert( - iftrue=(&MV_CASTABLOAD_RC=0), - desc=Check RC=0 when table is already in-memory (skip load) -) - - -/* ------------------------------------------------------------------------ */ -%put TEST 4 - explicit srcfile= where file name differs from table name; -/* ------------------------------------------------------------------------ */ -%mv_castabload( - caslib=&testcaslib, - table=&tab2, - srcfile=src_&tab2..sashdat, - mdebug=1 -) - -%mp_assert( - iftrue=(&MV_CASTABLOAD_RC=1), - desc=Check RC=1 when loading with explicit srcfile= parameter -) - - -/* ------------------------------------------------------------------------ */ -%put TEST 5 - load failure when srcfile does not exist in the caslib; -/* ------------------------------------------------------------------------ */ -%mv_castabload( - caslib=&testcaslib, - table=&tab3, - srcfile=doesnotexist_%mf_uid..sashdat, - mdebug=1 -) - -%mp_assert( - iftrue=(&MV_CASTABLOAD_RC=3), - desc=Check RC=3 when source file cannot be found / load fails -) - -/* reset so that a downstream failure RC does not break testterm */ -%let syscc=0; - - -/* ------------------------------------------------------------------------ */ -/* Teardown: drop promoted tables and remove source files */ -/* ------------------------------------------------------------------------ */ +/* Drop the table and reload - source file does not have the sentinel */ proc casutil; droptable casdata="&tab1" incaslib="&testcaslib" quiet; - droptable casdata="&tab2" incaslib="&testcaslib" quiet; - deletesource casdata="&tab1..sashdat" incaslib="&testcaslib" quiet; - deletesource casdata="src_&tab2..sashdat" incaslib="&testcaslib" quiet; + droptable casdata="&tab1" incaslib="&testcaslib" quiet; +quit; + +%mp_assertscope(SNAPSHOT) + +%mv_castabload(lib=mylib, table=&tab1, mdebug=1) + +%mp_assertscope(COMPARE, + desc=Check mv_castabload does not leak macro variables into GLOBAL scope +) + +%let _after=0; +proc sql noprint; + select count(*) into :_after + from mylib.&tab1 + where name='TESTROW'; +quit; + +%mp_assert( + iftrue=(&_after=0), + desc=Check sentinel row is absent after reload from source +) + + +/* -------------------------------------------------------------------- */ +/* Teardown */ +/* -------------------------------------------------------------------- */ +libname mylib clear; + +proc casutil; + droptable casdata="&tab1" incaslib="&testcaslib" quiet; + droptable casdata="&tab1" incaslib="&testcaslib" quiet; + deletesource casdata="&tab1..sashdat" + incaslib="&testcaslib" quiet; quit; cas mysess terminate; diff --git a/viya/mv_castabload.sas b/viya/mv_castabload.sas index c7f4fd2..4bdb13a 100644 --- a/viya/mv_castabload.sas +++ b/viya/mv_castabload.sas @@ -1,73 +1,78 @@ /** @file mv_castabload.sas - @brief Checks if a CAS table exists in a CASLIB; if not, loads & promotes it - @details Runs in SPRE against an active CAS session. Uses - `table.tableExists` to check whether the table is already in-memory, - and PROC CASUTIL LOAD with the PROMOTE option to load it if not. - CASUTIL infers the file type from the source file extension. + @brief Checks if a CAS table is loaded; if not, loads and promotes it + @details Runs in SPRE against an active CAS session. Accepts a + SAS libref, derives the CAS caslib and session UUID from + sashelp.vlibnam, then checks whether the table is already + in-memory. If not, locates the owning CAS server via the + casManagement REST API, queries the table endpoint to discover + the source file and caslib, then loads and promotes the table. A CAS session must already be established by the caller, eg: cas mysess; - %mv_castabload(caslib=Public, table=BASEBALL) + libname mylib cas caslib=Public; + %mv_castabload(lib=mylib, table=BASEBALL) - or (if not a hdat source with the same name as the table): - - %mv_castabload(caslib=Public, table=BASEBALL, - srcfile=MYBASEBALL.parquet) - - @param [in] caslib= CASLIB containing the source file - @param [in] table= Name to give the in-memory CAS table - @param [in] srcfile= (0) Source file name.ext in the caslib. If not provided, - the code assumes that srcfile=&table..sashdat - @param [in] mdebug= (0) Set to 1 to enable verbose logging: + @param [in] lib= SAS libref for the CAS caslib + @param [in] table= Name of the CAS table to load + @param [in] mdebug= (0) Set to 1 to enable verbose logging: - echoes resolved parameters - prints tableExists result - enables mprint/notes during PROC calls - @returns Sets global macro variable `MV_CASTABLOAD_RC`: - 0 = table already existed (no load performed) - 1 = table was loaded & promoted successfully - 3 = action failed (including source file missing) -

SAS Macros

- @li mfv_existsashdat.sas + @li mf_getplatform.sas + @li mf_getuniquefileref.sas + @li mf_getuniquelibref.sas + @li mp_abort.sas **/ %macro mv_castabload( - caslib= + lib= ,table= - ,srcfile=0 ,mdebug=0 ); -%global MV_CASTABLOAD_RC; -%let MV_CASTABLOAD_RC=3; - -%local _sysopts; +%local _sysopts base_uri caslib uuid server + srcfile srccaslib fname1 libref1 ftmp i _svcount _exists; %let _sysopts=%sysfunc(getoption(mprint)) %sysfunc(getoption(notes)); /* ---- input validation -------------------------------------------------- */ -%if "&caslib"="" or "&table"="" or "&srcfile"="" %then %do; - %put %str(ERR)OR: caslib=, table= and srcfile= are all required; - %return; -%end; -%if "&srcfile"="0" %then %let srcfile=&table..sashdat; +%mp_abort( + iftrue=("&lib"="" or "&table"=""), + msg=%str(lib= and table= are required) +) %if &mdebug=1 %then %do; - %put &=caslib; + %put &=lib; %put &=table; - %put &=srcfile; options mprint notes; %end; -/* ---- check source file exists ------------------------------------------ */ -%if not %mfv_existsashdat(&caslib..%scan(&srcfile,1,.)) %then %do; - %put %str(ERR)OR: Source file "&srcfile" not found in caslib "&caslib"; - %let MV_CASTABLOAD_RC=3; - %return; -%end; +/* ---- derive caslib and session UUID from sashelp.vlibnam --------------- */ +data _null_; + set sashelp.vlibnam( + where=(libname="%upcase(&lib)" + and sysname in ("Caslib","Session UUID")) + ); + if sysname="Caslib" then call symputx('caslib',sysvalue,'L'); + else call symputx('uuid',sysvalue,'L'); + %if &mdebug=1 %then %do; + putlog sysname sysvalue; + %end; +run; + +%mp_abort( + iftrue=("&caslib"=""), + msg=%str(&lib is not an assigned CAS libref) +) + +%mp_abort( + iftrue=("&uuid"=""), + msg=%str(No session UUID found for libref &lib) +) /* ---- existence check --------------------------------------------------- */ proc cas; @@ -77,37 +82,126 @@ proc cas; %if &mdebug=1 %then %do; print r; %end; - if r.exists = 0 then rc = 9; - else rc = 0; - symputx('MV_CASTABLOAD_RC', rc, 'G'); + if r.exists > 0 then call symputx('_exists', '1', 'L'); + else call symputx('_exists', '0', 'L'); quit; - -/* ---- load if absent ---------------------------------------------------- */ -%if &MV_CASTABLOAD_RC=9 %then %do; - - proc casutil; - load casdata="&srcfile" - incaslib="&caslib" - casout="&table" - outcaslib="&caslib" - promote; - quit; - - %if &syserr=0 %then %let MV_CASTABLOAD_RC=1; - %else %let MV_CASTABLOAD_RC=3; - +/* ---- already loaded: skip ---------------------------------------------- */ +%if &_exists=1 %then %do; + %put NOTE: Table &caslib..&table already loaded - skipping; + %return; %end; -%if &MV_CASTABLOAD_RC=0 %then - %put NOTE: Table &caslib..&table already loaded - skipping; -%else %if &MV_CASTABLOAD_RC=1 %then - %put NOTE: Table &caslib..&table loaded and promoted; -%else %put %str(ERR)OR: load failed for &caslib..&table; +/* ---- get list of CAS servers ----------------------------------------- */ +%let base_uri=%mf_getplatform(VIYARESTAPI); +%let fname1=%mf_getuniquefileref(); +%let libref1=%mf_getuniquelibref(); + +proc http method='GET' out=&fname1 oauth_bearer=sas_services + url="&base_uri/casManagement/servers"; +run; + +%mp_abort( + iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200), + msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) +) + +libname &libref1 JSON fileref=&fname1; + +data _null_; + set &libref1..items; + call symputx(cats('_sv_', _n_), name, 'L'); + call symputx('_svcount', _n_, 'L'); +run; + +libname &libref1 clear; +filename &fname1 clear; + +/* ---- find which server owns this session ------------------------------ */ +%do i=1 %to &_svcount; + %if "&server"="" %then %do; + %if &mdebug=1 %then %put checking server: &&_sv_&i; + %let ftmp=%mf_getuniquefileref(); + proc http method='GET' out=&ftmp oauth_bearer=sas_services + url="&base_uri/casManagement/servers/&&_sv_&i/sessions/&uuid"; + run; + %if &SYS_PROCHTTP_STATUS_CODE=200 + %then %let server=&&_sv_&i; + filename &ftmp clear; + %end; +%end; + +%mp_abort( + iftrue=("&server"=""), + msg=%str(Could not find owning server for CAS session &uuid) +) + +%if &mdebug=1 %then %put &=server; + +/* ---- discover source file from REST endpoint -------------------------- */ +%let fname1=%mf_getuniquefileref(); +%let libref1=%mf_getuniquelibref(); + +proc http method='GET' out=&fname1 oauth_bearer=sas_services + url="&base_uri/casManagement/servers/&server/caslibs/&caslib/tables/&table"; +run; + +%if &mdebug=1 %then %do; + %put &=SYS_PROCHTTP_STATUS_CODE &=SYS_PROCHTTP_STATUS_PHRASE; + data _null_; + infile &fname1; + input; + putlog _infile_; + run; +%end; + +%mp_abort( + iftrue=(&SYS_PROCHTTP_STATUS_CODE=404), + msg=%str(&caslib..&table not found - is a source file registered?) +) +%mp_abort( + iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200), + msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) +) + +libname &libref1 JSON fileref=&fname1; + +data _null_; + set &libref1..tablereference; + call symputx('srcfile', sourceTableName, 'L'); + call symputx('srccaslib', sourceCaslibName, 'L'); + stop; +run; + +libname &libref1 clear; +filename &fname1 clear; + +%mp_abort( + iftrue=("&srcfile"="" or "&srccaslib"=""), + msg=%str(No sourceTableName/sourceCaslibName for &caslib..&table) +) + +%if &mdebug=1 %then %put &=srcfile &=srccaslib; + +/* ---- load from discovered source -------------------------------------- */ +proc casutil; + load casdata="&srcfile" + incaslib="&srccaslib" + casout="&table" + outcaslib="&caslib" + promote; +quit; + +%mp_abort( + iftrue=(&syscc ne 0), + msg=%str(Load failed for &caslib..&table) +) + +%put NOTE: Table &caslib..&table loaded and promoted from &srcfile; /* ---- restore options --------------------------------------------------- */ %if &mdebug=1 %then %do; options &_sysopts; %end; -%mend mv_castabload; \ No newline at end of file +%mend mv_castabload;