From 1cd8ba03c506a26121f8847deb90490ffa559783 Mon Sep 17 00:00:00 2001 From: Trevor Moody Date: Tue, 18 Nov 2025 23:56:54 +0000 Subject: [PATCH] chore: improved file-type handling efficiency --- .../viyaonly/mv_getviyafileextparms.test.sas | 6 +- viya/mv_getviyafileextparms.sas | 257 +++++++++++------- 2 files changed, 165 insertions(+), 98 deletions(-) diff --git a/tests/viyaonly/mv_getviyafileextparms.test.sas b/tests/viyaonly/mv_getviyafileextparms.test.sas index 86b2c9e..d99887e 100644 --- a/tests/viyaonly/mv_getviyafileextparms.test.sas +++ b/tests/viyaonly/mv_getviyafileextparms.test.sas @@ -35,7 +35,7 @@ options mprint; ) %mp_assert( - iftrue=(not %mf_isBlank(&viyaProperties)), + iftrue=(not %mf_isBlank(%superq(viyaProperties))), desc=Check the requested macro variable viyaProperties is not blank. ) @@ -66,7 +66,7 @@ options mprint; %mp_assert( iftrue=(not ( %mf_isBlank(&cssViyaTypeDefName) or - %mf_isBlank(&cssViyaProperties) or + %mf_isBlank(%superq(cssViyaProperties)) or %mf_isBlank(&cssViyaMediaType) ) ), desc=Check a full set of requested macro variables are not blank. ) @@ -88,7 +88,7 @@ options mprint; %mp_assert( iftrue=( %mf_isBlank(&invalidTypeDefName) and - %mf_isBlank(&invalidProperties) and + %mf_isBlank(%superq(invalidProperties)) and %mf_isBlank(&invalidMediaType) ), desc=Check the requested macro variables are all blank. diff --git a/viya/mv_getviyafileextparms.sas b/viya/mv_getviyafileextparms.sas index 09a9623..f7f40ab 100644 --- a/viya/mv_getviyafileextparms.sas +++ b/viya/mv_getviyafileextparms.sas @@ -13,10 +13,13 @@ the 'typeDefName' property value, if found, else blank. @param [out] mediaTypeVar= SAS macro variable name that will contain the 'mediaType' property value, if found, else blank. + @param [out] viyaFileExtRespLibDs (work.mv_getViyaFileExtParmsResponse) + Library.name of the dataset to receive the local working copy of the initial + response that requests all file extension details. Created once per session + to avoid multiple api calls. @param [in] mdebug= (0) Set to 1 to enable DEBUG messages

SAS Macros

- @li mf_abort.sas @li mf_existds.sas @li mf_getplatform.sas @li mf_getuniquefileref.sas @@ -24,22 +27,39 @@ @li mf_getvalue.sas @li mf_getvarlist.sas @li mf_getvartype.sas + @li mp_abort.sas */ -%macro mv_getViyaFileExtParms(ext,typeDefNameVar=,propertiesVar=,mediaTypeVar=,mdebug=0); +%macro mv_getViyaFileExtParms( + ext, + typeDefNameVar=, + propertiesVar=, + mediaTypeVar=, + viyaFileExtRespLibDs=work.mv_getViyaFileExtParmsResponse, + mdebug=0 + ); %local base_uri; /* location of rest apis */ - %local viyatypedef; /* temp fileref to json response */ %local url; /* File extension info end-point */ - %mf_abort(iftrue=(%mf_isBlank(&ext)) - ,msg=%str(MV_GETVIYAFILEEXTPARMS - No file extension provided.) + %mp_abort( + iftrue=(%mf_isBlank(&ext)) + ,msg=%str(No file extension provided.) + ,mac=MV_GETVIYAFILEEXTPARMS ); - %mf_abort(iftrue=(%mf_isBlank(&typeDefNameVar) and - %mf_isBlank(&propertiesVar) and - %mf_isBlank(&mediaTypeVar)) + %mp_abort( + iftrue=(%mf_isBlank(&typeDefNameVar) and + %mf_isBlank(&propertiesVar) and + %mf_isBlank(&mediaTypeVar)) ,msg=%str(MV_GETVIYAFILEEXTPARMS - No parameter was requested.) + ,mac=MV_GETVIYAFILEEXTPARMS + ); + + %mp_abort( + iftrue=(%mf_isBlank(&viyaFileExtRespLibDs)) + ,msg=%str(No dataset name provided to cache inital response.) + ,mac=MV_GETVIYAFILEEXTPARMS ); /* Declare requested parameters as global macro vars and initialize blank */ @@ -58,122 +78,169 @@ %let base_uri=%mf_getplatform(VIYARESTAPI); %if &mdebug=1 %then %do; - %put DEBUG: &SYSMACRONAME &=base_uri; + %put DEBUG: &=base_uri; %end; %let ext=%lowcase(&ext); - /* Create a temp file and fill with JSON that declares */ - /* VIYA file-type details for the given file extension */ - %let viyatypedef=%mf_getuniquefileref(); - filename &viyatypedef temp; + /* Create a local copy of the Viya response containing all file type info, if + it does not already exist. */ + %if not %mf_existds(&viyaFileExtRespLibDs) %then %do; + /* Create a temp file and fill with JSON that declares */ + /* VIYA file-type details for the given file extension */ + %local viyatypedefs; + %let viyatypedefs=%mf_getuniquefileref(); + filename &viyatypedefs temp; - %let url = &base_uri/types/types?filter=contains(extensions,%str(%')&ext%str(%')); + %let url = &base_uri/types/types?limit=999999; - proc http oauth_bearer=sas_services out=&viyatypedef - url="&url"; - run; + proc http oauth_bearer=sas_services out=&viyatypedefs + url="&url"; + run; - %if &mdebug=1 %then %put DEBUG: &SYSMACRONAME &=url - &=SYS_PROCHTTP_STATUS_CODE &=SYS_PROCHTTP_STATUS_PHRASE; + %if &mdebug=1 %then %put DEBUG: &sysmacroname &=url + &=SYS_PROCHTTP_STATUS_CODE &=SYS_PROCHTTP_STATUS_PHRASE; - %if (&SYS_PROCHTTP_STATUS_CODE ne 200) %then %do; - /* To avoid a breaking change, exit early if the request failed. - The calling process will proceed with empty requested macro variables. */ - %put INFO: &sysmacroname A response was not returned.; - filename &viyatypedef clear; + %if (&SYS_PROCHTTP_STATUS_CODE ne 200) %then %do; + /* To avoid a breaking change, exit early if the request failed. + The calling process will proceed with empty requested macro variables. */ + %put INFO: &sysmacroname File extension details were not retrieved.; + filename &viyatypedefs clear; + %return; + %end; + + %if &mdebug=1 %then %do; + /* Dump the response to the log */ + data _null_; + length line $120; + null=byte(0); + infile &viyatypedefs dlm=null lrecl=120 recfm=n; + input line $120.; + if _n_ = 1 then put "DEBUG:"; + put line; + run; + %end; + + /* Convert the content of that JSON into SAS datasets */ + /* First prepare a new WORK-based folder to receive the datasets */ + %local jsonworkfolder jsonlib opt_dlcreatedir; + %let jsonworkfolder=%sysfunc(pathname(work))/%mf_getuniquename(prefix=json_); + %let jsonlib=%mf_getuniquelibref(prefix=json); + /* And point a libname at it */ + %let opt_dlcreatedir = %sysfunc(getoption(dlcreatedir)); + options dlcreatedir; libname &jsonlib "&jsonworkfolder"; options &opt_dlcreatedir; + + /* Read the json output once and copy datasets to its work folder */ + %local libref1; + %let libref1=%mf_getuniquelibref(); + libname &libref1 JSON fileref=&viyatypedefs automap=create; + proc copy in=&libref1 out=&jsonlib; run; + + libname &libref1 clear; + + /* Now give all rows belonging to the same items array a grouping value */ + data &viyaFileExtRespLibDs; + length _viyaItemIdx 8; + set &jsonlib..alldata; + retain _viyaItemIdx 0; + /* Increment the row group index when a new 'items' group is observed */ + if P=1 and P1='items' then _viyaItemIdx + 1; + run; + + %if &mdebug=0 %then %do; + /* Tidy up, unless debug=1 */ + proc datasets library=&jsonlib nolist kill; quit; + libname &jsonlib clear; + %end; + + filename &viyatypedefs clear; + + %end; /* If initial filetype query response didn't exist */ + + /* Find the row-group for the current file extension */ + %local itemRowGroup; + %let itemRowGroup = + %mf_getValue( + &viyaFileExtRespLibDs + ,_viyaItemIdx + ,filter=%quote(p1='items' and p2='extensions' and value="&ext") + ); + + %if &mdebug %then %put DEBUG: &=itemRowGroup; + + %if %mf_isBlank(&itemRowGroup) %then %do; + /* extension was not found */ + %if(&mdebug=1) %then %put DEBUG: No type details found for extension "&ext".; %return; %end; - %if &mdebug=1 %then %do; - data _null_; - infile &viyatypedef; - input; - put "DEBUG: &SYSMACRONAME" _infile_; - run; - %end; - - /* Convert the content of that JSON into SAS datasets */ - /* First prepare a new WORK-based folder to receive the datasets */ - %local jsonworkfolder jsonlib opt_dlcreatedir; - %let jsonworkfolder=%sysfunc(pathname(work))/%mf_getuniquename(prefix=json_); - %let jsonlib=%mf_getuniquelibref(prefix=json); - /* And point a libname at it */ - %let opt_dlcreatedir = %sysfunc(getoption(dlcreatedir)); - options dlcreatedir; libname &jsonlib "&jsonworkfolder"; options &opt_dlcreatedir; - - /* Read the json output once and copy datasets to its work folder */ - %local libref1; - %let libref1=%mf_getuniquelibref(); - libname &libref1 JSON fileref=&viyatypedef automap=create; - proc copy in=&libref1 out=&jsonlib; run; + /* Filter the cached response data down to the required file extension */ + %local dsItems; + %let dsItems = %mf_getuniquename(prefix=dsItems_); + data work.&dsItems; + set &viyaFileExtRespLibDs; + where _viyaItemIdx = &itemRowGroup; + run; /* Populate typeDefName, if requested */ %if (not %mf_isBlank(&typeDefNameVar)) %then %do; - %let &typeDefNameVar = %mf_getvalue(&jsonlib..alldata,value,filter=%quote(p1="items" and p2="name")); - %if &mdebug=1 %then %put DEBUG: &SYSMACRONAME &=typeDefNameVar &typeDefNameVar=&&&typeDefNameVar; + %let &typeDefNameVar = %mf_getvalue(&dsItems,value,filter=%quote(p1="items" and p2="name")); + %if &mdebug=1 %then %put DEBUG: &=typeDefNameVar &typeDefNameVar=&&&typeDefNameVar; %end; /* Populate mediaType, if requested */ %if (not %mf_isBlank(&mediaTypeVar)) %then %do; - %let &mediaTypeVar = %mf_getvalue(&jsonlib..alldata,value,filter=%quote(p1="items" and p2="mediaType")); - %if &mdebug=1 %then %put DEBUG: &SYSMACRONAME &=mediaTypeVar &mediaTypeVar=&&&mediaTypeVar; + %let &mediaTypeVar = %mf_getvalue(&dsItems,value,filter=%quote(p1="items" and p2="mediaType")); + %if &mdebug=1 %then %put DEBUG: &=mediaTypeVar &mediaTypeVar=&&&mediaTypeVar; %end; /* Populate properties macro variable, if requested */ %if not %mf_isBlank(&propertiesVar) %then %do; - /* Check for the items_properties table */ - %if ( not %mf_existds(&jsonlib..ITEMS_PROPERTIES) ) %then %do; + + /* Filter dsItems down to the properties */ + %local dsProperties; + %let dsProperties = %mf_getuniquename(prefix=dsProperties_); + data work.&dsProperties ( rename=(p3 = propertyName) ); + set work.&dsItems; + where p2="properties" and v=1; + run; + + /* Check for 1+ properties */ + %if ( %mf_nobs(&dsProperties) = 0 ) %then %do; %let &propertiesVar = %str(); %if &mdebug=1 %then %put DEBUG: &SYSMACRONAME - No Viya properties found for file suffix %str(%')&ext%str(%'); %end; %else %do; - /* Properties potentially span multiple rows in the ITEMS_PROPERTIES table */ - /* First remove some unwanted variables from the items_properties dataset. */ - %let dsTemplate=%mf_getuniquename(prefix=dsTemplate_); - data work.&dsTemplate; - stop; - set &jsonlib..ITEMS_PROPERTIES(drop=ordinal:); + /* Properties potentially span multiple rows in the input table */ + data _null_; + length + line $32767 + properties $32767 + ; + retain properties; + set &dsProperties end=last; + if _n_ = 1 then properties = '{'; + + line = cats(quote(trim(propertyName)),':'); + /* Only strings and bools appear in properties */ + if value not in ("true","false") then value = quote(trim(value)); + line = catx(' ',line,value); + /* Add a comma separator to all except the last line */ + if not last then line = cats(line,','); + + /* Add this line to the output value */ + properties = catx(' ',properties,line); + + if last then do; + /* Close off the properties object and output to the macro variable */ + properties=catx(' ',properties,'}'); + call symputx("&propertiesVar",properties); + end; run; - /* Retrieve the names of the remaining variables */ - /* These are the names of the properties. */ - %let varlist = %mf_getvarlist(work.&dsTemplate); - %if &mdebug %then %put DEBUG: &SYSMACRONAME &=varlist; - - %let &propertiesVar = %quote({); - - %let nvars = %sysfunc(countw(&varlist)); - %do i = 1 %to &nvars; - /* Use the name of each variable in the dataset as the property 'key' */ - %let key = %scan(&varlist,&i); - %let value = %mf_getvalue(&jsonlib..ITEMS_PROPERTIES,&key); - /* The data type determines if value should be quoted in the output*/ - %if %mf_getvartype(&jsonlib..ITEMS_PROPERTIES,&key) = C %then %do; - %let value = "&value"; - %end; - /* Transform the character '_', to '.' if found in the key */ - %let key = %sysfunc(translate(&key,.,_)); - /* Build the line to output */ - %let line="&key": &value; - /* ...adding a comma to all but the final line in the object */ - %if &i < &nvars %then %let line = &line%str(,); - %if &mdebug=1 %then %put DEBUG: &SYSMACRONAME line=%quote(&line); - %let &propertiesVar = &&&propertiesVar %quote(&line); - %end; - /* Close off the properties object */ - %let &propertiesVar = &&&propertiesVar %quote(}); - %if &mdebug=1 %then %put DEBUG: &SYSMACRONAME &=propertiesVar &propertiesVar=&&&propertiesVar; + %if &mdebug=1 %then %put DEBUG: &=propertiesVar &propertiesVar=&&&propertiesVar; %end; %end; - %if &mdebug=0 %then %do; - proc datasets library=&jsonlib nolist kill; quit; - libname &jsonlib clear; - %end; - - libname &libref1 clear; - filename &viyatypedef clear; - -%mend; \ No newline at end of file +%mend mv_getViyaFileExtParms; \ No newline at end of file