From 9f815c73e965c7f985793719cbf2ff80526e4f8a Mon Sep 17 00:00:00 2001 From: munja Date: Thu, 23 Dec 2021 13:50:58 +0000 Subject: [PATCH] feat: new mp_applyformats macro (and test), plus new addition to mp_validatecol (is_format) --- base/mf_getengine.sas | 7 +- base/mp_applyformats.sas | 181 +++++++++++++++++++ base/mp_getformats.sas | 1 + base/mp_validatecol.sas | 17 ++ tests/crossplatform/mp_applyformats.test.sas | 45 +++++ tests/crossplatform/mp_validatecol.test.sas | 33 ++++ 6 files changed, 281 insertions(+), 3 deletions(-) create mode 100644 base/mp_applyformats.sas create mode 100644 tests/crossplatform/mp_applyformats.test.sas diff --git a/base/mf_getengine.sas b/base/mf_getengine.sas index 06e3fc8..5049e4e 100755 --- a/base/mf_getengine.sas +++ b/base/mf_getengine.sas @@ -12,9 +12,10 @@ contributors of Chris Hemedingers blog [post]( http://blogs.sas.com/content/sasdummy/2013/06/04/find-a-sas-library-engine/) - @param libref Library reference (also accepts a 2 level libds ref). + @param [in] libref Library reference (also accepts a 2 level libds ref). - @return output returns the library engine for the FIRST library encountered. + @return output returns the library engine (uppercase) for the FIRST library + encountered. @warning will only return the FIRST library engine - for concatenated libraries, with different engines, inconsistent results may be encountered. @@ -46,7 +47,7 @@ %let rc= %sysfunc(close(&dsid)); %end; - &engine + %upcase(&engine) %mend mf_getengine; diff --git a/base/mp_applyformats.sas b/base/mp_applyformats.sas new file mode 100644 index 0000000..f79cb20 --- /dev/null +++ b/base/mp_applyformats.sas @@ -0,0 +1,181 @@ +/** + @file + @brief Apply a set of formats to a table + @details Applies a set of formats to one or more SAS datasets. Can be used + to migrate formats from one table to another. The input table must contain + the following columns: + + @li lib - the libref of the table to be updated + @li ds - the dataset to be updated + @li var - the variable to be updated + @li fmt - the format to apply. Missing or default ($CHAR, 8.) formats are + ignored. + + The macro will abort in the following scenarios: + + @li Libref not assigned + @li Dataset does not exist + @li Input table contains null or invalid values + + Example usage: + + data work.example; + set sashelp.prdsale; + format _all_ clear; + run; + + %mp_getcols(sashelp.prdsale,outds=work.cols) + + data work.cols2; + set work.cols; + lib='WORK'; + ds='EXAMPLE'; + var=name; + fmt=format; + keep lib ds var fmt; + run; + + %mp_applyformats(work.cols2) + + @param [in] inds The input dataset containing the formats to apply (and where + to apply them). Example structure: + |LIB:$4.|DS:$7.|VAR:$32.|FMT:$49.| + |---|---|---|---| + |`WORK `|`EXAMPLE `|`ACTUAL `|`DOLLAR12.2 `| + |`WORK `|`EXAMPLE `|`COUNTRY `|`$CHAR10. `| + |`WORK `|`EXAMPLE `|`DIVISION `|`$CHAR10. `| + |`WORK `|`EXAMPLE `|`MONTH `|`MONNAME3. `| + |`WORK `|`EXAMPLE `|`PREDICT `|`DOLLAR12.2 `| + |`WORK `|`EXAMPLE `|`PRODTYPE `|`$CHAR10. `| + |`WORK `|`EXAMPLE `|`PRODUCT `|`$CHAR10. `| + |`WORK `|`EXAMPLE `|`QUARTER `|`8. `| + |`WORK `|`EXAMPLE `|`REGION `|`$CHAR10. `| + |`WORK `|`EXAMPLE `|`YEAR `|`8. `| + + @param [out] errds= (0) Provide a libds reference here to export the + error messages to a table. In this case, they will not be printed to the + log. + +

SAS Macros

+ @li mf_getengine.sas + @li mf_getuniquefileref.sas + @li mf_getuniquename.sas + @li mf_nobs.sas + @li mp_validatecol.sas + + +

Related Macros

+ @li mp_getformats.sas + + @version 9.2 + @author Allan Bowe + +**/ + +%macro mp_applyformats(inds,errds=0 +)/*/STORE SOURCE*/; +%local outds liblist i engine lib msg ; + +/** + * Validations + */ +proc sort data=&inds; + by lib ds var fmt; +run; + +%if &errds=0 %then %let outds=%mf_getuniquename(prefix=mp_applyformats); +%else %let outds=&errds; + +data &outds; + set &inds; + where fmt not in ('','.', '$', '$CHAR.','8.'); + length msg $128; + by lib ds var fmt; + if libref(lib) ne 0 then do; + msg=catx(' ','libref',lib,'is not assigned!'); + %if &errds=0 %then %do; + putlog "%str(ERR)OR: " msg; + %end; + output; + return; + end; + if exist(cats(lib,'.',ds)) ne 1 then do; + msg=catx(' ','libds',lib,'.',ds,'does not exist!'); + %if &errds=0 %then %do; + putlog "%str(ERR)OR: " msg; + %end; + output; + return; + end; + %mp_validatecol(fmt,FORMAT,is_fmt) + if is_fmt=0 then do; + msg=catx(' ','format',fmt,'on libds',lib,'.',ds,'.',var,'is not valid!'); + %if &errds=0 %then %do; + putlog "%str(ERR)OR: " msg; + %end; + output; + return; + end; + + if first.ds then do; + retain dsid; + dsid=open(cats(lib,'.',ds)); + if dsid=0 then do; + msg=catx(' ','libds',lib,'.',ds,' could not be opened!'); + %if &errds=0 %then %do; + putlog "%str(ERR)OR: " msg; + %end; + output; + return; + end; + if varnum(dsid,var)<1 then do; + msg=catx(' ','Variable',lib,'.',ds,'.',var,' was not found!'); + %if &errds=0 %then %do; + putlog "%str(ERR)OR: " msg; + %end; + output; + end; + end; + if last.ds then rc=close(dsid); +run; + +proc sql noprint; +select distinct lib into: liblist separated by ' ' from &inds; +%put &=liblist; +%do i=1 %to %sysfunc(countw(&liblist)); + %let lib=%scan(&liblist,1); + %let engine=%mf_getengine(&lib); + %if &engine ne V9 and &engine ne BASE %then %do; + %let msg=&lib has &engine engine - formats cannot be applied; + proc sql; + insert into &outds set lib="&lib",ds="_all_",var="_all", msg="&msg" ; + %if &errds=0 %then %put %str(ERR)OR: &msg; + %end; +%end; + +%if %mf_nobs(&outds)>0 %then %return; + +/** + * Validations complete - now apply the actual formats! + */ +%let fref=%mf_getuniquefileref(); +data _null_; + set &inds; + by lib ds var fmt; + where fmt not in ('','.', '$', '$CHAR.','8.'); + file &fref; + if first.lib then put 'proc datasets nolist lib=' lib ';'; + if first.ds then put ' modify ' ds ';'; + put ' format ' var fmt ';'; + if last.ds then put ' run;'; + if last.lib then put 'quit;'; +run; + +%inc &fref/source2; + +%if &errds=0 %then %do; + proc sql; + drop table &outds; +%end; + +%mend mp_applyformats; \ No newline at end of file diff --git a/base/mp_getformats.sas b/base/mp_getformats.sas index b50373c..55ead0a 100644 --- a/base/mp_getformats.sas +++ b/base/mp_getformats.sas @@ -48,6 +48,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#

Related Macros

+ @li mp_applyformats.sas @li mp_getformats.test.sas @version 9.2 diff --git a/base/mp_validatecol.sas b/base/mp_validatecol.sas index f544853..bef24e7 100644 --- a/base/mp_validatecol.sas +++ b/base/mp_validatecol.sas @@ -20,10 +20,13 @@ ;;;; run; + Tip - when contributing, use https://regex101.com to test the regex validity! + @param [in] incol The column to be validated @param [in] rule The rule to apply. Current rules: @li ISNUM - checks if the variable is numeric @li LIBDS - matches LIBREF.DATASET format + @li FORMAT - checks if the provided format is syntactically valid @param [out] outcol The variable to create, with the results of the match

SAS Macros

@@ -62,5 +65,19 @@ if prxmatch(&tempcol, trim(&incol)) then &outcol=1; else &outcol=0; %end; +%else %if &rule=FORMAT %then %do; + /* match valid format - regex could probably be improved */ + if _n_=1 then do; + retain &tempcol; + &tempcol=prxparse('/^[_a-z\$]\w{0,31}\.[0-9]*$/i'); + if missing(&tempcol) then do; + putlog "%str(ERR)OR: Invalid expression for FORMAT"; + stop; + end; + drop &tempcol; + end; + if prxmatch(&tempcol, trim(&incol)) then &outcol=1; + else &outcol=0; +%end; %mend mp_validatecol; diff --git a/tests/crossplatform/mp_applyformats.test.sas b/tests/crossplatform/mp_applyformats.test.sas new file mode 100644 index 0000000..d6a5fdf --- /dev/null +++ b/tests/crossplatform/mp_applyformats.test.sas @@ -0,0 +1,45 @@ +/** + @file + @brief Testing mp_applyformats.sas macro + +

SAS Macros

+ @li mf_getvarformat.sas + @li mp_applyformats.sas + @li mp_assert.sas + @li mp_getcols.sas + +**/ + +/** + * Test 1 Base case + */ + +data work.example; + set sashelp.prdsale; + format _all_; +run; +%let origfmt=%mf_getvarformat(work.example,month); + +%mp_getcols(sashelp.prdsale,outds=work.cols) + +data work.cols2; + set work.cols; + lib='WORK'; + ds='EXAMPLE'; + var=name; + fmt=format; + keep lib ds var fmt; +run; + +%mp_applyformats(work.cols2) + +%mp_assert( + iftrue=("&orig_fmt"=""), + desc=Check that formats were cleared, + outds=work.test_results +) +%mp_assert( + iftrue=("%mf_getvarformat(work.example,month)"="MONNAME3."), + desc=Check that formats were applied, + outds=work.test_results +) \ No newline at end of file diff --git a/tests/crossplatform/mp_validatecol.test.sas b/tests/crossplatform/mp_validatecol.test.sas index 7a37916..2fdae62 100644 --- a/tests/crossplatform/mp_validatecol.test.sas +++ b/tests/crossplatform/mp_validatecol.test.sas @@ -59,4 +59,37 @@ run; desc=Test2 - ISNUM, test=EQUALS 4, outds=work.test_results +) + +/** + * Test 3 - FORMAT + */ +data test3; + infile datalines4 dsd; + input; + infile=_infile_; + %mp_validatecol(infile,FORMAT,is_format) + if is_format=1; +datalines4; +$. +$format. +$format12.2 +somenum. +somenum12.4 +above are good +the rest are bad +%abort +1&somethingverybad. +& ++-1 +. +a.A +$format12.1b +$format12.1b1 +;;;; +run; +%mp_assertdsobs(work.test3, + desc=Test3 - ISFORMAT, + test=EQUALS 5, + outds=work.test_results ) \ No newline at end of file