sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2026-01-16 13:00:05 +00:00
Code Issues Projects Releases Wiki Activity

fix: adjustments to ensure the tests work, also building all.sas

Browse Source
This commit is contained in:
munja
2022-01-24 12:53:36 +01:00
parent c3b89c7f7d
commit 0e03b06a4b
5 changed files with 223 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

368
all.sas
View File

@@ -29,7 +29,7 @@ options noquotelenmax;
@cond @cond
**/ **/
%macro mf_abort(mac=mf_abort.sas, type=deprecated, msg=, iftrue=%str(1=1) %macro mf_abort(mac=mf_abort.sas, msg=, iftrue=%str(1=1)
)/*/STORE SOURCE*/; )/*/STORE SOURCE*/;
%if not(%eval(%unquote(&iftrue))) %then %return; %if not(%eval(%unquote(&iftrue))) %then %return;
@@ -3036,7 +3036,7 @@ run;
@li COMPARE - compare the current macro variables against previous values @li COMPARE - compare the current macro variables against previous values
@param [in] scope= (GLOBAL) The scope of the variables to be checked. This @param [in] scope= (GLOBAL) The scope of the variables to be checked. This
corresponds to the values in the SCOPE column in `sashelp.vmacro`. corresponds to the values in the SCOPE column in `sashelp.vmacro`.
@param [in] desc= (Testing variable scope) The user provided test description @param [in] desc= (Testing scope leakage) The user provided test description
@param [in,out] scopeds= (work.mp_assertscope) The dataset to contain the @param [in,out] scopeds= (work.mp_assertscope) The dataset to contain the
scope snapshot scope snapshot
@param [out] outds= (work.test_results) The output dataset to contain the @param [out] outds= (work.test_results) The output dataset to contain the
@@ -3058,7 +3058,7 @@ run;
**/ **/
%macro mp_assertscope(action, %macro mp_assertscope(action,
desc=0, desc=Testing Scope Leakage,
scope=GLOBAL, scope=GLOBAL,
scopeds=work.mp_assertscope, scopeds=work.mp_assertscope,
outds=work.test_results outds=work.test_results
@@ -5028,6 +5028,123 @@ run;
%end; %end;
%mend mp_ds2md;/** %mend mp_ds2md;/**
@file
@brief Create a smaller version of a dataset, without data loss
@details This macro will scan the input dataset and create a new one, that
has the minimum variable lengths needed to store the data without data loss.
Inspiration was taken from [How to Reduce the Disk Space Required by a
SAS® Data Set](https://www.lexjansen.com/nesug/nesug06/io/io18.pdf) by
Selvaratnam Sridharma. The end of the referenced paper presents a macro named
"squeeze", hence the nomenclature.
Usage:
data big;
length my big $32000;
do i=1 to 1e4;
my=repeat('oh my',100);
big='dawg';
special=._;
output;
end;
run;
%mp_ds2squeeze(work.big,outds=work.smaller)
The following will also be printed to the log (exact values may differ
depending on your OS and COMPRESS settings):
> MP_DS2SQUEEZE: work.big was 625MB
> MP_DS2SQUEEZE: work.smaller is 5MB
@param [in] libds The library.dataset to be squeezed
@param [out] outds= (work.mp_ds2squeeze) The squeezed dataset to create
@param [in] mdebug= (0) Set to 1 to enable DEBUG messages
<h4> SAS Macros </h4>
@li mf_getfilesize.sas
@li mf_getuniquefileref.sas
@li mf_getuniquename.sas
@li mp_getmaxvarlengths.sas
<h4> Related Programs </h4>
@li mp_ds2squeeze.test.sas
@version 9.3
@author Allan Bowe
**/
%macro mp_ds2squeeze(
libds,
outds=work.work.mp_ds2squeeze,
mdebug=0
)/*/STORE SOURCE*/;
%local dbg source;
%if &mdebug=1 %then %do;
%put &sysmacroname entry vars:;
%put _local_;
%end;
%else %do;
%let dbg=*;
%let source=/source2;
%end;
%local optval ds fref;
%let ds=%mf_getuniquename();
%let fref=%mf_getuniquefileref();
%mp_getmaxvarlengths(&libds,outds=&ds)
data _null_;
set &ds end=last;
file &fref;
/* grab the types */
retain dsid;
if _n_=1 then dsid=open("&libds",'is');
if dsid le 0 then do;
msg=sysmsg();
put msg=;
stop;
end;
type=vartype(dsid,varnum(dsid, name));
if last then rc=close(dsid);
/* write out the length statement */
if _n_=1 then put 'length ';
length len $6;
if type='C' then do;
if maxlen=0 then len='$1';
else len=cats('$',maxlen);
end;
else do;
if maxlen=0 then len='3';
else len=cats(maxlen);
end;
put ' ' name ' ' len;
if last then put ';';
run;
/* configure varlenchk - as we are explicitly shortening the variables */
%let optval=%sysfunc(getoption(varlenchk));
options varlenchk=NOWARN;
data &outds;
%inc &fref &source;
set &libds;
run;
options varlenchk=&optval;
%if &mdebug=0 %then %do;
proc sql;
drop table &ds;
filename &fref clear;
%end;
%put &sysmacroname: &libds was %mf_getfilesize(libds=&libds,format=yes);
%put &sysmacroname: &outds is %mf_getfilesize(libds=&outds,format=yes);
%mend mp_ds2squeeze;/**
@file @file
@brief Checks an input filter table for validity @brief Checks an input filter table for validity
@details Performs checks on the input table to ensure it arrives in the @details Performs checks on the input table to ensure it arrives in the
@@ -6703,30 +6820,48 @@ create table &outsummary as
%end; %end;
%mend mp_getformats;/** %mend mp_getformats;/**
@file mp_getmaxvarlengths.sas @file
@brief Scans a dataset to find the max length of the variable values @brief Scans a dataset to find the max length of the variable values
@details @details
This macro will scan a base dataset and produce an output dataset with two This macro will scan a base dataset and produce an output dataset with two
columns: columns:
- NAME Name of the base dataset column - NAME Name of the base dataset column
- MAXLEN Maximum length of the data contained therein. - MAXLEN Maximum length of the data contained therein.
Character fields may be allocated very large widths (eg 32000) of which the Character fields are often allocated very large widths (eg 32000) of which the
maximum value is likely to be much narrower. This macro was designed to maximum value is likely to be much narrower. Identifying such cases can be
enable a HTML table to be appropriately sized however this could be used as helpful in the following scenarios:
part of a data audit to ensure we aren't over-sizing our tables in relation to
the data therein. @li Enabling a HTML table to be appropriately sized (`num2char=YES`)
@li Reducing the size of a dataset to save on storage (mp_ds2squeeze.sas)
@li Identifying columns containing nothing but missing values (`MAXLEN=0` in
the output table)
If the entire column is made up of (non-special) missing values then a value
of 0 is returned.
Numeric fields are converted using the relevant format to determine the width.
Usage: Usage:
%mp_getmaxvarlengths(sashelp.class,outds=work.myds) %mp_getmaxvarlengths(sashelp.class,outds=work.myds)
@param libds Two part dataset (or view) reference. @param [in] libds Two part dataset (or view) reference.
@param outds= The output dataset to create @param [in] num2char= (NO) When set to NO, numeric fields are sized according
to the number of bytes used (or set to zero in the case of non-special
missings). When YES, the numeric field is converted to character (using the
format, if available), and that is sized instead, using `lengthn()`.
@param [out] outds= The output dataset to create, eg:
|NAME:$8.|MAXLEN:best.|
|---|---|
|`Name `|`7 `|
|`Sex `|`1 `|
|`Age `|`3 `|
|`Height `|`8 `|
|`Weight `|`3 `|
<h4> SAS Macros </h4> <h4> SAS Macros </h4>
@li mcf_length.sas
@li mf_getuniquename.sas
@li mf_getvarlist.sas @li mf_getvarlist.sas
@li mf_getvartype.sas @li mf_getvartype.sas
@li mf_getvarformat.sas @li mf_getvarformat.sas
@@ -6734,20 +6869,32 @@ create table &outsummary as
@version 9.2 @version 9.2
@author Allan Bowe @author Allan Bowe
<h4> Related Macros </h4>
@li mp_ds2squeeze.sas
@li mp_getmaxvarlengths.test.sas
**/ **/
%macro mp_getmaxvarlengths( %macro mp_getmaxvarlengths(
libds /* libref.dataset to analyse */ libds
,outds=work.mp_getmaxvarlengths /* name of output dataset to create */ ,num2char=NO
,outds=work.mp_getmaxvarlengths
)/*/STORE SOURCE*/; )/*/STORE SOURCE*/;
%local vars x var fmt; %local vars prefix x var fmt;
%let vars=%mf_getvarlist(libds=&libds); %let vars=%mf_getvarlist(libds=&libds);
%let prefix=%substr(%mf_getuniquename(),1,25);
%let num2char=%upcase(&num2char);
%if &num2char=NO %then %do;
/* compile length function for numeric fields */
%mcf_length(wrap=YES, insert_cmplib=YES)
%end;
proc sql; proc sql;
create table &outds (rename=( create table &outds (rename=(
%do x=1 %to %sysfunc(countw(&vars,%str( ))); %do x=1 %to %sysfunc(countw(&vars,%str( )));
________&x=%scan(&vars,&x) &prefix.&x=%scan(&vars,&x)
%end; %end;
)) ))
as select as select
@@ -6755,18 +6902,21 @@ create table &outds (rename=(
%let var=%scan(&vars,&x); %let var=%scan(&vars,&x);
%if &x>1 %then ,; %if &x>1 %then ,;
%if %mf_getvartype(&libds,&var)=C %then %do; %if %mf_getvartype(&libds,&var)=C %then %do;
max(length(&var)) as ________&x max(lengthn(&var)) as &prefix.&x
%end; %end;
%else %do; %else %if &num2char=YES %then %do;
%let fmt=%mf_getvarformat(&libds,&var); %let fmt=%mf_getvarformat(&libds,&var);
%put fmt=&fmt; %put fmt=&fmt;
%if %str(&fmt)=%str() %then %do; %if %str(&fmt)=%str() %then %do;
max(length(cats(&var))) as ________&x max(lengthn(cats(&var))) as &prefix.&x
%end; %end;
%else %do; %else %do;
max(length(put(&var,&fmt))) as ________&x max(lengthn(put(&var,&fmt))) as &prefix.&x
%end; %end;
%end; %end;
%else %do;
max(mcf_length(&var)) as &prefix.&x
%end;
%end; %end;
from &libds; from &libds;
@@ -7616,38 +7766,42 @@ filename &tempref clear;
%macro mp_init(prefix=SASJS %macro mp_init(prefix=SASJS
)/*/STORE SOURCE*/; )/*/STORE SOURCE*/;
%global %if %symexist(SASJS_PREFIX) %then %return; /* only run once */
&prefix._INIT_NUM /* initialisation time as numeric */
&prefix._INIT_DTTM /* initialisation time in E8601DT26.6 format */
&prefix.WORK /* avoid typing %sysfunc(pathname(work)) every time */
;
%if %eval(&&&prefix._INIT_NUM>0) %then %return; /* only run once */
data _null_; %global
dttm=datetime(); SASJS_PREFIX /* the ONLY hard-coded global macro variable in SASjs */
call symputx("&prefix._init_num",dttm,'g'); &prefix._INIT_NUM /* initialisation time as numeric */
call symputx("&prefix._init_dttm",put(dttm,E8601DT26.6),'g'); &prefix._INIT_DTTM /* initialisation time in E8601DT26.6 format */
call symputx("&prefix.work",pathname('WORK'),'g'); &prefix.WORK /* avoid typing %sysfunc(pathname(work)) every time */
run; ;
options %let sasjs_prefix=&prefix;
noautocorrect /* disallow misspelled procedure names */
compress=CHAR /* default is none so ensure we have something! */ data _null_;
datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets */ dttm=datetime();
dsoptions=note2err /* undocumented - convert bad NOTEs to ERRs */ call symputx("&sasjs_prefix._init_num",dttm,'g');
%str(err)orcheck=STRICT /* catch errs in libname/filename statements */ call symputx("&sasjs_prefix._init_dttm",put(dttm,E8601DT26.6),'g');
fmterr /* ensure err when a format cannot be found */ call symputx("&sasjs_prefix.work",pathname('WORK'),'g');
mergenoby=%str(ERR)OR /* throw err when a merge has no BY variables */ run;
missing=. /* changing this can cause hard to detect errs */
noquotelenmax /* avoid warnings for long strings */ options
noreplace /* avoid overwriting permanent datasets */ noautocorrect /* disallow misspelled procedure names */
ps=max /* reduce log size slightly */ compress=CHAR /* default is none so ensure we have something! */
ls=max /* reduce log even more and avoid word truncation */ datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets */
validmemname=COMPATIBLE /* avoid special characters etc in table names */ dsoptions=note2err /* undocumented - convert bad NOTEs to ERRs */
validvarname=V7 /* avoid special characters etc in variable names */ %str(err)orcheck=STRICT /* catch errs in libname/filename statements */
varinitchk=%str(ERR)OR /* avoid data mistakes from variable name typos */ fmterr /* ensure err when a format cannot be found */
varlenchk=%str(ERR)OR /* fail hard if truncation (data loss) can result */ mergenoby=%str(ERR)OR /* throw err when a merge has no BY variables */
; missing=. /* changing this can cause hard to detect errs */
noquotelenmax /* avoid warnings for long strings */
noreplace /* avoid overwriting permanent datasets */
ps=max /* reduce log size slightly */
ls=max /* reduce log even more and avoid word truncation */
validmemname=COMPATIBLE /* avoid special characters etc in table names */
validvarname=V7 /* avoid special characters etc in variable names */
varinitchk=%str(ERR)OR /* avoid data mistakes from variable name typos */
varlenchk=%str(ERR)OR /* fail hard if truncation (data loss) can result */
;
%mend mp_init;/** %mend mp_init;/**
@file mp_jsonout.sas @file mp_jsonout.sas
@@ -16389,9 +16543,6 @@ run;
@param [in] stpcode= the source file (or fileref) containing the SAS code to load @param [in] stpcode= the source file (or fileref) containing the SAS code to load
into the stp. For multiple files, they should simply be concatenated first. into the stp. For multiple files, they should simply be concatenated first.
@param [in] minify= set to YES in order to strip comments, blank lines, and CRLFs. @param [in] minify= set to YES in order to strip comments, blank lines, and CRLFs.
@param frefin= deprecated - a unique fileref is now always used
@param frefout= deprecated - a unique fileref is now always used
@param mDebug= set to 1 to show debug messages in the log @param mDebug= set to 1 to show debug messages in the log
@version 9.3 @version 9.3
@@ -16406,16 +16557,8 @@ run;
,stpcode= ,stpcode=
,minify=NO ,minify=NO
,mdebug=0 ,mdebug=0
/* deprecated */
,frefin=inmeta
,frefout=outmeta
); );
%if &frefin ne inmeta or &frefout ne outmeta %then %do;
%put %str(WARN)ING: the frefin and frefout parameters will be deprecated in
an upcoming release.;
%end;
/* first, check if STP exists */ /* first, check if STP exists */
%local tsuri; %local tsuri;
%let tsuri=stopifempty ; %let tsuri=stopifempty ;
@@ -18671,6 +18814,7 @@ libname &libref1a JSON fileref=&fname1a;
%let found=0; %let found=0;
%put Getting object uri from &libref1a..items; %put Getting object uri from &libref1a..items;
data _null_; data _null_;
length contenttype name $1000;
set &libref1a..items; set &libref1a..items;
if contenttype="&contenttype" and upcase(name)="%upcase(&name)" then do; if contenttype="&contenttype" and upcase(name)="%upcase(&name)" then do;
call symputx('uri',uri,'l'); call symputx('uri',uri,'l');
@@ -18818,6 +18962,7 @@ libname &libref1a JSON fileref=&fname1a;
%let found=0; %let found=0;
%put Getting object uri from &libref1a..items; %put Getting object uri from &libref1a..items;
data _null_; data _null_;
length contenttype name $1000;
set &libref1a..items; set &libref1a..items;
if contenttype='jobDefinition' and upcase(name)="%upcase(&name)" then do; if contenttype='jobDefinition' and upcase(name)="%upcase(&name)" then do;
call symputx('uri',cats("&base_uri",uri),'l'); call symputx('uri',cats("&base_uri",uri),'l');
@@ -18990,59 +19135,6 @@ filename &fname2 clear;
libname &libref1 clear; libname &libref1 clear;
%mend mv_deleteviyafolder;/** %mend mv_deleteviyafolder;/**
@file mv_getaccesstoken.sas
@brief deprecated - replaced by mv_tokenrefresh.sas
@version VIYA V.03.04
@author Allan Bowe, source: https://github.com/sasjs/core
<h4> SAS Macros </h4>
@li mv_tokenrefresh.sas
**/
%macro mv_getaccesstoken(client_id=someclient
,client_secret=somesecret
,grant_type=authorization_code
,code=
,user=
,pass=
,access_token_var=ACCESS_TOKEN
,refresh_token_var=REFRESH_TOKEN
);
%mv_tokenrefresh(client_id=&client_id
,client_secret=&client_secret
,grant_type=&grant_type
,user=&user
,pass=&pass
,access_token_var=&access_token_var
,refresh_token_var=&refresh_token_var
)
%mend mv_getaccesstoken;/**
@file
@brief deprecated - replaced by mv_registerclient.sas
@version VIYA V.03.04
@author Allan Bowe, source: https://github.com/sasjs/core
<h4> SAS Macros </h4>
@li mv_registerclient.sas
**/
%macro mv_getapptoken(client_id=someclient
,client_secret=somesecret
,grant_type=authorization_code
);
%mv_registerclient(client_id=&client_id
,client_secret=&client_secret
,grant_type=&grant_type
)
%mend mv_getapptoken;/**
@file mv_getclients.sas @file mv_getclients.sas
@brief Get a list of Viya Clients @brief Get a list of Viya Clients
@details First, be sure you have an access token (which requires an app token). @details First, be sure you have an access token (which requires an app token).
@@ -20369,38 +20461,6 @@ filename &fname0 clear;
%mend mv_getjobstate; %mend mv_getjobstate;
/** /**
@file mv_getrefreshtoken.sas
@brief deprecated - replaced by mv_tokenauth.sas
@version VIYA V.03.04
@author Allan Bowe, source: https://github.com/sasjs/core
<h4> SAS Macros </h4>
@li mv_tokenauth.sas
**/
%macro mv_getrefreshtoken(client_id=someclient
,client_secret=somesecret
,grant_type=authorization_code
,code=
,user=
,pass=
,access_token_var=ACCESS_TOKEN
,refresh_token_var=REFRESH_TOKEN
);
%mv_tokenauth(client_id=&client_id
,client_secret=&client_secret
,grant_type=&grant_type
,code=&code
,user=&user
,pass=&pass
,access_token_var=&access_token_var
,refresh_token_var=&refresh_token_var
)
%mend mv_getrefreshtoken;/**
@file mv_getusergroups.sas @file mv_getusergroups.sas
@brief Creates a dataset with a list of groups for a particular user @brief Creates a dataset with a list of groups for a particular user
@details If using outside of Viya SPRE, then an access token is needed. @details If using outside of Viya SPRE, then an access token is needed.
@@ -22718,6 +22778,9 @@ run;
@param [out] pkg= (utils) The output package in which to create the function. @param [out] pkg= (utils) The output package in which to create the function.
Uses a 3 part format: libref.catalog.package Uses a 3 part format: libref.catalog.package
<h4> SAS Macros </h4>
@li mf_existfunction.sas
<h4> Related Macros </h4> <h4> Related Macros </h4>
@li mcf_length.test.sas @li mcf_length.test.sas
@@ -22730,13 +22793,15 @@ run;
,pkg=UTILS ,pkg=UTILS
)/*/STORE SOURCE*/; )/*/STORE SOURCE*/;
%if %mf_existfunction(mcf_length)=1 %then %return;
%if &wrap=YES %then %do; %if &wrap=YES %then %do;
proc fcmp outlib=&lib..&cat..&pkg; proc fcmp outlib=&lib..&cat..&pkg;
%end; %end;
function mcf_length(var); function mcf_length(var);
if missing(var) then len=0; if var=. then len=0;
else if trunc(var,3)=var then len=3; else if missing(var) or trunc(var,3)=var then len=3;
else if trunc(var,4)=var then len=4; else if trunc(var,4)=var then len=4;
else if trunc(var,5)=var then len=5; else if trunc(var,5)=var then len=5;
else if trunc(var,6)=var then len=6; else if trunc(var,6)=var then len=6;
@@ -22892,6 +22957,9 @@ endsub;
@param [out] pkg= (utils) The output package in which to create the function. @param [out] pkg= (utils) The output package in which to create the function.
Uses a 3 part format: libref.catalog.package Uses a 3 part format: libref.catalog.package
<h4> SAS Macros </h4>
@li mf_existfunction.sas
**/ **/
%macro mcf_string2file(wrap=NO %macro mcf_string2file(wrap=NO
@@ -22901,6 +22969,8 @@ endsub;
,pkg=UTILS ,pkg=UTILS
)/*/STORE SOURCE*/; )/*/STORE SOURCE*/;
%if %mf_existfunction(mcf_string2file)=1 %then %return;
%if &wrap=YES %then %do; %if &wrap=YES %then %do;
proc fcmp outlib=&lib..&cat..&pkg; proc fcmp outlib=&lib..&cat..&pkg;
%end; %end;

2
base/mp_ds2squeeze.sas
View File

@@ -89,7 +89,7 @@ data _null_;
end; end;
else do; else do;
if maxlen=0 then len='3'; if maxlen=0 then len='3';
else len=maxlen; else len=cats(maxlen);
end; end;
put ' ' name ' ' len; put ' ' name ' ' len;
if last then put ';'; if last then put ';';

2
tests/crossplatform/mcf_length.test.sas
View File

@@ -12,7 +12,7 @@
data test; data test;
call symputx('null',mcf_length(.)); call symputx('null',mcf_length(.));
call symputx('special',mcf_length(._)) call symputx('special',mcf_length(._));
call symputx('three',mcf_length(1)); call symputx('three',mcf_length(1));
call symputx('four',mcf_length(10000000)); call symputx('four',mcf_length(10000000));
call symputx('five',mcf_length(12345678)); call symputx('five',mcf_length(12345678));

1
viya/mv_deletefoldermember.sas
View File

@@ -117,6 +117,7 @@ libname &libref1a JSON fileref=&fname1a;
%let found=0; %let found=0;
%put Getting object uri from &libref1a..items; %put Getting object uri from &libref1a..items;
data _null_; data _null_;
length contenttype name $1000;
set &libref1a..items; set &libref1a..items;
if contenttype="&contenttype" and upcase(name)="%upcase(&name)" then do; if contenttype="&contenttype" and upcase(name)="%upcase(&name)" then do;
call symputx('uri',uri,'l'); call symputx('uri',uri,'l');

1
viya/mv_deletejes.sas
View File

@@ -114,6 +114,7 @@ libname &libref1a JSON fileref=&fname1a;
%let found=0; %let found=0;
%put Getting object uri from &libref1a..items; %put Getting object uri from &libref1a..items;
data _null_; data _null_;
length contenttype name $1000;
set &libref1a..items; set &libref1a..items;
if contenttype='jobDefinition' and upcase(name)="%upcase(&name)" then do; if contenttype='jobDefinition' and upcase(name)="%upcase(&name)" then do;
call symputx('uri',cats("&base_uri",uri),'l'); call symputx('uri',cats("&base_uri",uri),'l');