sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2026-04-10 18:33:14 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'main' into fix/non_stp_syscc_propagation

Browse Source
This commit is contained in:
Allan Bowe
2021-05-10 14:57:24 +03:00
committed by GitHub
parent db2531e0b3 5e77494aa6
commit 265389befc
164 changed files with 9521 additions and 2137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
base/mf_abort.sas
View File

@@ -3,7 +3,7 @@
@brief abort gracefully according to context
@details Do not use directly! See bottom of explanation for details.
Configures an abort mechanism according to site specific policies or the
Configures an abort mechanism according to site specific policies or the
particulars of an environment. For instance, can stream custom
results back to the client in an STP Web App context, or completely stop
in the case of a batch run.
@@ -49,7 +49,10 @@
input; putlog _infile_;
i=1;
retain logonce 0;
if (_infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR") and logonce=0 then do;
if (
_infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR"
) and logonce=0
then do;
call symputx('logline',_n_);
logonce+1;
end;
@@ -112,21 +115,22 @@
%let syscc=0;
%if %symexist(SYS_JES_JOB_URI) %then %do;
/* refer web service output to file service in one hit */
filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json";
filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
name="_webout.json";
%let rc=%sysfunc(fcopy(_web,_webout));
%end;
%else %do;
data _null_;
if symexist('sysprocessmode')
then if symget("sysprocessmode")="SAS Stored Process Server"
then rc=stpsrvset('program error', 0);
then if symget("sysprocessmode")="SAS Stored Process Server"
then rc=stpsrvset('program error', 0);
run;
%end;
/**
* endsas is reliable but kills some deployments.
* Abort variants are ungraceful (non zero return code)
* This approach lets SAS run silently until the end :-)
*/
* endsas is reliable but kills some deployments.
* Abort variants are ungraceful (non zero return code)
* This approach lets SAS run silently until the end :-)
*/
%put _all_;
filename skip temp;
data _null_;

6
base/mf_existvarlist.sas
View File

@@ -2,9 +2,9 @@
@file
@brief Checks if a set of variables ALL exist in a data set.
@details Returns 0 if ANY of the variables do not exist, or 1 if they ALL do.
Usage:
Usage:
%put %mf_existVarList(sashelp.class, age sex name dummyvar)
%put %mf_existVarList(sashelp.class, age sex name dummyvar);
<h4> SAS Macros </h4>
@li mf_abort.sas
@@ -29,7 +29,7 @@
%let dsid=%sysfunc(open(&libds,is));
%if &dsid=0 %then %do;
%put WARNING: unable to open &libds in mf_existvarlist (&dsid);
%put %str(WARN)ING: unable to open &libds in mf_existvarlist (&dsid);
%end;
%if %sysfunc(attrn(&dsid,NVARS))=0 %then %do;

4
base/mf_getattrc.sas
View File

@@ -17,13 +17,13 @@
**/
%macro mf_getattrc(
libds
libds
,attr
)/*/STORE SOURCE*/;
%local dsid rc;
%let dsid=%sysfunc(open(&libds,is));
%if &dsid = 0 %then %do;
%put WARNING: Cannot open %trim(&libds), system message below;
%put %str(WARN)ING: Cannot open %trim(&libds), system message below;
%put %sysfunc(sysmsg());
-1
%end;

4
base/mf_getattrn.sas
View File

@@ -8,7 +8,7 @@
@param libds library.dataset
@param attr Common values are NLOBS and NVARS, full list in [documentation](
http://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000212040.htm)
http://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000212040.htm)
@return output returns result of the attrn value supplied, or -1 and log
message if error.
@@ -17,7 +17,7 @@
**/
%macro mf_getattrn(
libds
libds
,attr
)/*/STORE SOURCE*/;
%local dsid rc;

9
base/mf_getengine.sas
View File

@@ -22,6 +22,9 @@
@version 9.2
@author Allan Bowe
<h4> Related Macros </h4>
@li mf_getxengine.sas
**/
/** @cond */
@@ -32,7 +35,9 @@
/* in case the parameter is a libref.tablename, pull off just the libref */
%let libref = %upcase(%scan(&libref, 1, %str(.)));
%let dsid=%sysfunc(open(sashelp.vlibnam(where=(libname="%upcase(&libref)")),i));
%let dsid=%sysfunc(
open(sashelp.vlibnam(where=(libname="%upcase(&libref)")),i)
);
%if (&dsid ^= 0) %then %do;
%let engnum=%sysfunc(varnum(&dsid,ENGINE));
%let rc=%sysfunc(fetch(&dsid));
@@ -41,7 +46,7 @@
%let rc= %sysfunc(close(&dsid));
%end;
&engine
&engine
%mend;

2
base/mf_getfilesize.sas
View File

@@ -38,7 +38,7 @@
%let rc=%sysfunc(filename(fref));
%if &format=NO %then %do;
&bytes
&bytes
%end;
%else %do;
%sysfunc(INPUTN(&bytes, best.),sizekmg.)

2
base/mf_getkeyvalue.sas
View File

@@ -16,7 +16,7 @@
%macro mf_getkeyvalue(key,libds=work.mp_setkeyvalue
)/*/STORE SOURCE*/;
%local ds dsid key valc valn type rc;
%local ds dsid key valc valn type rc;
%let dsid=%sysfunc(open(&libds(where=(key="&key"))));
%syscall set(dsid);
%let rc = %sysfunc(fetch(&dsid));

12
base/mf_getplatform.sas
View File

@@ -23,25 +23,27 @@
%local a b c;
%if &switch.NONE=NONE %then %do;
%if %symexist(sysprocessmode) %then %do;
%if "&sysprocessmode"="SAS Object Server"
%if "&sysprocessmode"="SAS Object Server"
or "&sysprocessmode"= "SAS Compute Server" %then %do;
SASVIYA
%end;
%else %if "&sysprocessmode"="SAS Stored Process Server" %then %do;
%else %if "&sysprocessmode"="SAS Stored Process Server"
or "&sysprocessmode"="SAS Workspace Server"
%then %do;
SASMETA
%return;
%end;
%else %do;
SAS
BASESAS
%return;
%end;
%end;
%else %if %symexist(_metaport) %then %do;
%else %if %symexist(_metaport) or %symexist(_metauser) %then %do;
SASMETA
%return;
%end;
%else %do;
SAS
BASESAS
%return;
%end;
%end;

4
base/mf_getuniquename.sas
View File

@@ -18,5 +18,5 @@
%macro mf_getuniquename(prefix=MC);
&prefix.%substr(%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32-%length(&prefix))
%mend;
&prefix.%substr(%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32-%length(&prefix))
%mend mf_getuniquename;

4
base/mf_getuser.sas
View File

@@ -2,7 +2,7 @@
@file
@brief Returns a userid according to session context
@details In a workspace session, a user is generally represented by <code>
&sysuserid</code> or <code>SYS_COMPUTE_SESSION_OWNER</code> if it exists.
&sysuserid</code> or <code>SYS_COMPUTE_SESSION_OWNER</code> if it exists.
In a Stored Process session, <code>&sysuserid</code>
resolves to a system account (default=sassrv) and instead there are several
metadata username variables to choose from (_metauser, _metaperson
@@ -12,7 +12,7 @@
%let user= %mf_getUser();
%put &user;
@param type - do not use, may be deprecated in a future release
@return SYSUSERID (if workspace server)

2
base/mf_getvalue.sas
View File

@@ -20,7 +20,7 @@
%macro mf_getvalue(libds,variable,filter=1
)/*/STORE SOURCE*/;
%if %mf_getattrn(&libds,NLOBS)>0 %then %do;
%if %mf_getattrn(&libds,NLOBS)>0 %then %do;
%local dsid rc &variable;
%let dsid=%sysfunc(open(&libds(where=(&filter))));
%syscall set(dsid);

22
base/mf_getvarformat.sas
View File

@@ -5,9 +5,9 @@
Usage:
data test;
format str1 $1. num1 datetime19.;
str2='hello mum!'; num2=666;
stop;
format str1 $1. num1 datetime19.;
str2='hello mum!'; num2=666;
stop;
run;
%put %mf_getVarFormat(test,str1);
%put %mf_getVarFormat(work.test,num1);
@@ -23,9 +23,9 @@
8.
NOTE: Variable renegade does not exist in test
@param libds Two part dataset (or view) reference.
@param var Variable name for which a format should be returned
@param force Set to 1 to supply a default if the variable has no format
@param [in] libds Two part dataset (or view) reference.
@param [in] var Variable name for which a format should be returned
@param [in] force=(0) Set to 1 to supply a default if the variable has no format
@returns outputs format
@author Allan Bowe
@@ -45,9 +45,9 @@
/* Get variable format */
%if(&vnum > 0) %then %let vformat=%sysfunc(varfmt(&dsid, &vnum));
%else %do;
%put NOTE: Variable &var does not exist in &libds;
%let rc = %sysfunc(close(&dsid));
%return;
%put NOTE: Variable &var does not exist in &libds;
%let rc = %sysfunc(close(&dsid));
%return;
%end;
%end;
%else %do;
@@ -60,7 +60,7 @@
%let vlen = %sysfunc(varlen(&dsid, &vnum));
%let vtype = %sysfunc(vartype(&dsid, &vnum.));
%if &vtype=C %then %let vformat=$&vlen..;
%else %let vformat=8.;
%else %let vformat=best.;
%end;
@@ -68,4 +68,4 @@
%let rc = %sysfunc(close(&dsid));
/* Return variable format */
&vformat
%mend;
%mend mf_getVarFormat;

8
base/mf_getvarlen.sas
View File

@@ -5,8 +5,8 @@
Usage:
data test;
format str $1. num datetime19.;
stop;
format str $1. num datetime19.;
stop;
run;
%put %mf_getVarLen(test,str);
%put %mf_getVarLen(work.test,num);
@@ -39,8 +39,8 @@
/* Get variable format */
%if(&vnum > 0) %then %let vlen = %sysfunc(varlen(&dsid, &vnum));
%else %do;
%put NOTE: Variable &var does not exist in &libds;
%let vlen = %str( );
%put NOTE: Variable &var does not exist in &libds;
%let vlen = %str( );
%end;
%end;
%else %put dataset &libds not opened! (rc=&dsid);

39
base/mf_getvarlist.sas
View File

@@ -10,14 +10,21 @@
returns:
> List of Variables=Name Sex Age Height Weight
For a seperated list of column values:
%put %mf_getvarlist(sashelp.class,dlm=%str(,),quote=double);
returns:
> "Name","Sex","Age","Height","Weight"
@param libds Two part dataset (or view) reference.
@param dlm= provide a delimiter (eg comma or space) to separate the vars
@param quote= use either DOUBLE or SINGLE to quote the results
@param [in] libds Two part dataset (or view) reference.
@param [in] dlm= ( ) Provide a delimiter (eg comma or space) to separate the
variables
@param [in] quote= (none) use either DOUBLE or SINGLE to quote the results
@param [in] typefilter= (A) Filter for certain types of column. Valid values:
@li A Return All columns
@li C Return Character columns
@li N Return Numeric columns
@version 9.2
@author Allan Bowe
@@ -27,9 +34,10 @@
%macro mf_getvarlist(libds
,dlm=%str( )
,quote=no
,typefilter=A
)/*/STORE SOURCE*/;
/* declare local vars */
%local outvar dsid nvars x rc dlm q var;
%local outvar dsid nvars x rc dlm q var vtype;
/* credit Rowland Hale - byte34 is double quote, 39 is single quote */
%if %upcase(&quote)=DOUBLE %then %let q=%qsysfunc(byte(34));
@@ -37,21 +45,22 @@
/* open dataset in macro */
%let dsid=%sysfunc(open(&libds));
%if &dsid %then %do;
%let nvars=%sysfunc(attrn(&dsid,NVARS));
%if &nvars>0 %then %do;
/* add first dataset variable to global macro variable */
%let outvar=&q.%sysfunc(varname(&dsid,1))&q.;
/* add remaining variables with supplied delimeter */
/* add variables with supplied delimeter */
%do x=1 %to &nvars;
%let var=&q.%sysfunc(varname(&dsid,&x))&q.;
%if &var=&q&q %then %do;
%put &sysmacroname: Empty column found in &libds!;
%let var=&q. &q.;
/* get variable type */
%let vtype=%sysfunc(vartype(&dsid,&x));
%if &vtype=&typefilter or &typefilter=A %then %do;
%let var=&q.%sysfunc(varname(&dsid,&x))&q.;
%if &var=&q&q %then %do;
%put &sysmacroname: Empty column found in &libds!;
%let var=&q. &q.;
%end;
%if %quote(&outvar)=%quote() %then %let outvar=&var;
%else %let outvar=&outvar.&dlm.&var.;
%end;
%if &x=1 %then %let outvar=&var;
%else %let outvar=&outvar.&dlm.&var.;
%end;
%end;
%let rc=%sysfunc(close(&dsid));
@@ -61,4 +70,4 @@
%let rc=%sysfunc(close(&dsid));
%end;
&outvar
%mend;
%mend mf_getvarlist;

8
base/mf_getvarnum.sas
View File

@@ -6,8 +6,8 @@
Usage:
data work.test;
format str $1. num datetime19.;
stop;
format str $1. num datetime19.;
stop;
run;
%put %mf_getVarNum(work.test,str);
%put %mf_getVarNum(work.test,num);
@@ -39,8 +39,8 @@ returns:
/* Get variable number */
%let vnum = %sysfunc(varnum(&dsid, &var));
%if(&vnum <= 0) %then %do;
%put NOTE: Variable &var does not exist in &libds;
%let vnum = %str( );
%put NOTE: Variable &var does not exist in &libds;
%let vnum = %str( );
%end;
%end;
%else %put dataset &ds not opened! (rc=&dsid);

10
base/mf_getvartype.sas
View File

@@ -5,8 +5,8 @@
Usage:
data test;
length str $1. num 8.;
stop;
length str $1. num 8.;
stop;
run;
%put %mf_getvartype(test,str);
%put %mf_getvartype(work.test,num);
@@ -35,8 +35,8 @@ Usage:
/* Get variable type (C/N) */
%if(&vnum. > 0) %then %let vtype = %sysfunc(vartype(&dsid, &vnum.));
%else %do;
%put NOTE: Variable &var does not exist in &libds;
%let vtype = %str( );
%put NOTE: Variable &var does not exist in &libds;
%let vtype = %str( );
%end;
%end;
%else %put dataset &libds not opened! (rc=&dsid);
@@ -45,4 +45,4 @@ Usage:
%let rc = %sysfunc(close(&dsid));
/* Return variable type */
&vtype
%mend;
%mend mf_getvartype;

43
base/mf_getxengine.sas Normal file
View File

@@ -0,0 +1,43 @@
/**
@file
@brief Returns the engine type of a SAS fileref
@details Queries sashelp.vextfl to get the xengine value.
Usage:
filename feng temp;
%put %mf_getxengine(feng);
returns:
> TEMP
@param fref The fileref to check
@returns The XENGINE value in sashelp.vextfl or 0 if not found.
@version 9.2
@author Allan Bowe
<h4> Related Macros </h4>
@li mf_getengine.sas
**/
%macro mf_getxengine(fref
)/*/STORE SOURCE*/;
%local dsid engnum rc engine;
%let dsid=%sysfunc(
open(sashelp.vextfl(where=(fileref="%upcase(&fref)")),i)
);
%if (&dsid ^= 0) %then %do;
%let engnum=%sysfunc(varnum(&dsid,XENGINE));
%let rc=%sysfunc(fetch(&dsid));
%let engine=%sysfunc(getvarc(&dsid,&engnum));
%* put &fref. ENGINE is &engine.;
%let rc= %sysfunc(close(&dsid));
%end;
%else %let engine=0;
&engine
%mend;

9
base/mf_isblank.sas
View File

@@ -6,12 +6,13 @@
%sysevalf(%superq(param)=,boolean)
Usage:
%put mf_isblank(&var);
inspiration: https://support.sas.com/resources/papers/proceedings09/022-2009.pdf
%put mf_isblank(&var);
@param param VALUE to be checked
inspiration:
https://support.sas.com/resources/papers/proceedings09/022-2009.pdf
@param param VALUE to be checked
@return output returns 1 (if blank) else 0

23
base/mf_isdir.sas
View File

@@ -6,7 +6,8 @@
%let isdir=%mf_isdir(/tmp);
With thanks and full credit to Andrea Defronzo - https://www.linkedin.com/in/andrea-defronzo-b1a47460/
With thanks and full credit to Andrea Defronzo -
https://www.linkedin.com/in/andrea-defronzo-b1a47460/
@param path full path of the file/directory to be checked
@@ -17,17 +18,17 @@
%macro mf_isdir(path
)/*/STORE SOURCE*/;
%local rc did is_directory fref_t;
%local rc did is_directory fref_t;
%let is_directory = 0;
%let rc = %sysfunc(filename(fref_t, %superq(path)));
%let did = %sysfunc(dopen(&fref_t.));
%if &did. ^= 0 %then %do;
%let is_directory = 1;
%let rc = %sysfunc(dclose(&did.));
%end;
%let rc = %sysfunc(filename(fref_t));
%let is_directory = 0;
%let rc = %sysfunc(filename(fref_t, %superq(path)));
%let did = %sysfunc(dopen(&fref_t.));
%if &did. ^= 0 %then %do;
%let is_directory = 1;
%let rc = %sysfunc(dclose(&did.));
%end;
%let rc = %sysfunc(filename(fref_t));
&is_directory
&is_directory
%mend;

10
base/mf_mkdir.sas
View File

@@ -46,8 +46,8 @@ Usage:
*/
%if (%length(&dir) gt %length(&child)) %then %do;
%let parent = %substr(&dir, 1, %length(&dir)-%length(&child));
%mf_mkdir(&parent)
%let parent = %substr(&dir, 1, %length(&dir)-%length(&child));
%mf_mkdir(&parent)
%end;
/*
@@ -56,11 +56,11 @@ Usage:
%let dname = %sysfunc(dcreate(&child, &parent));
%if (%bquote(&dname) eq ) %then %do;
%put %str(ERR)OR: could not create &parent + &child;
%abort cancel;
%put %str(ERR)OR: could not create &parent + &child;
%abort cancel;
%end;
%else %do;
%put Directory created: &dir;
%put Directory created: &dir;
%end;
%end;
/* exit quietly if directory did exist.*/

5
base/mf_mval.sas
View File

@@ -1,8 +1,9 @@
/**
@file mf_mval.sas
@brief Returns a macro variable value if the variable exists
@details Use this macro to avoid repetitive use of `%if %symexist(MACVAR) %then`
type logic.
@details
Use this macro to avoid repetitive use of `%if %symexist(MACVAR) %then`
type logic.
Usage:
%if %mf_mval(maynotexist)=itdid %then %do;

5
base/mf_trimstr.sas
View File

@@ -1,7 +1,7 @@
/**
@file mf_trimstr.sas
@brief Removes character(s) from the end, if they exist
@details If the designated characters exist at the end of the string, they
@details If the designated characters exist at the end of the string, they
are removed
%put %mf_trimstr(/blah/,/); * /blah;
@@ -12,7 +12,8 @@
@param basestr The string to be modified
@param trimstr The string to be removed from the end of `basestr`, if it exists
@param trimstr The string to be removed from the end of `basestr`, if it
exists
@return output returns result with the value of `trimstr` removed from the end

8
base/mf_uid.sas
View File

@@ -1,11 +1,11 @@
/**
@file
@brief Creates a Unique ID based on system time in a friendly format
@brief Creates a unique ID based on system time in friendly format
@details format = YYYYMMDD_HHMMSSmmm_<sysjobid>_<3randomDigits>
%put %mf_uid();
@version 9.2
@version 9.3
@author Allan Bowe
**/
@@ -14,8 +14,8 @@
)/*/STORE SOURCE*/;
%local today now;
%let today=%sysfunc(today(),yymmddn8.);
%let now=%sysfunc(compress(%sysfunc(time(),time12.3),:.));
%let now=%sysfunc(compress(%sysfunc(time(),tod12.3),:.));
&today._&now._&sysjobid._%sysevalf(%sysfunc(ranuni(0))*999,CEIL)
%mend;
%mend mf_uid;

2
base/mf_verifymacvars.sas
View File

@@ -35,7 +35,7 @@
%macro mf_verifymacvars(
verifyVars /* list of macro variable NAMES */
verifyVars /* list of macro variable NAMES */
,makeUpcase=NO /* set to YES to make all the variable VALUES uppercase */
,mAbort=SOFT
)/*/STORE SOURCE*/;

8
base/mf_wordsinstr1butnotstr2.sas
View File

@@ -6,7 +6,7 @@
Usage:
%let x= %mf_wordsInStr1ButNotStr2(
Str1=blah sss blaaah brah bram boo
Str1=blah sss blaaah brah bram boo
,Str2= blah blaaah brah ssss
);
@@ -24,13 +24,13 @@
**/
%macro mf_wordsInStr1ButNotStr2(
Str1= /* string containing words to extract */
,Str2= /* used to compare with the extract string */
Str1= /* string containing words to extract */
,Str2= /* used to compare with the extract string */
)/*/STORE SOURCE*/;
%local count_base count_extr i i2 extr_word base_word match outvar;
%if %length(&str1)=0 or %length(&str2)=0 %then %do;
%put WARNING: empty string provided!;
%put %str(WARN)ING: empty string provided!;
%put base string (str1)= &str1;
%put compare string (str2) = &str2;
%return;

23
base/mp_abort.sas
View File

@@ -6,11 +6,11 @@
results back to the client in an STP Web App context, or completely stop
in the case of a batch run.
Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored Process
environments. This macro takes a unique approach - we set the SAS syscc to 0,
run `stpsrvset('program error', 0)` (if SAS 9) and then - we open a macro
but don't close it! This provides a graceful abort for SAS web services in all
web enabled environments.
Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored
Process environments. This macro takes a unique approach - we set the SAS
syscc to 0, run `stpsrvset('program error', 0)` (if SAS 9) and then - we open
a macro but don't close it! This provides a graceful abort for SAS web
services in all web enabled environments.
@param mac= to contain the name of the calling macro
@param msg= message to be returned
@@ -48,7 +48,10 @@
input; putlog _infile_;
i=1;
retain logonce 0;
if (_infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR") and logonce=0 then do;
if (
_infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR"
) and logonce=0 then
do;
call symputx('logline',_n_);
logonce+1;
end;
@@ -135,10 +138,10 @@
run;
%end;
/**
* endsas is reliable but kills some deployments.
* Abort variants are ungraceful (non zero return code)
* This approach lets SAS run silently until the end :-)
*/
* endsas is reliable but kills some deployments.
* Abort variants are ungraceful (non zero return code)
* This approach lets SAS run silently until the end :-)
*/
%put _all_;
filename skip temp;
data _null_;

56
base/mp_assert.sas Normal file
View File

@@ -0,0 +1,56 @@
/**
@file
@brief Generic assertion
@details Useful in the context of writing sasjs tests. The results of the
test are _appended_ to the &outds. table.
Example usage:
%mp_assert(iftrue=(1=1),
desc=Obviously true
)
%mp_assert(iftrue=(1=0),
desc=Will fail
)
@param [in] iftrue= (1=1) A condition where, if true, the test is a PASS.
Else, the test is a fail.
@param [in] desc= (Testing observations) The user provided test description
@param [out] outds= (work.test_results) The output dataset to contain the
results. If it does not exist, it will be created, with the following format:
|TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
|---|---|---|
|User Provided description|PASS|Column &inds contained ALL columns|
@version 9.2
@author Allan Bowe
**/
%macro mp_assert(iftrue=(1=1),
desc=0,
outds=work.test_results
)/*/STORE SOURCE*/;
data ;
length test_description $256 test_result $4 test_comments $256;
test_description=symget('desc');
test_comments="&sysmacroname: Test result of "!!symget('iftrue');
%if %eval(%unquote(&iftrue)) %then %do;
test_result='PASS';
%end;
%else %do;
test_result='FAIL';
%end;
run;
%local ds ;
%let ds=&syslast;
proc append base=&outds data=&ds;
run;
proc sql;
drop table &ds;
%mend mp_assert;

145
base/mp_assertcols.sas Normal file
View File

@@ -0,0 +1,145 @@
/**
@file
@brief Asserts the existence (or not) of columns
@details Useful in the context of writing sasjs tests. The results of the
test are _appended_ to the &outds. table.
Example usage:
%mp_assertcols(sashelp.class,
cols=name age sex,
test=ALL,
desc=check all columns exist
)
%mp_assertcols(sashelp.class,
cols=a b c,
test=NONE
)
%mp_assertcols(sashelp.class,
cols=age depth,
test=ANY
)
<h4> SAS Macros </h4>
@li mf_existds.sas
@li mf_existvarlist.sas
@li mf_getvarlist.sas
@li mf_wordsinstr1butnotstr2.sas
@li mp_abort.sas
@param [in] inds The input library.dataset to test for values
@param [in] cols= The list of columns to check for
@param [in] desc= (Testing observations) The user provided test description
@param [in] test= (ALL) The test to apply. Valid values are:
@li ALL - Test is a PASS if ALL columns exist in &inds
@li ANY - Test is a PASS if ANY of the columns exist in &inds
@li NONE - Test is a PASS if NONE of the columns exist in &inds
@param [out] outds= (work.test_results) The output dataset to contain the
results. If it does not exist, it will be created, with the following format:
|TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
|---|---|---|
|User Provided description|PASS|Column &inds contained ALL columns|
<h4> Related Macros </h4>
@li mp_assertdsobs.sas
@li mp_assertcolvals.sas
@li mp_assertdsobs.sas
@version 9.2
@author Allan Bowe
**/
%macro mp_assertcols(inds,
cols=0,
test=ALL,
desc=0,
outds=work.test_results
)/*/STORE SOURCE*/;
%mp_abort(iftrue= (&syscc ne 0)
,mac=&sysmacroname
,msg=%str(syscc=&syscc - on macro entry)
)
%local lib ds ;
%let lib=%scan(&inds,1,%str(.));
%let ds=%scan(&inds,2,%str(.));
%let cols=%upcase(&cols);
%mp_abort(iftrue= (%mf_existds(&lib..&ds)=0)
,mac=&sysmacroname
,msg=%str(&lib..&ds not found!)
)
%mp_abort(iftrue= (&cols=0)
,mac=&sysmacroname
,msg=%str(No cols provided)
)
%let test=%upcase(&test);
%if &test ne ANY and &test ne ALL and &test ne NONE %then %do;
%mp_abort(
mac=&sysmacroname,
msg=%str(Invalid test - &test)
)
%end;
/**
* now do the actual test!
*/
%local result;
%if %mf_existVarList(&inds,&cols)=1 %then %let result=ALL;
%else %do;
%local targetcols compare;
%let targetcols=%upcase(%mf_getvarlist(&inds));
%let compare=%mf_wordsinstr1butnotstr2(
Str1=&cols,
Str2=&targetcols
);
%if %cmpres(&compare)=%cmpres(&cols) %then %let result=NONE;
%else %let result=SOME;
%end;
data;
length test_description $256 test_result $4 test_comments $256;
test_description=symget('desc');
if test_description='0'
then test_description="Testing &inds for existence of &test of: &cols";
test_result='FAIL';
test_comments="&sysmacroname: &inds has &result columns ";
%if &test=ALL %then %do;
%if &result=ALL %then %do;
test_result='PASS';
%end;
%end;
%else %if &test=ANY %then %do;
%if &result=SOME %then %do;
test_result='PASS';
%end;
%end;
%else %if &test=NONE %then %do;
%if &result=NONE %then %do;
test_result='PASS';
%end;
%end;
%else %do;
test_comments="&sysmacroname: Unsatisfied test condition - &test";
%end;
run;
%local ds;
%let ds=&syslast;
proc append base=&outds data=&ds;
run;
proc sql;
drop table &ds;
%mend;

147
base/mp_assertcolvals.sas Normal file
View File

@@ -0,0 +1,147 @@
/**
@file
@brief Asserts the values in a column
@details Useful in the context of writing sasjs tests. The results of the
test are _appended_ to the &outds. table.
Example usage:
data work.checkds;
do checkval='Jane','James','Jill';
output;
end;
run;
%mp_assertcolvals(sashelp.class.name,
checkvals=work.checkds.checkval,
desc=At least one value has a match,
test=ANYVAL
)
data work.check;
do val='M','F';
output;
end;
run;
%mp_assertcolvals(sashelp.class.sex,
checkvals=work.check.val,
desc=All values have a match,
test=ALLVALS
)
<h4> SAS Macros </h4>
@li mf_existds.sas
@li mf_nobs.sas
@li mp_abort.sas
@param [in] indscol The input library.dataset.column to test for values
@param [in] checkvals= A library.dataset.column value containing a UNIQUE
list of values to be compared against the source (indscol).
@param [in] desc= (Testing observations) The user provided test description
@param [in] test= (ALLVALS) The test to apply. Valid values are:
@li ALLVALS - Test is a PASS if ALL values have a match in checkvals
@li ANYVAL - Test is a PASS if at least 1 value has a match in checkvals
@param [out] outds= (work.test_results) The output dataset to contain the
results. If it does not exist, it will be created, with the following format:
|TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
|---|---|---|
|User Provided description|PASS|Column &indscol contained ALL target vals|
<h4> Related Macros </h4>
@li mp_assertdsobs.sas
@version 9.2
@author Allan Bowe
**/
%macro mp_assertcolvals(indscol,
checkvals=0,
test=ALLVALS,
desc=mp_assertcolvals - no desc provided,
outds=work.test_results
)/*/STORE SOURCE*/;
%mp_abort(iftrue= (&syscc ne 0)
,mac=&sysmacroname
,msg=%str(syscc=&syscc - on macro entry)
)
%local lib ds col clib cds ccol nobs;
%let lib=%scan(&indscol,1,%str(.));
%let ds=%scan(&indscol,2,%str(.));
%let col=%scan(&indscol,3,%str(.));
%mp_abort(iftrue= (%mf_existds(&lib..&ds)=0)
,mac=&sysmacroname
,msg=%str(&lib..&ds not found!)
)
%mp_abort(iftrue= (&checkvals=0)
,mac=&sysmacroname
,msg=%str(Set CHECKVALS to a library.dataset.column containing check vals)
)
%let clib=%scan(&checkvals,1,%str(.));
%let cds=%scan(&checkvals,2,%str(.));
%let ccol=%scan(&checkvals,3,%str(.));
%mp_abort(iftrue= (%mf_existds(&clib..&cds)=0)
,mac=&sysmacroname
,msg=%str(&clib..&cds not found!)
)
%let nobs=%mf_nobs(&clib..&cds);
%mp_abort(iftrue= (&nobs=0)
,mac=&sysmacroname
,msg=%str(&clib..&cds is empty!)
)
%let test=%upcase(&test);
%if &test ne ALLVALS and &test ne ANYVAL %then %do;
%mp_abort(
mac=&sysmacroname,
msg=%str(Invalid test - &test)
)
%end;
%local result orig;
%let result=-1;
%let orig=-1;
proc sql noprint;
select count(*) into: result
from &lib..&ds
where &col not in (
select &ccol from &clib..&cds
);
select count(*) into: orig from &lib..&ds;
quit;
%mp_abort(iftrue= (&syscc ne 0)
,mac=&sysmacroname
,msg=%str(syscc=&syscc after macro query)
)
data;
length test_description $256 test_result $4 test_comments $256;
test_description=symget('desc');
test_result='FAIL';
test_comments="&sysmacroname: &lib..&ds..&col has &result values "
!!"not in &clib..&cds..&ccol ";
%if &test=ANYVAL %then %do;
if &result < &orig then test_result='PASS';
%end;
%else %if &test=ALLVALS %then %do;
if &result=0 then test_result='PASS';
%end;
%else %do;
test_comments="&sysmacroname: Unsatisfied test condition - &test";
%end;
run;
%local ds;
%let ds=&syslast;
proc append base=&outds data=&ds;
run;
proc sql;
drop table &ds;
%mend;

89
base/mp_assertdsobs.sas Normal file
View File

@@ -0,0 +1,89 @@
/**
@file
@brief Asserts the number of observations in a dataset
@details Useful in the context of writing sasjs tests. The results of the
test are _appended_ to the &outds. table.
Example usage:
%mp_assertdsobs(sashelp.class) %* tests if any observations are present;
<h4> SAS Macros </h4>
@li mf_nobs.sas
@li mp_abort.sas
@param [in] inds input dataset to test for presence of observations
@param [in] desc= (Testing observations) The user provided test description
@param [in] test= (HASOBS) The test to apply. Valid values are:
@li HASOBS - Test is a PASS if the input dataset has any observations
@li EMPTY - Test is a PASS if input dataset is empty
@li EQUALS [integer] - Test passes if obs count matches the provided integer
@param [out] outds= (work.test_results) The output dataset to contain the
results. If it does not exist, it will be created, with the following format:
|TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
|---|---|---|
|User Provided description|PASS|Dataset &inds has XX obs|
<h4> Related Macros </h4>
@li mp_assertcolvals.sas
@version 9.2
@author Allan Bowe
**/
%macro mp_assertdsobs(inds,
test=HASOBS,
desc=Testing observations,
outds=work.test_results
)/*/STORE SOURCE*/;
%local nobs;
%let nobs=%mf_nobs(&inds);
%let test=%upcase(&test);
%if %substr(&test.xxxxx,1,6)=EQUALS %then %do;
%let val=%scan(&test,2,%str( ));
%mp_abort(iftrue= (%DATATYP(&val)=CHAR)
,mac=&sysmacroname
,msg=%str(Invalid test - &test, expected EQUALS [integer])
)
%let test=EQUALS;
%end;
%else %if &test ne HASOBS and &test ne EMPTY %then %do;
%mp_abort(
mac=&sysmacroname,
msg=%str(Invalid test - &test)
)
%end;
data;
length test_description $256 test_result $4 test_comments $256;
test_description=symget('desc');
test_result='FAIL';
test_comments="&sysmacroname: Dataset &inds has &nobs observations";
%if &test=HASOBS %then %do;
if &nobs>0 then test_result='PASS';
%end;
%else %if &test=EMPTY %then %do;
if &nobs=0 then test_result='PASS';
%end;
%else %if &test=EQUALS %then %do;
if &nobs=&val then test_result='PASS';
%end;
%else %do;
test_comments="&sysmacroname: Unsatisfied test condition - &test";
%end;
run;
%local ds;
%let ds=&syslast;
proc append base=&outds data=&ds;
run;
proc sql;
drop table &ds;
%mend;

39
base/mp_binarycopy.sas
View File

@@ -2,9 +2,10 @@
@file
@brief Copy any file using binary input / output streams
@details Reads in a file byte by byte and writes it back out. Is an
os-independent method to copy files. In case of naming collision, the
default filerefs can be modified.
Based on http://stackoverflow.com/questions/13046116/using-sas-to-copy-a-text-file
os-independent method to copy files. In case of naming collision, the
default filerefs can be modified.
Based on:
https://stackoverflow.com/questions/13046116/using-sas-to-copy-a-text-file
%mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
@@ -19,12 +20,12 @@
**/
%macro mp_binarycopy(
inloc= /* full path and filename of the object to be copied */
inloc= /* full path and filename of the object to be copied */
,outloc= /* full path and filename of object to be created */
,inref=____in /* override default to use own filerefs */
,outref=____out /* override default to use own filerefs */
)/*/STORE SOURCE*/;
/* these IN and OUT filerefs can point to anything */
/* these IN and OUT filerefs can point to anything */
%if &inref = ____in %then %do;
filename &inref &inloc lrecl=1048576 ;
%end;
@@ -32,20 +33,20 @@
filename &outref &outloc lrecl=1048576 ;
%end;
/* copy the file byte-for-byte */
data _null_;
length filein 8 fileid 8;
filein = fopen("&inref",'I',1,'B');
fileid = fopen("&outref",'O',1,'B');
rec = '20'x;
do while(fread(filein)=0);
rc = fget(filein,rec,1);
rc = fput(fileid, rec);
rc =fwrite(fileid);
end;
rc = fclose(filein);
rc = fclose(fileid);
run;
/* copy the file byte-for-byte */
data _null_;
length filein 8 fileid 8;
filein = fopen("&inref",'I',1,'B');
fileid = fopen("&outref",'O',1,'B');
rec = '20'x;
do while(fread(filein)=0);
rc = fget(filein,rec,1);
rc = fput(fileid, rec);
rc =fwrite(fileid);
end;
rc = fclose(filein);
rc = fclose(fileid);
run;
%if &inref = ____in %then %do;
filename &inref clear;
%end;

16
base/mp_cleancsv.sas
View File

@@ -2,10 +2,10 @@
@file mp_cleancsv.sas
@brief Fixes embedded cr / lf / crlf in CSV
@details CSVs will sometimes contain lf or crlf within quotes (eg when
saved by excel). When the termstr is ALSO lf or crlf that can be tricky
to process using SAS defaults.
This macro converts any csv to follow the convention of a windows excel file,
applying CRLF line endings and converting embedded cr and crlf to lf.
saved by excel). When the termstr is ALSO lf or crlf that can be tricky
to process using SAS defaults.
This macro converts any csv to follow the convention of a windows excel file,
applying CRLF line endings and converting embedded cr and crlf to lf.
usage:
@@ -23,7 +23,7 @@
%macro mp_cleancsv(in=NOTPROVIDED,out=NOTPROVIDED,qchar='22'x);
%if "&in"="NOTPROVIDED" or "&out"="NOTPROVIDED" %then %do;
%put %str(ERR)OR: Please provide valid input (&in) and output (&out) locations;
%put %str(ERR)OR: Please provide valid input (&in) & output (&out) locations;
%return;
%end;
@@ -32,9 +32,9 @@
%if %index(&out,.) %then %let out="&out";
/**
* convert all cr and crlf within quotes to lf
* convert all other cr or lf to crlf
*/
* convert all cr and crlf within quotes to lf
* convert all other cr or lf to crlf
*/
data _null_;
infile &in recfm=n ;
file &out recfm=n;

4
base/mp_createconstraints.sas
View File

@@ -13,7 +13,7 @@
constraint unq unique(tx_from, dd_type),
constraint nnn not null(DD_SHORTDESC)
);
%mp_getconstraints(lib=work,ds=example,outds=work.constraints)
%mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
%mp_createconstraints(inds=work.constraints,outds=created,execute=YES)
@@ -48,7 +48,7 @@ data &outds;
else type=constraint_type;
create_statement=catx(" ","alter table",libref,".",table_name
,"add constraint",constraint_name,type,"(");
if last.constraint_name then
if last.constraint_name then
create_statement=cats(create_statement,column_name,");");
else create_statement=cats(create_statement,column_name,",");
if "&execute"="YES" then call execute(create_statement);

2
base/mp_createwebservice.sas
View File

@@ -14,7 +14,7 @@ Usage:
filename ft15f001 temp;
parmcards4;
%* fetch any data from frontend ;
%webout(FETCH)
%webout(FETCH)
data example1 example2;
set sashelp.class;
run;

6
base/mp_csv2ds.sas
View File

@@ -62,8 +62,8 @@
%local hasheader; %let hasheader=0;
data _null_;
if _N_ > 1 then do;
call symputx('hasheader',1,'l');
stop;
call symputx('hasheader',1,'l');
stop;
end;
infile &inref;
input;
@@ -131,7 +131,7 @@ run;
/* import the CSV */
data &outds
%if %upcase(&view)=YES %then %do;
/view=&outds
/view=&outds
%end;
;
infile &inref dsd firstobs=2;

2
base/mp_deleteconstraints.sas
View File

@@ -13,7 +13,7 @@
constraint unq unique(tx_from, dd_type),
constraint nnn not null(DD_SHORTDESC)
);
%mp_getconstraints(lib=work,ds=example,outds=work.constraints)
%mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)

41
base/mp_dirlist.sas
View File

@@ -38,13 +38,13 @@
@returns outds contains the following variables:
- directory (containing folder)
- file_or_folder (file / folder)
- filepath (path/to/file.name)
- filename (just the file name)
- ext (.extension)
- msg (system message if any issues)
- OS SPECIFIC variables, if <code>getattrs=</code> is used.
- directory (containing folder)
- file_or_folder (file / folder)
- filepath (path/to/file.name)
- filename (just the file name)
- ext (.extension)
- msg (system message if any issues)
- OS SPECIFIC variables, if <code>getattrs=</code> is used.
@version 9.2
@author Allan Bowe
@@ -57,8 +57,11 @@
)/*/STORE SOURCE*/;
%let getattrs=%upcase(&getattrs)XX;
data &outds (compress=no keep=file_or_folder filepath filename ext msg directory);
length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80 ext $20 msg $200;
data &outds(compress=no
keep=file_or_folder filepath filename ext msg directory
);
length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
ext $20 msg $200;
%if &fref=0 %then %do;
rc = filename(fref, "&path");
%end;
@@ -67,15 +70,15 @@ data &outds (compress=no keep=file_or_folder filepath filename ext msg directory
rc=0;
%end;
if rc = 0 then do;
did = dopen(fref);
directory=dinfo(did,'Directory');
if did=0 then do;
putlog "NOTE: This directory is empty - " directory;
msg=sysmsg();
put _all_;
stop;
end;
rc = filename(fref);
did = dopen(fref);
directory=dinfo(did,'Directory');
if did=0 then do;
putlog "NOTE: This directory is empty - " directory;
msg=sysmsg();
put _all_;
stop;
end;
rc = filename(fref);
end;
else do;
msg=sysmsg();
@@ -98,7 +101,7 @@ data &outds (compress=no keep=file_or_folder filepath filename ext msg directory
if index(fmsg,'File is in use') or index(dmsg,'is not a directory')
then file_or_folder='file';
else if index(fmsg, 'Insufficient authorization') then file_or_folder='file';
else if index(fmsg,'Insufficient authorization') then file_or_folder='file';
else if file_or_folder='' then file_or_folder='locked';
if file_or_folder='file' then do;

6
base/mp_distinctfmtvalues.sas
View File

@@ -19,7 +19,7 @@
**/
%macro mp_distinctfmtvalues(
libds=
libds=
,var=
,outvar=formatted_value
,outds=work.mp_distinctfmtvalues
@@ -34,7 +34,7 @@
create table &outds as
select distinct
%if &vtype=C & %trim(&fmt)=%str() %then %do;
&var
&var
%end;
%else %if &vtype=C %then %do;
put(&var,&fmt)
@@ -45,6 +45,6 @@
%else %do;
put(&var,&fmt)
%end;
as &outvar length=&varlen
as &outvar length=&varlen
from &libds;
%mend;

2
base/mp_dropmembers.sas
View File

@@ -22,7 +22,7 @@
**/
%macro mp_dropmembers(
list /* space separated list of datasets / views */
list /* space separated list of datasets / views */
,libref=WORK /* can only drop from a single library at a time */
)/*/STORE SOURCE*/;

56
base/mp_ds2cards.sas
View File

@@ -10,20 +10,22 @@
, maxobs=5)
TODO:
- labelling the dataset
- explicity setting a unix LF
- constraints / indexes etc
- labelling the dataset
- explicity setting a unix LF
- constraints / indexes etc
@param [in] base_ds= Should be two level - eg work.blah. This is the table that
is converted to a cards file.
@param [in] tgt_ds= Table that the generated cards file would create. Optional -
if omitted, will be same as BASE_DS.
@param [in] base_ds= Should be two level - eg work.blah. This is the table
that is converted to a cards file.
@param [in] tgt_ds= Table that the generated cards file would create.
Optional - if omitted, will be same as BASE_DS.
@param [out] cards_file= Location in which to write the (.sas) cards file
@param [in] maxobs= to limit output to the first <code>maxobs</code> observations
@param [in] showlog= whether to show generated cards file in the SAS log (YES/NO)
@param [in] maxobs= to limit output to the first <code>maxobs</code>
observations
@param [in] showlog= whether to show generated cards file in the SAS log
(YES/NO)
@param [in] outencoding= provide encoding value for file statement (eg utf-8)
@param [in] append= If NO then will rebuild the cards file if it already exists,
otherwise will append to it. Used by the mp_lib2cards.sas macro.
@param [in] append= If NO then will rebuild the cards file if it already
exists, otherwise will append to it. Used by the mp_lib2cards.sas macro.
@version 9.2
@@ -41,8 +43,8 @@
%local i setds nvars;
%if not %sysfunc(exist(&base_ds)) %then %do;
%put WARNING: &base_ds does not exist;
%return;
%put %str(WARN)ING: &base_ds does not exist;
%return;
%end;
%if %index(&base_ds,.)=0 %then %let base_ds=WORK.&base_ds;
@@ -59,15 +61,17 @@ select count(*) into: nvars from dictionary.columns
where libname="%scan(%upcase(&base_ds),1)"
and memname="%scan(%upcase(&base_ds),2)";
%if &nvars=0 %then %do;
%put WARNING: Dataset &base_ds has no variables! It will not be converted.;
%put %str(WARN)ING: Dataset &base_ds has no variables, will not be converted.;
%return;
%end;
/* get indexes */
proc sort data=sashelp.vindex
(where=(upcase(libname)="%scan(%upcase(&base_ds),1)"
and upcase(memname)="%scan(%upcase(&base_ds),2)"))
out=_data_;
proc sort
data=sashelp.vindex(
where=(upcase(libname)="%scan(%upcase(&base_ds),1)"
and upcase(memname)="%scan(%upcase(&base_ds),2)")
)
out=_data_;
by indxname indxpos;
run;
@@ -82,7 +86,7 @@ data _null_;
idxcnt+1;
nom='';
uni='';
vars=name;
vars=name;
end;
else vars=catx(' ',vars,name);
if last.indxname then do;
@@ -110,8 +114,8 @@ proc sql
;
reset outobs=max;
create table datalines1 as
select name,type,length,varnum,format,label from dictionary.columns
where libname="%upcase(%scan(&base_ds,1))"
select name,type,length,varnum,format,label from dictionary.columns
where libname="%upcase(%scan(&base_ds,1))"
and memname="%upcase(%scan(&base_ds,2))";
/**
@@ -126,7 +130,7 @@ create table datalines1 as
data datalines_2;
format dataline $32000.;
set datalines1 (where=(upcase(name) not in
set datalines1 (where=(upcase(name) not in
('PROCESSED_DTTM','VALID_FROM_DTTM','VALID_TO_DTTM')));
if type='num' then dataline=
cats('ifc(int(',name,')=',name,'
@@ -140,9 +144,9 @@ proc sql noprint;
select dataline into: datalines separated by ',' from datalines_2;
%local
process_dttm_flg
valid_from_dttm_flg
valid_to_dttm_flg
process_dttm_flg
valid_from_dttm_flg
valid_to_dttm_flg
;
%let process_dttm_flg = N;
%let valid_from_dttm_flg = N;
@@ -212,7 +216,7 @@ data _null_;
put "input ";
%do i = 1 %to &nvars.;
%if(%length(&&input_stmt_&i..)) %then
put " &&input_stmt_&i..";
put " &&input_stmt_&i..";
;
%end;
put ";";

28
base/mp_ds2csv.sas
View File

@@ -19,8 +19,8 @@
)/*/STORE SOURCE*/;
%if not %sysfunc(exist(&ds)) %then %do;
%put WARNING: &ds does not exist;
%return;
%put %str(WARN)ING: &ds does not exist;
%return;
%end;
%if %index(&ds,.)=0 %then %let ds=WORK.&ds;
@@ -36,22 +36,22 @@
/* first get headers */
data _null_;
file &outloc dlm=',' dsd &outencoding lrecl=32767;
length header $ 2000;
dsid=open("&ds.","i");
num=attrn(dsid,"nvars");
do i=1 to num;
header = trim(left(coalescec(varlabel(dsid,i),varname(dsid,i))));
put header @;
end;
rc=close(dsid);
file &outloc dlm=',' dsd &outencoding lrecl=32767;
length header $ 2000;
dsid=open("&ds.","i");
num=attrn(dsid,"nvars");
do i=1 to num;
header = trim(left(coalescec(varlabel(dsid,i),varname(dsid,i))));
put header @;
end;
rc=close(dsid);
run;
/* next, export data */
data _null_;
set &ds.;
file &outloc mod dlm=',' dsd &outencoding lrecl=32767;
put (_all_) (+0);
set &ds.;
file &outloc mod dlm=',' dsd &outencoding lrecl=32767;
put (_all_) (+0);
run;

98
base/mp_ds2fmtds.sas Normal file
View File

@@ -0,0 +1,98 @@
/**
@file
@brief Converts every value in a dataset to it's formatted value
@details Converts every value to it's formatted value. All variables will
become character, and will be in the same order.
Usage:
%mp_ds2fmtds(sashelp.cars,work.cars)
@param [in] libds The library.dataset to be converted
@param [out] outds The dataset to create.
<h4> Related Macros <h4>
@li mp_jsonout.sas
@version 9.2
@author Allan Bowe
**/
%macro mp_ds2fmtds(libds, outds
)/*/STORE SOURCE*/;
/* validations */
%if not %sysfunc(exist(&libds)) %then %do;
%put %str(WARN)ING: &libds does not exist;
%return;
%end;
%if %index(&libds,.)=0 %then %let libds=WORK.&libds;
/* grab metadata */
proc contents noprint data=&libds
out=_data_(keep=name type length format formatl formatd varnum);
run;
proc sort;
by varnum;
run;
/* prepare formats and varnames */
data _null_;
set &syslast end=last;
name=upcase(name);
/* fix formats */
if type=2 or type=6 then do;
length fmt $49.;
if format='' then fmt=cats('$',length,'.');
else if formatl=0 then fmt=cats(format,'.');
else fmt=cats(format,formatl,'.');
newlen=max(formatl,length);
end;
else do;
if format='' then fmt='best.';
else if formatl=0 then fmt=cats(format,'.');
else if formatd=0 then fmt=cats(format,formatl,'.');
else fmt=cats(format,formatl,'.',formatd);
/* needs to be wide, for datetimes etc */
newlen=max(length,formatl,24);
end;
/* 32 char unique name */
newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
call symputx(cats('name',_n_),name,'l');
call symputx(cats('newname',_n_),newname,'l');
call symputx(cats('len',_n_),newlen,'l');
call symputx(cats('fmt',_n_),fmt,'l');
call symputx(cats('type',_n_),type,'l');
if last then call symputx('nobs',_n_,'l');
run;
/* clean up */
proc sql;
drop table &syslast;
%if &nobs=0 %then %do;
%put Dataset &libds has no columns!
data &outds;
set &libds;
run;
%return;
%end;
data &outds;
/* rename on entry */
set &libds(rename=(
%local i;
%do i=1 %to &nobs;
&&name&i=&&newname&i
%end;
));
%do i=1 %to &nobs;
length &&name&i $&&len&i;
&&name&i=left(put(&&newname&i,&&fmt&i));
drop &&newname&i;
%end;
if _error_ then call symputx('syscc',1012);
run;
%mend mp_ds2fmtds;

188
base/mp_filtercheck.sas Normal file
View File

@@ -0,0 +1,188 @@
/**
@file
@brief Checks an input filter table for validity
@details Performs checks on the input table to ensure it arrives in the
correct format. This is necessary to prevent code injection. Will update
SYSCC to 1008 if bad records are found, and call mp_abort.sas for a
graceful service exit (configurable).
Used for dynamic filtering in [Data Controller for SAS&reg;](https://datacontroller.io).
Usage:
%mp_filtercheck(work.filter,targetds=sashelp.class,outds=work.badrecords)
The input table should have the following format:
|GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$4000|
|---|---|---|---|---|---|
|AND|AND|1|AGE|=|12|
|AND|AND|1|SEX|<=|'M'|
|AND|OR|2|Name|NOT IN|('Jane','Alfred')|
|AND|OR|2|Weight|>=|7|
Rules applied:
@li GROUP_LOGIC - only AND/OR
@li SUBGROUP_LOGIC - only AND/OR
@li SUBGROUP_ID - only integers
@li VARIABLE_NM - must be in the target table
@li OPERATOR_NM - only =/>/</<=/>=/BETWEEN/IN/NOT IN/NE/CONTAINS
@li RAW_VALUE - no unquoted values except integers, commas and spaces.
@returns The &outds table containing any bad rows, plus a REASON_CD column.
@param [in] inds The table to be checked, with the format above
@param [in] targetds= The target dataset against which to verify VARIABLE_NM
@param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
@param [out] outds= The output table, which is a copy of the &inds. table
plus a REASON_CD column, containing only bad records. If bad records found,
the SYSCC value will be set to 1008 (general data problem). Downstream
processes should check this table (and return code) before continuing.
<h4> SAS Macros </h4>
@li mp_abort.sas
@li mf_getuniquefileref.sas
@li mf_getvarlist.sas
@li mf_getvartype.sas
@li mf_nobs.sas
@li mp_filtergenerate.sas
@li mp_filtervalidate.sas
<h4> Related Macros </h4>
@li mp_filtergenerate.sas
@li mp_filtervalidate.sas
@version 9.3
@author Allan Bowe
@todo Support date / hex / name literals and exponents in RAW_VALUE field
**/
%macro mp_filtercheck(inds,targetds=,outds=work.badrecords,abort=YES);
%mp_abort(iftrue= (&syscc ne 0)
,mac=&sysmacroname
,msg=%str(syscc=&syscc - on macro entry)
)
/* Validate input column */
%local vtype;
%let vtype=%mf_getvartype(&inds,RAW_VALUE);
%mp_abort(iftrue=(&abort=YES and &vtype ne C),
mac=&sysmacroname,
msg=%str(%str(ERR)OR: RAW_VALUE must be character)
)
%if &vtype ne C %then %do;
%put &sysmacroname: RAW_VALUE must be character;
%let syscc=42;
%return;
%end;
/**
* Sanitise the values based on valid value lists, then strip out
* quotes, commas, periods and spaces.
* Only numeric values should remain
*/
%local reason_cd;
data &outds;
/*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
OPERATOR_NM $10 RAW_VALUE $4000;*/
set &inds;
length reason_cd $32;
/* closed list checks */
if GROUP_LOGIC not in ('AND','OR') then do;
REASON_CD='GROUP_LOGIC should be either AND or OR';
putlog REASON_CD= GROUP_LOGIC=;
output;
end;
if SUBGROUP_LOGIC not in ('AND','OR') then do;
REASON_CD='SUBGROUP_LOGIC should be either AND or OR';
putlog REASON_CD= SUBGROUP_LOGIC=;
output;
end;
if mod(SUBGROUP_ID,1) ne 0 then do;
REASON_CD='SUBGROUP_ID should be integer';
putlog REASON_CD= SUBGROUP_ID=;
output;
end;
if upcase(VARIABLE_NM) not in
(%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
then do;
REASON_CD="VARIABLE_NM not in &targetds";
putlog REASON_CD= VARIABLE_NM=;
output;
end;
if OPERATOR_NM not in
('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
then do;
REASON_CD='Invalid OPERATOR_NM';
putlog REASON_CD= OPERATOR_NM=;
output;
end;
/* special logic */
if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ','');
else if OPERATOR_NM in ('IN','NOT IN') then do;
if substr(raw_value,1,1) ne '('
or substr(cats(reverse(raw_value)),1,1) ne ')'
then do;
REASON_CD='Missing brackets in RAW_VALUE';
putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
output;
end;
else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
end;
else raw_value1=raw_value;
/* remove nested literals eg '' */
raw_value1=tranwrd(raw_value1,"''",'');
/* now match string literals (always single quotes) */
raw_value2=raw_value1;
regex = prxparse("s/(\').*?(\')//");
call prxchange(regex,-1,raw_value2);
/* remove commas and periods*/
raw_value3=compress(raw_value2,',.');
/* output records that contain values other than digits and spaces */
if notdigit(compress(raw_value3,' '))>0 then do;
putlog raw_value3= $hex32.;
REASON_CD='Invalid RAW_VALUE';
putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
output;
end;
run;
data _null_;
set &outds;
call symputx('REASON_CD',reason_cd,'l');
stop;
run;
%mp_abort(iftrue=(&abort=YES and %mf_nobs(&outds)>0),
mac=&sysmacroname,
msg=%str(Filter issues in &inds, reason: &reason_cd, details in &outds)
)
%if %mf_nobs(&outds)>0 %then %do;
%let syscc=1008;
%return;
%end;
/**
* syntax checking passed but it does not mean the filter is valid
* for that we can run a proc sql validate query
*/
%local fref1;
%let fref1=%mf_getuniquefileref();
%mp_filtergenerate(&inds,outref=&fref1)
/* this macro will also set syscc to 1008 if any issues found */
%mp_filtervalidate(&fref1,&targetds,outds=&outds,abort=&abort)
%mend mp_filtercheck;

102
base/mp_filtergenerate.sas Normal file
View File

@@ -0,0 +1,102 @@
/**
@file
@brief Generates a filter clause from an input table, to a fileref
@details Uses the input table to generate an output filter clause.
This feature is used to create dynamic dropdowns in [Data Controller for SAS&reg](
https://datacontroller.io). The input table should be in the format below:
|GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$4000|
|---|---|---|---|---|---|
|AND|AND|1|AGE|=|12|
|AND|AND|1|SEX|<=|'M'|
|AND|OR|2|Name|NOT IN|('Jane','Alfred')|
|AND|OR|2|Weight|>=|7|
Note - if the above table is received from an external client, the values
should first be validated using the mp_filtercheck.sas macro to avoid risk
of SQL injection.
To generate the filter, run the following code:
data work.filtertable;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,AGE,=,12
AND,AND,1,SEX,<=,"'M'"
AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
AND,OR,2,Weight,>=,7
;;;;
run;
%mp_filtergenerate(work.filtertable,outref=myfilter)
data _null_;
infile myfilter;
input;
put _infile_;
run;
Will write the following query to the log:
> (
> AGE = 12
> AND
> SEX <= 'M'
> ) AND (
> Name NOT IN ('Jane','Alfred')
> OR
> Weight >= 7
> )
@param [in] inds The input table with query values
@param [out] outref= The output fileref to contain the filter clause. Will
be created (or replaced).
<h4> Related Macros </h4>
@li mp_filtercheck.sas
@li mp_filtervalidate.sas
<h4> SAS Macros </h4>
@li mp_abort.sas
@li mf_nobs.sas
@version 9.3
@author Allan Bowe
**/
%macro mp_filtergenerate(inds,outref=filter);
%mp_abort(iftrue= (&syscc ne 0)
,mac=&sysmacroname
,msg=%str(syscc=&syscc - on macro entry)
)
filename &outref temp;
%if %mf_nobs(&inds)=0 %then %do;
/* ensure we have a default filter */
data _null_;
file &outref;
put '1=1';
run;
%end;
%else %do;
data _null_;
file &outref lrecl=32800;
set &inds end=last;
by SUBGROUP_ID;
if _n_=1 then put '((';
else if first.SUBGROUP_ID then put +1 GROUP_LOGIC '(';
else put +2 SUBGROUP_LOGIC;
put +4 VARIABLE_NM OPERATOR_NM RAW_VALUE;
if last.SUBGROUP_ID then put ')'@;
if last then put ')';
run;
%end;
%mend;

104
base/mp_filtervalidate.sas Normal file
View File

@@ -0,0 +1,104 @@
/**
@file
@brief Checks a generated filter query for validity
@details Runs a generated filter in proc sql with the validate option.
Used in mp_filtercheck.sas in an fcmp container.
Built to support dynamic filtering in
[Data Controller for SAS&reg;](https://datacontroller.io).
Usage:
data work.filtertable;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,AGE,=,12
AND,AND,1,SEX,<=,"'M'"
AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
AND,OR,2,Weight,>=,7
;;;;
run;
%mp_filtergenerate(work.filtertable,outref=myfilter)
%mp_filtervalidate(myfilter,sashelp.class)
@returns The SYSCC value will be 1008 if there are validation issues.
@param [in] inref The input fileref to validate (generated by
mp_filtergenerate.sas)
@param [in] targetds The target dataset against which to verify the query
@param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
@param [out] outds= (work.mp_filtervalidate) Output dataset containing the
error / warning message, if one exists. If this table contains any rows,
there are problems!
<h4> SAS Macros </h4>
@li mf_getuniquefileref.sas
@li mf_nobs.sas
@li mp_abort.sas
<h4> Related Macros </h4>
@li mp_filtercheck.sas
@li mp_filtergenerate.sas
@version 9.3
@author Allan Bowe
**/
%macro mp_filtervalidate(inref,targetds,abort=YES,outds=work.mp_filtervalidate);
%mp_abort(iftrue= (&syscc ne 0 or &syserr ne 0)
,mac=&sysmacroname
,msg=%str(syscc=&syscc / syserr=&syserr - on macro entry)
)
%local fref1;
%let fref1=%mf_getuniquefileref();
data _null_;
file &fref1;
infile &inref end=eof;
if _n_=1 then do;
put "proc sql;";
put "validate select * from &targetds";
put "where " ;
end;
input;
put _infile_;
putlog _infile_;
if eof then put ";quit;";
run;
%inc &fref1;
data &outds;
if &sqlrc or &syscc or &syserr then do;
REASON_CD=coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
output;
end;
else stop;
run;
filename &fref1 clear;
%if %mf_nobs(&outds)>0 %then %do;
%if &abort=YES %then %do;
data _null_;
set &outds;
call symputx('REASON_CD',reason_cd,'l');
stop;
run;
%mp_abort(
mac=&sysmacroname,
msg=%str(Filter issues in &inref: %quote(&reason_cd))
)
%end;
%let syscc=1008;
%end;
%mend;

6
base/mp_getconstraints.sas
View File

@@ -14,7 +14,7 @@
constraint unq unique(tx_from, dd_type),
constraint nnn not null(DD_SHORTDESC)
);
%mp_getconstraints(lib=work,ds=example,outds=work.constraints)
@param lib= The target library (default=WORK)
@@ -49,8 +49,8 @@ create table &outds as
on a.TABLE_CATALOG=b.TABLE_CATALOG
and a.TABLE_NAME=b.TABLE_NAME
and a.constraint_name=b.constraint_name
where a.TABLE_CATALOG="&lib"
and b.TABLE_CATALOG="&lib"
where a.TABLE_CATALOG="&lib"
and b.TABLE_CATALOG="&lib"
%if "&ds" ne "" %then %do;
and a.TABLE_NAME="&ds"
and b.TABLE_NAME="&ds"

27
base/mp_getdbml.sas
View File

@@ -133,7 +133,9 @@ run;
if notnull='yes' then notnul=' not null';
if notnull='no' and missing(label) then put ' ' name typ;
else if notnull='yes' and missing(label) then put ' ' name typ '[' notnul ']';
else if notnull='yes' and missing(label) then do;
put ' ' name typ '[' notnul ']';
end;
else if notnull='no' then put ' ' name typ '[' lab ']';
else put ' ' name typ '[' notnul ',' lab ']';
@@ -166,7 +168,7 @@ run;
call symputx('constcheck',1);
end;
if last then call symputx('constraints_used',cats(upcase(constraints_used)));
if last then call symput('constraints_used',cats(upcase(constraints_used)));
length curds const col $39;
curds="&curds";
@@ -176,7 +178,8 @@ run;
proc append base=&pkds data=&syslast;run;
/* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
/* Create Unique Indexes, but only if they were not already defined within
the Constraints section. */
data _data_(keep=curds const col);
set &idxinfo (where=(
libname="%scan(&curds,1,.)"
@@ -187,7 +190,7 @@ run;
file &outref mod;
by idxusage indxname;
name=upcase(name);
if &constcheck=1 then stop; /* in fact we only care about PKs so stop if we have */
if &constcheck=1 then stop; /* we only care about PKs so stop if we have */
if _n_=1 and &constcheck=0 then put / ' indexes {';
length cols $5000;
@@ -217,8 +220,8 @@ run;
%end;
/**
* now we need to figure out the relationships
*/
* now we need to figure out the relationships
*/
/* sort alphabetically so we can have one set of unique cols per table */
proc sort data=&pkds nodupkey;
@@ -226,7 +229,7 @@ proc sort data=&pkds nodupkey;
run;
data &pkds.1 (keep=curds col)
&pkds.2 (keep=curds cols);
&pkds.2 (keep=curds cols);
set &pkds;
by curds const;
length retconst $39 cols $5000;
@@ -261,7 +264,11 @@ run;
line='Ref: "'!!"&curds"
!!cats('".(',"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))",')')
!!' - '
!!cats(quote(trim(curds)),'.(',"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))",')');
!!cats(quote(trim(curds))
,'.('
,"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))"
,')'
);
put line;
run;
@@ -282,7 +289,9 @@ run;
create table &pkds.5b as
select curds,count(*) as cnt
from &pkds.5a
where curds not in (select curds from &pkds.2 where cols="&pkcols") /* not a one to one match */
where curds not in (
select curds from &pkds.2 where cols="&pkcols"
) /* not a one to one match */
and curds ne "&curds" /* exclude self */
group by 1;
create table &pkds.6 as

47
base/mp_getddl.sas
View File

@@ -27,7 +27,7 @@
@param schema= Choose a preferred schema name (default is to use actual schema
,else libref)
@param applydttm= for non SAS DDL, choose if columns are created with native
datetime2 format or regular decimal type
datetime2 format or regular decimal type
@version 9.3
@author Allan Bowe
**/
@@ -86,7 +86,9 @@ create table _data_ as
%global constraints_used;
data _null_;
length ctype $11 constraint_name_orig $256 constraints_used $5000;
set &colconst (where=(table_name="&curds" and constraint_type in ('PRIMARY','UNIQUE'))) end=last;
set &colconst(
where=(table_name="&curds" and constraint_type in ('PRIMARY','UNIQUE'))
) end=last;
file &fref mod;
by constraint_type constraint_name;
retain constraints_used;
@@ -161,10 +163,19 @@ run;
put ');';
run;
/* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
/* Create Unique Indexes, but only if they were not already defined within
the Constraints section. */
data _null_;
*length ds $128;
set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
set &idxinfo(
where=(
memname="&curds"
and unique='yes'
and indxname not in (
%sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
)
)
);
file &fref mod;
by idxusage indxname;
/* ds=cats(libname,'.',memname); */
@@ -228,10 +239,19 @@ run;
/* Extra step for data constraints */
%addConst()
/* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
/* Create Unique Indexes, but only if they were not already defined within
the Constraints section. */
data _null_;
*length ds $128;
set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
set &idxinfo(
where=(
memname="&curds"
and unique='yes'
and indxname not in (
%sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
)
)
);
file &fref mod;
by idxusage indxname;
*ds=cats(libname,'.',memname);
@@ -320,15 +340,24 @@ run;
put ');';
run;
/* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
/* Create Unique Indexes, but only if they were not already defined within
the Constraints section. */
data _null_;
*length ds $128;
set &idxinfo (where=(memname="&curds" and unique='yes' and indxname not in (%sysfunc(tranwrd("&constraints_used",%str( ),%str(","))))));
set &idxinfo(
where=(
memname="&curds"
and unique='yes'
and indxname not in (
%sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
)
)
);
file &fref mod;
by idxusage indxname;
/* ds=cats(libname,'.',memname); */
if first.indxname then do;
put 'CREATE UNIQUE INDEX "' indxname +(-1) '" ' "ON &schema..&curds (" ;
put 'CREATE UNIQUE INDEX "' indxname +(-1) '" ' "ON &schema..&curds(";
put ' "' name +(-1) '"' ;
end;
else put ' ,"' name +(-1) '"';

13
base/mp_getmaxvarlengths.sas
View File

@@ -1,17 +1,18 @@
/**
@file mp_getmaxvarlengths.sas
@brief Scans a dataset to find the max length of the variable values
@details
@details
This macro will scan a base dataset and produce an output dataset with two
columns:
- NAME Name of the base dataset column
- MAXLEN Maximum length of the data contained therein.
Character fields may be allocated very large widths (eg 32000) of which the maximum
value is likely to be much narrower. This macro was designed to enable a HTML
table to be appropriately sized however this could be used as part of a data
audit to ensure we aren't over-sizing our tables in relation to the data therein.
Character fields may be allocated very large widths (eg 32000) of which the
maximum value is likely to be much narrower. This macro was designed to
enable a HTML table to be appropriately sized however this could be used as
part of a data audit to ensure we aren't over-sizing our tables in relation to
the data therein.
Numeric fields are converted using the relevant format to determine the width.
Usage:
@@ -33,7 +34,7 @@
%macro mp_getmaxvarlengths(
libds /* libref.dataset to analyse */
,outds=work.mp_getmaxvarlengths /* name of output dataset to create */
,outds=work.mp_getmaxvarlengths /* name of output dataset to create */
)/*/STORE SOURCE*/;
%local vars x var fmt;

75
base/mp_hashdataset.sas Normal file
View File

@@ -0,0 +1,75 @@
/**
@file
@brief Returns a unique hash for a dataset
@details Ignores metadata attributes, used only to hash values. Compared
datasets must be in the same order.
%mp_hashdataset(sashelp.class,outds=myhash)
data _null_;
set work.myhash;
put hashkey=;
run;
![sas md5 hash dataset log results](https://i.imgur.com/MqF98vk.png)
<h4> SAS Macros </h4>
@li mf_getattrn.sas
@li mf_getuniquename.sas
@li mf_getvarlist.sas
@li mf_getvartype.sas
@param [in] libds dataset to hash
@param [out] outds= (work.mf_hashdataset) The output dataset to create. This
will contain one column (hashkey) with one observation (a hex32.
representation of the input hash)
|hashkey:$32.|
|---|
|28ABC74ABFC45F50794237BA5566E6CA|
@version 9.2
@author Allan Bowe
**/
%macro mp_hashdataset(
libds,
outds=
)/*/STORE SOURCE*/;
%if %mf_getattrn(&libds,NLOBS)=0 %then %do;
%put %str(WARN)ING: Dataset &libds is empty;, or is not a dataset;
%end;
%else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
%put %str(ERR)OR: Dataset &libds is not a dataset;
%end;
%else %do;
%local keyvar /* roll up the md5 */
prevkeyvar /* retain prev record md5 */
lastvar /* last var in input ds */
varlist var i;
/* avoid naming conflict for hash key vars */
%let keyvar=%mf_getuniquename();
%let prevkeyvar=%mf_getuniquename();
%let lastvar=%mf_getuniquename();
%let varlist=%mf_getvarlist(&libds);
data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
length &prevkeyvar &keyvar $32;
retain &prevkeyvar;
set &libds end=&lastvar;
/* hash should include previous row */
&keyvar=put(md5(&prevkeyvar
/* loop every column, hashing every individual value */
%do i=1 %to %sysfunc(countw(&varlist));
%let var=%scan(&varlist,&i,%str( ));
%if %mf_getvartype(&libds,&var)=C %then %do;
!!put(md5(trim(&var)),$hex32.)
%end;
%else %do;
!!put(md5(trim(put(&var*1,binary64.))),$hex32.)
%end;
%end;
),$hex32.);
&prevkeyvar=&keyvar;
if &lastvar then output;
run;
%end;
%mend;

112
base/mp_jsonout.sas
View File

@@ -4,8 +4,11 @@
@details PROC JSON is faster but will produce errs like the ones below if
special chars are encountered.
>An object or array close is not valid at this point in the JSON text.
>Date value out of range
> ERROR: Some code points did not transcode.
> An object or array close is not valid at this point in the JSON text.
> Date value out of range
If this happens, try running with ENGINE=DATASTEP.
@@ -13,8 +16,10 @@
filename tmp temp;
data class; set sashelp.class;run;
%mp_jsonout(OPEN,jref=tmp)
%mp_jsonout(OBJ,class,jref=tmp)
%mp_jsonout(CLOSE,jref=tmp)
data _null_;
infile tmp;
@@ -22,27 +27,29 @@
run;
If you are building web apps with SAS then you are strongly encouraged to use
the mX_createwebservice macros in combination with the
the mX_createwebservice macros in combination with the
[sasjs adapter](https://github.com/sasjs/adapter).
For more information see https://sasjs.io
@param action Valid values:
* OPEN - opens the JSON
* OBJ - sends a table with each row as an object
* ARR - sends a table with each row in an array
* CLOSE - closes the JSON
@li OPEN - opens the JSON
@li OBJ - sends a table with each row as an object
@li ARR - sends a table with each row in an array
@li CLOSE - closes the JSON
@param ds the dataset to send. Must be a work table.
@param jref= the fileref to which to send the JSON
@param dslabel= the name to give the table in the exported JSON
@param fmt= Whether to keep or strip formats from the table
@param engine= Which engine to use to send the JSON, options are:
* PROCJSON (default)
* DATASTEP
@param engine= Which engine to use to send the JSON, valid options are:
@li PROCJSON (default)
@li DATASTEP (more reliable when data has non standard characters)
@param dbg= DEPRECATED - was used to conditionally add PRETTY to
proc json but this can cause line truncation in large files.
<h4> Related Macros <h4>
@li mp_ds2fmtds.sas
@version 9.2
@author Allan Bowe
@@ -50,10 +57,11 @@
**/
%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=PROCJSON,dbg=0
%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0
)/*/STORE SOURCE*/;
%put output location=&jref;
%if &action=OPEN %then %do;
OPTIONS NOBOMFILE;
data _null_;file &jref encoding='utf-8';
put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
run;
@@ -66,7 +74,7 @@
%if &engine=PROCJSON %then %do;
data;run;%let tempds=&syslast;
proc sql;drop table &tempds;
data &tempds /view=&tempds;set &ds;
data &tempds /view=&tempds;set &ds;
%if &fmt=N %then format _numeric_ best32.;;
proc json out=&jref pretty
%if &action=ARR %then nokeys ;
@@ -81,13 +89,72 @@
%put &sysmacroname: &ds NOT FOUND!!!;
%return;
%end;
data _null_;file &jref mod ;
%if &fmt=Y %then %do;
%put converting every variable to a formatted variable;
/* see mp_ds2fmtds.sas for source */
proc contents noprint data=&ds
out=_data_(keep=name type length format formatl formatd varnum);
run;
proc sort;
by varnum;
run;
%local fmtds;
%let fmtds=%scan(&syslast,2,.);
/* prepare formats and varnames */
data _null_;
set &fmtds end=last;
name=upcase(name);
/* fix formats */
if type=2 or type=6 then do;
length fmt $49.;
if format='' then fmt=cats('$',length,'.');
else if formatl=0 then fmt=cats(format,'.');
else fmt=cats(format,formatl,'.');
newlen=max(formatl,length);
end;
else do;
if format='' then fmt='best.';
else if formatl=0 then fmt=cats(format,'.');
else if formatd=0 then fmt=cats(format,formatl,'.');
else fmt=cats(format,formatl,'.',formatd);
/* needs to be wide, for datetimes etc */
newlen=max(length,formatl,24);
end;
/* 32 char unique name */
newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
call symputx(cats('name',_n_),name,'l');
call symputx(cats('newname',_n_),newname,'l');
call symputx(cats('len',_n_),newlen,'l');
call symputx(cats('fmt',_n_),fmt,'l');
call symputx(cats('type',_n_),type,'l');
if last then call symputx('nobs',_n_,'l');
run;
data &fmtds;
/* rename on entry */
set &ds(rename=(
%local i;
%do i=1 %to &nobs;
&&name&i=&&newname&i
%end;
));
%do i=1 %to &nobs;
length &&name&i $&&len&i;
&&name&i=left(put(&&newname&i,&&fmt&i));
drop &&newname&i;
%end;
if _error_ then call symputx('syscc',1012);
run;
%let ds=&fmtds;
%end; /* &fmt=Y */
data _null_;file &jref mod ;
put "["; call symputx('cols',0,'l');
proc sort data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
proc sort
data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
out=_data_;
by varnum;
data _null_;
data _null_;
set _last_ end=last;
call symputx(cats('name',_n_),name,'l');
call symputx(cats('type',_n_),type,'l');
@@ -121,8 +188,9 @@
)))))!!'"';
%end;
%end;
run;
/* write to temp loc to avoid _webout truncation - https://support.sas.com/kb/49/325.html */
run;
/* write to temp loc to avoid _webout truncation
- https://support.sas.com/kb/49/325.html */
filename _sjs temp lrecl=131068 encoding='utf-8';
data _null_; file _sjs lrecl=131068 encoding='utf-8' mod;
set &tempds;
@@ -131,7 +199,7 @@
%do i=1 %to &cols;
%if &i>1 %then "," ;
%if &action=OBJ %then """&&name&i"":" ;
&&name&i
&&name&i
%end;
%if &action=ARR %then "]" ; %else "}" ; ;
proc sql;
@@ -158,8 +226,8 @@
%end;
%else %if &action=CLOSE %then %do;
data _null_;file &jref encoding='utf-8';
data _null_;file &jref encoding='utf-8' mod;
put "}";
run;
%end;
%mend;
%mend mp_jsonout;

98
base/mp_mdtablewrite.sas Normal file
View File

@@ -0,0 +1,98 @@
/**
@file
@brief Create a Markdown Table from a dataset
@details A markdown table is a simple table representation for use in
documents written in markdown format.
An online generator is available here:
https://www.tablesgenerator.com/markdown_tables
This structure is also used by the Macro Core library for documenting input/
output datasets, as well as the sasjs/cli tool for documenting inputs/outputs
for web services.
We take the standard definition one step further by embedding the informat
in the table header row, like so:
|var1:$|var2:best.|var3:date9.|
|---|---|---|
|some text|42|01JAN1960|
|blah|1|31DEC1999|
Which resolves to:
|var1:$|var2:best.|var3:date9.|
|---|---|---|
|some text|42|01JAN1960|
|blah|1|31DEC1999|
Usage:
%mp_mdtablewrite(libds=sashelp.class,showlog=YES)
<h4> SAS Macros </h4>
@li mf_getvarlist.sas
@li mf_getvarformat.sas
@param [in] libds= the library / dataset to create or read from.
@param [out] fref= Fileref to contain the markdown. Default=mdtable.
@param [out] showlog= set to YES to show the markdown in the log. Default=NO.
@version 9.3
@author Allan Bowe
**/
%macro mp_mdtablewrite(
libds=,
fref=mdtable,
showlog=NO
)/*/STORE SOURCE*/;
/* check fileref is assigned */
%if %sysfunc(fileref(&fref)) > 0 %then %do;
filename &fref temp;
%end;
%local vars;
%let vars=%mf_getvarlist(&libds);
/* create the header row */
data _null_;
file &fref;
length line $32767;
put '|'
%local i var fmt;
%do i=1 %to %sysfunc(countw(&vars));
%let var=%scan(&vars,&i);
%let fmt=%mf_getvarformat(&libds,&var,force=1);
"&var:&fmt|"
%end;
;
put '|'
%do i=1 %to %sysfunc(countw(&vars));
"---|"
%end;
;
run;
/* write out the data */
data _null_;
file &fref mod dlm='|' lrecl=32767;
set &libds ;
length line $32767;
line=cats('|',%mf_getvarlist(&libds,dlm=%str(,'|',)),'|');
put line;
run;
%if %upcase(&showlog)=YES %then %do;
options ps=max;
data _null_;
infile &fref;
input;
putlog _infile_;
run;
%end;
%mend mp_mdtablewrite;

40
base/mp_prevobs.sas
View File

@@ -16,13 +16,13 @@
%mp_prevobs(INIT,history=2)
if _n_ =10 then do;
%* fetch previous but 1 record;
%mp_prevobs(FETCH,-2)
put _n_= name= age= calc_var=;
%mp_prevobs(FETCH,-2)
put _n_= name= age= calc_var=;
%* fetch previous record;
%mp_prevobs(FETCH,-1)
put _n_= name= age= calc_var=;
%mp_prevobs(FETCH,-1)
put _n_= name= age= calc_var=;
%* reinstate current record ;
%mp_prevobs(FETCH,0)
%mp_prevobs(FETCH,0)
put _n_= name= age= calc_var=;
end;
run;
@@ -35,11 +35,11 @@
https://www.lexjansen.com/pharmasug/2008/cc/CC08.pdf
@param action Either FETCH a current or previous record, or INITialise.
@param record The relative (to current) position of the previous observation
to return.
@param record The relative (to current) position of the previous observation
to return.
@param history= The number of records to retain in the hash table. Default=5
@param prefix= the prefix to give to the variables used to store the hash name
and index. Default=mp_prevobs
and index. Default=mp_prevobs
@version 9.2
@author Allan Bowe
@@ -53,33 +53,33 @@
%let record=%eval((&record+0) * -1);
%if &action=INIT %then %do;
if _n_ eq 1 then do;
attrib &prefix._VAR length=$64;
if _n_ eq 1 then do;
attrib &prefix._VAR length=$64;
dcl hash &prefix._HASH(ordered:'Y');
&prefix._KEY=0;
&prefix._HASH.defineKey("&prefix._KEY");
do while(1);
call vnext(&prefix._VAR);
&prefix._HASH.defineKey("&prefix._KEY");
do while(1);
call vnext(&prefix._VAR);
if &prefix._VAR='' then leave;
if &prefix._VAR eq "&prefix._VAR" then continue;
else if &prefix._VAR eq "&prefix._KEY" then continue;
if &prefix._VAR eq "&prefix._VAR" then continue;
else if &prefix._VAR eq "&prefix._KEY" then continue;
&prefix._HASH.defineData(&prefix._VAR);
end;
&prefix._HASH.defineDone();
end;
&prefix._HASH.defineDone();
end;
/* this part has to happen before FETCHing */
&prefix._KEY+1;
&prefix._rc=&prefix._HASH.add();
if &prefix._rc then putlog 'adding' &prefix._rc=;
%if &history>0 %then %do;
if &prefix._key>&history+1 then
if &prefix._key>&history+1 then
&prefix._HASH.remove(key: &prefix._KEY - &history - 1);
if &prefix._rc then putlog 'removing' &prefix._rc=;
%end;
%end;
%else %if &action=FETCH %then %do;
if &record > &prefix._key then putlog "Not enough records in &Prefix._hash yet";
if &record>&prefix._key then putlog "Not enough records in &Prefix._hash yet";
else &prefix._rc=&prefix._HASH.find(key: &prefix._KEY - &record);
if &prefix._rc then putlog &prefix._rc= " when fetching " &prefix._KEY=
"with record &record and " _n_=;

6
base/mp_recursivejoin.sas
View File

@@ -32,9 +32,9 @@
@returns outds contains the following variables:
- level (0 = top level)
- &parentvar
- &childvar (null if none found)
- level (0 = top level)
- &parentvar
- &childvar (null if none found)
@version 9.2
@author Allan Bowe

3
base/mp_resetoption.sas
View File

@@ -1,7 +1,8 @@
/**
@file
@brief Reset an option to original value
@details Inspired by the SAS Jedi - https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options/
@details Inspired by the SAS Jedi -
https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options
Called as follows:
options obs=30;

12
base/mp_runddl.sas
View File

@@ -8,14 +8,14 @@
rootlib
|-- LIBREF1
| |__ mytable.ddl
| |__ someothertable.ddl
| |__ mytable.ddl
| |__ someothertable.ddl
|-- LIBREF2
| |__ table1.ddl
| |__ table2.ddl
| |__ table1.ddl
| |__ table2.ddl
|-- LIBREF3
|__ table3.ddl
|__ table4.ddl
|__ table3.ddl
|__ table4.ddl
Only files with the .ddl suffix are executed. The parent folder name is used
as the libref.

2
base/mp_searchcols.sas
View File

@@ -32,7 +32,7 @@ create table _data_ as
where upcase(libname) in ("IMPOSSIBLE",
%local x;
%do x=1 %to %sysfunc(countw(&libs));
"%upcase(%scan(&libs,&x))"
"%upcase(%scan(&libs,&x))"
%end;
)
%end;

23
base/mp_searchdata.sas
View File

@@ -3,7 +3,7 @@
@brief Searches all data in a library
@details
Scans an entire library and creates a copy of any table
containing a specific string OR numeric value. Only
containing a specific string OR numeric value. Only
matching records are written out.
If both a string and numval are provided, the string
will take precedence.
@@ -20,9 +20,10 @@
@param ds= the dataset to search (leave blank to search entire library)
@param string= the string value to search
@param numval= the numeric value to search (must be exact)
@param outloc= the directory in which to create the output datasets with matching
rows. Will default to a subfolder in the WORK library.
@param outobs= set to a positive integer to restrict the number of observations
@param outloc= the directory in which to create the output datasets with
matching rows. Will default to a subfolder in the WORK library.
@param outobs= set to a positive integer to restrict the number of
observations
@param filter_text= add a (valid) filter clause to further filter the results
<h4> SAS Macros </h4>
@@ -36,7 +37,7 @@
**/
%macro mp_searchdata(lib=sashelp
,ds=
,ds=
,string= /* the query will use a contains (?) operator */
,numval= /* numeric must match exactly */
,outloc=%sysfunc(pathname(work))/mpsearch
@@ -44,7 +45,8 @@
,filter_text=%str(1=1)
)/*/STORE SOURCE*/;
%local table_list table table_num table colnum col start_tm check_tm vars type coltype;
%local table_list table table_num table colnum col start_tm check_tm vars type
coltype;
%put process began at %sysfunc(datetime(),datetime19.);
%if &syscc ge 4 %then %do;
@@ -61,14 +63,14 @@ libname mpsearch "&outloc";
/* get the list of tables in the library */
proc sql noprint;
select distinct memname into: table_list separated by ' '
from dictionary.tables
from dictionary.tables
where upcase(libname)="%upcase(&lib)"
%if &ds ne %then %do;
and upcase(memname)=%upcase("&ds")
%end;
;
/* check that we have something to check */
proc sql
proc sql
%if &outobs>-1 %then %do;
outobs=&outobs
%end;
@@ -85,7 +87,7 @@ proc sql
%let check_tm=%sysfunc(datetime());
/* build sql statement */
create table mpsearch.&table as select * from &lib..&table
where %unquote(&filter_text) and
where %unquote(&filter_text) and
(0
/* loop through columns */
%do colnum=1 %to %sysfunc(countw(&vars,%str( )));
@@ -101,7 +103,8 @@ proc sql
%end;
%end;
);
%put Search query for &table took %sysevalf(%sysfunc(datetime())-&check_tm) seconds;
%put Search query for &table took
%sysevalf(%sysfunc(datetime())-&check_tm) seconds;
%if &sqlrc ne 0 %then %do;
%put %str(WAR)NING: SQLRC=&sqlrc when processing &table;
%return;

2
base/mp_setkeyvalue.sas
View File

@@ -12,7 +12,7 @@
@param key Provide a key on which to perform the lookup
@param value Provide a value
@param type= either C or N will populate valc and valn respectively. C is
default.
default.
@param libds= define the target table to hold the parameters
@version 9.2

5
base/mp_stprequests.sas
View File

@@ -1,12 +1,13 @@
/**
@file
@brief Capture session start / finish times and request details
@details For details, see http://www.rawsas.com/2015/09/logging-of-stored-process-server.html.
@details For details, see
https://rawsas.com/event-logging-of-stored-process-server-sessions.
Requires a base table in the following structure (name can be changed):
proc sql;
create table &libds(
request_dttm num not null format=datetime.
request_dttm num not null format=datetime.
,status_cd char(4) not null
,_metaperson varchar(100) not null
,_program varchar(500)

37
base/mp_streamfile.sas
View File

@@ -6,7 +6,8 @@
Usage:
filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
filename mc url
"https://raw.githubusercontent.com/sasjs/core/main/all.sas";
%inc mc;
%mp_streamfile(contenttype=csv,inloc=/some/where.txt,outname=myfile.txt)
@@ -35,8 +36,20 @@
%let contentype=%upcase(&contenttype);
%local platform; %let platform=%mf_getplatform();
/**
* check engine type to avoid the below err message:
* > Function is only valid for filerefs using the CACHE access method.
*/
%local streamweb;
%let streamweb=0;
data _null_;
set sashelp.vextfl(where=(upcase(fileref)="_WEBOUT"));
if xengine='STREAM' then call symputx('streamweb',1,'l');
run;
%if &contentype=ZIP %then %do;
%if &platform=SASMETA %then %do;
%if &platform=SASMETA and &streamweb=1 %then %do;
data _null_;
rc=stpsrv_header('Content-type','application/zip');
rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -50,7 +63,7 @@
%end;
%else %if &contentype=EXCEL %then %do;
/* suitable for XLS format */
%if &platform=SASMETA %then %do;
%if &platform=SASMETA and &streamweb=1 %then %do;
data _null_;
rc=stpsrv_header('Content-type','application/vnd.ms-excel');
rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -63,20 +76,22 @@
%end;
%end;
%else %if &contentype=XLSX %then %do;
%if &platform=SASMETA %then %do;
%if &platform=SASMETA and &streamweb=1 %then %do;
data _null_;
rc=stpsrv_header('Content-type','application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
rc=stpsrv_header('Content-type',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
run;
%end;
%else %if &platform=SASVIYA %then %do;
filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
contenttype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
contenttype=
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
contentdisp="attachment; filename=&outname";
%end;
%end;
%else %if &contentype=TEXT %then %do;
%if &platform=SASMETA %then %do;
%if &platform=SASMETA and &streamweb=1 %then %do;
data _null_;
rc=stpsrv_header('Content-type','application/text');
rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -89,7 +104,7 @@
%end;
%end;
%else %if &contentype=CSV %then %do;
%if &platform=SASMETA %then %do;
%if &platform=SASMETA and &streamweb=1 %then %do;
data _null_;
rc=stpsrv_header('Content-type','application/csv');
rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -113,10 +128,10 @@
%end;
%if &inref ne 0 %then %do;
%mp_binarycopy(inref=&inref,outref=_webout)
%mp_binarycopy(inref=&inref,outref=_webout)
%end;
%else %do;
%mp_binarycopy(inloc="&inloc",outref=_webout)
%mp_binarycopy(inloc="&inloc",outref=_webout)
%end;
%mend;
%mend;

2
base/mp_testjob.sas
View File

@@ -5,7 +5,7 @@
testing of arbitrary jobs.
%mp_testjob(
duration=60*5
duration=60*5
)
@param [in] duration= the time in seconds which the job should run for. Actual

259
base/mp_testservice.sas Normal file
View File

@@ -0,0 +1,259 @@
/**
@file mp_testservice.sas
@brief Will execute a test against a SASjs web service on SAS 9 or Viya
@details Prepares the input files and retrieves the resulting datasets from
the response JSON.
%mp_testjob(
duration=60*5
)
Note - the _webout fileref should NOT be assigned prior to running this macro.
@param [in] program The _PROGRAM endpoint to test
@param [in] inputfiles=(0) A list of space seperated fileref:filename pairs as
follows:
inputfiles=inref:filename inref2:filename2
@param [in] inputparams=(0) A dataset containing name/value pairs in the
following format:
|name:$32|value:$1000|
|---|---|
|stpmacname|some value|
|mustbevalidname|can be anything, oops, %abort!!|
@param [in] debug= (log) Provide the _debug value
@param [in] viyaresult=(WEBOUT_JSON) The Viya result type to return. For
more info, see mv_getjobresult.sas
@param [out] outlib= (0) Output libref to contain the final tables. Set to
0 if the service output is not in JSON format.
@param [out] outref= (0) Output fileref to create, to contain the full _webout
response.
<h4> SAS Macros </h4>
@li mf_getplatform.sas
@li mf_getuniquefileref.sas
@li mf_getuniquename.sas
@li mp_abort.sas
@li mp_binarycopy.sas
@li mv_getjobresult.sas
@li mv_jobflow.sas
@version 9.4
@author Allan Bowe
**/
%macro mp_testservice(program,
inputfiles=0,
inputparams=0,
debug=log,
outlib=0,
outref=0,
viyaresult=WEBOUT_JSON
)/*/STORE SOURCE*/;
%local mdebug;
%if &debug ne 0 %then %do;
%let mdebug=1;
%put &sysmacroname entry vars:;
%put _local_;
%end;
%else %let mdebug=0;
/* sanitise inputparams */
%local pcnt;
%let pcnt=0;
%if &inputparams ne 0 %then %do;
data _null_;
set &inputparams;
if not nvalid(name,'v7') then putlog (_all_)(=);
else if name in (
'program','inputfiles','inputparams','debug','outlib','outref'
) then putlog (_all_)(=);
else do;
x+1;
call symputx(name,quote(cats(value)),'l');
call symputx('pval'!!left(x),name,'l');
call symputx('pcnt',x,'l');
end;
run;
%mp_abort(iftrue= (%mf_nobs(&inputparams) ne &pcnt)
,mac=&sysmacroname
,msg=%str(Invalid values in &inputparams)
)
%end;
%local fref1 webref;
%let fref1=%mf_getuniquefileref();
%let webref=%mf_getuniquefileref();
%local platform;
%let platform=%mf_getplatform();
%if &platform=SASMETA %then %do;
/* parse the input files */
%local webcount i var;
%if %quote(&inputfiles) ne 0 %then %do;
%let webcount=%sysfunc(countw(&inputfiles));
%put &=webcount;
%do i=1 %to &webcount;
%let var=%scan(&inputfiles,&i,%str( ));
%local webfref&i webname&i;
%let webref&i=%scan(&var,1,%str(:));
%let webname&i=%scan(&var,2,%str(:));
%put webref&i=&&webref&i;
%put webname&i=&&webname&i;
%end;
%end;
%else %let webcount=0;
proc stp program="&program";
inputparam _program="&program"
%do i=1 %to &webcount;
%if &webcount=1 %then %do;
_webin_fileref="&&webref&i"
_webin_name="&&webname&i"
%end;
%else %do;
_webin_fileref&i="&&webref&i"
_webin_name&i="&&webname&i"
%end;
%end;
_webin_file_count="&webcount"
_debug="&debug"
%do i=1 %to &pcnt;
/* resolve name only, proc stp fetches value */
&&pval&i=&&&&&&pval&i
%end;
;
%do i=1 %to &webcount;
inputfile &&webref&i;
%end;
outputfile _webout=&webref;
run;
data _null_;
infile &webref;
file &fref1;
input;
length line $10000;
if index(_infile_,'>>weboutBEGIN<<') then do;
line=tranwrd(_infile_,'>>weboutBEGIN<<','');
put line;
end;
else if index(_infile_,'>>weboutEND<<') then do;
line=tranwrd(_infile_,'>>weboutEND<<','');
put line;
stop;
end;
else put _infile_;
run;
data _null_;
infile &fref1;
input;
put _infile_;
run;
%if &outlib ne 0 %then %do;
libname &outlib json (&fref1);
%end;
%if &outref ne 0 %then %do;
filename &outref temp;
%mp_binarycopy(inref=&webref,outref=&outref)
%end;
%end;
%else %if &platform=SASVIYA %then %do;
/* prepare inputparams */
%local ds1;
%let ds1=%mf_getuniquename();
%if "&inputparams" ne "0" %then %do;
proc transpose data=&inputparams out=&ds1;
id name;
var value;
run;
%end;
%else %do;
data &ds1;run;
%end;
/* parse the input files - convert to sasjs params */
%local webcount i var sasjs_tables;
%if %quote(&inputfiles) ne 0 %then %do;
%let webcount=%sysfunc(countw(&inputfiles));
%put &=webcount;
%do i=1 %to &webcount;
%let var=%scan(&inputfiles,&i,%str( ));
%local webfref&i webname&i sasjs&i.data;
%let webref&i=%scan(&var,1,%str(:));
%let webname&i=%scan(&var,2,%str(:));
%put webref&i=&&webref&i;
%put webname&i=&&webname&i;
%let sasjs_tables=&sasjs_tables &&webname&i;
data _null_;
infile &&webref&i lrecl=32767;
input;
if _n_=1 then call symputx("sasjs&i.data",_infile_);
else call symputx(
"sasjs&i.data",cats(symget("sasjs&i.data"),'0D0A'x,_infile_)
);
putlog "&sysmacroname infile: " _infile_;
run;
data &ds1;
set &ds1;
length sasjs&i.data $32767 sasjs_tables $1000;
sasjs&i.data=symget("sasjs&i.data");
sasjs_tables=symget("sasjs_tables");
run;
%end;
%end;
%else %let webcount=0;
data &ds1;
retain _program "&program";
set &ds1;
putlog "&sysmacroname inputparams:";
putlog (_all_)(=);
run;
%mv_jobflow(inds=&ds1
,maxconcurrency=1
,outds=work.results
,outref=&fref1
,mdebug=&mdebug
)
/* show the log */
data _null_;
infile &fref1;
input;
putlog _infile_;
run;
/* get the uri to fetch results */
data _null_;
set work.results;
call symputx('uri',uri);
putlog "&sysmacroname: fetching results for " uri;
run;
/* fetch results from webout.json */
%mv_getjobresult(uri=&uri,
result=&viyaresult,
outref=&outref,
outlib=&outlib,
mdebug=&mdebug
)
%end;
%else %do;
%put %str(ERR)OR: Unrecognised platform: &platform;
%end;
%if &mdebug=0 %then %do;
filename &webref clear;
%end;
%else %do;
%put &sysmacroname exit vars:;
%put _local_;
%end;
%mend mp_testservice;

4
base/mp_testwritespeedlibrary.sas
View File

@@ -1,10 +1,10 @@
/**
@file mp_testwritespeedlibrary.sas
@brief Tests the write speed of a new table in a SAS library
@details Will create a new table of a certain size in an
@details Will create a new table of a certain size in an
existing SAS library. The table will have one column,
and will be subsequently deleted.
%mp_testwritespeedlibrary(
lib=work
,size=0.5

21
base/mp_tree.sas
View File

@@ -8,8 +8,11 @@
Credits:
* Roger Deangelis, https://communities.sas.com/t5/SAS-Programming/listing-all-files-within-a-directory-and-subdirectories/m-p/332616/highlight/true#M74887
* Tom, https://communities.sas.com/t5/SAS-Programming/listing-all-files-of-all-types-from-all-subdirectories/m-p/334113/highlight/true#M75419
Roger Deangelis:
https://communities.sas.com/t5/SAS-Programming/listing-all-files-within-a-directory-and-subdirectories/m-p/332616/highlight/true#M74887
Tom:
https://communities.sas.com/t5/SAS-Programming/listing-all-files-of-all-types-from-all-subdirectories/m-p/334113/highlight/true#M75419
@param dir= Directory to be scanned (default=/tmp)
@@ -17,13 +20,13 @@
@returns outds contains the following variables:
- `dir`: a flag (1/0) to say whether it is a directory or not. This is not
reliable - folders that you do not have permission to open will be flagged
as directories.
- `ext`: file extension
- `filename`: file name
- `dirname`: directory name
- `fullpath`: directory + file name
- `dir`: a flag (1/0) to say whether it is a directory or not. This is not
reliable - folders that you do not have permission to open will be flagged
as directories.
- `ext`: file extension
- `filename`: file name
- `dirname`: directory name
- `fullpath`: directory + file name
@version 9.2
**/

66
base/mp_validatecol.sas Normal file
View File

@@ -0,0 +1,66 @@
/**
@file
@brief Used to validate variables in a dataset
@details Useful when sanitising inputs, to ensure that they arrive with a
certain pattern.
Usage:
data test;
infile datalines4 dsd;
input;
libds=_infile_;
%mp_validatecol(libds,LIBDS,is_libds)
datalines4;
some.libname
!lib.blah
%abort
definite.ok
not.ok!
nineletrs._
;;;;
run;
@param [in] incol The column to be validated
@param [in] rule The rule to apply. Current rules:
@li ISNUM - checks if the variable is numeric
@li LIBDS - matches LIBREF.DATASET format
@param [out] outcol The variable to create, with the results of the match
<h4> SAS Macros </h4>
@li mf_getuniquename.sas
@version 9.3
**/
%macro mp_validatecol(incol,rule,outcol);
/* tempcol is given a unique name with every invocation */
%local tempcol;
%let tempcol=%mf_getuniquename();
%if &rule=ISNUM %then %do;
/*
credit SØREN LASSEN
https://sasmacro.blogspot.com/2009/06/welcome-isnum-macro.html
*/
&tempcol=input(&incol,?? best32.);
if missing(&tempcol) then &outcol=0;
else &outcol=1;
drop &tempcol;
%end;
%else %if &rule=LIBDS %then %do;
/* match libref.dataset */
if _n_=1 then do;
retain &tempcol;
&tempcol=prxparse('/^[_a-z]\w{0,7}\.[_a-z]\w{0,31}$/i');
if missing(&tempcol) then do;
putlog "%str(ERR)OR: Invalid expression for LIBDS";
stop;
end;
drop &tempcol;
end;
if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
else &outcol=0;
%end;
%mend mp_validatecol;

8
base/mp_zip.sas
View File

@@ -2,11 +2,11 @@
@file
@brief Creates a zip file
@details For DIRECTORY usage, will ignore subfolders. For DATASET usage,
provide a column that contains the full file path to each file to be zipped.
provide a column that contains the full file path to each file to be zipped.
%mp_zip(in=myzips,type=directory,outname=myDir)
%mp_zip(in=/my/file/path.txt,type=FILE,outname=myFile)
%mp_zip(in=SOMEDS,incol=FPATH,type=DATASET,outname=myFile)
%mp_zip(in=myzips,type=directory,outname=myDir)
%mp_zip(in=/my/file/path.txt,type=FILE,outname=myFile)
%mp_zip(in=SOMEDS,incol=FPATH,type=DATASET,outname=myFile)
If you are sending zipped output to the _webout destination as part of an STP
be sure that _debug is not set (else the SPWA will send non zipped content