sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2026-01-10 10:50:04 +00:00
Code Issues Projects Releases Wiki Activity

fix: refreshed the testing toolkit, added debug options, updated documentation, included one new test (mv_getjobcode.sas)

Browse Source
This commit is contained in:
Allan Bowe
2021-05-08 23:27:55 +03:00
parent 298acc4e50
commit 4e564b5409
12 changed files with 691 additions and 248 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
viya/mv_createwebservice.sas
View File

@@ -39,23 +39,26 @@
@li mf_isblank.sas
@li mv_deletejes.sas
@param path= The full path (on SAS Drive) where the service will be created
@param name= The name of the service
@param desc= The description of the service
@param precode= Space separated list of filerefs, pointing to the code that
needs to be attached to the beginning of the service
@param code= Fileref(s) of the actual code to be added
@param access_token_var= The global macro variable to contain the access token
@param grant_type= valid values are "password" or "authorization_code"
@param [in] path= The full path (on SAS Drive) where the service will be
created
@param [in] name= The name of the service
@param [in] desc= The description of the service
@param [in] precode= Space separated list of filerefs, pointing to the code
that needs to be attached to the beginning of the service
@param [in] code= Fileref(s) of the actual code to be added
@param [in] access_token_var= The global macro variable to contain the access
token
@param [in] grant_type= valid values are "password" or "authorization_code"
(unquoted). The default is authorization_code.
@param replace= select NO to avoid replacing any existing service in that
location
@param adapter= the macro uses the sasjs adapter by default. To use another
adapter, add a (different) fileref here.
@param contextname= Choose a specific context on which to run the Job. Leave
@param [in] replace=(YES) Select NO to avoid replacing any existing service in
that location
@param [in] adapter= the macro uses the sasjs adapter by default. To use
another adapter, add a (different) fileref here.
@param [in] contextname= Choose a specific context on which to run the Job. Leave
blank to use the default context. From Viya 3.5 it is possible to configure
a shared context - see
https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5pyhn1wg3ja0drdl6h.htm&docsetVersion=3.5&locale=en
@param [in] mdebug=(0) set to 1 to enable DEBUG messages
@version VIYA V.03.04
@author Allan Bowe, source: https://github.com/sasjs/core
@@ -71,9 +74,17 @@ https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5p
,grant_type=sas_services
,replace=YES
,adapter=sasjs
,debug=0
,mdebug=0
,contextname=
,debug=0 /* @TODO - Deprecate */
);
%local dbg;
%if &mdebug=1 %then %do;
%put &sysmacroname entry vars:;
%put _local_;
%end;
%else %let dbg=*;
%local oauth_bearer;
%if &grant_type=detect %then %do;
%if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -126,7 +137,7 @@ proc http method='GET' out=&fname1 &oauth_bearer
headers "Authorization"="Bearer &&&access_token_var";
%end;
run;
%if &debug %then %do;
%if &mdebug=1 %then %do;
data _null_;
infile &fname1;
input;
@@ -165,7 +176,7 @@ proc http method='GET'
%end;
'Accept'='application/vnd.sas.collection+json'
'Accept-Language'='string';
%if &debug=1 %then %do;
%if &mdebug=1 %then %do;
debug level = 3;
%end;
run;
@@ -220,9 +231,9 @@ run;
* These put statements are auto generated - to change the macro, change the
* source (mv_webout) and run `build.py`
*/
filename sasjs temp lrecl=3000;
filename &adapter temp lrecl=3000;
data _null_;
file sasjs;
file &adapter;
put "/* Created on %sysfunc(datetime(),datetime19.) by &sysuserid */";
/* WEBOUT BEGIN */
put ' ';
@@ -561,11 +572,12 @@ data _null_;
run;
/* insert the code, escaping double quotes and carriage returns */
%&dbg.put &sysmacroname: Creating final input file;
%local x fref freflist;
%let freflist= &adapter &precode &code ;
%do x=1 %to %sysfunc(countw(&freflist));
%let fref=%scan(&freflist,&x);
%put &sysmacroname: adding &fref;
%&dbg.put &sysmacroname: adding &fref fileref;
data _null_;
length filein 8 fileid 8;
filein = fopen("&fref","I",1,"B");
@@ -617,7 +629,12 @@ data _null_;
put '"}';
run;
/* now we can create the job!! */
%if &mdebug=1 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then %do;
%put &sysmacroname: input about to be POSTed;
data _null_;infile &fname3;input;putlog _infile_;run;
%end;
%&dbg.put &sysmacroname: Creating the actual service!;
%local fname4;
%let fname4=%mf_getuniquefileref();
proc http method='POST'
@@ -630,22 +647,18 @@ proc http method='POST'
"Authorization"="Bearer &&&access_token_var"
%end;
"Accept"="application/vnd.sas.job.definition+json";
%if &debug=1 %then %do;
%if &mdebug=1 %then %do;
debug level = 3;
%end;
run;
/*data _null_;infile &fname4;input;putlog _infile_;run;*/
%if &mdebug=1 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then %do;
%put &sysmacroname: output from POSTing job definition;
data _null_;infile &fname4;input;putlog _infile_;run;
%end;
%mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 201)
,mac=&sysmacroname
,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
)
/* clear refs */
filename &fname1 clear;
filename &fname2 clear;
filename &fname3 clear;
filename &fname4 clear;
filename &adapter clear;
libname &libref1 clear;
/* get the url so we can give a helpful log message */
%local url;
@@ -660,6 +673,19 @@ data _null_;
call symputx('url',url);
run;
%if &mdebug=1 %then %do;
%put &sysmacroname exit vars:;
%put _local_;
%end;
%else %do;
/* clear refs */
filename &fname1 clear;
filename &fname2 clear;
filename &fname3 clear;
filename &fname4 clear;
filename &adapter clear;
libname &libref1 clear;
%end;
%put &sysmacroname: Job &name successfully created in &path;
%put &sysmacroname:;
@@ -669,4 +695,4 @@ run;
%put &sysmacroname:;
%put &sysmacroname:;
%mend;
%mend mv_createwebservice;

57
viya/mv_getjobcode.sas
View File

@@ -10,17 +10,20 @@
,outfile=/tmp/some_job.sas
)
@param [in] access_token_var= The global macro variable to contain the access token
@param [in] access_token_var= The global macro variable to contain the access
token
@param [in] grant_type= valid values:
* password
* authorization_code
* detect - will check if access_token exists, if not will use sas_services if
a SASStudioV session else authorization_code. Default option.
* sas_services - will use oauth_bearer=sas_services
@li password
@liauthorization_code
@li detect - will check if access_token exists, if not will use sas_services
if a SASStudioV session else authorization_code. Default option.
@li sas_services - will use oauth_bearer=sas_services
@param [in] path= The SAS Drive path of the job
@param [in] name= The name of the job
@param [out] outref= A fileref to which to write the source code
@param [out] outfile= A file to which to write the source code
@param [in] mdebug=(0) set to 1 to enable DEBUG messages
@param [out] outref=(0) A fileref to which to write the source code (will be
created with a TEMP engine)
@param [out] outfile=(0) A file to which to write the source code
@version VIYA V.03.04
@author Allan Bowe, source: https://github.com/sasjs/core
@@ -39,7 +42,15 @@
,contextName=SAS Job Execution compute context
,access_token_var=ACCESS_TOKEN
,grant_type=sas_services
,mdebug=0
);
%local dbg;
%if &mdebug=1 %then %do;
%put &sysmacroname entry vars:;
%put _local_;
%end;
%else %let dbg=*;
%local oauth_bearer;
%if &grant_type=detect %then %do;
%if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -133,18 +144,30 @@ data _null_;
run;
%inc "&fpath3..lua";
/* export to desired destination */
data _null_;
%if &outref=0 %then %do;
%if "&outref"="0" %then %do;
data _null_;
file "&outfile" lrecl=32767;
%end;
%else %do;
%end;
%else %do;
filename &outref temp;
data _null_;
file &outref;
%end;
%end;
infile &fname2;
input;
put _infile_;
&dbg. putlog _infile_;
run;
filename &fname1 clear;
filename &fname2 clear;
filename &fname3 clear;
%mend;
%if &mdebug=1 %then %do;
%put &sysmacroname exit vars:;
%put _local_;
%end;
%else %do;
/* clear refs */
filename &fname1 clear;
filename &fname2 clear;
filename &fname3 clear;
%end;
%mend mv_getjobcode;

10
viya/mv_getjoblog.sas
View File

@@ -86,6 +86,13 @@
,grant_type=sas_services
,mdebug=0
);
%local dbg;
%if &mdebug=1 %then %do;
%put &sysmacroname entry vars:;
%put _local_;
%end;
%else %let dbg=*;
%local oauth_bearer;
%if &grant_type=detect %then %do;
%if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -261,9 +268,10 @@ run;
filename &fname3 clear;
%end;
%else %do;
%put &sysmacroname exit vars:;
%put _local_;
%end;
%mend;
%mend mv_getjoblog;

31
viya/mv_getjobresult.sas
View File

@@ -69,7 +69,9 @@
@param [out] result= (WEBOUT_JSON) The result type to capture. Resolves
to "_[column name]" from the results table when parsed with the JSON libname
engine.
engine. Example values:
@li WEBOUT_JSON
@li WEBOUT_TXT
@param [out] outref= (0) The output fileref to which to write the results
@param [out] outlib= (0) The output library to which to assign the results
@@ -96,6 +98,13 @@
,outref=0
,outlib=0
);
%local dbg;
%if &mdebug=1 %then %do;
%put &sysmacroname entry vars:;
%put _local_;
%end;
%else %let dbg=*;
%local oauth_bearer;
%if &grant_type=detect %then %do;
%if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -161,6 +170,13 @@ run;
,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
)
%end;
%if &mdebug=1 %then %do;
data _null_;
infile &fname1 lrecl=32767;
input;
putlog _infile_;
run;
%end;
/* extract results link */
%local lib1 resuri;
@@ -169,7 +185,7 @@ libname &lib1 JSON fileref=&fname1;
data _null_;
set &lib1..results;
call symputx('resuri',_&result,'l');
putlog (_all_)(=);
&dbg putlog "&sysmacroname results: " (_all_)(=);
run;
%mp_abort(iftrue=("&resuri"=".")
,mac=&sysmacroname
@@ -187,6 +203,13 @@ proc http method='GET' out=&fname2 &oauth_bearer
%end;
;
run;
%if &mdebug=1 %then %do;
data _null_;
infile &fname2 lrecl=32767;
input;
putlog _infile_;
run;
%end;
%if &outref ne 0 %then %do;
filename &outref temp;
@@ -202,6 +225,8 @@ run;
libname &lib1 clear;
%end;
%else %do;
%put &sysmacroname exit vars:;
%put _local_;
%end;
%mend;
%mend mv_getjobresult;

48
viya/mv_jobexecute.sas
View File

@@ -23,24 +23,25 @@
,paramstring=%str("macvarname":"macvarvalue","answer":42)
)
@param [in] access_token_var= The global macro variable to contain the access token
@param [in] access_token_var= The global macro variable to contain the access
token
@param [in] grant_type= valid values:
* password
* authorization_code
* detect - will check if access_token exists, if not will use sas_services if
a SASStudioV session else authorization_code. Default option.
* sas_services - will use oauth_bearer=sas_services
@li password
@li authorization_code
@li detect - will check if access_token exists, if not will use sas_services
if a SASStudioV session else authorization_code. Default option.
@li sas_services - will use oauth_bearer=sas_services
@param [in] path= The SAS Drive path to the job being executed
@param [in] name= The name of the job to execute
@param [in] paramstring= A JSON fragment with name:value pairs, eg: `"name":"value"`
or "name":"value","name2":42`. This will need to be wrapped in `%str()`.
@param [in] paramstring= A JSON fragment with name:value pairs, eg:
`"name":"value"` or "name":"value","name2":42`. This will need to be
wrapped in `%str()`.
@param [in] contextName= Context name with which to run the job.
Default = `SAS Job Execution compute context`
@param [out] outds= The output dataset containing links (Default=work.mv_jobexecute)
@param [in] mdebug= set to 1 to enable DEBUG messages
@param [out] outds= (work.mv_jobexecute) The output dataset containing links
@version VIYA V.03.04
@@ -62,7 +63,15 @@
,grant_type=sas_services
,paramstring=0
,outds=work.mv_jobexecute
,mdebug=0
);
%local dbg;
%if &mdebug=1 %then %do;
%put &sysmacroname entry vars:;
%put _local_;
%end;
%else %let dbg=*;
%local oauth_bearer;
%if &grant_type=detect %then %do;
%if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -164,9 +173,14 @@ data &outds;
_program="&path/&name";
run;
/* clear refs */
filename &fname0 clear;
filename &fname1 clear;
libname &libref;
%mend;
%if &mdebug=1 %then %do;
%put &sysmacroname exit vars:;
%put _local_;
%end;
%else %do;
/* clear refs */
filename &fname0 clear;
filename &fname1 clear;
libname &libref;
%end;
%mend mv_jobexecute;

20
viya/mv_jobflow.sas
View File

@@ -136,6 +136,13 @@
,raise_err=0
,mdebug=0
);
%local dbg;
%if &mdebug=1 %then %do;
%put &sysmacroname entry vars:;
%put _local_;
%end;
%else %let dbg=*;
%local oauth_bearer;
%if &grant_type=detect %then %do;
%if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -293,6 +300,7 @@ data;run;%let jdswaitfor=&syslast;
,name=&jobname
,paramstring=%superq(jparams&jid)
,outds=&jdsapp
,mdebug=&mdebug
)
data &jdsapp;
format jobparams $32767.;
@@ -313,8 +321,13 @@ data;run;%let jdswaitfor=&syslast;
%end;
%if &jid=&jcnt %then %do;
/* we are at the end of the loop - time to see which jobs have finished */
%mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref
,raise_err=&raise_err)
%mv_jobwaitfor(ANY
,inds=&jdsrunning
,outds=&jdswaitfor
,outref=&outref
,raise_err=&raise_err
,mdebug=&mdebug
)
%local done;
%let done=%mf_nobs(&jdswaitfor);
%if &done>0 %then %do;
@@ -346,7 +359,8 @@ data;run;%let jdswaitfor=&syslast;
%end;
%if &mdebug=1 %then %do;
%put &sysmacroname exit vars:;
%put _local_;
%end;
%mend;
%mend mv_jobflow;

48
viya/mv_jobwaitfor.sas
View File

@@ -1,6 +1,6 @@
/**
@file
@brief Takes a dataset of running jobs and waits for ANY or ALL of them to complete
@brief Takes a table of running jobs and waits for ANY/ALL of them to complete
@details Will poll `/jobs/{jobId}/state` at set intervals until ANY or ALL
jobs are completed. Completion is determined by reference to the returned
_state_, as per the following table:
@@ -55,13 +55,14 @@
%mv_deletejes(path=/Public/temp,name=demo)
@param [in] access_token_var= The global macro variable to contain the access token
@param [in] access_token_var= The global macro variable to contain the access
token
@param [in] grant_type= valid values:
- password
- authorization_code
- detect - will check if access_token exists, if not will use sas_services if
a SASStudioV session else authorization_code. Default option.
- detect - will check if access_token exists, if not will use sas_services
if a SASStudioV session else authorization_code. Default option.
- sas_services - will use oauth_bearer=sas_services
@param [in] action=Either ALL (to wait for every job) or ANY (if one job
@@ -72,9 +73,11 @@
should be in a `_program` variable.
@param [in] raise_err=0 Set to 1 to raise SYSCC when a job does not complete
succcessfully
@param [in] mdebug= set to 1 to enable DEBUG messages
@param [out] outds= The output dataset containing the list of states by job
(default=work.mv_jobexecute)
@param [out] outref= A fileref to which the spawned job logs should be appended.
@param [out] outref= A fileref to which the spawned job logs should be
appended.
@version VIYA V.03.04
@author Allan Bowe, source: https://github.com/sasjs/core
@@ -97,7 +100,15 @@
,outds=work.mv_jobwaitfor
,outref=0
,raise_err=0
,mdebug=0
);
%local dbg;
%if &mdebug=1 %then %do;
%put &sysmacroname entry vars:;
%put _local_;
%end;
%else %let dbg=*;
%local oauth_bearer;
%if &grant_type=detect %then %do;
%if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -155,7 +166,8 @@ run;
%let fname0=%mf_getuniquefileref();
data &outds;
format _program uri $128. state $32. stateDetails $32. timestamp datetime19. jobparams $32767.;
format _program uri $128. state $32. stateDetails $32. timestamp datetime19.
jobparams $32767.;
stop;
run;
@@ -168,8 +180,8 @@ run;
"Authorization"="Bearer &&&access_token_var"
%end; ;
run;
%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
%do;
%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201
%then %do;
data _null_;infile &fname0;input;putlog _infile_;run;
%mp_abort(mac=&sysmacroname
,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
@@ -205,7 +217,7 @@ run;
%let joburi&i=0; /* do not re-check */
/* fetch log */
%if %str(&outref) ne 0 %then %do;
%mv_getjoblog(uri=&plainuri,outref=&outref)
%mv_getjoblog(uri=&plainuri,outref=&outref,mdebug=&mdebug)
%end;
%end;
%else %if &status=idle or &status=pending or &status=running %then %do;
@@ -220,10 +232,11 @@ run;
%end;
%if (&raise_err) %then %do;
%if (&status = canceled or &status = failed or %length(&stateDetails)>0) %then %do;
%if (&status = canceled or &status = failed or %length(&stateDetails)>0)
%then %do;
%if ("&stateDetails" = "%str(war)ning") %then %let SYSCC=4;
%else %let SYSCC=5;
%put %str(ERR)OR: Job &&jobname&i. did not complete successfully. &stateDetails;
%put %str(ERR)OR: Job &&jobname&i. did not complete. &stateDetails;
%return;
%end;
%end;
@@ -238,7 +251,12 @@ run;
%end;
%end;
/* clear refs */
filename &fname0 clear;
%mend;
%if &mdebug=1 %then %do;
%put &sysmacroname exit vars:;
%put _local_;
%end;
%else %do;
/* clear refs */
filename &fname0 clear;
%end;
%mend mv_jobwaitfor;