From 223bdd5983b987955a208a48b9b69264741acc35 Mon Sep 17 00:00:00 2001 From: Allan Bowe Date: Tue, 11 May 2021 11:58:27 +0300 Subject: [PATCH 01/11] fix: mp_abort cleanup --- all.sas | 27 +++++++++++++++++---------- base/mp_abort.sas | 27 +++++++++++++++++---------- tests/base/mp_abort.test.1.sas | 28 ++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 20 deletions(-) create mode 100644 tests/base/mp_abort.test.1.sas diff --git a/all.sas b/all.sas index a30009a..c1dfc10 100644 --- a/all.sas +++ b/all.sas @@ -1748,19 +1748,26 @@ Usage: if debug ge '"131"' then put '>>weboutEND<<'; run; - %if %symexist(_metaport) %then %do; - data _null_; - if symexist('sysprocessmode') then - if symget("sysprocessmode")="SAS Stored Process Server" then do; - rc=stpsrvset('program error', 0); - call symputx("syscc",0,"g"); - end; - run; + data _null_; + if symexist('sysprocessmode') then + if symget("sysprocessmode")="SAS Stored Process Server" then do; + putlog 'stpsrvset program error and syscc'; + rc=stpsrvset('program error', 0); + call symputx("syscc",0,"g"); + end; + run; + %if "%substr(&sysvlong.xxxxxxx,1,9)" ne "9.04.01M3" %then %do; + %put NOTE: Ending SAS session due to:; + %put NOTE- &msg; + endsas; %end; + SYSVLONG=9.04.01M7P080520 /** - * endsas is reliable but kills some deployments. + * endsas is reliable but kills 9.4m3 deployments by hanging multibridges. * Abort variants are ungraceful (non zero return code) * This approach lets SAS run silently until the end :-) + * Caution - fails when called within a %include within a macro + * See tests/mp_abort.test.1 for an example case. */ %put _all_; filename skip temp; @@ -1774,7 +1781,7 @@ Usage: %put _all_; %abort cancel; %end; -%mend; +%mend mp_abort; /** @endcond *//** @file diff --git a/base/mp_abort.sas b/base/mp_abort.sas index c9e9043..e73f31a 100644 --- a/base/mp_abort.sas +++ b/base/mp_abort.sas @@ -128,19 +128,26 @@ if debug ge '"131"' then put '>>weboutEND<<'; run; - %if %symexist(_metaport) %then %do; - data _null_; - if symexist('sysprocessmode') then - if symget("sysprocessmode")="SAS Stored Process Server" then do; - rc=stpsrvset('program error', 0); - call symputx("syscc",0,"g"); - end; - run; + data _null_; + if symexist('sysprocessmode') then + if symget("sysprocessmode")="SAS Stored Process Server" then do; + putlog 'stpsrvset program error and syscc'; + rc=stpsrvset('program error', 0); + call symputx("syscc",0,"g"); + end; + run; + %if "%substr(&sysvlong.xxxxxxx,1,9)" ne "9.04.01M3" %then %do; + %put NOTE: Ending SAS session due to:; + %put NOTE- &msg; + endsas; %end; + SYSVLONG=9.04.01M7P080520 /** - * endsas is reliable but kills some deployments. + * endsas is reliable but kills 9.4m3 deployments by hanging multibridges. * Abort variants are ungraceful (non zero return code) * This approach lets SAS run silently until the end :-) + * Caution - fails when called within a %include within a macro + * See tests/mp_abort.test.1 for an example case. */ %put _all_; filename skip temp; @@ -154,6 +161,6 @@ %put _all_; %abort cancel; %end; -%mend; +%mend mp_abort; /** @endcond */ \ No newline at end of file diff --git a/tests/base/mp_abort.test.1.sas b/tests/base/mp_abort.test.1.sas new file mode 100644 index 0000000..cbff2ce --- /dev/null +++ b/tests/base/mp_abort.test.1.sas @@ -0,0 +1,28 @@ +/** + @file + @brief Testing mp_abort macro + @details This is an unfixed problem with mp_abort. When called from within + a macro, within a %include, and that macro contains subsequent logic, the + service does not end cleanly - rather, we see: + + ERROR: %EVAL function has no expression to evaluate, or %IF statement has no condition. + ERROR: The macro TEST will stop executing. + +

SAS Macros

+ @li mp_abort.sas + +**/ + +%macro test(); + +filename blah temp; +data _null_; + file blah; + put '%mp_abort();'; +run; +%inc blah; + +%if 1=1 %then %put Houston - we have a problem here; +%mend test; + +%test() \ No newline at end of file From 583c7e0c83446b9c82db90fee8cda0c9ba9a226a Mon Sep 17 00:00:00 2001 From: Allan Bowe Date: Tue, 11 May 2021 12:14:09 +0300 Subject: [PATCH 02/11] fix: removing string --- all.sas | 3 ++- base/mp_abort.sas | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/all.sas b/all.sas index c1dfc10..2940d88 100644 --- a/all.sas +++ b/all.sas @@ -1756,12 +1756,13 @@ Usage: call symputx("syscc",0,"g"); end; run; + %if "%substr(&sysvlong.xxxxxxx,1,9)" ne "9.04.01M3" %then %do; %put NOTE: Ending SAS session due to:; %put NOTE- &msg; endsas; %end; - SYSVLONG=9.04.01M7P080520 + /** * endsas is reliable but kills 9.4m3 deployments by hanging multibridges. * Abort variants are ungraceful (non zero return code) diff --git a/base/mp_abort.sas b/base/mp_abort.sas index e73f31a..05bb339 100644 --- a/base/mp_abort.sas +++ b/base/mp_abort.sas @@ -136,12 +136,13 @@ call symputx("syscc",0,"g"); end; run; + %if "%substr(&sysvlong.xxxxxxx,1,9)" ne "9.04.01M3" %then %do; %put NOTE: Ending SAS session due to:; %put NOTE- &msg; endsas; %end; - SYSVLONG=9.04.01M7P080520 + /** * endsas is reliable but kills 9.4m3 deployments by hanging multibridges. * Abort variants are ungraceful (non zero return code) From e6146dcbcf939d78574b0b6618c12bbe0bfb105f Mon Sep 17 00:00:00 2001 From: Allan Bowe Date: Tue, 11 May 2021 12:39:10 +0300 Subject: [PATCH 03/11] fix: more logic to improve robustness --- base/mp_abort.sas | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/base/mp_abort.sas b/base/mp_abort.sas index 05bb339..19ae598 100644 --- a/base/mp_abort.sas +++ b/base/mp_abort.sas @@ -24,6 +24,8 @@ %macro mp_abort(mac=mp_abort.sas, type=, msg=, iftrue=%str(1=1) )/*/STORE SOURCE*/; + %global sysprocessmode sysprocessname; + %if not(%eval(%unquote(&iftrue))) %then %return; %put NOTE: /// mp_abort macro executing //; @@ -31,9 +33,7 @@ %put NOTE - &msg; /* Stored Process Server web app context */ - %if %symexist(_metaperson) - or (%symexist(SYSPROCESSNAME) and "&SYSPROCESSNAME"="Compute Server" ) - %then %do; + %if %symexist(_metaperson) or "&SYSPROCESSNAME"="Compute Server" %then %do; options obs=max replace nosyntaxcheck mprint; /* extract log errs / warns, if exist */ %local logloc logline; @@ -128,14 +128,13 @@ if debug ge '"131"' then put '>>weboutEND<<'; run; - data _null_; - if symexist('sysprocessmode') then - if symget("sysprocessmode")="SAS Stored Process Server" then do; - putlog 'stpsrvset program error and syscc'; - rc=stpsrvset('program error', 0); - call symputx("syscc",0,"g"); - end; - run; + %if "&sysprocessmode " = "SAS Stored Process Server " %then %do; + data _null_; + putlog 'stpsrvset program error and syscc'; + rc=stpsrvset('program error', 0); + call symputx("syscc",0,"g"); + run; + %end; %if "%substr(&sysvlong.xxxxxxx,1,9)" ne "9.04.01M3" %then %do; %put NOTE: Ending SAS session due to:; @@ -154,7 +153,10 @@ filename skip temp; data _null_; file skip; - put '%macro skip(); %macro skippy();'; + put '%macro skip();'; + comment '%mend skip; -> fix lint '; + put '%macro skippy();'; + comment '%mend skippy; -> fix lint '; run; %inc skip; %end; From 9a5574ea0e528b245366fa9dbd6dcca0f65a5147 Mon Sep 17 00:00:00 2001 From: Allan Bowe Date: Tue, 11 May 2021 13:35:21 +0300 Subject: [PATCH 04/11] fix: all --- all.sas | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/all.sas b/all.sas index 2940d88..85c6f4e 100644 --- a/all.sas +++ b/all.sas @@ -1644,6 +1644,8 @@ Usage: %macro mp_abort(mac=mp_abort.sas, type=, msg=, iftrue=%str(1=1) )/*/STORE SOURCE*/; + %global sysprocessmode sysprocessname; + %if not(%eval(%unquote(&iftrue))) %then %return; %put NOTE: /// mp_abort macro executing //; @@ -1651,9 +1653,7 @@ Usage: %put NOTE - &msg; /* Stored Process Server web app context */ - %if %symexist(_metaperson) - or (%symexist(SYSPROCESSNAME) and "&SYSPROCESSNAME"="Compute Server" ) - %then %do; + %if %symexist(_metaperson) or "&SYSPROCESSNAME"="Compute Server" %then %do; options obs=max replace nosyntaxcheck mprint; /* extract log errs / warns, if exist */ %local logloc logline; @@ -1748,14 +1748,13 @@ Usage: if debug ge '"131"' then put '>>weboutEND<<'; run; - data _null_; - if symexist('sysprocessmode') then - if symget("sysprocessmode")="SAS Stored Process Server" then do; - putlog 'stpsrvset program error and syscc'; - rc=stpsrvset('program error', 0); - call symputx("syscc",0,"g"); - end; - run; + %if "&sysprocessmode " = "SAS Stored Process Server " %then %do; + data _null_; + putlog 'stpsrvset program error and syscc'; + rc=stpsrvset('program error', 0); + call symputx("syscc",0,"g"); + run; + %end; %if "%substr(&sysvlong.xxxxxxx,1,9)" ne "9.04.01M3" %then %do; %put NOTE: Ending SAS session due to:; @@ -1774,7 +1773,10 @@ Usage: filename skip temp; data _null_; file skip; - put '%macro skip(); %macro skippy();'; + put '%macro skip();'; + comment '%mend skip; -> fix lint '; + put '%macro skippy();'; + comment '%mend skippy; -> fix lint '; run; %inc skip; %end; From 58a0cce39edcb5d4d8c61ff6f79cb7f5a579e022 Mon Sep 17 00:00:00 2001 From: Allan Bowe Date: Tue, 11 May 2021 13:36:25 +0300 Subject: [PATCH 05/11] chore: automated commit --- all.sas | 2 +- base/mp_abort.sas | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/all.sas b/all.sas index 85c6f4e..40d9f7b 100644 --- a/all.sas +++ b/all.sas @@ -1653,7 +1653,7 @@ Usage: %put NOTE - &msg; /* Stored Process Server web app context */ - %if %symexist(_metaperson) or "&SYSPROCESSNAME"="Compute Server" %then %do; + %if %symexist(_metaperson) or "&SYSPROCESSNAME "="Compute Server " %then %do; options obs=max replace nosyntaxcheck mprint; /* extract log errs / warns, if exist */ %local logloc logline; diff --git a/base/mp_abort.sas b/base/mp_abort.sas index 19ae598..b7d3c44 100644 --- a/base/mp_abort.sas +++ b/base/mp_abort.sas @@ -33,7 +33,7 @@ %put NOTE - &msg; /* Stored Process Server web app context */ - %if %symexist(_metaperson) or "&SYSPROCESSNAME"="Compute Server" %then %do; + %if %symexist(_metaperson) or "&SYSPROCESSNAME "="Compute Server " %then %do; options obs=max replace nosyntaxcheck mprint; /* extract log errs / warns, if exist */ %local logloc logline; From 15f903aa42eea708b14d0ed7b945e8cf2e00751a Mon Sep 17 00:00:00 2001 From: Allan Bowe Date: Tue, 11 May 2021 18:14:33 +0300 Subject: [PATCH 06/11] fix: updating mv_getjoblog to deal with endsas'd sessions, and removing endsas from viya mp_abort --- all.sas | 102 ++++++++++++++++++++++--------- base/mp_abort.sas | 10 ++- tests/base/mp_abort.test.1.sas | 1 + tests/viya/mv_getjoblog.test.sas | 64 +++++++++++++++++++ viya/mv_getjoblog.sas | 92 ++++++++++++++++++++-------- 5 files changed, 213 insertions(+), 56 deletions(-) create mode 100644 tests/viya/mv_getjoblog.test.sas diff --git a/all.sas b/all.sas index 40d9f7b..9c3b333 100644 --- a/all.sas +++ b/all.sas @@ -1754,9 +1754,15 @@ Usage: rc=stpsrvset('program error', 0); call symputx("syscc",0,"g"); run; + endsas; %end; - - %if "%substr(&sysvlong.xxxxxxx,1,9)" ne "9.04.01M3" %then %do; + %else %if "&sysprocessmode " = "SAS Compute Server " %then %do; + /* endsas kills the session making it harder to fetch results */ + data _null_; + abort; + run; + %end; + %else %if "%substr(&sysvlong.xxxxxxxxx,1,9)" ne "9.04.01M3" %then %do; %put NOTE: Ending SAS session due to:; %put NOTE- &msg; endsas; @@ -15064,21 +15070,19 @@ run; /** @file @brief Extract the log from a completed SAS Viya Job - @details Extracts log from a Viya job and writes it out to a fileref + @details Extracts log from a Viya job and writes it out to a fileref. To query the job, you need the URI. Sample code for achieving this is provided below. ## Example - First, compile the macros: - + %* First, compile the macros; filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; %inc mc; - Next, create a job (in this case, a web service): - + %* Next, create a job (in this case, a web service); filename ft15f001 temp; parmcards4; data ; @@ -15094,28 +15098,40 @@ run; ;;;; %mv_createwebservice(path=/Public/temp,name=demo) - Execute it: - + %* Execute it; %mv_jobexecute(path=/Public/temp ,name=demo ,outds=work.info ) - Wait for it to finish, and grab the uri: - - data _null_; + %* Wait for it to finish; + data work.info; set work.info; - if method='GET' and rel='self'; + where method='GET' and rel='state'; + run; + %mv_jobwaitfor(ALL,inds=work.info,outds=work.jobstates) + + %* and grab the uri; + data _null_; + set work.jobstates; call symputx('uri',uri); run; - Finally, fetch the log: - + %* Finally, fetch the log; %mv_getjoblog(uri=&uri,outref=mylog) This macro is used by the mv_jobwaitfor.sas macro, which is generally a more convenient way to wait for the job to finish before fetching the log. + If the remote session calls `endsas` then it is not possible to get the log + from the provided uri, and so the log from the parent session is fetched + instead. This happens for a 400 response, eg below: + + ErrorResponse[version=2,status=400,err=5113,id=,message=The session + requested is currently in a failed or stopped state.,detail=[path: + /compute/sessions/LONGURI-ses0006/jobs/LONGURI/log/content, traceId: 63 + 51aa617d01fd2b],remediation=Correct the errors in the session request, + and create a new session.,targetUri=,errors=[],links=[]] @param [in] access_token_var= The global macro variable to contain the access token @@ -15127,7 +15143,7 @@ run; if a SASStudioV session else authorization_code. Default option. @li sas_services - will use oauth_bearer=sas_services. @param [in] uri= The uri of the running job for which to fetch the status, - in the format `/jobExecution/jobs/$UUID/state` (unquoted). + in the format `/jobExecution/jobs/$UUID` (unquoted). @param [out] outref= The output fileref to which to APPEND the log (is always appended). @@ -15185,7 +15201,7 @@ data _null_; call symputx('errflg',1); call symputx('errmsg', "URI should be in format /jobExecution/jobs/$$$$UUID$$$$" - !!" but is actually like: &uri",'l'); + !!" but is actually like:"!!uri,'l'); end; run; @@ -15218,6 +15234,10 @@ proc http method='GET' out=&fname1 &oauth_bearer %end; ; run; +%if &mdebug=1 %then %do; + %put &sysmacroname: fetching log loc from &uri; + data _null_;infile &fname1;input;putlog _infile_;run; +%end; %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then %do; data _null_;infile &fname1;input;putlog _infile_;run; @@ -15255,43 +15275,69 @@ run; data _null_; infile &fname2; input; - uri=_infile_; + uri=cats(_infile_); if length(uri)<12 then do; call symputx('errflg',1); call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l'); end; - if scan(uri,1) ne 'files' or scan(uri,2) ne 'files' then do; + else if (scan(uri,1,'/') ne 'compute' or scan(uri,2,'/') ne 'sessions') + and (scan(uri,1,'/') ne 'files' or scan(uri,2,'/') ne 'files') + then do; call symputx('errflg',1); call symputx('errmsg', - "URI should be in format /files/files/$$$$UUID$$$$" - !!" but is actually like: &uri",'l'); + "URI should be in format /compute/sessions/$$$$UUID$$$$/jobs/$$$$UUID$$$$" + !!" or /files/files/$$$$UUID$$$$" + !!" but is actually like:"!!uri,'l'); + end; + else do; + call symputx('errflg',0,'l'); + call symputx('logloc',uri,'l'); end; - call symputx('errflg',0,'l'); - call symputx('logloc',uri,'l'); run; -%mp_abort(iftrue=(&errflg=1) +%mp_abort(iftrue=(%str(&errflg)=1) ,mac=&sysmacroname ,msg=%str(&errmsg) ) /* we have a log uri - now fetch the log */ +%&dbg.put &sysmacroname: querying &base_uri&logloc/content; proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri&logloc/content"; + url="&base_uri&logloc/content?limit=10000"; headers %if &grant_type=authorization_code %then %do; "Authorization"="Bearer &&&access_token_var" %end; ; run; -%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then -%do; +%if &mdebug=1 %then %do; + %put &sysmacroname: fetching log content from &base_uri&logloc/content; + data _null_;infile &fname1;input;putlog _infile_;run; +%end; +%if &SYS_PROCHTTP_STATUS_CODE=400 %then %do; + /* fetch log from parent session */ + %let logloc=%substr(&logloc,1,%index(&logloc,%str(/jobs/))-1); + %&dbg.put &sysmacroname: Now querying &base_uri&logloc/log/content; + proc http method='GET' out=&fname1 &oauth_bearer + url="&base_uri&logloc/log/content?limit=10000"; + headers + %if &grant_type=authorization_code %then %do; + "Authorization"="Bearer &&&access_token_var" + %end; + ; + run; + %if &mdebug=1 %then %do; + %put &sysmacroname: fetching log content from &base_uri&logloc/log/content; + data _null_;infile &fname1;input;putlog _infile_;run; + %end; +%end; +%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 +%then %do; data _null_;infile &fname1;input;putlog _infile_;run; %mp_abort(mac=&sysmacroname ,msg=%str(logfetch: &SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) ) %end; - data _null_; file "&fpath3..lua"; put ' diff --git a/base/mp_abort.sas b/base/mp_abort.sas index b7d3c44..4c5ea00 100644 --- a/base/mp_abort.sas +++ b/base/mp_abort.sas @@ -134,9 +134,15 @@ rc=stpsrvset('program error', 0); call symputx("syscc",0,"g"); run; + endsas; %end; - - %if "%substr(&sysvlong.xxxxxxx,1,9)" ne "9.04.01M3" %then %do; + %else %if "&sysprocessmode " = "SAS Compute Server " %then %do; + /* endsas kills the session making it harder to fetch results */ + data _null_; + abort; + run; + %end; + %else %if "%substr(&sysvlong.xxxxxxxxx,1,9)" ne "9.04.01M3" %then %do; %put NOTE: Ending SAS session due to:; %put NOTE- &msg; endsas; diff --git a/tests/base/mp_abort.test.1.sas b/tests/base/mp_abort.test.1.sas index cbff2ce..85d9bed 100644 --- a/tests/base/mp_abort.test.1.sas +++ b/tests/base/mp_abort.test.1.sas @@ -10,6 +10,7 @@

SAS Macros

@li mp_abort.sas + @li mp_assert.sas **/ diff --git a/tests/viya/mv_getjoblog.test.sas b/tests/viya/mv_getjoblog.test.sas new file mode 100644 index 0000000..499a2c2 --- /dev/null +++ b/tests/viya/mv_getjoblog.test.sas @@ -0,0 +1,64 @@ +/** + @file + @brief Testing mv_createwebservice macro + +

SAS Macros

+ @li mp_assert.sas + @li mv_createjob.sas + @li mv_jobexecute.sas + @li mv_jobwaitfor.sas + @li mv_getjoblog.sas + +**/ + +/** + * Test Case 1 + */ + +/* create a service */ +filename testref temp; +data _null_; + file testref; + put 'endsas;'; +run; +%mv_createjob( + path=&mcTestAppLoc/jobs/temp, + code=testref, + name=testjob +) + +%* Execute it; +%mv_jobexecute( + path=&mcTestAppLoc/jobs/temp, + name=testjob, + outds=work.info, +) + +%* Wait for it to finish; +data work.info; + set work.info; + where method='GET' and rel='state'; +run; +%mv_jobwaitfor(ALL,inds=work.info,outds=work.jobstates) + +%* and grab the uri; +data _null_; + set work.jobstates; + call symputx('uri',uri); +run; + +%* Finally, fetch the log; +%mv_getjoblog(uri=%str(&uri),outref=mylog) + + +data _null_; + infile mylog; + input; + if index(_infile_,'endsas;') then call symputx('found',1); + else call symputx('found',0); +run; + +%mp_assert( + iftrue=(%str(&found)=1), + desc=Check if the log was still fetched even though endsas was submitted +) \ No newline at end of file diff --git a/viya/mv_getjoblog.sas b/viya/mv_getjoblog.sas index 7c4ce0a..dc44c9b 100644 --- a/viya/mv_getjoblog.sas +++ b/viya/mv_getjoblog.sas @@ -1,21 +1,19 @@ /** @file @brief Extract the log from a completed SAS Viya Job - @details Extracts log from a Viya job and writes it out to a fileref + @details Extracts log from a Viya job and writes it out to a fileref. To query the job, you need the URI. Sample code for achieving this is provided below. ## Example - First, compile the macros: - + %* First, compile the macros; filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; %inc mc; - Next, create a job (in this case, a web service): - + %* Next, create a job (in this case, a web service); filename ft15f001 temp; parmcards4; data ; @@ -31,28 +29,40 @@ ;;;; %mv_createwebservice(path=/Public/temp,name=demo) - Execute it: - + %* Execute it; %mv_jobexecute(path=/Public/temp ,name=demo ,outds=work.info ) - Wait for it to finish, and grab the uri: - - data _null_; + %* Wait for it to finish; + data work.info; set work.info; - if method='GET' and rel='self'; + where method='GET' and rel='state'; + run; + %mv_jobwaitfor(ALL,inds=work.info,outds=work.jobstates) + + %* and grab the uri; + data _null_; + set work.jobstates; call symputx('uri',uri); run; - Finally, fetch the log: - + %* Finally, fetch the log; %mv_getjoblog(uri=&uri,outref=mylog) This macro is used by the mv_jobwaitfor.sas macro, which is generally a more convenient way to wait for the job to finish before fetching the log. + If the remote session calls `endsas` then it is not possible to get the log + from the provided uri, and so the log from the parent session is fetched + instead. This happens for a 400 response, eg below: + + ErrorResponse[version=2,status=400,err=5113,id=,message=The session + requested is currently in a failed or stopped state.,detail=[path: + /compute/sessions/LONGURI-ses0006/jobs/LONGURI/log/content, traceId: 63 + 51aa617d01fd2b],remediation=Correct the errors in the session request, + and create a new session.,targetUri=,errors=[],links=[]] @param [in] access_token_var= The global macro variable to contain the access token @@ -64,7 +74,7 @@ if a SASStudioV session else authorization_code. Default option. @li sas_services - will use oauth_bearer=sas_services. @param [in] uri= The uri of the running job for which to fetch the status, - in the format `/jobExecution/jobs/$UUID/state` (unquoted). + in the format `/jobExecution/jobs/$UUID` (unquoted). @param [out] outref= The output fileref to which to APPEND the log (is always appended). @@ -122,7 +132,7 @@ data _null_; call symputx('errflg',1); call symputx('errmsg', "URI should be in format /jobExecution/jobs/$$$$UUID$$$$" - !!" but is actually like: &uri",'l'); + !!" but is actually like:"!!uri,'l'); end; run; @@ -155,6 +165,10 @@ proc http method='GET' out=&fname1 &oauth_bearer %end; ; run; +%if &mdebug=1 %then %do; + %put &sysmacroname: fetching log loc from &uri; + data _null_;infile &fname1;input;putlog _infile_;run; +%end; %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then %do; data _null_;infile &fname1;input;putlog _infile_;run; @@ -192,43 +206,69 @@ run; data _null_; infile &fname2; input; - uri=_infile_; + uri=cats(_infile_); if length(uri)<12 then do; call symputx('errflg',1); call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l'); end; - if scan(uri,1) ne 'files' or scan(uri,2) ne 'files' then do; + else if (scan(uri,1,'/') ne 'compute' or scan(uri,2,'/') ne 'sessions') + and (scan(uri,1,'/') ne 'files' or scan(uri,2,'/') ne 'files') + then do; call symputx('errflg',1); call symputx('errmsg', - "URI should be in format /files/files/$$$$UUID$$$$" - !!" but is actually like: &uri",'l'); + "URI should be in format /compute/sessions/$$$$UUID$$$$/jobs/$$$$UUID$$$$" + !!" or /files/files/$$$$UUID$$$$" + !!" but is actually like:"!!uri,'l'); + end; + else do; + call symputx('errflg',0,'l'); + call symputx('logloc',uri,'l'); end; - call symputx('errflg',0,'l'); - call symputx('logloc',uri,'l'); run; -%mp_abort(iftrue=(&errflg=1) +%mp_abort(iftrue=(%str(&errflg)=1) ,mac=&sysmacroname ,msg=%str(&errmsg) ) /* we have a log uri - now fetch the log */ +%&dbg.put &sysmacroname: querying &base_uri&logloc/content; proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri&logloc/content"; + url="&base_uri&logloc/content?limit=10000"; headers %if &grant_type=authorization_code %then %do; "Authorization"="Bearer &&&access_token_var" %end; ; run; -%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then -%do; +%if &mdebug=1 %then %do; + %put &sysmacroname: fetching log content from &base_uri&logloc/content; + data _null_;infile &fname1;input;putlog _infile_;run; +%end; +%if &SYS_PROCHTTP_STATUS_CODE=400 %then %do; + /* fetch log from parent session */ + %let logloc=%substr(&logloc,1,%index(&logloc,%str(/jobs/))-1); + %&dbg.put &sysmacroname: Now querying &base_uri&logloc/log/content; + proc http method='GET' out=&fname1 &oauth_bearer + url="&base_uri&logloc/log/content?limit=10000"; + headers + %if &grant_type=authorization_code %then %do; + "Authorization"="Bearer &&&access_token_var" + %end; + ; + run; + %if &mdebug=1 %then %do; + %put &sysmacroname: fetching log content from &base_uri&logloc/log/content; + data _null_;infile &fname1;input;putlog _infile_;run; + %end; +%end; +%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 +%then %do; data _null_;infile &fname1;input;putlog _infile_;run; %mp_abort(mac=&sysmacroname ,msg=%str(logfetch: &SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) ) %end; - data _null_; file "&fpath3..lua"; put ' From 4ee13c9389b2e18ab8961db5bf720488b1735e3a Mon Sep 17 00:00:00 2001 From: Allan Bowe Date: Tue, 11 May 2021 20:25:39 +0300 Subject: [PATCH 07/11] fix: 400 log repeat, refactor mp_abort abortions, updated doc header --- all.sas | 89 ++++++++++++++++++++++++++++--------------- base/mp_abort.sas | 82 ++++++++++++++++++++++++--------------- viya/mv_getjoblog.sas | 7 +++- 3 files changed, 116 insertions(+), 62 deletions(-) diff --git a/all.sas b/all.sas index 9c3b333..b73f606 100644 --- a/all.sas +++ b/all.sas @@ -1624,13 +1624,24 @@ Usage: @details Configures an abort mechanism according to site specific policies or the particulars of an environment. For instance, can stream custom results back to the client in an STP Web App context, or completely stop - in the case of a batch run. + in the case of a batch run. For STP sessions + + The method used varies according to the context. Important points: + + @li should not use endsas or abort cancel in 9.4m3 environments as this can + cause hung multibridge sessions and result in a frozen STP server + @li should not use endsas in viya 3.5 as this destroys the session and cannot + fetch results (although both mv_getjoblog.sas and the @sasjs/adapter will + recognise this and fetch the log of the parent session instead) + @li STP environments must finish cleanly to avoid the log being sent to + _webout. To assist with this, we also run stpsrvset('program error', 0) + and set SYSCC=0. For 9.4m3 we take a unique approach - we open a macro + but don't close it! This provides a graceful abort, EXCEPT when called + called within a %include within a macro. See tests/mp_abort.test.1 for an + example case. + If you know of another way to gracefully abort a 9.4m3 STP session, we'd + love to hear about it! - Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored - Process environments. This macro takes a unique approach - we set the SAS - syscc to 0, run `stpsrvset('program error', 0)` (if SAS 9) and then - we open - a macro but don't close it! This provides a graceful abort for SAS web - services in all web enabled environments. @param mac= to contain the name of the calling macro @param msg= message to be returned @@ -1748,43 +1759,54 @@ Usage: if debug ge '"131"' then put '>>weboutEND<<'; run; + %put _all_; + %if "&sysprocessmode " = "SAS Stored Process Server " %then %do; data _null_; putlog 'stpsrvset program error and syscc'; rc=stpsrvset('program error', 0); call symputx("syscc",0,"g"); run; - endsas; + %if "%substr(&sysvlong.xxxxxxxxx,1,9)" ne "9.04.01M3" %then %do; + %put NOTE: Ending SAS session due to:; + %put NOTE- &msg; + endsas; + %end; %end; %else %if "&sysprocessmode " = "SAS Compute Server " %then %do; /* endsas kills the session making it harder to fetch results */ data _null_; - abort; + syswarningtext=symget('syswarningtext'); + syserrortext=symget('syserrortext'); + abort_msg=symget('msg'); + syscc=symget('syscc'); + sysuserid=symget('sysuserid'); + iftrue=symget('iftrue'); + put (_all_)(/=); + abort cancel nolist; run; %end; - %else %if "%substr(&sysvlong.xxxxxxxxx,1,9)" ne "9.04.01M3" %then %do; - %put NOTE: Ending SAS session due to:; - %put NOTE- &msg; - endsas; + %else %if "%substr(&sysvlong.xxxxxxxxx,1,9)" = "9.04.01M3" %then %do; + /** + * endsas kills 9.4m3 deployments by orphaning multibridges. + * Abort variants are ungraceful (non zero return code) + * This approach lets SAS run silently until the end :-) + * Caution - fails when called within a %include within a macro + * See tests/mp_abort.test.1 for an example case. + */ + filename skip temp; + data _null_; + file skip; + put '%macro skip();'; + comment '%mend skip; -> fix lint '; + put '%macro skippy();'; + comment '%mend skippy; -> fix lint '; + run; + %inc skip; + %end; + %else %do; + %abort cancel; %end; - - /** - * endsas is reliable but kills 9.4m3 deployments by hanging multibridges. - * Abort variants are ungraceful (non zero return code) - * This approach lets SAS run silently until the end :-) - * Caution - fails when called within a %include within a macro - * See tests/mp_abort.test.1 for an example case. - */ - %put _all_; - filename skip temp; - data _null_; - file skip; - put '%macro skip();'; - comment '%mend skip; -> fix lint '; - put '%macro skippy();'; - comment '%mend skippy; -> fix lint '; - run; - %inc skip; %end; %else %do; %put _all_; @@ -15310,10 +15332,12 @@ proc http method='GET' out=&fname1 &oauth_bearer %end; ; run; + %if &mdebug=1 %then %do; %put &sysmacroname: fetching log content from &base_uri&logloc/content; data _null_;infile &fname1;input;putlog _infile_;run; %end; + %if &SYS_PROCHTTP_STATUS_CODE=400 %then %do; /* fetch log from parent session */ %let logloc=%substr(&logloc,1,%index(&logloc,%str(/jobs/))-1); @@ -15331,9 +15355,12 @@ run; data _null_;infile &fname1;input;putlog _infile_;run; %end; %end; + %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then %do; - data _null_;infile &fname1;input;putlog _infile_;run; + %if &mdebug ne 1 %then %do; /* have already output above */ + data _null_;infile &fname1;input;putlog _infile_;run; + %end; %mp_abort(mac=&sysmacroname ,msg=%str(logfetch: &SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) ) diff --git a/base/mp_abort.sas b/base/mp_abort.sas index 4c5ea00..f8189b1 100644 --- a/base/mp_abort.sas +++ b/base/mp_abort.sas @@ -4,13 +4,24 @@ @details Configures an abort mechanism according to site specific policies or the particulars of an environment. For instance, can stream custom results back to the client in an STP Web App context, or completely stop - in the case of a batch run. + in the case of a batch run. For STP sessions + + The method used varies according to the context. Important points: + + @li should not use endsas or abort cancel in 9.4m3 environments as this can + cause hung multibridge sessions and result in a frozen STP server + @li should not use endsas in viya 3.5 as this destroys the session and cannot + fetch results (although both mv_getjoblog.sas and the @sasjs/adapter will + recognise this and fetch the log of the parent session instead) + @li STP environments must finish cleanly to avoid the log being sent to + _webout. To assist with this, we also run stpsrvset('program error', 0) + and set SYSCC=0. For 9.4m3 we take a unique approach - we open a macro + but don't close it! This provides a graceful abort, EXCEPT when called + called within a %include within a macro. See tests/mp_abort.test.1 for an + example case. + If you know of another way to gracefully abort a 9.4m3 STP session, we'd + love to hear about it! - Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored - Process environments. This macro takes a unique approach - we set the SAS - syscc to 0, run `stpsrvset('program error', 0)` (if SAS 9) and then - we open - a macro but don't close it! This provides a graceful abort for SAS web - services in all web enabled environments. @param mac= to contain the name of the calling macro @param msg= message to be returned @@ -128,43 +139,54 @@ if debug ge '"131"' then put '>>weboutEND<<'; run; + %put _all_; + %if "&sysprocessmode " = "SAS Stored Process Server " %then %do; data _null_; putlog 'stpsrvset program error and syscc'; rc=stpsrvset('program error', 0); call symputx("syscc",0,"g"); run; - endsas; + %if "%substr(&sysvlong.xxxxxxxxx,1,9)" ne "9.04.01M3" %then %do; + %put NOTE: Ending SAS session due to:; + %put NOTE- &msg; + endsas; + %end; %end; %else %if "&sysprocessmode " = "SAS Compute Server " %then %do; /* endsas kills the session making it harder to fetch results */ data _null_; - abort; + syswarningtext=symget('syswarningtext'); + syserrortext=symget('syserrortext'); + abort_msg=symget('msg'); + syscc=symget('syscc'); + sysuserid=symget('sysuserid'); + iftrue=symget('iftrue'); + put (_all_)(/=); + abort cancel nolist; run; %end; - %else %if "%substr(&sysvlong.xxxxxxxxx,1,9)" ne "9.04.01M3" %then %do; - %put NOTE: Ending SAS session due to:; - %put NOTE- &msg; - endsas; + %else %if "%substr(&sysvlong.xxxxxxxxx,1,9)" = "9.04.01M3" %then %do; + /** + * endsas kills 9.4m3 deployments by orphaning multibridges. + * Abort variants are ungraceful (non zero return code) + * This approach lets SAS run silently until the end :-) + * Caution - fails when called within a %include within a macro + * See tests/mp_abort.test.1 for an example case. + */ + filename skip temp; + data _null_; + file skip; + put '%macro skip();'; + comment '%mend skip; -> fix lint '; + put '%macro skippy();'; + comment '%mend skippy; -> fix lint '; + run; + %inc skip; + %end; + %else %do; + %abort cancel; %end; - - /** - * endsas is reliable but kills 9.4m3 deployments by hanging multibridges. - * Abort variants are ungraceful (non zero return code) - * This approach lets SAS run silently until the end :-) - * Caution - fails when called within a %include within a macro - * See tests/mp_abort.test.1 for an example case. - */ - %put _all_; - filename skip temp; - data _null_; - file skip; - put '%macro skip();'; - comment '%mend skip; -> fix lint '; - put '%macro skippy();'; - comment '%mend skippy; -> fix lint '; - run; - %inc skip; %end; %else %do; %put _all_; diff --git a/viya/mv_getjoblog.sas b/viya/mv_getjoblog.sas index dc44c9b..1db9896 100644 --- a/viya/mv_getjoblog.sas +++ b/viya/mv_getjoblog.sas @@ -241,10 +241,12 @@ proc http method='GET' out=&fname1 &oauth_bearer %end; ; run; + %if &mdebug=1 %then %do; %put &sysmacroname: fetching log content from &base_uri&logloc/content; data _null_;infile &fname1;input;putlog _infile_;run; %end; + %if &SYS_PROCHTTP_STATUS_CODE=400 %then %do; /* fetch log from parent session */ %let logloc=%substr(&logloc,1,%index(&logloc,%str(/jobs/))-1); @@ -262,9 +264,12 @@ run; data _null_;infile &fname1;input;putlog _infile_;run; %end; %end; + %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then %do; - data _null_;infile &fname1;input;putlog _infile_;run; + %if &mdebug ne 1 %then %do; /* have already output above */ + data _null_;infile &fname1;input;putlog _infile_;run; + %end; %mp_abort(mac=&sysmacroname ,msg=%str(logfetch: &SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) ) From e26af5c09a518be71739b7730408ded21247d293 Mon Sep 17 00:00:00 2001 From: Allan Bowe Date: Tue, 11 May 2021 20:32:00 +0300 Subject: [PATCH 08/11] chore: automated commit --- base/mp_abort.sas | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/mp_abort.sas b/base/mp_abort.sas index f8189b1..8eb3f12 100644 --- a/base/mp_abort.sas +++ b/base/mp_abort.sas @@ -17,8 +17,8 @@ _webout. To assist with this, we also run stpsrvset('program error', 0) and set SYSCC=0. For 9.4m3 we take a unique approach - we open a macro but don't close it! This provides a graceful abort, EXCEPT when called - called within a %include within a macro. See tests/mp_abort.test.1 for an - example case. + called within a %include within a macro. See tests/mp_abort.test.1.sas for + an example case. If you know of another way to gracefully abort a 9.4m3 STP session, we'd love to hear about it! From 44069e9867de9c80a62fe937c539bccde6b9e2b9 Mon Sep 17 00:00:00 2001 From: Allan Bowe Date: Tue, 11 May 2021 20:32:12 +0300 Subject: [PATCH 09/11] chore: automated commit --- all.sas | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/all.sas b/all.sas index b73f606..82568ec 100644 --- a/all.sas +++ b/all.sas @@ -1637,8 +1637,8 @@ Usage: _webout. To assist with this, we also run stpsrvset('program error', 0) and set SYSCC=0. For 9.4m3 we take a unique approach - we open a macro but don't close it! This provides a graceful abort, EXCEPT when called - called within a %include within a macro. See tests/mp_abort.test.1 for an - example case. + called within a %include within a macro. See tests/mp_abort.test.1.sas for + an example case. If you know of another way to gracefully abort a 9.4m3 STP session, we'd love to hear about it! From 28209950ab4d485d68206c0e585cf00c9bd7a2d0 Mon Sep 17 00:00:00 2001 From: Allan Bowe Date: Tue, 11 May 2021 20:49:57 +0300 Subject: [PATCH 10/11] chore: automated commit --- base/mp_abort.sas | 4 ++-- tests/base/mp_abort.test.1.sas | 29 ------------------------ tests/base/mp_abort.test.nofix.sas | 36 ++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 31 deletions(-) delete mode 100644 tests/base/mp_abort.test.1.sas create mode 100644 tests/base/mp_abort.test.nofix.sas diff --git a/base/mp_abort.sas b/base/mp_abort.sas index 8eb3f12..ea4b6f3 100644 --- a/base/mp_abort.sas +++ b/base/mp_abort.sas @@ -17,8 +17,8 @@ _webout. To assist with this, we also run stpsrvset('program error', 0) and set SYSCC=0. For 9.4m3 we take a unique approach - we open a macro but don't close it! This provides a graceful abort, EXCEPT when called - called within a %include within a macro. See tests/mp_abort.test.1.sas for - an example case. + called within a %include within a macro (and that macro contains additional + logic). See mp_abort.test.nofix.sas for the example case. If you know of another way to gracefully abort a 9.4m3 STP session, we'd love to hear about it! diff --git a/tests/base/mp_abort.test.1.sas b/tests/base/mp_abort.test.1.sas deleted file mode 100644 index 85d9bed..0000000 --- a/tests/base/mp_abort.test.1.sas +++ /dev/null @@ -1,29 +0,0 @@ -/** - @file - @brief Testing mp_abort macro - @details This is an unfixed problem with mp_abort. When called from within - a macro, within a %include, and that macro contains subsequent logic, the - service does not end cleanly - rather, we see: - - ERROR: %EVAL function has no expression to evaluate, or %IF statement has no condition. - ERROR: The macro TEST will stop executing. - -

SAS Macros

- @li mp_abort.sas - @li mp_assert.sas - -**/ - -%macro test(); - -filename blah temp; -data _null_; - file blah; - put '%mp_abort();'; -run; -%inc blah; - -%if 1=1 %then %put Houston - we have a problem here; -%mend test; - -%test() \ No newline at end of file diff --git a/tests/base/mp_abort.test.nofix.sas b/tests/base/mp_abort.test.nofix.sas new file mode 100644 index 0000000..c8e40ce --- /dev/null +++ b/tests/base/mp_abort.test.nofix.sas @@ -0,0 +1,36 @@ +/** + @file + @brief Testing mp_abort macro + @details This is an unfixed problem with mp_abort when using the + 'unclosed macro' technique. This is only relevant for 9.4m3 environments, + which can suffer from hung multibridge sessions from %abort and endsas. + + The issue is that when called within a macro, within a %include, AND that + macro contains subsequent logic, the service does not end cleanly - rather, + we see: + + ERROR: %EVAL function has no expression to evaluate, or %IF statement has no condition. + ERROR: The macro TEST will stop executing. + + We are not able to test this without a 9.4m3 environment, it is marked as + nofix. + +

SAS Macros

+ @li mp_abort.sas + @li mp_assert.sas + +**/ + +%macro test(); + +filename blah temp; +data _null_; + file blah; + put '%mp_abort();'; +run; +%inc blah; + +%if 1=1 %then %put Houston - we have a problem here; +%mend test; + +%test() \ No newline at end of file From 2c901831b7682d2eaf5ccb74fa24f2665d32f880 Mon Sep 17 00:00:00 2001 From: Allan Bowe Date: Tue, 11 May 2021 21:11:24 +0300 Subject: [PATCH 11/11] chore: automated commit --- all.sas | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/all.sas b/all.sas index 82568ec..cf6000b 100644 --- a/all.sas +++ b/all.sas @@ -1637,8 +1637,8 @@ Usage: _webout. To assist with this, we also run stpsrvset('program error', 0) and set SYSCC=0. For 9.4m3 we take a unique approach - we open a macro but don't close it! This provides a graceful abort, EXCEPT when called - called within a %include within a macro. See tests/mp_abort.test.1.sas for - an example case. + called within a %include within a macro (and that macro contains additional + logic). See mp_abort.test.nofix.sas for the example case. If you know of another way to gracefully abort a 9.4m3 STP session, we'd love to hear about it!