Merge pull request #25 from sasjs/dcfixes

fix: more logging in mp_abort, fixing job test, better return values …

all.sas

@@ -18,24 +18,12 @@
 options noquotelenmax;
 /**
   @file
-  @brief abort gracefully according to context
+  @brief to be deprecated
-  @details Do not use directly!  See bottom of explanation for details.
+  @details We will deprecate this macro in 2022
 
-  Configures an abort mechanism according to site specific policies or the
+  As you can see, it's not a macro function.
-    particulars of an environment.  For instance, can stream custom
-    results back to the client in an STP Web App context, or completely stop
-    in the case of a batch run.
 
-  For the sharp eyed readers - this is no longer a macro function!! It became
+  Use mp_abort.sas instead.
-  a macro procedure during a project and now it's kinda stuck that way until
-  that project is updated (if it's ever updated).  In the meantime we created
-  `mp_abort` which is just a wrapper for this one, and so we recomend you use
-  that for forwards compatibility reasons.
-
-  @param mac= to contain the name of the calling macro
-  @param type= deprecated.  Not used.
-  @param msg= message to be returned
-  @param iftrue= supply a condition under which the macro should be executed.
 
   @version 9.2
   @author Allan Bowe
@@ -1636,13 +1624,24 @@ Usage:
   @details Configures an abort mechanism according to site specific policies or
     the particulars of an environment.  For instance, can stream custom
     results back to the client in an STP Web App context, or completely stop
-    in the case of a batch run.
+    in the case of a batch run.  For STP sessions
+
+  The method used varies according to the context.  Important points:
+
+  @li should not use endsas or abort cancel in 9.4m3 environments as this can
+    cause hung multibridge sessions and result in a frozen STP server
+  @li should not use endsas in viya 3.5 as this destroys the session and cannot
+    fetch results (although both mv_getjoblog.sas and the @sasjs/adapter will
+    recognise this and fetch the log of the parent session instead)
+  @li STP environments must finish cleanly to avoid the log being sent to
+    _webout.  To assist with this, we also run stpsrvset('program error', 0)
+    and set SYSCC=0.  For 9.4m3 we take a unique approach - we open a macro
+    but don't close it!  This provides a graceful abort, EXCEPT when called
+    called within a %include within a macro (and that macro contains additional
+    logic).  See mp_abort.test.nofix.sas for the example case.
+    If you know of another way to gracefully abort a 9.4m3 STP session, we'd
+    love to hear about it!
 
-  Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored
-  Process environments.  This macro takes a unique approach - we set the SAS
-  syscc to 0, run `stpsrvset('program error', 0)` (if SAS 9) and then - we open
-  a macro but don't close it!  This provides a graceful abort for SAS web
-  services in all web enabled environments.
 
   @param mac= to contain the name of the calling macro
   @param msg= message to be returned
@@ -1656,6 +1655,8 @@ Usage:
 %macro mp_abort(mac=mp_abort.sas, type=, msg=, iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
+  %global sysprocessmode sysprocessname;
+
   %if not(%eval(%unquote(&iftrue))) %then %return;
 
   %put NOTE: ///  mp_abort macro executing //;
@@ -1663,9 +1664,7 @@ Usage:
   %put NOTE - &msg;
 
   /* Stored Process Server web app context */
-  %if %symexist(_metaperson)
+  %if %symexist(_metaperson) or "&SYSPROCESSNAME "="Compute Server " %then %do;
-  or (%symexist(SYSPROCESSNAME) and "&SYSPROCESSNAME"="Compute Server" )
-  %then %do;
     options obs=max replace nosyntaxcheck mprint;
     /* extract log errs / warns, if exist */
     %local logloc logline;
@@ -1760,32 +1759,60 @@ Usage:
       if debug ge '"131"' then put '>>weboutEND<<';
     run;
 
-    %let syscc=0;
+    %put _all_;
-    %if %symexist(_metaport) %then %do;
+
+    %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
       data _null_;
-        if symexist('sysprocessmode')
+        putlog 'stpsrvset program error and syscc';
-          then if symget("sysprocessmode")="SAS Stored Process Server"
+        rc=stpsrvset('program error', 0);
-            then rc=stpsrvset('program error', 0);
+        call symputx("syscc",0,"g");
+      run;
+      %if "%substr(&sysvlong.xxxxxxxxx,1,9)" ne "9.04.01M3" %then %do;
+        %put NOTE: Ending SAS session due to:;
+        %put NOTE- &msg;
+        endsas;
+      %end;
+    %end;
+    %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
+      /* endsas kills the session making it harder to fetch results */
+      data _null_;
+        syswarningtext=symget('syswarningtext');
+        syserrortext=symget('syserrortext');
+        abort_msg=symget('msg');
+        syscc=symget('syscc');
+        sysuserid=symget('sysuserid');
+        iftrue=symget('iftrue');
+        put (_all_)(/=);
+        abort cancel nolist;
       run;
     %end;
+    %else %if "%substr(&sysvlong.xxxxxxxxx,1,9)" = "9.04.01M3" %then %do;
       /**
-      * endsas is reliable but kills some deployments.
+        * endsas kills 9.4m3 deployments by orphaning multibridges.
         * Abort variants are ungraceful (non zero return code)
         * This approach lets SAS run silently until the end :-)
+        * Caution - fails when called within a %include within a macro
+        * See tests/mp_abort.test.1 for an example case.
         */
-    %put _all_;
       filename skip temp;
       data _null_;
         file skip;
-      put '%macro skip(); %macro skippy();';
+        put '%macro skip();';
+        comment '%mend skip; -> fix lint ';
+        put '%macro skippy();';
+        comment '%mend skippy; -> fix lint ';
       run;
       %inc skip;
     %end;
+    %else %do;
+      %abort cancel;
+    %end;
+  %end;
   %else %do;
     %put _all_;
     %abort cancel;
   %end;
-%mend;
+%mend mp_abort;
 
 /** @endcond *//**
   @file
@@ -3397,7 +3424,6 @@ run;
   @li mf_getuniquefileref.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
-  @li mf_nobs.sas
   @li mp_filtergenerate.sas
   @li mp_filtervalidate.sas
 
@@ -3510,18 +3536,23 @@ data &outds;
 
 run;
 
+%local nobs;
+%let nobs=0;
 data _null_;
-  set &outds;
+  set &outds end=last;
+  putlog (_all_)(=);
+  if last then do;
     call symputx('REASON_CD',reason_cd,'l');
-  stop;
+    call symputx('nobs',_n_,'l');
+  end;
 run;
 
-%mp_abort(iftrue=(&abort=YES and %mf_nobs(&outds)>0),
+%mp_abort(iftrue=(&abort=YES and &nobs>0),
   mac=&sysmacroname,
-  msg=%str(Filter issues in &inds, reason: &reason_cd, details in &outds)
+  msg=%str(&nobs filter issues in &inds, reason: &reason_cd, details in &outds)
 )
 
-%if %mf_nobs(&outds)>0 %then %do;
+%if &nobs>0 %then %do;
   %let syscc=1008;
   %return;
 %end;
@@ -10338,9 +10369,8 @@ run;
   such as dangling metadata, embedded passwords, security issues and more.
 
   @param [in] libname= the metadata name of the library to be compared
-  @param [out] outlib= The output library in which to store the output tables.
+  @param [out] outlib=(work) The library in which to store the output tables.
-  Default=WORK.
+  @param [out] prefix=(metadiff) The prefix for the four tables created.
-  @param [out] prefix The prefix for the four tables created. Default=metadiff.
 
   @version 9.3
   @author Allan Bowe
@@ -15062,21 +15092,19 @@ run;
 /**
   @file
   @brief Extract the log from a completed SAS Viya Job
-  @details Extracts log from a Viya job and writes it out to a fileref
+  @details Extracts log from a Viya job and writes it out to a fileref.
 
   To query the job, you need the URI.  Sample code for achieving this
   is provided below.
 
   ## Example
 
-  First, compile the macros:
+      %* First, compile the macros;
-
       filename mc url
         "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
 
-  Next, create a job (in this case, a web service):
+      %* Next, create a job (in this case, a web service);
-
       filename ft15f001 temp;
       parmcards4;
         data ;
@@ -15092,28 +15120,40 @@ run;
       ;;;;
       %mv_createwebservice(path=/Public/temp,name=demo)
 
-  Execute it:
+      %* Execute it;
-
       %mv_jobexecute(path=/Public/temp
         ,name=demo
         ,outds=work.info
       )
 
-  Wait for it to finish, and grab the uri:
+      %* Wait for it to finish;
-
+      data work.info;
-      data _null_;
         set work.info;
-        if method='GET' and rel='self';
+        where method='GET' and rel='state';
+      run;
+      %mv_jobwaitfor(ALL,inds=work.info,outds=work.jobstates)
+
+      %* and grab the uri;
+      data _null_;
+        set work.jobstates;
         call symputx('uri',uri);
       run;
 
-  Finally, fetch the log:
+      %* Finally, fetch the log;
-
       %mv_getjoblog(uri=&uri,outref=mylog)
 
   This macro is used by the mv_jobwaitfor.sas macro, which is generally a more
   convenient way to wait for the job to finish before fetching the log.
 
+  If the remote session calls `endsas` then it is not possible to get the log
+  from the provided uri, and so the log from the parent session is fetched
+  instead.  This happens for a 400 response, eg below:
+
+      ErrorResponse[version=2,status=400,err=5113,id=,message=The session
+      requested is currently in a failed or stopped state.,detail=[path:
+      /compute/sessions/LONGURI-ses0006/jobs/LONGURI/log/content, traceId: 63
+      51aa617d01fd2b],remediation=Correct the errors in the session request,
+      and create a new session.,targetUri=<null>,errors=[],links=[]]
 
   @param [in] access_token_var= The global macro variable to contain the access
     token
@@ -15125,7 +15165,7 @@ run;
       if a SASStudioV session else authorization_code.  Default option.
     @li sas_services - will use oauth_bearer=sas_services.
   @param [in] uri= The uri of the running job for which to fetch the status,
-    in the format `/jobExecution/jobs/$UUID/state` (unquoted).
+    in the format `/jobExecution/jobs/$UUID` (unquoted).
   @param [out] outref= The output fileref to which to APPEND the log (is always
   appended).
 
@@ -15183,7 +15223,7 @@ data _null_;
     call symputx('errflg',1);
     call symputx('errmsg',
       "URI should be in format /jobExecution/jobs/$$$$UUID$$$$"
-      !!" but is actually like: &uri",'l');
+      !!" but is actually like:"!!uri,'l');
   end;
 run;
 
@@ -15216,6 +15256,10 @@ proc http method='GET' out=&fname1 &oauth_bearer
   %end;
   ;
 run;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname: fetching log loc from &uri;
+  data _null_;infile &fname1;input;putlog _infile_;run;
+%end;
 %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
 %do;
   data _null_;infile &fname1;input;putlog _infile_;run;
@@ -15253,43 +15297,74 @@ run;
 data _null_;
   infile &fname2;
   input;
-  uri=_infile_;
+  uri=cats(_infile_);
   if length(uri)<12 then do;
     call symputx('errflg',1);
     call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
   end;
-  if scan(uri,1) ne 'files' or scan(uri,2) ne 'files' then do;
+  else if (scan(uri,1,'/') ne 'compute' or scan(uri,2,'/') ne 'sessions')
+    and (scan(uri,1,'/') ne 'files' or scan(uri,2,'/') ne 'files')
+  then do;
     call symputx('errflg',1);
     call symputx('errmsg',
-      "URI should be in format /files/files/$$$$UUID$$$$"
+      "URI should be in format /compute/sessions/$$$$UUID$$$$/jobs/$$$$UUID$$$$"
-      !!" but is actually like: &uri",'l');
+      !!" or /files/files/$$$$UUID$$$$"
+      !!" but is actually like:"!!uri,'l');
   end;
+  else do;
     call symputx('errflg',0,'l');
     call symputx('logloc',uri,'l');
+  end;
 run;
 
-%mp_abort(iftrue=(&errflg=1)
+%mp_abort(iftrue=(%str(&errflg)=1)
   ,mac=&sysmacroname
   ,msg=%str(&errmsg)
 )
 
 /* we have a log uri - now fetch the log */
+%&dbg.put &sysmacroname: querying &base_uri&logloc/content;
 proc http method='GET' out=&fname1 &oauth_bearer
-  url="&base_uri&logloc/content";
+  url="&base_uri&logloc/content?limit=10000";
   headers
   %if &grant_type=authorization_code %then %do;
           "Authorization"="Bearer &&&access_token_var"
   %end;
   ;
 run;
-%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+
-%do;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname: fetching log content from &base_uri&logloc/content;
   data _null_;infile &fname1;input;putlog _infile_;run;
+%end;
+
+%if &SYS_PROCHTTP_STATUS_CODE=400 %then %do;
+  /* fetch log from parent session */
+  %let logloc=%substr(&logloc,1,%index(&logloc,%str(/jobs/))-1);
+  %&dbg.put &sysmacroname: Now querying &base_uri&logloc/log/content;
+  proc http method='GET' out=&fname1 &oauth_bearer
+    url="&base_uri&logloc/log/content?limit=10000";
+    headers
+    %if &grant_type=authorization_code %then %do;
+            "Authorization"="Bearer &&&access_token_var"
+    %end;
+    ;
+  run;
+  %if &mdebug=1 %then %do;
+    %put &sysmacroname: fetching log content from &base_uri&logloc/log/content;
+    data _null_;infile &fname1;input;putlog _infile_;run;
+  %end;
+%end;
+
+%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201
+%then %do;
+  %if &mdebug ne 1 %then %do; /* have already output above */
+    data _null_;infile &fname1;input;putlog _infile_;run;
+  %end;
   %mp_abort(mac=&sysmacroname
     ,msg=%str(logfetch: &SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
   )
 %end;
-
 data _null_;
   file "&fpath3..lua";
   put '

base/mf_abort.sas

@@ -1,23 +1,11 @@
 /**
   @file
-  @brief abort gracefully according to context
+  @brief to be deprecated
-  @details Do not use directly!  See bottom of explanation for details.
+  @details We will deprecate this macro in 2022
 
-  Configures an abort mechanism according to site specific policies or the
+  As you can see, it's not a macro function.
-    particulars of an environment.  For instance, can stream custom
-    results back to the client in an STP Web App context, or completely stop
-    in the case of a batch run.
 
-  For the sharp eyed readers - this is no longer a macro function!! It became
+  Use mp_abort.sas instead.
-  a macro procedure during a project and now it's kinda stuck that way until
-  that project is updated (if it's ever updated).  In the meantime we created
-  `mp_abort` which is just a wrapper for this one, and so we recomend you use
-  that for forwards compatibility reasons.
-
-  @param mac= to contain the name of the calling macro
-  @param type= deprecated.  Not used.
-  @param msg= message to be returned
-  @param iftrue= supply a condition under which the macro should be executed.
 
   @version 9.2
   @author Allan Bowe

base/mp_abort.sas

@@ -4,13 +4,24 @@
   @details Configures an abort mechanism according to site specific policies or
     the particulars of an environment.  For instance, can stream custom
     results back to the client in an STP Web App context, or completely stop
-    in the case of a batch run.
+    in the case of a batch run.  For STP sessions
+
+  The method used varies according to the context.  Important points:
+
+  @li should not use endsas or abort cancel in 9.4m3 environments as this can
+    cause hung multibridge sessions and result in a frozen STP server
+  @li should not use endsas in viya 3.5 as this destroys the session and cannot
+    fetch results (although both mv_getjoblog.sas and the @sasjs/adapter will
+    recognise this and fetch the log of the parent session instead)
+  @li STP environments must finish cleanly to avoid the log being sent to
+    _webout.  To assist with this, we also run stpsrvset('program error', 0)
+    and set SYSCC=0.  For 9.4m3 we take a unique approach - we open a macro
+    but don't close it!  This provides a graceful abort, EXCEPT when called
+    called within a %include within a macro (and that macro contains additional
+    logic).  See mp_abort.test.nofix.sas for the example case.
+    If you know of another way to gracefully abort a 9.4m3 STP session, we'd
+    love to hear about it!
 
-  Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored
-  Process environments.  This macro takes a unique approach - we set the SAS
-  syscc to 0, run `stpsrvset('program error', 0)` (if SAS 9) and then - we open
-  a macro but don't close it!  This provides a graceful abort for SAS web
-  services in all web enabled environments.
 
   @param mac= to contain the name of the calling macro
   @param msg= message to be returned
@@ -24,6 +35,8 @@
 %macro mp_abort(mac=mp_abort.sas, type=, msg=, iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
+  %global sysprocessmode sysprocessname;
+
   %if not(%eval(%unquote(&iftrue))) %then %return;
 
   %put NOTE: ///  mp_abort macro executing //;
@@ -31,9 +44,7 @@
   %put NOTE - &msg;
 
   /* Stored Process Server web app context */
-  %if %symexist(_metaperson)
+  %if %symexist(_metaperson) or "&SYSPROCESSNAME "="Compute Server " %then %do;
-  or (%symexist(SYSPROCESSNAME) and "&SYSPROCESSNAME"="Compute Server" )
-  %then %do;
     options obs=max replace nosyntaxcheck mprint;
     /* extract log errs / warns, if exist */
     %local logloc logline;
@@ -63,7 +74,7 @@
           input;
           i=1;
           stoploop=0;
-          if _n_ ge &logline-5 and stoploop=0 then do until (i>12);
+          if _n_ ge &logline-15 and stoploop=0 then do until (i>22);
             call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
             input;
             i+1;
@@ -128,32 +139,59 @@
       if debug ge '"131"' then put '>>weboutEND<<';
     run;
 
-    %if %symexist(_metaport) %then %do;
+    %put _all_;
+
+    %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
       data _null_;
-        if symexist('sysprocessmode') then
+        putlog 'stpsrvset program error and syscc';
-          if symget("sysprocessmode")="SAS Stored Process Server" then do;
         rc=stpsrvset('program error', 0);
         call symputx("syscc",0,"g");
-          end;
+      run;
+      %if "%substr(&sysvlong.xxxxxxxxx,1,9)" ne "9.04.01M3" %then %do;
+        %put NOTE: Ending SAS session due to:;
+        %put NOTE- &msg;
+        endsas;
+      %end;
+    %end;
+    %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
+      /* endsas kills the session making it harder to fetch results */
+      data _null_;
+        syswarningtext=symget('syswarningtext');
+        syserrortext=symget('syserrortext');
+        abort_msg=symget('msg');
+        syscc=symget('syscc');
+        sysuserid=symget('sysuserid');
+        iftrue=symget('iftrue');
+        put (_all_)(/=);
+        abort cancel nolist;
       run;
     %end;
+    %else %if "%substr(&sysvlong.xxxxxxxxx,1,9)" = "9.04.01M3" %then %do;
       /**
-      * endsas is reliable but kills some deployments.
+        * endsas kills 9.4m3 deployments by orphaning multibridges.
         * Abort variants are ungraceful (non zero return code)
         * This approach lets SAS run silently until the end :-)
+        * Caution - fails when called within a %include within a macro
+        * See tests/mp_abort.test.1 for an example case.
         */
-    %put _all_;
       filename skip temp;
       data _null_;
         file skip;
-      put '%macro skip(); %macro skippy();';
+        put '%macro skip();';
+        comment '%mend skip; -> fix lint ';
+        put '%macro skippy();';
+        comment '%mend skippy; -> fix lint ';
       run;
       %inc skip;
     %end;
+    %else %do;
+      %abort cancel;
+    %end;
+  %end;
   %else %do;
     %put _all_;
     %abort cancel;
   %end;
-%mend;
+%mend mp_abort;
 
 /** @endcond */

base/mp_filtercheck.sas

@@ -33,7 +33,8 @@
   @returns The &outds table containing any bad rows, plus a REASON_CD column.
 
   @param [in] inds The table to be checked, with the format above
-  @param [in] targetds= The target dataset against which to verify VARIABLE_NM
+  @param [in] targetds= The target dataset against which to verify VARIABLE_NM.
+    This must be available (ie, the library must be assigned).
   @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
   @param [out] outds= The output table, which is a copy of the &inds. table
     plus a REASON_CD column, containing only bad records.  If bad records found,
@@ -45,7 +46,6 @@
   @li mf_getuniquefileref.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
-  @li mf_nobs.sas
   @li mp_filtergenerate.sas
   @li mp_filtervalidate.sas
 
@@ -85,41 +85,52 @@
   * quotes, commas, periods and spaces.
   * Only numeric values should remain
   */
-%local reason_cd;
+%local reason_cd nobs;
+%let nobs=0;
 data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
-  length reason_cd $32;
+  length reason_cd $4032;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
-    REASON_CD='GROUP_LOGIC should be either AND or OR';
+    REASON_CD='GROUP_LOGIC should be AND/OR, not:'!!cats(GROUP_LOGIC);
     putlog REASON_CD= GROUP_LOGIC=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if SUBGROUP_LOGIC not in ('AND','OR') then do;
-    REASON_CD='SUBGROUP_LOGIC should be either AND or OR';
+    REASON_CD='SUBGROUP_LOGIC should be AND/OR, not:'!!cats(SUBGROUP_LOGIC);
     putlog REASON_CD= SUBGROUP_LOGIC=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if mod(SUBGROUP_ID,1) ne 0 then do;
-    REASON_CD='SUBGROUP_ID should be integer';
+    REASON_CD='SUBGROUP_ID should be integer, not '!!left(subgroup_id);
     putlog REASON_CD= SUBGROUP_ID=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if upcase(VARIABLE_NM) not in
   (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
   then do;
-    REASON_CD="VARIABLE_NM not in &targetds";
+    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
     putlog REASON_CD= VARIABLE_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
   if OPERATOR_NM not in
   ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
   then do;
-    REASON_CD='Invalid OPERATOR_NM';
+    REASON_CD='Invalid OPERATOR_NM: '!!left(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
 
@@ -129,8 +140,10 @@ data &outds;
     if substr(raw_value,1,1) ne '('
     or substr(cats(reverse(raw_value)),1,1) ne ')'
     then do;
-      REASON_CD='Missing brackets in RAW_VALUE';
+      REASON_CD='Missing start/end bracket in RAW_VALUE';
       putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
       output;
     end;
     else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
@@ -151,25 +164,27 @@ data &outds;
   /* output records that contain values other than digits and spaces */
   if notdigit(compress(raw_value3,' '))>0 then do;
     putlog raw_value3= $hex32.;
-    REASON_CD='Invalid RAW_VALUE';
+    REASON_CD=cats('Invalid RAW_VALUE:',raw_value);
     putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
     output;
   end;
 
 run;
 
+
 data _null_;
-  set &outds;
+  set &outds end=last;
-  call symputx('REASON_CD',reason_cd,'l');
+  putlog (_all_)(=);
-  stop;
 run;
 
-%mp_abort(iftrue=(&abort=YES and %mf_nobs(&outds)>0),
+%mp_abort(iftrue=(&abort=YES and &nobs>0),
   mac=&sysmacroname,
-  msg=%str(Filter issues in &inds, reason: &reason_cd, details in &outds)
+  msg=%str(Data issue: %superq(reason_cd))
 )
 
-%if %mf_nobs(&outds)>0 %then %do;
+%if &nobs>0 %then %do;
   %let syscc=1008;
   %return;
 %end;

base/mp_filtervalidate.sas

@@ -95,7 +95,8 @@ filename &fref1 clear;
     run;
     %mp_abort(
       mac=&sysmacroname,
-      msg=%str(Filter issues in &inref: %quote(&reason_cd))
+      msg=%str(Filter validation issues. ERR=%superq(SYSERRORTEXT)
+        , WARN=%superq(SYSWARNINGTEXT) )
     )
   %end;
   %let syscc=1008;

base/mp_jsonout.sas

@@ -61,7 +61,7 @@
 )/*/STORE SOURCE*/;
 %put output location=&jref;
 %if &action=OPEN %then %do;
-  OPTIONS NOBOMFILE;
+  options nobomfile;
   data _null_;file &jref encoding='utf-8';
     put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
   run;

meta/mm_getlibmetadiffs.sas

@@ -80,9 +80,8 @@
   such as dangling metadata, embedded passwords, security issues and more.
 
   @param [in] libname= the metadata name of the library to be compared
-  @param [out] outlib= The output library in which to store the output tables.
+  @param [out] outlib=(work) The library in which to store the output tables.
-  Default=WORK.
+  @param [out] prefix=(metadiff) The prefix for the four tables created.
-  @param [out] prefix The prefix for the four tables created. Default=metadiff.
 
   @version 9.3
   @author Allan Bowe

tests/base/mp_abort.test.nofix.sas

@@ -0,0 +1,36 @@
+/**
+  @file
+  @brief Testing mp_abort macro
+  @details This is an unfixed problem with mp_abort when using the
+  'unclosed macro' technique.  This is only relevant for 9.4m3 environments,
+  which can suffer from hung multibridge sessions from %abort and endsas.
+
+  The issue is that when called within a macro, within a %include, AND that
+  macro contains subsequent logic, the service does not end cleanly - rather,
+  we see:
+
+      ERROR: %EVAL function has no expression to evaluate, or %IF statement has no condition.
+      ERROR: The macro TEST will stop executing.
+
+  We are not able to test this without a 9.4m3 environment, it is marked as
+  nofix.
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_assert.sas
+
+**/
+
+%macro test();
+
+filename blah temp;
+data _null_;
+  file blah;
+  put '%mp_abort();';
+run;
+%inc blah;
+
+%if 1=1 %then %put Houston - we have a problem here;
+%mend test;
+
+%test()

tests/viya/mv_getjoblog.test.sas

@@ -0,0 +1,70 @@
+/**
+  @file
+  @brief Testing mv_createwebservice macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mv_createjob.sas
+  @li mv_jobexecute.sas
+  @li mv_jobwaitfor.sas
+  @li mv_getjoblog.sas
+
+**/
+
+/**
+  * Test Case 1
+  */
+
+/* create a service */
+filename testref temp;
+data _null_;
+  file testref;
+  put 'data;run;';
+  put 'endsas;';
+run;
+%mv_createjob(
+  path=&mcTestAppLoc/jobs/temp,
+  code=testref,
+  name=testjob
+)
+
+%* Execute it;
+%mv_jobexecute(
+  path=&mcTestAppLoc/jobs/temp,
+  name=testjob,
+  outds=work.info
+)
+
+%* Wait for it to finish;
+data work.info;
+  set work.info;
+  where method='GET' and rel='state';
+run;
+%mv_jobwaitfor(ALL,inds=work.info,outds=work.jobstates)
+
+%* and grab the uri;
+data _null_;
+  set work.jobstates;
+  call symputx('uri',uri);
+run;
+
+%* Finally, fetch the log;
+%mv_getjoblog(uri=%str(&uri),outref=mylog)
+
+
+data _null_;
+  infile mylog end=eof;
+  input;
+  putlog _infile_;
+  retain found 0;
+  if index(_infile_,'endsas;') then do;
+    found=1;
+    call symputx('found',found);
+  end;
+  else if eof and found ne 1 then call symputx('found',0);
+run;
+
+%mp_assert(
+  iftrue=(%str(&found)=1),
+  desc=Check if the log was still fetched even though endsas was submitted
+)

viya/mv_getjoblog.sas

@@ -1,21 +1,19 @@
 /**
   @file
   @brief Extract the log from a completed SAS Viya Job
-  @details Extracts log from a Viya job and writes it out to a fileref
+  @details Extracts log from a Viya job and writes it out to a fileref.
 
   To query the job, you need the URI.  Sample code for achieving this
   is provided below.
 
   ## Example
 
-  First, compile the macros:
+      %* First, compile the macros;
-
       filename mc url
         "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
 
-  Next, create a job (in this case, a web service):
+      %* Next, create a job (in this case, a web service);
-
       filename ft15f001 temp;
       parmcards4;
         data ;
@@ -31,28 +29,40 @@
       ;;;;
       %mv_createwebservice(path=/Public/temp,name=demo)
 
-  Execute it:
+      %* Execute it;
-
       %mv_jobexecute(path=/Public/temp
         ,name=demo
         ,outds=work.info
       )
 
-  Wait for it to finish, and grab the uri:
+      %* Wait for it to finish;
-
+      data work.info;
-      data _null_;
         set work.info;
-        if method='GET' and rel='self';
+        where method='GET' and rel='state';
+      run;
+      %mv_jobwaitfor(ALL,inds=work.info,outds=work.jobstates)
+
+      %* and grab the uri;
+      data _null_;
+        set work.jobstates;
         call symputx('uri',uri);
       run;
 
-  Finally, fetch the log:
+      %* Finally, fetch the log;
-
       %mv_getjoblog(uri=&uri,outref=mylog)
 
   This macro is used by the mv_jobwaitfor.sas macro, which is generally a more
   convenient way to wait for the job to finish before fetching the log.
 
+  If the remote session calls `endsas` then it is not possible to get the log
+  from the provided uri, and so the log from the parent session is fetched
+  instead.  This happens for a 400 response, eg below:
+
+      ErrorResponse[version=2,status=400,err=5113,id=,message=The session
+      requested is currently in a failed or stopped state.,detail=[path:
+      /compute/sessions/LONGURI-ses0006/jobs/LONGURI/log/content, traceId: 63
+      51aa617d01fd2b],remediation=Correct the errors in the session request,
+      and create a new session.,targetUri=<null>,errors=[],links=[]]
 
   @param [in] access_token_var= The global macro variable to contain the access
     token
@@ -64,7 +74,7 @@
       if a SASStudioV session else authorization_code.  Default option.
     @li sas_services - will use oauth_bearer=sas_services.
   @param [in] uri= The uri of the running job for which to fetch the status,
-    in the format `/jobExecution/jobs/$UUID/state` (unquoted).
+    in the format `/jobExecution/jobs/$UUID` (unquoted).
   @param [out] outref= The output fileref to which to APPEND the log (is always
   appended).
 
@@ -122,7 +132,7 @@ data _null_;
     call symputx('errflg',1);
     call symputx('errmsg',
       "URI should be in format /jobExecution/jobs/$$$$UUID$$$$"
-      !!" but is actually like: &uri",'l');
+      !!" but is actually like:"!!uri,'l');
   end;
 run;
 
@@ -155,6 +165,10 @@ proc http method='GET' out=&fname1 &oauth_bearer
   %end;
   ;
 run;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname: fetching log loc from &uri;
+  data _null_;infile &fname1;input;putlog _infile_;run;
+%end;
 %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
 %do;
   data _null_;infile &fname1;input;putlog _infile_;run;
@@ -192,43 +206,74 @@ run;
 data _null_;
   infile &fname2;
   input;
-  uri=_infile_;
+  uri=cats(_infile_);
   if length(uri)<12 then do;
     call symputx('errflg',1);
     call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
   end;
-  if scan(uri,1) ne 'files' or scan(uri,2) ne 'files' then do;
+  else if (scan(uri,1,'/') ne 'compute' or scan(uri,2,'/') ne 'sessions')
+    and (scan(uri,1,'/') ne 'files' or scan(uri,2,'/') ne 'files')
+  then do;
     call symputx('errflg',1);
     call symputx('errmsg',
-      "URI should be in format /files/files/$$$$UUID$$$$"
+      "URI should be in format /compute/sessions/$$$$UUID$$$$/jobs/$$$$UUID$$$$"
-      !!" but is actually like: &uri",'l');
+      !!" or /files/files/$$$$UUID$$$$"
+      !!" but is actually like:"!!uri,'l');
   end;
+  else do;
     call symputx('errflg',0,'l');
     call symputx('logloc',uri,'l');
+  end;
 run;
 
-%mp_abort(iftrue=(&errflg=1)
+%mp_abort(iftrue=(%str(&errflg)=1)
   ,mac=&sysmacroname
   ,msg=%str(&errmsg)
 )
 
 /* we have a log uri - now fetch the log */
+%&dbg.put &sysmacroname: querying &base_uri&logloc/content;
 proc http method='GET' out=&fname1 &oauth_bearer
-  url="&base_uri&logloc/content";
+  url="&base_uri&logloc/content?limit=10000";
   headers
   %if &grant_type=authorization_code %then %do;
           "Authorization"="Bearer &&&access_token_var"
   %end;
   ;
 run;
-%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+
-%do;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname: fetching log content from &base_uri&logloc/content;
   data _null_;infile &fname1;input;putlog _infile_;run;
+%end;
+
+%if &SYS_PROCHTTP_STATUS_CODE=400 %then %do;
+  /* fetch log from parent session */
+  %let logloc=%substr(&logloc,1,%index(&logloc,%str(/jobs/))-1);
+  %&dbg.put &sysmacroname: Now querying &base_uri&logloc/log/content;
+  proc http method='GET' out=&fname1 &oauth_bearer
+    url="&base_uri&logloc/log/content?limit=10000";
+    headers
+    %if &grant_type=authorization_code %then %do;
+            "Authorization"="Bearer &&&access_token_var"
+    %end;
+    ;
+  run;
+  %if &mdebug=1 %then %do;
+    %put &sysmacroname: fetching log content from &base_uri&logloc/log/content;
+    data _null_;infile &fname1;input;putlog _infile_;run;
+  %end;
+%end;
+
+%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201
+%then %do;
+  %if &mdebug ne 1 %then %do; /* have already output above */
+    data _null_;infile &fname1;input;putlog _infile_;run;
+  %end;
   %mp_abort(mac=&sysmacroname
     ,msg=%str(logfetch: &SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
   )
 %end;
-
 data _null_;
   file "&fpath3..lua";
   put '