Merge pull request #17 from sasjs/jobresult

feat: new mv_getjobresult.sas macro, corresponding test, and additional fixes

base/mp_filtercheck.sas

@@ -70,7 +70,7 @@
   * quotes, commas, periods and spaces.
   * Only numeric values should remain
   */
-
+%local reason_cd;
 data &outds;
   set &inds;
   length reason_cd $32;
@@ -147,9 +147,9 @@ data _null_;
   stop;
 run;
 
-%mp_abort(iftrue=(&abort=YES),
+%mp_abort(iftrue=(&abort=YES and %mf_nobs(&outds)>0),
   mac=&sysmacroname,
-  msg=%str(Filter issues in &inds, first was &reason_cd, details in &outds)
+  msg=%str(Filter issues in &inds, reason: &reason_cd, details in &outds)
 )
 
 %if %mf_nobs(&outds)>0 %then %do;

base/mp_hashdataset.sas

@@ -56,7 +56,7 @@
       retain &prevkeyvar;
       set &libds end=&lastvar;
       /* hash should include previous row */
-      if _n_>1 then &keyvar=put(md5(&prevkeyvar
+      &keyvar=put(md5(&prevkeyvar
       /* loop every column, hashing every individual value */
     %do i=1 %to %sysfunc(countw(&varlist));
       %let var=%scan(&varlist,&i,%str( ));

base/mp_testservice.sas

@@ -0,0 +1,189 @@
+/**
+  @file mp_testservice.sas
+  @brief Will execute a test against a SASjs web service on SAS 9 or Viya
+  @details Prepares the input files and retrieves the resulting datasets from
+  the response JSON.
+
+      %mp_testjob(
+        duration=60*5
+      )
+
+  Note - the _webout fileref should NOT be assigned prior to running this macro.
+
+  @param [in] program The _PROGRAM endpoint to test
+  @param [in] inputfiles=(0) A list of space seperated fileref:filename pairs as
+    follows:
+        inputfiles=inref:filename inref2:filename2
+  @param [in] inputparams=(0) A dataset containing name/value pairs in the
+    following format:
+    |name:$32|value:$1000|
+    |---|---|
+    |stpmacname|some value|
+    |mustbevalidname|can be anything, oops, %abort!!|
+
+  @param [in] debug= (log) Provide the _debug value
+  @param [out] outlib= (0) Output libref to contain the final tables.  Set to
+    0 if the service output is not in JSON format.
+  @param [out] outref= (0) Output fileref to create, to contain the full _webout
+    response.
+
+  <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+  @li mv_getjobresult.sas
+  @li mv_jobflow.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mp_testservice(program,
+  inputfiles=0,
+  inputparams=0,
+  debug=log,
+  outlib=0,
+  outref=0
+)/*/STORE SOURCE*/;
+
+/* sanitise inputparams */
+%local pcnt;
+%let pcnt=0;
+%if &inputparams ne 0 %then %do;
+  data _null_;
+    set &inputparams;
+    if not nvalid(name,'v7') then putlog (_all_)(=);
+    else if name in (
+      'program','inputfiles','inputparams','debug','outlib','outref'
+    ) then putlog (_all_)(=);
+    else do;
+      x+1;
+      call symputx(name,quote(cats(value)),'l');
+      call symputx('pval'!!left(x),name,'l');
+      call symputx('pcnt',x,'l');
+    end;
+  run;
+  %mp_abort(iftrue= (%mf_nobs(&inputparams) ne &pcnt)
+    ,mac=&sysmacroname
+    ,msg=%str(Invalid values in &inputparams)
+  )
+%end;
+
+/* parse the input files */
+%local webcount i var;
+%if %quote(&inputfiles) ne 0 %then %do;
+  %let webcount=%sysfunc(countw(&inputfiles));
+  %put &=webcount;
+  %do i=1 %to &webcount;
+    %let var=%scan(&inputfiles,&i,%str( ));
+    %local webfref&i webname&i;
+    %let webref&i=%scan(&var,1,%str(:));
+    %let webname&i=%scan(&var,2,%str(:));
+    %put webref&i=&&webref&i;
+    %put webname&i=&&webname&i;
+  %end;
+%end;
+%else %let webcount=0;
+
+
+%local fref1 webref;
+%let fref1=%mf_getuniquefileref();
+%let webref=%mf_getuniquefileref();
+
+%local platform;
+%let platform=%mf_getplatform();
+%if &platform=SASMETA %then %do;
+  proc stp program="&program";
+    inputparam _program="&program"
+  %do i=1 %to &webcount;
+    %if &webcount=1 %then %do;
+      _webin_fileref="&&webref&i"
+      _webin_name="&&webname&i"
+    %end;
+    %else %do;
+      _webin_fileref&i="&&webref&i"
+      _webin_name&i="&&webname&i"
+    %end;
+  %end;
+    _webin_file_count="&webcount"
+    _debug="&debug"
+  %do i=1 %to &pcnt;
+    /* resolve name only, proc stp fetches value */
+    &&pval&i=&&&&&&pval&i
+  %end;
+    ;
+  %do i=1 %to &webcount;
+    inputfile &&webref&i;
+  %end;
+    outputfile _webout=&webref;
+  run;
+
+  data _null_;
+    infile &webref;
+    file &fref1;
+    input;
+    length line $10000;
+    if index(_infile_,'>>weboutBEGIN<<') then do;
+        line=tranwrd(_infile_,'>>weboutBEGIN<<','');
+        put line;
+    end;
+    else if index(_infile_,'>>weboutEND<<') then do;
+        line=tranwrd(_infile_,'>>weboutEND<<','');
+        put line;
+        stop;
+    end;
+    else put _infile_;
+  run;
+  data _null_;
+    infile &fref1;
+    input;
+    put _infile_;
+  run;
+  %if &outlib ne 0 %then %do;
+    libname &outlib json (&fref1);
+  %end;
+  %if &outref ne 0 %then %do;
+    filename &outref temp;
+    %mp_binarycopy(inref=&webref,outref=&outref)
+  %end;
+
+%end;
+%else %if &platform=SASVIYA %then %do;
+  data ;
+    _program="&program";
+  run;
+
+  %mv_jobflow(inds=&syslast
+    ,maxconcurrency=1
+    ,outds=work.results
+    ,outref=&fref1
+  )
+  /* show the log */
+  data _null_;
+    infile &fref1;
+    input;
+    putlog _infile_;
+  run;
+  /* get the uri to fetch results */
+  data _null_;
+    set work.results;
+    call symputx('uri',uri);
+  run;
+  /* fetch results from webout.json */
+  %mv_getjobresult(uri=&uri,
+    result=WEBOUT_JSON,
+    outref=&outref,
+    outlib=&outlib
+  )
+
+%end;
+%else %do;
+  %put %str(ERR)OR: Unrecognised platform:  &platform;
+%end;
+
+filename &webref clear;
+
+%mend;

tests/testinit.sas

@@ -5,4 +5,4 @@
 **/
 
 /* location in metadata or SAS Drive for temporary files */
-%let mcTestAppLoc=/Public/temp/test;
+%let mcTestAppLoc=/Public/temp/macrocore;

tests/viya/mv_createwebservice.test.sas

@@ -19,23 +19,27 @@ data _null_;
   put '01'x;
 run;
 %mv_createwebservice(
-  path=&mcTestAppLoc/tests/macros,
+  path=&mcTestAppLoc/temp/macros,
   code=testref,
   name=mv_createwebservice
 )
 
 filename compare temp;
 %mv_getjobcode(
-  path=&mcTestAppLoc/tests/macros
+  path=&mcTestAppLoc/temp/macros
   ,name=mv_createwebservice
   ,outref=compare;
 )
 
 data test_results;
   length test_description $256 test_result $4 test_comments $256;
-  infile compare;
+  infile compare end=eof;
   input;
-  if _infile_='01'x then test_result='PASS';
-  else test_result='FAIL';
-  test_description="Creating web service with invisible character";
+  if eof then do;
+    if _infile_='01'x then test_result='PASS';
+    else test_result='FAIL';
+    test_description="Creating web service with invisible character";
+    output;
+    stop;
+  end;
 run;

tests/viya/mv_getjobresult.test.sas

@@ -0,0 +1,74 @@
+/**
+  @file
+  @brief Testing mv_createwebservice macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assertdsobs.sas
+  @li mv_createwebservice.sas
+  @li mv_getjobresult.sas
+  @li mv_jobflow.sas
+
+**/
+
+/**
+  * Test Case 1
+  */
+
+/* create a service */
+filename testref temp;
+data _null_;
+  file testref;
+  put 'data test; set sashelp.class;run;';
+  put '%webout(OPEN)';
+  put '%webout(OBJ,test)';
+  put '%webout(CLOSE)';
+run;
+%mv_createwebservice(
+  path=&mcTestAppLoc/services/temp,
+  code=testref,
+  name=testsvc
+)
+
+/* trigger and wait for it to finish */
+data work.inputjobs;
+  _program="&mcTestAppLoc/services/temp/testsvc";
+run;
+%mv_jobflow(inds=work.inputjobs
+  ,maxconcurrency=4
+  ,outds=work.results
+  ,outref=myjoblog
+)
+/* stream the log */
+data _null_;
+  infile myjoblog;
+  input;
+  put _infile_;
+run;
+
+/* fetch the uri */
+data _null_;
+  set work.results;
+  call symputx('uri',uri);
+  put (_all_)(=);
+run;
+
+/* now get the results */
+%mv_getjobresult(uri=&uri
+  ,result=WEBOUT_JSON
+  ,outref=myweb
+  ,outlib=myweblib
+)
+data _null_;
+  infile myweb;
+  input;
+  putlog _infile_;
+run;
+data work.out;
+  set myweblib.test;
+  put (_all_)(=);
+run;
+%mp_assertdsobs(work.out,
+  desc=Test1 - 19 obs from sashelp.class in service result,
+  test=EQUALS 19,
+  outds=work.test_results
+)

viya/mv_getjoblog.sas

@@ -54,13 +54,14 @@
   convenient way to wait for the job to finish before fetching the log.
 
 
-  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] access_token_var= The global macro variable to contain the access
+    token
   @param [in] mdebug= set to 1 to enable DEBUG messages
   @param [in] grant_type= valid values:
     @li password
     @li authorization_code
-    @li detect - will check if access_token exists, if not will use sas_services if
-        a SASStudioV session else authorization_code.  Default option.
+    @li detect - will check if access_token exists, if not will use sas_services
+      if a SASStudioV session else authorization_code.  Default option.
     @li sas_services - will use oauth_bearer=sas_services.
   @param [in] uri= The uri of the running job for which to fetch the status,
     in the format `/jobExecution/jobs/$UUID/state` (unquoted).

viya/mv_getjobresult.sas

@@ -0,0 +1,207 @@
+/**
+  @file
+  @brief Extract the result from a completed SAS Viya Job
+  @details Extracts result from a Viya job and writes it out to a fileref
+  and/or a JSON-engine library.
+
+  To query the job, you need the URI.  Sample code for achieving this
+  is provided below.
+
+  ## Example
+
+  First, compile the macros:
+
+      filename mc url
+        "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+  Next, create a job (in this case, a web service):
+
+      filename ft15f001 temp;
+      parmcards4;
+        data test;
+          rand=ranuni(0)*1000;
+          do x=1 to rand;
+            y=rand*4;
+            output;
+          end;
+        run;
+        proc sort data=&syslast
+          by descending y;
+        run;
+        %webout(OPEN)
+        %webout(OBJ, test)
+        %webout(CLOSE)
+      ;;;;
+      %mv_createwebservice(path=/Public/temp,name=demo)
+
+  Execute it:
+
+      %mv_jobexecute(path=/Public/temp
+        ,name=demo
+        ,outds=work.info
+      )
+
+  Wait for it to finish, and grab the uri:
+
+      data _null_;
+        set work.info;
+        if method='GET' and rel='self';
+        call symputx('uri',uri);
+      run;
+
+  Finally, fetch the result (In this case, WEBOUT):
+
+      %mv_getjobresult(uri=&uri,result=WEBOUT_JSON,outref=myweb,outlib=myweblib)
+
+
+  @param [in] access_token_var= The global macro variable containing the access
+    token
+  @param [in] mdebug= set to 1 to enable DEBUG messages
+  @param [in] grant_type= valid values:
+    @li password
+    @li authorization_code
+    @li detect - will check if access_token exists, if not will use sas_services
+        if a SASStudioV session else authorization_code.  Default option.
+    @li sas_services - will use oauth_bearer=sas_services.
+  @param [in] uri= The uri of the running job for which to fetch the status,
+    in the format `/jobExecution/jobs/$UUID` (unquoted).
+
+  @param [out] result= (WEBOUT_JSON) The result type to capture.  Resolves
+  to "_[column name]" from the results table when parsed with the JSON libname
+  engine.
+
+  @param [out] outref= (0) The output fileref to which to write the results
+  @param [out] outlib= (0) The output library to which to assign the results
+    (assumes the data is in JSON format)
+
+
+  @version VIYA V.03.05
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+  @li mf_getplatform.sas
+  @li mf_existfileref.sas
+
+**/
+
+%macro mv_getjobresult(uri=0
+    ,contextName=SAS Job Execution compute context
+    ,access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+    ,mdebug=0
+    ,result=WEBOUT_JSON
+    ,outref=0
+    ,outlib=0
+  );
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+
+
+/* validation in datastep for better character safety */
+%local errmsg errflg;
+data _null_;
+  uri=symget('uri');
+  if length(uri)<12 then do;
+    call symputx('errflg',1);
+    call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
+  end;
+  if scan(uri,-1)='state' or scan(uri,1) ne 'jobExecution' then do;
+    call symputx('errflg',1);
+    call symputx('errmsg',
+      "URI should be in format /jobExecution/jobs/$$$$UUID$$$$"
+      !!" but is actually like: &uri",'l');
+  end;
+run;
+
+%mp_abort(iftrue=(&errflg=1)
+  ,mac=&sysmacroname
+  ,msg=%str(&errmsg)
+)
+
+%if &outref ne 0 and %mf_existfileref(&outref) ne 1 %then %do;
+  filename &outref temp;
+%end;
+
+options noquotelenmax;
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+/* fetch job info */
+%local fname1;
+%let fname1=%mf_getuniquefileref();
+proc http method='GET' out=&fname1 &oauth_bearer
+  url="&base_uri&uri";
+  headers "Accept"="application/json"
+  %if &grant_type=authorization_code %then %do;
+          "Authorization"="Bearer &&&access_token_var"
+  %end;
+  ;
+run;
+%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+%do;
+  data _null_;infile &fname1;input;putlog _infile_;run;
+  %mp_abort(mac=&sysmacroname
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+%end;
+
+/* extract results link */
+%local lib1 resuri;
+%let lib1=%mf_getuniquelibref();
+libname &lib1 JSON fileref=&fname1;
+data _null_;
+  set &lib1..results;
+  call symputx('resuri',_&result,'l');
+  putlog (_all_)(=);
+run;
+%mp_abort(iftrue=("&resuri"=".")
+  ,mac=&sysmacroname
+  ,msg=%str(Variable _&result did not exist in the response json)
+)
+
+/* extract results */
+%local fname2;
+%let fname2=%mf_getuniquefileref();
+proc http method='GET' out=&fname2 &oauth_bearer
+  url="&base_uri&resuri/content?limit=10000";
+  headers "Accept"="application/json"
+  %if &grant_type=authorization_code %then %do;
+          "Authorization"="Bearer &&&access_token_var"
+  %end;
+  ;
+run;
+
+%if &outref ne 0 %then %do;
+  filename &outref temp;
+  %mp_binarycopy(inref=&fname2,outref=&outref)
+%end;
+%if &outlib ne 0 %then %do;
+  libname &outlib JSON fileref=&fname2;
+%end;
+
+%if &mdebug=0 %then %do;
+  filename &fname1 clear;
+  filename &fname2 clear;
+  libname &lib1 clear;
+%end;
+%else %do;
+  %put _local_;
+%end;
+%mend;