feat: adding mv_getjobstate macro to fetch the state of a running SAS Viya job

all.sas

@@ -12374,7 +12374,7 @@ data _null_;
     infile = io.open (sas.symget("fpath1"), "r")
     outfile = io.open (sas.symget("fpath2"), "w")
     io.input(infile)
-    local resp=json2sas.decode(io.read())
+    local resp=json.decode(io.read())
     local job=resp["code"]
     outfile:write(job)
     io.close(infile)
@@ -12396,6 +12396,161 @@ data _null_;
 run;
 filename &fname1 clear;
 filename &fname2 clear;
+%mend;
+/**
+  @file
+  @brief Extract the status from a running SAS Viya job
+  @details Extracts the status from a running job and writes it to a fileref.
+  An output dataset is created like this:
+
+      | uri                                                           | state   | timestamp          |
+      |---------------------------------------------------------------|---------|--------------------|
+      | /jobExecution/jobs/5cebd840-2063-42c1-be0c-421ec3e1c175/state | running | 15JAN2021:12:35:08 |
+
+  ## Example
+
+  First, compile the macros:
+
+      filename mc url
+      "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+  Create a long running job (in this case, a web service):
+
+      filename ft15f001 temp;
+      parmcards4;
+        data ;
+          rand=ranuni(0)*1000;
+          do x=1 to rand;
+            y=rand*4;
+            output;
+          end;
+        run;
+        data _null_;
+          call sleep(5,1);
+        run;
+      ;;;;
+      %mv_createwebservice(path=/Public/temp,name=demo)
+
+  Execute it, grab the uri, and check status:
+
+      %mv_jobexecute(path=/Public/temp
+        ,name=demo
+        ,outds=work.info
+      )
+
+      data _null_;
+        set work.info;
+        if method='GET' and rel='state';
+        call symputx('uri',uri);
+      run;
+
+      %mv_getjobstate(uri=&uri,outds=results)
+
+
+  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] grant_type= valid values:
+      * password
+      * authorization_code
+      * detect - will check if access_token exists, if not will use sas_services if
+        a SASStudioV session else authorization_code.  Default option.
+      * sas_services - will use oauth_bearer=sas_services
+  @param [in] uri= The uri of the running job for which to fetch the status,
+    in the format `/jobExecution/jobs/$UUID/state` (unquoted).
+  @param [out] outds= The output dataset in which to APPEND the status. Three
+    fields are appended:  `CHECK_TM`, `URI` and `STATE`. If the dataset does not
+    exist, it is created.
+
+
+  @version VIYA V.03.04
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+
+**/
+
+%macro mv_getjobstate(uri=0,outds=work.mv_getjobstate
+    ,contextName=SAS Job Execution compute context
+    ,access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+  );
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+%put &sysmacroname: grant_type=&grant_type;
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+
+/* validation in datastep for better character safety */
+%local errmsg errflg;
+data _null_;
+  uri=symget('uri');
+  if length(uri)<12 then do;
+    call symputx('errflg',1);
+    call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
+  end;
+  if scan(uri,-1) ne 'state' or scan(uri,1) ne 'jobExecution' then do;
+
+    call symputx('errflg',1);
+    call symputx('errmsg',
+      "URI should be in format /jobExecution/jobs/$$$$UUID$$$$/state"
+      !!" but is actually like: &uri",'l');
+  end;
+run;
+
+%mp_abort(iftrue=(&errflg=1)
+  ,mac=&sysmacroname
+  ,msg=%str(&errmsg)
+)
+
+options noquotelenmax;
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+%local fname0;
+%let fname0=%mf_getuniquefileref();
+
+proc http method='GET' out=&fname0 &oauth_bearer url="&base_uri/&uri";
+  headers "Accept"="text/plain"
+  %if &grant_type=authorization_code %then %do;
+          "Authorization"="Bearer &&&access_token_var"
+  %end;  ;
+run;
+%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+%do;
+  data _null_;infile &fname0;input;putlog _infile_;run;
+  %mp_abort(mac=&sysmacroname
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+%end;
+
+data;
+  format uri $128. state $32. timestamp datetime19.;
+  infile &fname0;
+  uri="&uri";
+  timestamp=datetime();
+  input;
+  state=_infile_;
+run;
+
+proc append base=&outds data=&syslast;
+run;
+
+filename &fname0 clear;
+
 %mend;
  /**
    @file mv_getrefreshtoken.sas
@@ -12660,23 +12815,24 @@ libname &libref1 clear;
         ,paramstring=%str("macvarname":"macvarvalue","answer":42)
       )
 
-  @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values:
-   * password
-   * authorization_code
-   * detect - will check if access_token exists, if not will use sas_services if
-    a SASStudioV session else authorization_code.  Default option.
-   * sas_services - will use oauth_bearer=sas_services
+  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] grant_type= valid values:
 
-  @param path= The SAS Drive path to the job being executed
-  @param name= The name of the job to execute
-  @param paramstring= A JSON fragment with name:value pairs, eg: `"name":"value"`
+    * password
+    * authorization_code
+    * detect - will check if access_token exists, if not will use sas_services if
+      a SASStudioV session else authorization_code.  Default option.
+    * sas_services - will use oauth_bearer=sas_services
+
+  @param [in] path= The SAS Drive path to the job being executed
+  @param [in] name= The name of the job to execute
+  @param [in] paramstring= A JSON fragment with name:value pairs, eg: `"name":"value"`
   or "name":"value","name2":42`.  This will need to be wrapped in `%str()`.
 
-  @param contextName= Context name with which to run the job.
+  @param [in] contextName= Context name with which to run the job.
     Default = `SAS Job Execution compute context`
 
-  @param outds= The output dataset containing links (Default=work.mv_jobexecute)
+  @param [out] outds= The output dataset containing links (Default=work.mv_jobexecute)
 
 
   @version VIYA V.03.04
@@ -12762,9 +12918,11 @@ data _null_;
   length joburi contextname $128 paramstring $32765;
   joburi=quote(trim(symget('joburi')));
   contextname=quote(trim(symget('contextname')));
+  _program=quote("&path/&name");
   paramstring=symget('paramstring');
   put '{"jobDefinitionUri":' joburi ;
   put '  ,"arguments":{"_contextName":' contextname;
+  put '    ,"_program":' _program;
   if paramstring ne "0" then do;
     put '    ,' paramstring;
   end;
@@ -12795,6 +12953,7 @@ libname &libref JSON fileref=&fname1;
 
 data &outds;
   set &libref..links;
+  _program="&path/&name";
 run;
 
 /* clear refs */
@@ -12802,6 +12961,402 @@ filename &fname0 clear;
 filename &fname1 clear;
 libname &libref;
 
+%mend;/**
+  @file
+  @brief Execute a series of job flows
+  @details Very (very) simple flow manager.  Jobs execute in sequential waves,
+  all previous waves must finish successfully.
+
+  The input table is formed as per below.  Each observation represents one job.
+  Each variable is converted into a macro variable with the same name.
+  The FLOW column provides the sequential ordering capability.
+
+
+  ## Example
+
+  First, compile the macros:
+
+      filename mc url
+      "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+  Next, create some jobs (in this case, as web services):
+
+      filename ft15f001 temp;
+      parmcards4;
+        data ;
+          rand=ranuni(0)*&macrovar1;
+          do x=1 to rand;
+            y=rand*&macrovar2;
+            output;
+          end;
+        run;
+      ;;;;
+      %mv_createwebservice(path=/Public/temp,name=demo1)
+      %mv_createwebservice(path=/Public/temp,name=demo2)
+
+  Prepare an input table with 60 executions:
+
+      data work.inputjobs;
+        do flow=1 to 3;
+          do job=1 to 10;
+            _program='/Public/temp/name/demo1';
+            macrovar1=10*job;
+            macrovar2=4*job;
+            output;
+            _program='/Public/temp/name/demo2';
+            macrovar1=40*job;
+            macrovar2=44*job;
+            output;
+          end;
+        end;
+      run;
+
+  Trigger the flow
+
+      %mv_jobflow(inds=work.inputjobs,outds=work.results,maxconcurrency=4)
+
+
+  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] grant_type= valid values:
+      * password
+      * authorization_code
+      * detect - will check if access_token exists, if not will use sas_services if
+        a SASStudioV session else authorization_code.  Default option.
+      * sas_services - will use oauth_bearer=sas_services
+  @param [in] inds= The input dataset containing a list of jobs and parameters
+  @param [in] maxconcurrency= The max number of parallel jobs to run
+  @param [out] outds= The output dataset containing the results
+
+  @version VIYA V.03.05
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mv_getfoldermembers.sas
+  @li ml_json.sas
+
+**/
+
+%macro mv_jobflow(outref=0,outfile=0
+    ,name=0,path=0
+    ,contextName=SAS Job Execution compute context
+    ,access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+  );
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+%put &sysmacroname: grant_type=&grant_type;
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+%mp_abort(iftrue=("&path"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(Job Path not provided)
+)
+%mp_abort(iftrue=("&name"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(Job Name not provided)
+)
+%mp_abort(iftrue=("&outfile"="0" and "&outref"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(Output destination (file or fileref) must be provided)
+)
+options noquotelenmax;
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+data;run;
+%local foldermembers;
+%let foldermembers=&syslast;
+%mv_getfoldermembers(root=&path
+    ,access_token_var=&access_token_var
+    ,grant_type=&grant_type
+    ,outds=&foldermembers
+)
+%local joburi;
+%let joburi=0;
+data _null_;
+  set &foldermembers;
+  if name="&name" and uri=:'/jobDefinitions/definitions'
+    then call symputx('joburi',uri);
+run;
+%mp_abort(iftrue=("&joburi"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(Job &path/&name not found)
+)
+
+/* prepare request*/
+%local  fname1;
+%let fname1=%mf_getuniquefileref();
+proc http method='GET' out=&fname1 &oauth_bearer
+  url="&base_uri&joburi";
+  headers "Accept"="application/vnd.sas.job.definition+json"
+  %if &grant_type=authorization_code %then %do;
+          "Authorization"="Bearer &&&access_token_var"
+  %end;
+  ;
+run;
+%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+%do;
+  data _null_;infile &fname1;input;putlog _infile_;run;
+  %mp_abort(mac=&sysmacroname
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+%end;
+%local  fname2 fname3 fpath1 fpath2 fpath3;
+%let fname2=%mf_getuniquefileref();
+%let fname3=%mf_getuniquefileref();
+%let fpath1=%sysfunc(pathname(&fname1));
+%let fpath2=%sysfunc(pathname(&fname2));
+%let fpath3=%sysfunc(pathname(&fname2));
+
+/* compile the lua JSON module */
+%ml_json()
+/* read using LUA - this allows the code to be of any length */
+data _null_;
+  file "&fpath3..lua";
+  put '
+    infile = io.open (sas.symget("fpath1"), "r")
+    outfile = io.open (sas.symget("fpath2"), "w")
+    io.input(infile)
+    local resp=json.decode(io.read())
+    local job=resp["code"]
+    outfile:write(job)
+    io.close(infile)
+    io.close(outfile)
+   ';
+run;
+%inc "&fpath3..lua";
+/* export to desired destination */
+data _null_;
+  %if &outref=0 %then %do;
+    file "&outfile" lrecl=32767;
+  %end;
+  %else %do;
+    file &outref;
+  %end;
+  infile &fname2;
+  input;
+  put _infile_;
+run;
+filename &fname1 clear;
+filename &fname2 clear;
+%mend;
+/**
+  @file
+  @brief Takes a dataset of running jobs and waits for them to complete
+  @details Will poll `/jobs/{jobId}/state` at set intervals until they are all
+  completed.  Completion is determined by reference to the returned _state_, as
+  per the following table:
+
+  | state     | Wait? | Notes|
+  |-----------|-------|------|
+  | idle      | yes   | We assume processing will continue. Beware of idle sessions with no code submitted! |
+  | pending   | yes   | Job is preparing to run |
+  | running   | yes   | Job is running|
+  | canceled  | no    | Job was cancelled|
+  | completed | no    | Job finished - does not mean it was successful.  Check stateDetails|
+  | failed    | no    | Job failed to execute, could be a problem when calling the apis|
+
+
+  ## Example
+
+  First, compile the macros:
+
+      filename mc url
+      "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+  Next, create a job (in this case, as a web service):
+
+      filename ft15f001 temp;
+      parmcards4;
+        data ;
+          rand=ranuni(0)*1000000;
+          do x=1 to rand;
+            y=rand*x;
+            output;
+          end;
+        run;
+      ;;;;
+      %mv_createwebservice(path=/Public/temp,name=demo)
+
+  Then, execute the job,multiple times, and wait for them all to finish:
+
+      %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds1)
+      %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds2)
+      %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds3)
+      %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds4)
+
+      data work.jobs;
+        set work.ds1 work.ds2 work.ds3 work.ds4;
+        where method='GET' and rel='state';
+      run;
+
+      %mv_jobwaitfor(inds=work.jobs,outds=work.jobstates)
+
+
+  Delete the job:
+
+      %mv_deletejes(path=/Public/temp,name=demo)
+
+  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] grant_type= valid values:
+
+      - password
+      - authorization_code
+      - detect - will check if access_token exists, if not will use sas_services if
+        a SASStudioV session else authorization_code.  Default option.
+      - sas_services - will use oauth_bearer=sas_services
+
+  @param [in] inds= The input dataset containing the list of job uris, in the
+    following format:  `/jobExecution/jobs/&JOBID./state` and the corresponding
+    job name.  The uri should be in a `uri` variable, and the job path/name
+    should be in a `_program` variable.
+  @param [out] outds= The output dataset containing the list of states by job
+    (default=work.mv_jobexecute)
+
+
+  @version VIYA V.03.04
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> Dependencies </h4>
+  @li mp_abort.sas
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mf_existvar.sas
+  @li mf_nobs.sas
+
+**/
+
+%macro mv_jobwaitfor(
+     access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+    ,inds=0
+    ,outds=work.mv_jobwaitfor
+  );
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+%put &sysmacroname: grant_type=&grant_type;
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+
+%mp_abort(iftrue=("&inds"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(input dataset not provided)
+)
+%mp_abort(iftrue=(%mf_existvar(&inds,uri)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(The URI variable was not found in the input dataset(&inds))
+)
+%mp_abort(iftrue=(%mf_existvar(&inds,_program)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(The _PROGRAM variable was not found in the input dataset(&inds))
+)
+
+%if %mf_nobs(&inds)=0 %then %do;
+  %put NOTE: Zero observations in &inds, &sysmacroname will now exit;
+  %return;
+%end;
+
+options noquotelenmax;
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+data _null_;
+  set &inds end=last;
+  call symputx(cats('joburi',_n_),uri,'l');
+  call symputx(cats('jobname',_n_),_program,'l');
+  if last then call symputx('uricnt',_n_,'l');
+run;
+
+%local fname0 ;
+%let fname0=%mf_getuniquefileref();
+
+data &outds;
+  format _program uri $128. state $32. timestamp datetime19.;
+  stop;
+run;
+
+%local i;
+%do i=1 %to &uricnt;
+  %if "&&joburi&i" ne "0" %then %do;
+    proc http method='GET' out=&fname0 &oauth_bearer url="&base_uri/&&joburi&i";
+      headers "Accept"="text/plain"
+      %if &grant_type=authorization_code %then %do;
+              "Authorization"="Bearer &&&access_token_var"
+      %end;  ;
+    run;
+    %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+    %do;
+      data _null_;infile &fname0;input;putlog _infile_;run;
+      %mp_abort(mac=&sysmacroname
+        ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+      )
+    %end;
+
+    %let status=notset;
+    data _null_;
+      infile &fname0;
+      input;
+      call symputx('status',_infile_,'l');
+    run;
+
+    %if &status=completed or &status=failed or &status=canceled %then %do;
+      proc sql;
+      insert into &outds set
+        _program="&&jobname&i",
+        uri="&&joburi&i",
+        state="&status",
+        timestamp=datetime();
+      %let joburi&i=0;
+    %end;
+    %else %if &status=idle or &status=pending or &status=running %then %do;
+      data _null_;
+        call sleep(1,1);
+      run;
+    %end;
+    %else %do;
+      %mp_abort(mac=&sysmacroname
+        ,msg=%str(status &status not expected!!)
+      )
+    %end;
+  %end;
+  %if &i=&uricnt %then %do;
+    %local goback;
+    %let goback=0;
+    proc sql noprint;
+    select count(*) into:goback from &outds;
+    %if &goback ne &uricnt %then %let i=0;
+  %end;
+%end;
+
+/* clear refs */
+filename &fname0 clear;
+
 %mend;/**
   @file mv_registerclient.sas
   @brief Register Client and Secret (admin task)
@@ -13549,7 +14104,7 @@ filename &fref1 clear;
 data _null_;
   file "%sysfunc(pathname(work))/ml_json.lua";
   put '-- ';
-  put '-- json.lua  (modified from json.lua) ';
+  put '-- json.lua ';
   put '-- ';
   put '-- Copyright (c) 2019 rxi ';
   put '-- ';
@@ -13572,7 +14127,7 @@ data _null_;
   put '-- SOFTWARE. ';
   put '-- ';
   put ' ';
-  put 'local json = { _version = "0.1.2" } ';
+  put 'json = { _version = "0.1.2" } ';
   put ' ';
   put '------------------------------------------------------------------------------- ';
   put '-- Encode ';
@@ -13920,6 +14475,7 @@ data _null_;
   put ' ';
   put 'return json ';
 run;
-%mend;
 
 %inc "%sysfunc(pathname(work))/ml_json.lua";
+
+%mend;

build.py

@@ -21,9 +21,10 @@ for file in files:
     with open(file) as infile:
         for line in infile:
           ml.write("  put '" + line.rstrip().replace("'","''") + " ';\n")
-    ml.write("run;\n")
-    ml.write("%mend;\n\n")
-    ml.write("%inc \"%sysfunc(pathname(work))/" + name + ".lua\";\n")
+    ml.write("run;\n\n")
+    ml.write("%inc \"%sysfunc(pathname(work))/" + name + ".lua\";\n\n")
+    ml.write("%mend;\n")
+
 ml.close()
 
 # prepare web files

doxy/Doxyfile

@@ -19,7 +19,7 @@ HTML_FOOTER            = ./doxy/new_footer.html
 HTML_EXTRA_STYLESHEET  = ./doxy/new_stylesheet.css
 INHERIT_DOCS           = NO
 INLINE_INFO            = NO
-INPUT                  = base meta metax viya
+INPUT                  = base meta metax viya lua
 LAYOUT_FILE            = ./doxy/DoxygenLayout.xml
 MAX_INITIALIZER_LINES  = 0
 PROJECT_NAME           = Macro Core

doxy/build.sh

@@ -16,6 +16,7 @@ mkdir $BUILD_FOLDER
 cp -r base $BUILD_FOLDER
 cp -r meta $BUILD_FOLDER
 cp -r metax $BUILD_FOLDER
+cp -r lua $BUILD_FOLDER
 cp -r viya $BUILD_FOLDER
 cp -r doxy $BUILD_FOLDER
 cp main.dox $BUILD_FOLDER

lua/json.lua

@@ -1,5 +1,5 @@
 --
--- json.lua  (modified from json.lua)
+-- json.lua
 --
 -- Copyright (c) 2019 rxi
 --
@@ -22,7 +22,7 @@
 -- SOFTWARE.
 --
 
-local json = { _version = "0.1.2" }
+json = { _version = "0.1.2" }
 
 -------------------------------------------------------------------------------
 -- Encode
@@ -368,4 +368,4 @@ function json.decode(str)
   return res
 end
 
-return json
+return json

lua/ml_json.sas

@@ -13,7 +13,7 @@
 data _null_;
   file "%sysfunc(pathname(work))/ml_json.lua";
   put '-- ';
-  put '-- json.lua  (modified from json.lua) ';
+  put '-- json.lua ';
   put '-- ';
   put '-- Copyright (c) 2019 rxi ';
   put '-- ';
@@ -36,7 +36,7 @@ data _null_;
   put '-- SOFTWARE. ';
   put '-- ';
   put ' ';
-  put 'local json = { _version = "0.1.2" } ';
+  put 'json = { _version = "0.1.2" } ';
   put ' ';
   put '------------------------------------------------------------------------------- ';
   put '-- Encode ';
@@ -384,6 +384,7 @@ data _null_;
   put ' ';
   put 'return json ';
 run;
-%mend;
 
 %inc "%sysfunc(pathname(work))/ml_json.lua";
+
+%mend;

main.dox

@@ -50,4 +50,15 @@
         * No X command
         * Prefixes: _mv_
 
+ */
+
+ /*! \dir lua
+ *  \brief Lua macros
+ *  \details These macros have the following attributes:
+
+        * OS independent
+        * Work as LUA functions (they are immediately executed/compiled)
+        * Auto-generated from the plain source `.lua` files in the same directory
+        * Prefixes: _ml_
+
  */

viya/mv_getjobcode.sas

@@ -125,7 +125,7 @@ data _null_;
     infile = io.open (sas.symget("fpath1"), "r")
     outfile = io.open (sas.symget("fpath2"), "w")
     io.input(infile)
-    local resp=json2sas.decode(io.read())
+    local resp=json.decode(io.read())
     local job=resp["code"]
     outfile:write(job)
     io.close(infile)

viya/mv_getjobstate.sas

@@ -0,0 +1,155 @@
+/**
+  @file
+  @brief Extract the status from a running SAS Viya job
+  @details Extracts the status from a running job and writes it to a fileref.
+  An output dataset is created like this:
+
+      | uri                                                           | state   | timestamp          |
+      |---------------------------------------------------------------|---------|--------------------|
+      | /jobExecution/jobs/5cebd840-2063-42c1-be0c-421ec3e1c175/state | running | 15JAN2021:12:35:08 |
+
+  ## Example
+
+  First, compile the macros:
+
+      filename mc url
+      "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+  Create a long running job (in this case, a web service):
+
+      filename ft15f001 temp;
+      parmcards4;
+        data ;
+          rand=ranuni(0)*1000;
+          do x=1 to rand;
+            y=rand*4;
+            output;
+          end;
+        run;
+        data _null_;
+          call sleep(5,1);
+        run;
+      ;;;;
+      %mv_createwebservice(path=/Public/temp,name=demo)
+
+  Execute it, grab the uri, and check status:
+
+      %mv_jobexecute(path=/Public/temp
+        ,name=demo
+        ,outds=work.info
+      )
+
+      data _null_;
+        set work.info;
+        if method='GET' and rel='state';
+        call symputx('uri',uri);
+      run;
+
+      %mv_getjobstate(uri=&uri,outds=results)
+
+
+  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] grant_type= valid values:
+      * password
+      * authorization_code
+      * detect - will check if access_token exists, if not will use sas_services if
+        a SASStudioV session else authorization_code.  Default option.
+      * sas_services - will use oauth_bearer=sas_services
+  @param [in] uri= The uri of the running job for which to fetch the status,
+    in the format `/jobExecution/jobs/$UUID/state` (unquoted).
+  @param [out] outds= The output dataset in which to APPEND the status. Three
+    fields are appended:  `CHECK_TM`, `URI` and `STATE`. If the dataset does not
+    exist, it is created.
+
+
+  @version VIYA V.03.04
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+
+**/
+
+%macro mv_getjobstate(uri=0,outds=work.mv_getjobstate
+    ,contextName=SAS Job Execution compute context
+    ,access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+  );
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+%put &sysmacroname: grant_type=&grant_type;
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+
+/* validation in datastep for better character safety */
+%local errmsg errflg;
+data _null_;
+  uri=symget('uri');
+  if length(uri)<12 then do;
+    call symputx('errflg',1);
+    call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
+  end;
+  if scan(uri,-1) ne 'state' or scan(uri,1) ne 'jobExecution' then do;
+
+    call symputx('errflg',1);
+    call symputx('errmsg',
+      "URI should be in format /jobExecution/jobs/$$$$UUID$$$$/state"
+      !!" but is actually like: &uri",'l');
+  end;
+run;
+
+%mp_abort(iftrue=(&errflg=1)
+  ,mac=&sysmacroname
+  ,msg=%str(&errmsg)
+)
+
+options noquotelenmax;
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+%local fname0;
+%let fname0=%mf_getuniquefileref();
+
+proc http method='GET' out=&fname0 &oauth_bearer url="&base_uri/&uri";
+  headers "Accept"="text/plain"
+  %if &grant_type=authorization_code %then %do;
+          "Authorization"="Bearer &&&access_token_var"
+  %end;  ;
+run;
+%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+%do;
+  data _null_;infile &fname0;input;putlog _infile_;run;
+  %mp_abort(mac=&sysmacroname
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+%end;
+
+data;
+  format uri $128. state $32. timestamp datetime19.;
+  infile &fname0;
+  uri="&uri";
+  timestamp=datetime();
+  input;
+  state=_infile_;
+run;
+
+proc append base=&outds data=&syslast;
+run;
+
+filename &fname0 clear;
+
+%mend;

viya/mv_jobexecute.sas

@@ -23,23 +23,24 @@
         ,paramstring=%str("macvarname":"macvarvalue","answer":42)
       )
 
-  @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values:
-   * password
-   * authorization_code
-   * detect - will check if access_token exists, if not will use sas_services if
-    a SASStudioV session else authorization_code.  Default option.
-   * sas_services - will use oauth_bearer=sas_services
+  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] grant_type= valid values:
 
-  @param path= The SAS Drive path to the job being executed
-  @param name= The name of the job to execute
-  @param paramstring= A JSON fragment with name:value pairs, eg: `"name":"value"`
+    * password
+    * authorization_code
+    * detect - will check if access_token exists, if not will use sas_services if
+      a SASStudioV session else authorization_code.  Default option.
+    * sas_services - will use oauth_bearer=sas_services
+
+  @param [in] path= The SAS Drive path to the job being executed
+  @param [in] name= The name of the job to execute
+  @param [in] paramstring= A JSON fragment with name:value pairs, eg: `"name":"value"`
   or "name":"value","name2":42`.  This will need to be wrapped in `%str()`.
 
-  @param contextName= Context name with which to run the job.
+  @param [in] contextName= Context name with which to run the job.
     Default = `SAS Job Execution compute context`
 
-  @param outds= The output dataset containing links (Default=work.mv_jobexecute)
+  @param [out] outds= The output dataset containing links (Default=work.mv_jobexecute)
 
 
   @version VIYA V.03.04
@@ -125,9 +126,11 @@ data _null_;
   length joburi contextname $128 paramstring $32765;
   joburi=quote(trim(symget('joburi')));
   contextname=quote(trim(symget('contextname')));
+  _program=quote("&path/&name");
   paramstring=symget('paramstring');
   put '{"jobDefinitionUri":' joburi ;
   put '  ,"arguments":{"_contextName":' contextname;
+  put '    ,"_program":' _program;
   if paramstring ne "0" then do;
     put '    ,' paramstring;
   end;
@@ -158,6 +161,7 @@ libname &libref JSON fileref=&fname1;
 
 data &outds;
   set &libref..links;
+  _program="&path/&name";
 run;
 
 /* clear refs */

viya/mv_jobwaitfor.sas

@@ -0,0 +1,204 @@
+/**
+  @file
+  @brief Takes a dataset of running jobs and waits for them to complete
+  @details Will poll `/jobs/{jobId}/state` at set intervals until they are all
+  completed.  Completion is determined by reference to the returned _state_, as
+  per the following table:
+
+  | state     | Wait? | Notes|
+  |-----------|-------|------|
+  | idle      | yes   | We assume processing will continue. Beware of idle sessions with no code submitted! |
+  | pending   | yes   | Job is preparing to run |
+  | running   | yes   | Job is running|
+  | canceled  | no    | Job was cancelled|
+  | completed | no    | Job finished - does not mean it was successful.  Check stateDetails|
+  | failed    | no    | Job failed to execute, could be a problem when calling the apis|
+
+
+  ## Example
+
+  First, compile the macros:
+
+      filename mc url
+      "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+  Next, create a job (in this case, as a web service):
+
+      filename ft15f001 temp;
+      parmcards4;
+        data ;
+          rand=ranuni(0)*1000000;
+          do x=1 to rand;
+            y=rand*x;
+            output;
+          end;
+        run;
+      ;;;;
+      %mv_createwebservice(path=/Public/temp,name=demo)
+
+  Then, execute the job,multiple times, and wait for them all to finish:
+
+      %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds1)
+      %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds2)
+      %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds3)
+      %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds4)
+
+      data work.jobs;
+        set work.ds1 work.ds2 work.ds3 work.ds4;
+        where method='GET' and rel='state';
+      run;
+
+      %mv_jobwaitfor(inds=work.jobs,outds=work.jobstates)
+
+
+  Delete the job:
+
+      %mv_deletejes(path=/Public/temp,name=demo)
+
+  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] grant_type= valid values:
+
+      - password
+      - authorization_code
+      - detect - will check if access_token exists, if not will use sas_services if
+        a SASStudioV session else authorization_code.  Default option.
+      - sas_services - will use oauth_bearer=sas_services
+
+  @param [in] inds= The input dataset containing the list of job uris, in the
+    following format:  `/jobExecution/jobs/&JOBID./state` and the corresponding
+    job name.  The uri should be in a `uri` variable, and the job path/name
+    should be in a `_program` variable.
+  @param [out] outds= The output dataset containing the list of states by job
+    (default=work.mv_jobexecute)
+
+
+  @version VIYA V.03.04
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> Dependencies </h4>
+  @li mp_abort.sas
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mf_existvar.sas
+  @li mf_nobs.sas
+
+**/
+
+%macro mv_jobwaitfor(
+     access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+    ,inds=0
+    ,outds=work.mv_jobwaitfor
+  );
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+%put &sysmacroname: grant_type=&grant_type;
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+
+%mp_abort(iftrue=("&inds"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(input dataset not provided)
+)
+%mp_abort(iftrue=(%mf_existvar(&inds,uri)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(The URI variable was not found in the input dataset(&inds))
+)
+%mp_abort(iftrue=(%mf_existvar(&inds,_program)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(The _PROGRAM variable was not found in the input dataset(&inds))
+)
+
+%if %mf_nobs(&inds)=0 %then %do;
+  %put NOTE: Zero observations in &inds, &sysmacroname will now exit;
+  %return;
+%end;
+
+options noquotelenmax;
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+data _null_;
+  set &inds end=last;
+  call symputx(cats('joburi',_n_),uri,'l');
+  call symputx(cats('jobname',_n_),_program,'l');
+  if last then call symputx('uricnt',_n_,'l');
+run;
+
+%local fname0 ;
+%let fname0=%mf_getuniquefileref();
+
+data &outds;
+  format _program uri $128. state $32. timestamp datetime19.;
+  stop;
+run;
+
+%local i;
+%do i=1 %to &uricnt;
+  %if "&&joburi&i" ne "0" %then %do;
+    proc http method='GET' out=&fname0 &oauth_bearer url="&base_uri/&&joburi&i";
+      headers "Accept"="text/plain"
+      %if &grant_type=authorization_code %then %do;
+              "Authorization"="Bearer &&&access_token_var"
+      %end;  ;
+    run;
+    %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+    %do;
+      data _null_;infile &fname0;input;putlog _infile_;run;
+      %mp_abort(mac=&sysmacroname
+        ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+      )
+    %end;
+
+    %let status=notset;
+    data _null_;
+      infile &fname0;
+      input;
+      call symputx('status',_infile_,'l');
+    run;
+
+    %if &status=completed or &status=failed or &status=canceled %then %do;
+      proc sql;
+      insert into &outds set
+        _program="&&jobname&i",
+        uri="&&joburi&i",
+        state="&status",
+        timestamp=datetime();
+      %let joburi&i=0;
+    %end;
+    %else %if &status=idle or &status=pending or &status=running %then %do;
+      data _null_;
+        call sleep(1,1);
+      run;
+    %end;
+    %else %do;
+      %mp_abort(mac=&sysmacroname
+        ,msg=%str(status &status not expected!!)
+      )
+    %end;
+  %end;
+  %if &i=&uricnt %then %do;
+    %local goback;
+    %let goback=0;
+    proc sql noprint;
+    select count(*) into:goback from &outds;
+    %if &goback ne &uricnt %then %let i=0;
+  %end;
+%end;
+
+/* clear refs */
+filename &fname0 clear;
+
+%mend;