feat: mv_jobwaitfor macro, similar to waitfor statement (in concept) - will wait for ALL of a set of viya jobs to finish executing

all.sas

@@ -12374,7 +12374,7 @@ data _null_;
     infile = io.open (sas.symget("fpath1"), "r")
     outfile = io.open (sas.symget("fpath2"), "w")
     io.input(infile)
-    local resp=json2sas.decode(io.read())
+    local resp=json.decode(io.read())
     local job=resp["code"]
     outfile:write(job)
     io.close(infile)
@@ -12660,23 +12660,24 @@ libname &libref1 clear;
         ,paramstring=%str("macvarname":"macvarvalue","answer":42)
       )
 
-  @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values:
-   * password
-   * authorization_code
-   * detect - will check if access_token exists, if not will use sas_services if
-    a SASStudioV session else authorization_code.  Default option.
-   * sas_services - will use oauth_bearer=sas_services
+  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] grant_type= valid values:
 
-  @param path= The SAS Drive path to the job being executed
-  @param name= The name of the job to execute
-  @param paramstring= A JSON fragment with name:value pairs, eg: `"name":"value"`
+    * password
+    * authorization_code
+    * detect - will check if access_token exists, if not will use sas_services if
+      a SASStudioV session else authorization_code.  Default option.
+    * sas_services - will use oauth_bearer=sas_services
+
+  @param [in] path= The SAS Drive path to the job being executed
+  @param [in] name= The name of the job to execute
+  @param [in] paramstring= A JSON fragment with name:value pairs, eg: `"name":"value"`
   or "name":"value","name2":42`.  This will need to be wrapped in `%str()`.
 
-  @param contextName= Context name with which to run the job.
+  @param [in] contextName= Context name with which to run the job.
     Default = `SAS Job Execution compute context`
 
-  @param outds= The output dataset containing links (Default=work.mv_jobexecute)
+  @param [out] outds= The output dataset containing links (Default=work.mv_jobexecute)
 
 
   @version VIYA V.03.04
@@ -12795,6 +12796,7 @@ libname &libref JSON fileref=&fname1;
 
 data &outds;
   set &libref..links;
+  _program="&path/&name";
 run;
 
 /* clear refs */
@@ -12802,6 +12804,205 @@ filename &fname0 clear;
 filename &fname1 clear;
 libname &libref;
 
+%mend;/**
+  @file
+  @brief Takes a dataset of running jobs and waits for them to complete
+  @details Will poll `/jobs/{jobId}/state` at set intervals until they are all
+  completed.  Completion is determined by reference to the returned _state_, as
+  per the following table:
+
+  | state     | Wait? | Notes|
+  |-----------|-------|------|
+  | idle      | yes   | We assume processing will continue. Beware of idle sessions with no code submitted! |
+  | pending   | yes   | Job is preparing to run |
+  | running   | yes   | Job is running|
+  | canceled  | no    | Job was cancelled|
+  | completed | no    | Job finished - does not mean it was successful.  Check stateDetails|
+  | failed    | no    | Job failed to execute, could be a problem when calling the apis|
+
+
+  ## Example
+
+  First, compile the macros:
+
+      filename mc url
+      "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+  Next, create a job (in this case, as a web service):
+
+      filename ft15f001 temp;
+      parmcards4;
+        data ;
+          rand=ranuni(0)*1000000;
+          do x=1 to rand;
+            y=rand*x;
+            output;
+          end;
+        run;
+      ;;;;
+      %mv_createwebservice(path=/Public/temp,name=demo)
+
+  Then, execute the job,multiple times, and wait for them all to finish:
+
+      %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds1)
+      %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds2)
+      %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds3)
+      %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds4)
+
+      data work.jobs;
+        set work.ds1 work.ds2 work.ds3 work.ds4;
+        where method='GET' and rel='state';
+      run;
+
+      %mv_jobwaitfor(inds=work.jobs,outds=work.jobstates)
+
+
+  Delete the job:
+
+      %mv_deletejes(path=/Public/temp,name=demo)
+
+  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] grant_type= valid values:
+
+    * password
+    * authorization_code
+    * detect - will check if access_token exists, if not will use sas_services if
+      a SASStudioV session else authorization_code.  Default option.
+    * sas_services - will use oauth_bearer=sas_services
+
+  @param [in] inds= The input dataset containing the list of job uris, in the
+    following format:  `/jobExecution/jobs/&JOBID./state` and the corresponding
+    job name.  The uri should be in a `uri` variable, and the job path/name
+    should be in a `_program` variable.
+  @param [out] outds= The output dataset containing the list of states by job
+    (default=mv_jobexecute)
+
+
+  @version VIYA V.03.04
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> Dependencies </h4>
+  @li mp_abort.sas
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mf_existvar.sas
+  @li mf_nobs.sas
+
+**/
+
+%macro mv_jobwaitfor(
+     access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+    ,inds=0
+    ,outds=work.mv_jobwaitfor
+  );
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+%put &sysmacroname: grant_type=&grant_type;
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+
+%mp_abort(iftrue=("&inds"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(input dataset not provided)
+)
+%mp_abort(iftrue=(%mf_existvar(&inds,uri)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(The URI variable was not found in the input dataset(&inds))
+)
+
+%if %mf_nobs(&inds)=0 %then %do;
+  %put NOTE: Zero observations in &inds, &sysmacroname will now exit;
+  %return;
+%end;
+
+options noquotelenmax;
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+data _null_;
+  set &inds end=last;
+  call symputx(cats('joburi',_n_),uri,'l');
+  call symputx(cats('jobname',_n_),_program,'l');
+  if last then call symputx('uricnt',_n_,'l');
+run;
+
+%local fname0 ;
+%let fname0=%mf_getuniquefileref();
+
+data &outds;
+  format _program uri $128. state $32. timestamp datetime19.;
+  stop;
+run;
+
+%local i;
+%do i=1 %to &uricnt;
+  %if "&&joburi&i" ne "0" %then %do;
+    proc http method='GET' out=&fname0 &oauth_bearer url="&base_uri/&&joburi&i";
+      headers "Accept"="text/plain"
+      %if &grant_type=authorization_code %then %do;
+              "Authorization"="Bearer &&&access_token_var"
+      %end;  ;
+    run;
+    %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+    %do;
+      data _null_;infile &fname0;input;putlog _infile_;run;
+      %mp_abort(mac=&sysmacroname
+        ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+      )
+    %end;
+
+    %let status=notset;
+    data _null_;
+      infile &fname0;
+      input;
+      call symputx('status',_infile_,'l');
+    run;
+
+    %if &status=completed or &status=failed or &status=canceled %then %do;
+      proc sql;
+      insert into &outds set
+        _program="&&jobname&i",
+        uri="&&joburi&i",
+        state="&status",
+        timestamp=datetime();
+      %let joburi&i=0;
+    %end;
+    %else %if &status=idle or &status=pending or &status=running %then %do;
+      data _null_;
+        call sleep(1,1);
+      run;
+    %end;
+    %else %do;
+      %mp_abort(mac=&sysmacroname
+        ,msg=%str(status &status not expected!!)
+      )
+    %end;
+  %end;
+  %if &i=&uricnt %then %do;
+    %local goback;
+    %let goback=0;
+    proc sql noprint;
+    select count(*) into:goback from &outds;
+    %if &goback ne &uricnt %then %let i=0;
+  %end;
+%end;
+
+/* clear refs */
+filename &fname0 clear;
+
 %mend;/**
   @file mv_registerclient.sas
   @brief Register Client and Secret (admin task)
@@ -13549,7 +13750,7 @@ filename &fref1 clear;
 data _null_;
   file "%sysfunc(pathname(work))/ml_json.lua";
   put '-- ';
-  put '-- json.lua  (modified from json.lua) ';
+  put '-- json.lua ';
   put '-- ';
   put '-- Copyright (c) 2019 rxi ';
   put '-- ';
@@ -13572,7 +13773,7 @@ data _null_;
   put '-- SOFTWARE. ';
   put '-- ';
   put ' ';
-  put 'local json = { _version = "0.1.2" } ';
+  put 'json = { _version = "0.1.2" } ';
   put ' ';
   put '------------------------------------------------------------------------------- ';
   put '-- Encode ';
@@ -13920,6 +14121,7 @@ data _null_;
   put ' ';
   put 'return json ';
 run;
-%mend;
 
 %inc "%sysfunc(pathname(work))/ml_json.lua";
+
+%mend;

build.py

@@ -21,9 +21,10 @@ for file in files:
     with open(file) as infile:
         for line in infile:
           ml.write("  put '" + line.rstrip().replace("'","''") + " ';\n")
-    ml.write("run;\n")
-    ml.write("%mend;\n\n")
-    ml.write("%inc \"%sysfunc(pathname(work))/" + name + ".lua\";\n")
+    ml.write("run;\n\n")
+    ml.write("%inc \"%sysfunc(pathname(work))/" + name + ".lua\";\n\n")
+    ml.write("%mend;\n")
+
 ml.close()
 
 # prepare web files

doxy/Doxyfile

@@ -19,7 +19,7 @@ HTML_FOOTER            = ./doxy/new_footer.html
 HTML_EXTRA_STYLESHEET  = ./doxy/new_stylesheet.css
 INHERIT_DOCS           = NO
 INLINE_INFO            = NO
-INPUT                  = base meta metax viya
+INPUT                  = base meta metax viya lua
 LAYOUT_FILE            = ./doxy/DoxygenLayout.xml
 MAX_INITIALIZER_LINES  = 0
 PROJECT_NAME           = Macro Core

doxy/build.sh

@@ -16,6 +16,7 @@ mkdir $BUILD_FOLDER
 cp -r base $BUILD_FOLDER
 cp -r meta $BUILD_FOLDER
 cp -r metax $BUILD_FOLDER
+cp -r lua $BUILD_FOLDER
 cp -r viya $BUILD_FOLDER
 cp -r doxy $BUILD_FOLDER
 cp main.dox $BUILD_FOLDER

lua/json.lua

@@ -1,5 +1,5 @@
 --
--- json.lua  (modified from json.lua)
+-- json.lua
 --
 -- Copyright (c) 2019 rxi
 --
@@ -22,7 +22,7 @@
 -- SOFTWARE.
 --
 
-local json = { _version = "0.1.2" }
+json = { _version = "0.1.2" }
 
 -------------------------------------------------------------------------------
 -- Encode
@@ -368,4 +368,4 @@ function json.decode(str)
   return res
 end
 
-return json
+return json

lua/ml_json.sas

@@ -13,7 +13,7 @@
 data _null_;
   file "%sysfunc(pathname(work))/ml_json.lua";
   put '-- ';
-  put '-- json.lua  (modified from json.lua) ';
+  put '-- json.lua ';
   put '-- ';
   put '-- Copyright (c) 2019 rxi ';
   put '-- ';
@@ -36,7 +36,7 @@ data _null_;
   put '-- SOFTWARE. ';
   put '-- ';
   put ' ';
-  put 'local json = { _version = "0.1.2" } ';
+  put 'json = { _version = "0.1.2" } ';
   put ' ';
   put '------------------------------------------------------------------------------- ';
   put '-- Encode ';
@@ -384,6 +384,7 @@ data _null_;
   put ' ';
   put 'return json ';
 run;
-%mend;
 
 %inc "%sysfunc(pathname(work))/ml_json.lua";
+
+%mend;

main.dox

@@ -50,4 +50,15 @@
         * No X command
         * Prefixes: _mv_
 
+ */
+
+ /*! \dir lua
+ *  \brief Lua macros
+ *  \details These macros have the following attributes:
+
+        * OS independent
+        * Work as LUA functions (they are immediately executed/compiled)
+        * Auto-generated from the plain source `.lua` files in the same directory
+        * Prefixes: _ml_
+
  */

viya/mv_getjobcode.sas

@@ -125,7 +125,7 @@ data _null_;
     infile = io.open (sas.symget("fpath1"), "r")
     outfile = io.open (sas.symget("fpath2"), "w")
     io.input(infile)
-    local resp=json2sas.decode(io.read())
+    local resp=json.decode(io.read())
     local job=resp["code"]
     outfile:write(job)
     io.close(infile)

viya/mv_jobexecute.sas

@@ -23,23 +23,24 @@
         ,paramstring=%str("macvarname":"macvarvalue","answer":42)
       )
 
-  @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values:
-   * password
-   * authorization_code
-   * detect - will check if access_token exists, if not will use sas_services if
-    a SASStudioV session else authorization_code.  Default option.
-   * sas_services - will use oauth_bearer=sas_services
+  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] grant_type= valid values:
 
-  @param path= The SAS Drive path to the job being executed
-  @param name= The name of the job to execute
-  @param paramstring= A JSON fragment with name:value pairs, eg: `"name":"value"`
+    * password
+    * authorization_code
+    * detect - will check if access_token exists, if not will use sas_services if
+      a SASStudioV session else authorization_code.  Default option.
+    * sas_services - will use oauth_bearer=sas_services
+
+  @param [in] path= The SAS Drive path to the job being executed
+  @param [in] name= The name of the job to execute
+  @param [in] paramstring= A JSON fragment with name:value pairs, eg: `"name":"value"`
   or "name":"value","name2":42`.  This will need to be wrapped in `%str()`.
 
-  @param contextName= Context name with which to run the job.
+  @param [in] contextName= Context name with which to run the job.
     Default = `SAS Job Execution compute context`
 
-  @param outds= The output dataset containing links (Default=work.mv_jobexecute)
+  @param [out] outds= The output dataset containing links (Default=work.mv_jobexecute)
 
 
   @version VIYA V.03.04
@@ -158,6 +159,7 @@ libname &libref JSON fileref=&fname1;
 
 data &outds;
   set &libref..links;
+  _program="&path/&name";
 run;
 
 /* clear refs */

viya/mv_jobwaitfor.sas

@@ -0,0 +1,200 @@
+/**
+  @file
+  @brief Takes a dataset of running jobs and waits for them to complete
+  @details Will poll `/jobs/{jobId}/state` at set intervals until they are all
+  completed.  Completion is determined by reference to the returned _state_, as
+  per the following table:
+
+  | state     | Wait? | Notes|
+  |-----------|-------|------|
+  | idle      | yes   | We assume processing will continue. Beware of idle sessions with no code submitted! |
+  | pending   | yes   | Job is preparing to run |
+  | running   | yes   | Job is running|
+  | canceled  | no    | Job was cancelled|
+  | completed | no    | Job finished - does not mean it was successful.  Check stateDetails|
+  | failed    | no    | Job failed to execute, could be a problem when calling the apis|
+
+
+  ## Example
+
+  First, compile the macros:
+
+      filename mc url
+      "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+  Next, create a job (in this case, as a web service):
+
+      filename ft15f001 temp;
+      parmcards4;
+        data ;
+          rand=ranuni(0)*1000000;
+          do x=1 to rand;
+            y=rand*x;
+            output;
+          end;
+        run;
+      ;;;;
+      %mv_createwebservice(path=/Public/temp,name=demo)
+
+  Then, execute the job,multiple times, and wait for them all to finish:
+
+      %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds1)
+      %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds2)
+      %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds3)
+      %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds4)
+
+      data work.jobs;
+        set work.ds1 work.ds2 work.ds3 work.ds4;
+        where method='GET' and rel='state';
+      run;
+
+      %mv_jobwaitfor(inds=work.jobs,outds=work.jobstates)
+
+
+  Delete the job:
+
+      %mv_deletejes(path=/Public/temp,name=demo)
+
+  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] grant_type= valid values:
+
+    * password
+    * authorization_code
+    * detect - will check if access_token exists, if not will use sas_services if
+      a SASStudioV session else authorization_code.  Default option.
+    * sas_services - will use oauth_bearer=sas_services
+
+  @param [in] inds= The input dataset containing the list of job uris, in the
+    following format:  `/jobExecution/jobs/&JOBID./state` and the corresponding
+    job name.  The uri should be in a `uri` variable, and the job path/name
+    should be in a `_program` variable.
+  @param [out] outds= The output dataset containing the list of states by job
+    (default=mv_jobexecute)
+
+
+  @version VIYA V.03.04
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> Dependencies </h4>
+  @li mp_abort.sas
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mf_existvar.sas
+  @li mf_nobs.sas
+
+**/
+
+%macro mv_jobwaitfor(
+     access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+    ,inds=0
+    ,outds=work.mv_jobwaitfor
+  );
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+%put &sysmacroname: grant_type=&grant_type;
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+
+%mp_abort(iftrue=("&inds"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(input dataset not provided)
+)
+%mp_abort(iftrue=(%mf_existvar(&inds,uri)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(The URI variable was not found in the input dataset(&inds))
+)
+
+%if %mf_nobs(&inds)=0 %then %do;
+  %put NOTE: Zero observations in &inds, &sysmacroname will now exit;
+  %return;
+%end;
+
+options noquotelenmax;
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+data _null_;
+  set &inds end=last;
+  call symputx(cats('joburi',_n_),uri,'l');
+  call symputx(cats('jobname',_n_),_program,'l');
+  if last then call symputx('uricnt',_n_,'l');
+run;
+
+%local fname0 ;
+%let fname0=%mf_getuniquefileref();
+
+data &outds;
+  format _program uri $128. state $32. timestamp datetime19.;
+  stop;
+run;
+
+%local i;
+%do i=1 %to &uricnt;
+  %if "&&joburi&i" ne "0" %then %do;
+    proc http method='GET' out=&fname0 &oauth_bearer url="&base_uri/&&joburi&i";
+      headers "Accept"="text/plain"
+      %if &grant_type=authorization_code %then %do;
+              "Authorization"="Bearer &&&access_token_var"
+      %end;  ;
+    run;
+    %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+    %do;
+      data _null_;infile &fname0;input;putlog _infile_;run;
+      %mp_abort(mac=&sysmacroname
+        ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+      )
+    %end;
+
+    %let status=notset;
+    data _null_;
+      infile &fname0;
+      input;
+      call symputx('status',_infile_,'l');
+    run;
+
+    %if &status=completed or &status=failed or &status=canceled %then %do;
+      proc sql;
+      insert into &outds set
+        _program="&&jobname&i",
+        uri="&&joburi&i",
+        state="&status",
+        timestamp=datetime();
+      %let joburi&i=0;
+    %end;
+    %else %if &status=idle or &status=pending or &status=running %then %do;
+      data _null_;
+        call sleep(1,1);
+      run;
+    %end;
+    %else %do;
+      %mp_abort(mac=&sysmacroname
+        ,msg=%str(status &status not expected!!)
+      )
+    %end;
+  %end;
+  %if &i=&uricnt %then %do;
+    %local goback;
+    %let goback=0;
+    proc sql noprint;
+    select count(*) into:goback from &outds;
+    %if &goback ne &uricnt %then %let i=0;
+  %end;
+%end;
+
+/* clear refs */
+filename &fname0 clear;
+
+%mend;