fix: lua feature discovery logic fix

initial commit

.gitignore

@@ -1,2 +1,3 @@
 node_modules
-.DS_Store
+.DS_Store
+sasjsbuild/

CONTRIBUTING.md

@@ -1,83 +1,32 @@
 # Contributing
 
-Contributions to the macrocore library are warmly welcomed!  To avoid any
-misunderstandings, do please first discuss the change you wish to make via issue,
-email, or any other method with the owners of this repository before submitting
-a PR.
+Contributions are warmly welcomed!  To avoid any misunderstandings, do please first discuss the change you wish to make via issue, email, or any other method with the owners of this repository before submitting a PR.
 
-Please note we have a code of conduct, please follow it in all your interactions
-with the project.
+Please note we have a [code of conduct](https://www.contributor-covenant.org/version/2/0/code_of_conduct/), please follow it in all your interactions with the project.
+
+# Environment Setup
+
+This repository makes use of the [SASjs](https://sasjs.io) framework for code organisation, compilation, documentation, and deployment.  The following tools are highly recommended:
+
+* [NPM](https://sasjs.io/windows/#npm) - the runtime and dependency manager for [SASjs CLI](https://cli.sasjs.io) (batteries included)
+* [VSCode](https://sasjs.io/windows/#vscode) - feature packed IDE for code editing (warning - highly effective!)
+* [GIT](https://sasjs.io/windows/#git) - a safety net you cannot (and should not) do without.
+
+For generating the documentation (`sasjs doc`) it is also necessary to install [doxygen](https://www.doxygen.nl/manual/install.html).
 
 
-## Code of Conduct
+To get configured:
 
-### Our Pledge
+1.  Clone the repository
+2.  Install local dependencies (`npm install`)
+3.  Install the SASjs CLI globally (`npm install -g @sasjs/cli`)
+4.  Add a target, and authentication (`npm add`).  See [docs](https://cli.sasjs.io/add/).
 
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to making participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, gender identity and expression, level of experience,
-nationality, personal appearance, race, religion, or sexual identity and
-orientation.
+To contribute:
 
-### Our Standards
+1.  Create your feature branch (`git checkout -b myfeature`)
+2.  Make your change
+3.  Update the `all.sas` file (`python3 build.py`)
+4.  Commit the change, using the [conventional commit](https://www.conventionalcommits.org/en/v1.0.0) standard
+5.  Push and make a PR
 
-Examples of behavior that contributes to creating a positive environment
-include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-### Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-### Scope
-
-This Code of Conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
-address, posting via an official social media account, or acting as an appointed
-representative at an online or offline event. Representation of a project may be
-further defined and clarified by project maintainers.
-
-### Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at support@macropeople.com. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-### Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at [http://contributor-covenant.org/version/1/4][version]
-
-[homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/4/

all.sas

@@ -218,9 +218,11 @@ options noquotelenmax;
     %put Supported features:  PROCLUA;
   %end;
   %else %if &feature=PROCLUA %then %do;
+    /* https://blogs.sas.com/content/sasdummy/2015/08/03/using-lua-within-your-sas-programs */
     %if &platform=SASVIYA %then 1;
-    %else %if "&sysver"="9.3" or "&sysver"="9.4" %then 1;
-    %else 0;
+    %else %if "&sysver"="9.2" or "&sysver"="9.3" %then 0;
+    %else %if "&SYSVLONG" < "9.04.01M3" %then 0;
+    %else 1;
   %end;
   %else %do;
     -1
@@ -404,7 +406,7 @@ options noquotelenmax;
   %local dsid rc;
   %let dsid=%sysfunc(open(&libds,is));
   %if &dsid = 0 %then %do;
-    %put WARNING: Cannot open %trim(&libds), system message below;
+    %put %str(WARN)ING: Cannot open %trim(&libds), system message below;
     %put %sysfunc(sysmsg());
     -1
   %end;
@@ -510,8 +512,8 @@ options noquotelenmax;
   @brief retrieves a key value pair from a control dataset
   @details By default, control dataset is work.mp_setkeyvalue.  Usage:
 
-    %mp_setkeyvalue(someindex,22,type=N)
-    %put %mf_getkeyvalue(someindex)
+      %mp_setkeyvalue(someindex,22,type=N)
+      %put %mf_getkeyvalue(someindex)
 
 
   @param key Provide a key on which to perform the lookup
@@ -4548,8 +4550,8 @@ proc sql
   @brief Logs a key value pair a control dataset
   @details If the dataset does not exist, it is created.  Usage:
 
-    %mp_setkeyvalue(someindex,22,type=N)
-    %mp_setkeyvalue(somenewindex,somevalue)
+      %mp_setkeyvalue(someindex,22,type=N)
+      %mp_setkeyvalue(somenewindex,somevalue)
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
@@ -4571,7 +4573,7 @@ proc sql
 
   %if not (%mf_existds(&libds)) %then %do;
     data &libds (index=(key/unique));
-      length key $32 valc $256 valn 8 type $1;
+      length key $64 valc $2048 valn 8 type $1;
       call missing(of _all_);
       stop;
     run;
@@ -10659,7 +10661,6 @@ run;
   @details Expects oauth token in a global macro variable (default
   ACCESS_TOKEN).
 
-      options mprint;
       %mv_createfolder(path=/Public)
 
 
@@ -11509,7 +11510,7 @@ run;
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
@@ -11764,7 +11765,6 @@ libname &libref1a clear;
   @details If not running in Studo 5 +, will expect an oauth token in a global
   macro variable (default ACCESS_TOKEN).
 
-      options mprint;
       %mv_createfolder(path=/Public/test/blah)
       %mv_deleteviyafolder(path=/Public/test)
 
@@ -12094,7 +12094,7 @@ libname &libref1 clear;
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
@@ -12329,7 +12329,7 @@ filename &fname1 clear;
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
@@ -12422,7 +12422,6 @@ libname &libref1 clear;
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
@@ -12573,7 +12572,7 @@ filename &fname3 clear;
 
       %mv_getjoblog(uri=&uri,outref=mylog)
 
-  This macro is used by the mv_jobwaitfor macro, which is generally a more
+  This macro is used by the mv_jobwaitfor.sas macro, which is generally a more
   convenient way to wait for the job to finish before fetching the log.
 
 
@@ -12617,7 +12616,7 @@ filename &fname3 clear;
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
@@ -12769,7 +12768,7 @@ data _null_;
   end;
   input;
   put _infile_;
-  %if &mdebug=0 %then %do;
+  %if &mdebug=1 %then %do;
     putlog _infile_;
   %end;
   if last then do;
@@ -12892,7 +12891,7 @@ run;
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
@@ -13378,18 +13377,19 @@ libname &libref;
 
   ## Input table (minimum variables needed)
 
-  @li FLOW_ID - Numeric value, provides sequential ordering capability
-  @li _CONTEXTNAME - Dictates which context should be used to run the job. If
-    blank, will default to `SAS Job Execution compute context`.
   @li _PROGRAM - Provides the path to the job itself
+  @li FLOW_ID - Numeric value, provides sequential ordering capability. Is
+    optional, will default to 0 if not provided.
+  @li _CONTEXTNAME - Dictates which context should be used to run the job. If
+    blank (or not provided), will default to `SAS Job Execution compute context`.
 
   Any additional variables provided in this table are converted into macro
   variables and passed into the relevant job.
 
-  | FLOW_ID| _CONTEXTNAME   |_PROGRAM|
+  |_PROGRAM| FLOW_ID (optional)| _CONTEXTNAME (optional) |
   |---|---|---|
-  |0|SAS Job Execution compute context|/Public/jobs/somejob1|
-  |0|SAS Job Execution compute context|/Public/jobs/somejob2|
+  |/Public/jobs/somejob1|0|SAS Job Execution compute context|
+  |/Public/jobs/somejob2|0|SAS Job Execution compute context|
 
   ## Output table (minimum variables produced)
 
@@ -13402,6 +13402,9 @@ libname &libref;
 
   ![https://i.imgur.com/nZE9PvT.png](https://i.imgur.com/nZE9PvT.png)
 
+  To avoid hammering the box with many hits in rapid succession, a one
+  second pause is made between every request.
+
 
   ## Example
 
@@ -13450,7 +13453,16 @@ libname &libref;
 
   Trigger the flow
 
-      %mv_jobflow(inds=work.inputjobs,outds=work.results,maxconcurrency=4)
+      %mv_jobflow(inds=work.inputjobs
+        ,maxconcurrency=4
+        ,outds=work.results
+        ,outref=myjoblog
+      )
+
+      data _null_;
+        infile myjoblog;
+        input; put _infile_;
+      run;
 
 
   @param [in] access_token_var= The global macro variable to contain the access token
@@ -13462,7 +13474,9 @@ libname &libref;
       @li sas_services - will use oauth_bearer=sas_services
   @param [in] inds= The input dataset containing a list of jobs and parameters
   @param [in] maxconcurrency= The max number of parallel jobs to run.  Default=8.
+  @param [in] mdebug= set to 1 to enable DEBUG messages
   @param [out] outds= The output dataset containing the results
+  @param [out] outref= The output fileref to which to append the log file(s).
 
   @version VIYA V.03.05
   @author Allan Bowe, source: https://github.com/sasjs/core
@@ -13482,6 +13496,8 @@ libname &libref;
     ,maxconcurrency=8
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
+    ,outref=0
+    ,mdebug=0
   );
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
@@ -13504,16 +13520,29 @@ libname &libref;
   ,mac=&sysmacroname
   ,msg=%str(Input dataset was not provided)
 )
-%mp_abort(iftrue=(%mf_existVarList(&inds,_CONTEXTNAME FLOW_ID _PROGRAM)=0)
+%mp_abort(iftrue=(%mf_existVarList(&inds,_PROGRAM)=0)
   ,mac=&sysmacroname
-  ,msg=%str(The following columns must exist on input dataset &inds:
-    _CONTEXTNAME FLOW_ID _PROGRAM)
+  ,msg=%str(The _PROGRAM column must exist on input dataset &inds)
 )
 %mp_abort(iftrue=(&maxconcurrency<1)
   ,mac=&sysmacroname
   ,msg=%str(The maxconcurrency variable should be a positive integer)
 )
 
+/* set defaults if not provided */
+%if %mf_existVarList(&inds,_CONTEXTNAME FLOW_ID)=0 %then %do;
+  data &inds;
+    %if %mf_existvarList(&inds,_CONTEXTNAME)=0 %then %do;
+      length _CONTEXTNAME $128;
+      retain _CONTEXTNAME "SAS Job Execution compute context";
+    %end;
+    %if %mf_existvarList(&inds,FLOW_ID)=0 %then %do;
+      retain FLOW_ID 0;
+    %end;
+    set &inds;
+  run;
+%end;
+
 %local missings;
 proc sql noprint;
 select count(*) into: missings
@@ -13591,8 +13620,8 @@ data;run;%let jdswaitfor=&syslast;
       jparams='jparams'!!left(symget('jid'));
       call symputx(jparams,substr(_infile_,3,length(_infile_)-4));
     run;
-    %local joburi&jid;
-    %let joburi&jid=0; /* used in next loop */
+    %local jobuid&jid;
+    %let jobuid&jid=0; /* used in next loop */
   %end;
   %local concurrency completed;
   %let concurrency=0;
@@ -13603,8 +13632,21 @@ data;run;%let jdswaitfor=&syslast;
       * now we can execute the jobs up to the maxconcurrency setting
       */
     %if "&&job&jid" ne "0" %then %do; /* this var is zero if job finished */
-      %if "&&joburi&jid"="0" and &concurrency<&maxconcurrency %then %do;
-        /* job has not been triggered and we have free slots */
+
+      /* check to see if the job finished in the previous round */
+      %if %sysfunc(exist(&outds))=1 %then %do;
+        %local jobcheck;  %let jobcheck=0;
+        proc sql noprint;
+        select count(*) into: jobcheck
+          from &outds where uuid="&&jobuid&jid";
+        %if &jobcheck>0 %then %do;
+          %put &&job&jid in flow &fid with uid &&jobuid&jid completed!;
+          %let job&jid=0;
+        %end;
+      %end;
+
+      /* check if job was triggered and if so, if we have enough slots to run */
+      %if "&&jobuid&jid"="0" and &concurrency<&maxconcurrency %then %do;
         %local jobname jobpath;
         %let jobname=%scan(&&job&jid,-1,/);
         %let jobpath=%substr(&&job&jid,1,%length(&&job&jid)-%length(&jobname)-1);
@@ -13618,27 +13660,22 @@ data;run;%let jdswaitfor=&syslast;
           format jobparams $32767.;
           set &jdsapp(where=(method='GET' and rel='state'));
           jobparams=symget("jparams&jid");
-          call symputx("joburi&jid",uri,'l');
+          /* uri here has the /state suffix */
+          uuid=scan(uri,-2,'/');
+          call symputx("jobuid&jid",uuid,'l');
         run;
         proc append base=&jdsrunning data=&jdsapp;
         run;
         %let concurrency=%eval(&concurrency+1);
-      %end;
-      %else %if %sysfunc(exist(&outds))=1 %then %do;
-        /* check to see if the job has finished as was previously executed */
-        %local jobcheck;  %let jobcheck=0;
-        proc sql noprint;
-        select count(*) into: jobcheck
-          from &outds where uri="&&joburi&jid";
-        %if &jobcheck>0 %then %do;
-          %put &&job&jid in flow &fid with uri &&joburi&jid completed!;
-          %let job&jid=0;
-        %end;
+        /* sleep one second after every request to smooth the impact */
+        data _null_;
+          call sleep(1,1);
+        run;
       %end;
     %end;
     %if &jid=&jcnt %then %do;
       /* we are at the end of the loop - time to see which jobs have finished */
-      %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor)
+      %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref)
       %local done;
       %let done=%mf_nobs(&jdswaitfor);
       %if &done>0 %then %do;
@@ -13647,13 +13684,14 @@ data;run;%let jdswaitfor=&syslast;
         data &jdsapp;
           set &jdswaitfor;
           flow_id=&&flow&fid;
+          uuid=scan(uri,-1,'/');
         run;
         proc append base=&outds data=&jdsapp;
         run;
       %end;
       proc sql;
       delete from &jdsrunning
-        where uri in (select uri from &outds
+        where uuid in (select uuid from &outds
           where state in ('canceled','completed','failed')
         );
 
@@ -13667,6 +13705,9 @@ data;run;%let jdswaitfor=&syslast;
   /* back up and execute the next flow */
 %end;
 
+%if &mdebug=1 %then %do;
+  %put _local_;
+%end;
 
 %mend;
 /**

base/mf_existfeature.sas

@@ -32,9 +32,11 @@
     %put Supported features:  PROCLUA;
   %end;
   %else %if &feature=PROCLUA %then %do;
+    /* https://blogs.sas.com/content/sasdummy/2015/08/03/using-lua-within-your-sas-programs */
     %if &platform=SASVIYA %then 1;
-    %else %if "&sysver"="9.3" or "&sysver"="9.4" %then 1;
-    %else 0;
+    %else %if "&sysver"="9.2" or "&sysver"="9.3" %then 0;
+    %else %if "&SYSVLONG" < "9.04.01M3" %then 0;
+    %else 1;
   %end;
   %else %do;
     -1

base/mf_getattrn.sas

@@ -23,7 +23,7 @@
   %local dsid rc;
   %let dsid=%sysfunc(open(&libds,is));
   %if &dsid = 0 %then %do;
-    %put WARNING: Cannot open %trim(&libds), system message below;
+    %put %str(WARN)ING: Cannot open %trim(&libds), system message below;
     %put %sysfunc(sysmsg());
     -1
   %end;

base/mf_getkeyvalue.sas

@@ -3,8 +3,8 @@
   @brief retrieves a key value pair from a control dataset
   @details By default, control dataset is work.mp_setkeyvalue.  Usage:
 
-    %mp_setkeyvalue(someindex,22,type=N)
-    %put %mf_getkeyvalue(someindex)
+      %mp_setkeyvalue(someindex,22,type=N)
+      %put %mf_getkeyvalue(someindex)
 
 
   @param key Provide a key on which to perform the lookup

base/mp_setkeyvalue.sas

@@ -3,8 +3,8 @@
   @brief Logs a key value pair a control dataset
   @details If the dataset does not exist, it is created.  Usage:
 
-    %mp_setkeyvalue(someindex,22,type=N)
-    %mp_setkeyvalue(somenewindex,somevalue)
+      %mp_setkeyvalue(someindex,22,type=N)
+      %mp_setkeyvalue(somenewindex,somevalue)
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
@@ -26,7 +26,7 @@
 
   %if not (%mf_existds(&libds)) %then %do;
     data &libds (index=(key/unique));
-      length key $32 valc $256 valn 8 type $1;
+      length key $64 valc $2048 valn 8 type $1;
       call missing(of _all_);
       stop;
     run;

doxy/build.sh

@@ -1,45 +0,0 @@
-#!/bin/bash
-####################################################################
-# PROJECT: Macro Core Docs Build                                   #
-####################################################################
-
-BUILD_FOLDER="/tmp/macrocore_docs"
-
-# move to project root
-cd ..
-
-# create build directory
-rm -rf $BUILD_FOLDER
-mkdir $BUILD_FOLDER
-
-# copy relevant files
-cp -r base $BUILD_FOLDER
-cp -r meta $BUILD_FOLDER
-cp -r metax $BUILD_FOLDER
-cp -r lua $BUILD_FOLDER
-cp -r viya $BUILD_FOLDER
-cp -r doxy $BUILD_FOLDER
-cp main.dox $BUILD_FOLDER
-cp doxy/Doxyfile $BUILD_FOLDER
-
-# update Doxyfile and generate
-cd $BUILD_FOLDER
-echo "OUTPUT_DIRECTORY=$BUILD_FOLDER/out" >> $BUILD_FOLDER/Doxyfile
-echo "INPUT+=main.dox" >> $BUILD_FOLDER/Doxyfile
-doxygen Doxyfile
-
-# refresh github pages site
-git clone git@github.com:sasjs/core.github.io.git
-cd core.github.io
-rm -r *
-mv $BUILD_FOLDER/out/doxy/* .
-echo 'core.sasjs.io' > CNAME
-git add .
-git commit -m "build.sh build on $(date +%F:%H:%M:%S)"
-git push
-npx sitemap-generator-cli https://core.sasjs.io
-git add .
-git commit -m "adding sitemap"
-git push
-
-echo "check it out:  https://sasjs.github.io/core.github.io/files.html"

doxy/new_footer.html

@@ -1,23 +0,0 @@
-<!-- HTML footer for doxygen 1.8.17-->
-<!-- start footer part -->
-<!--BEGIN GENERATE_TREEVIEW-->
-<div id="nav-path" class="navpath"><!-- id is needed for treeview function! -->
-  <ul>
-    $navpath
-    <li class="footer">$generatedby
-    <a href="https://www.doxygen.org/index.html">
-      <img class="footer" src="$relpath^doxygen.png" alt="doxygen"/></a> $doxygenversion </li>
-   <i> For more information visit the </i> <a href="https://github.com/sasjs/core">Macro Core library</a>.</li>
-
-   </ul>
- </div>
-<!--END GENERATE_TREEVIEW-->
-<!--BEGIN !GENERATE_TREEVIEW-->
-<hr class="footer"/><address class="footer"><small>
-  $generatedby &#160;<a href="http://www.doxygen.org/index.html">
-  <img class="footer" src="$relpath^doxygen.png" alt="doxygen"/>
-  </a> $doxygenversion
-  </small></address>
-<!--END !GENERATE_TREEVIEW-->
-</body>
-</html>

doxy/new_header.html

@@ -1,72 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<!-- HTML header for doxygen 1.8.17-->
-<html xmlns="https://www.w3.org/1999/xhtml" lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
-<meta http-equiv="X-UA-Compatible" content="IE=9"/>
-<meta name="generator" content="Doxygen $doxygenversion"/>
-<meta name="viewport" content="width=device-width, initial-scale=1"/>
-<!--BEGIN PROJECT_NAME--><title>$projectname: $title</title><!--END PROJECT_NAME-->
-<!--BEGIN !PROJECT_NAME--><title>$title</title><!--END !PROJECT_NAME-->
-<link href="$relpath^tabs.css" rel="stylesheet" type="text/css"/>
-<script type="text/javascript" src="$relpath^jquery.js"></script>
-<script type="text/javascript" src="$relpath^dynsections.js"></script>
-$treeview
-$search
-$mathjax
-<link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
-<link REL="icon" HREF="https://sasjs.io/img/runningman.jpg">
-$extrastylesheet
-</head>
-<body>
-<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
-
-<!--BEGIN TITLEAREA-->
-<div id="titlearea">
-<table cellspacing="0" cellpadding="0">
- <tbody>
-  <tr style="height: 56px;">
-  <!--BEGIN PROJECT_LOGO-->
-  <td id="projectlogo">
-    <img alt="Logo" src="$relpath^$projectlogo"/></td>
-  <!--END PROJECT_LOGO-->
-<!--BEGIN PROJECT_NAME-->
-<td id="projectalign" style="padding-left: 0.5em;">
-  <div id="projectname">
-   <!--BEGIN PROJECT_NUMBER-->&#160;<span id="projectnumber">$projectnumber</span><!--END PROJECT_NUMBER-->
-   </div>
-   <!--BEGIN PROJECT_BRIEF--><div id="projectbrief">
-    Production Ready Macros for SAS Application Developers</br>
-    <a href="https://github.com/sasjs/core">
-        https://github.com/sasjs/core
-    </a>
-
-   </div>
-   <meta name="Description" content="$projectbrief">
-   <!--END PROJECT_BRIEF-->
-  </td>
-  <!--END PROJECT_NAME-->
-  <!--BEGIN !PROJECT_NAME-->
-   <!--BEGIN PROJECT_BRIEF-->
-   <td style="padding-left: 0.5em;">
-    <div id="projectbrief">$projectbrief</div>
-    <table style="padding-left: 2em;" cellspacing="0" cellpadding="0">
-      <tr><td> Production Ready Macros for SAS Application Developers</td></tr>
-      <tr><td><a href="https://github.com/sasjs/core">
-          https://github.com/sasjs/core
-      </a></td></tr>
-    </table>
-    </td>
-   <!--END PROJECT_BRIEF-->
-  <!--END !PROJECT_NAME-->
-  <!--BEGIN DISABLE_INDEX-->
-   <!--BEGIN SEARCHENGINE-->
-   <td>$searchbox</td>
-   <!--END SEARCHENGINE-->
-  <!--END DISABLE_INDEX-->
- </tr>
- </tbody>
-</table>
-</div>
-<!--END TITLEAREA-->
-<!-- end header part -->

package.json

@@ -10,13 +10,27 @@
   "author": "Allan Bowe <support@macropeople.com>",
   "repository": {
     "type": "git",
-    "url": "https://github.com/sasjs/core"
+    "url": "git+https://github.com/sasjs/core.git"
   },
   "release": {
-    "branches": ["main"]
+    "branches": [
+      "main"
+    ]
   },
   "publishConfig": {
     "access": "public"
   },
-  "devDependencies": {}
+  "bugs": {
+    "url": "https://github.com/sasjs/core/issues"
+  },
+  "homepage": "https://github.com/sasjs/core#readme",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "docs": "./sasjs/utils/build.sh"
+  },
+  "devDependencies": {
+    "@sasjs/cli": "^2.4.0"
+  }
 }

sasjs/doxy/Doxyfile

@@ -1,5 +1,5 @@
 ALPHABETICAL_INDEX     = NO
-DISABLE_INDEX          = YES
+DISABLE_INDEX          = NO
 ENABLE_PREPROCESSING   = NO
 EXTENSION_MAPPING      = sas=Java ddl=Java
 EXTRACT_LOCAL_CLASSES  = NO
@@ -13,17 +13,20 @@ HIDE_IN_BODY_DOCS      = YES
 HIDE_SCOPE_NAMES       = YES
 HIDE_UNDOC_CLASSES     = YES
 HIDE_UNDOC_MEMBERS     = YES
-HTML_OUTPUT            = doxy
-HTML_HEADER            = ./doxy/new_header.html
-HTML_FOOTER            = ./doxy/new_footer.html
-HTML_EXTRA_STYLESHEET  = ./doxy/new_stylesheet.css
+HTML_OUTPUT            = $(DOXY_HTML_OUTPUT)
+HTML_HEADER            = $(DOXY_CONTENT)new_header.html
+HTML_FOOTER            = $(DOXY_CONTENT)new_footer.html
+HTML_EXTRA_STYLESHEET  = $(DOXY_CONTENT)new_stylesheet.css
 INHERIT_DOCS           = NO
 INLINE_INFO            = NO
-INPUT                  = base meta metax viya lua
-LAYOUT_FILE            = ./doxy/DoxygenLayout.xml
+INPUT                  = $(DOXY_CONTENT)../../README.md \
+                         $(DOXY_CONTENT)../../CONTRIBUTING.md \
+                         $(DOXY_INPUT)
+USE_MDFILE_AS_MAINPAGE = README.md
+LAYOUT_FILE            = $(DOXY_CONTENT)DoxygenLayout.xml
 MAX_INITIALIZER_LINES  = 0
 PROJECT_NAME           = Macro Core
-PROJECT_LOGO           = doxy/Macro_core_website_1.png
+PROJECT_LOGO           = $(DOXY_CONTENT)Macro_core_website_1.png
 PROJECT_BRIEF          = "Production Ready Macros for SAS Application Developers"
 RECURSIVE              = YES
 REPEAT_BRIEF           = NO

sasjs/doxy/DoxygenLayout.xml

@@ -2,7 +2,8 @@
   <!-- Generated by doxygen 1.8.14 -->
   <!-- Navigation index tabs for HTML output -->
   <navindex>
-    <tab type="mainpage" visible="no" title=""/>
+    <tab type="mainpage" visible="yes" title="Home"/>
+    <tab type="user" url="/contributing" title="Contributing"/>
     <tab type="pages" visible="no" title="" intro=""/>
     <tab type="modules" visible="no" title="" intro=""/>
     <tab type="namespaces" visible="no" title="">
@@ -108,4 +109,4 @@
       <files visible="yes"/>
     </memberdecl>
   </directory>
-</doxygenlayout>
+</doxygenlayout>

sasjs/doxy/new_footer.html

@@ -0,0 +1,32 @@
+<!-- HTML footer for doxygen 1.8.17-->
+<!-- start footer part -->
+<!--BEGIN GENERATE_TREEVIEW-->
+<div id="nav-path" class="navpath">
+  <!-- id is needed for treeview function! -->
+  <ul>
+    $navpath
+    <li class="footer">
+      $generatedby
+      <a href="https://www.doxygen.org/index.html">
+        <img class="footer" src="$relpath^doxygen.png" alt="doxygen"
+      /></a>
+      $doxygenversion
+    </li>
+    <li>
+      <i> For more information visit the </i>
+      <a href="https://github.com/sasjs/core">Macro Core library</a>.
+    </li>
+  </ul>
+</div>
+<!--END GENERATE_TREEVIEW-->
+<!--BEGIN !GENERATE_TREEVIEW-->
+<hr class="footer" />
+<address class="footer">
+  <small>
+    $generatedby &#160;<a href="http://www.doxygen.org/index.html">
+      <img class="footer" src="$relpath^doxygen.png" alt="doxygen" />
+    </a>
+    $doxygenversion
+  </small>
+</address>
+<!--END !GENERATE_TREEVIEW-->

sasjs/doxy/new_header.html

@@ -0,0 +1,93 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!-- HTML header for doxygen 1.8.17-->
+<html xmlns="https://www.w3.org/1999/xhtml" lang="en">
+  <head>
+    <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=9" />
+    <meta name="generator" content="Doxygen $doxygenversion" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <!--BEGIN PROJECT_NAME-->
+    <title>$projectname: $title</title>
+    <!--END PROJECT_NAME-->
+    <!--BEGIN !PROJECT_NAME-->
+    <title>$title</title>
+    <!--END !PROJECT_NAME-->
+    <link href="$relpath^tabs.css" rel="stylesheet" type="text/css" />
+    <script type="text/javascript" src="$relpath^jquery.js"></script>
+    <script type="text/javascript" src="$relpath^dynsections.js"></script>
+    $treeview $search $mathjax
+    <link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
+    <link rel="icon" href="https://sasjs.io/img/runningman.jpg" />
+    $extrastylesheet
+  </head>
+  <body>
+    <div id="top">
+      <!-- do not remove this div, it is closed by doxygen! -->
+
+      <!--BEGIN TITLEAREA-->
+      <div id="titlearea">
+        <table cellspacing="0" cellpadding="0">
+          <tbody>
+            <tr style="height: 56px">
+              <!--BEGIN PROJECT_LOGO-->
+              <td id="projectlogo">
+                <img alt="Logo" src="$relpath^$projectlogo" />
+              </td>
+              <!--END PROJECT_LOGO-->
+              <!--BEGIN PROJECT_NAME-->
+              <td id="projectalign" style="padding-left: 0.5em">
+                <div id="projectname">
+                  <!--BEGIN PROJECT_NUMBER-->&#160;<span id="projectnumber"
+                    >$projectnumber</span
+                  ><!--END PROJECT_NUMBER-->
+                </div>
+                <!--BEGIN PROJECT_BRIEF-->
+                <div id="projectbrief">
+                  Production Ready Macros for SAS Application Developers<br />
+                  <a href="https://github.com/sasjs/core">
+                    https://github.com/sasjs/core
+                  </a>
+                </div>
+                <meta name="Description" content="$projectbrief" />
+                <!--END PROJECT_BRIEF-->
+              </td>
+              <!--END PROJECT_NAME-->
+              <!--BEGIN !PROJECT_NAME-->
+              <!--BEGIN PROJECT_BRIEF-->
+              <td style="padding-left: 0.5em">
+                <div id="projectbrief">$projectbrief</div>
+                <table
+                  style="padding-left: 2em"
+                  cellspacing="0"
+                  cellpadding="0"
+                >
+                  <tr>
+                    <td>
+                      Production Ready Macros for SAS Application Developers
+                    </td>
+                  </tr>
+                  <tr>
+                    <td>
+                      <a href="https://github.com/sasjs/core">
+                        https://github.com/sasjs/core
+                      </a>
+                    </td>
+                  </tr>
+                </table>
+              </td>
+              <!--END PROJECT_BRIEF-->
+              <!--END !PROJECT_NAME-->
+              <!--BEGIN DISABLE_INDEX-->
+              <!--BEGIN SEARCHENGINE-->
+              <td>$searchbox</td>
+              <!--END SEARCHENGINE-->
+              <!--END DISABLE_INDEX-->
+            </tr>
+          </tbody>
+        </table>
+      </div>
+      <!--END TITLEAREA-->
+      <!-- end header part -->
+    </div>
+  </body>
+</html>

sasjs/doxy/new_stylesheet.css

@@ -1,5 +1,5 @@
-#projectlogo img
-{
-	border: 0px none;
-	max-height:70px
+#projectlogo img
+{
+	border: 0px none;
+	max-height:70px
 }

sasjs/sasjsconfig.json

@@ -0,0 +1,7 @@
+{
+  "$schema": "https://cli.sasjs.io/sasjsconfig-schema.json",
+  "macroFolders": ["../base", "../meta", "../metax", "../viya", "../lua"],
+  "docConfig":{
+    "displayMacroCore": false
+  }
+}

sasjs/utils/build.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+####################################################################
+# PROJECT: Macro Core Docs Build                                   #
+####################################################################
+
+sasjs doc
+
+# refresh github pages site
+rm -rf sasjsbuild/docsite
+git clone git@github.com:sasjs/core.github.io.git sasjsbuild/docsite
+rm -rf sasjsbuild/docsite/*
+mv sasjsbuild/docs/* sasjsbuild/docsite/
+cd sasjsbuild/docsite/
+echo 'core.sasjs.io' > CNAME
+git add .
+git commit -m "build.sh build on $(date +%F:%H:%M:%S)"
+git push
+npx sitemap-generator-cli https://core.sasjs.io
+git add .
+git commit -m "adding sitemap"
+git push
+
+echo "check it out:  https://sasjs.github.io/core.github.io/files.html"

viya/mv_createfolder.sas

@@ -4,7 +4,6 @@
   @details Expects oauth token in a global macro variable (default
   ACCESS_TOKEN).
 
-      options mprint;
       %mv_createfolder(path=/Public)
 
 

viya/mv_deletefoldermember.sas

@@ -46,7 +46,7 @@
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )

viya/mv_deleteviyafolder.sas

@@ -4,7 +4,6 @@
   @details If not running in Studo 5 +, will expect an oauth token in a global
   macro variable (default ACCESS_TOKEN).
 
-      options mprint;
       %mv_createfolder(path=/Public/test/blah)
       %mv_deleteviyafolder(path=/Public/test)
 

viya/mv_getfoldermembers.sas

@@ -40,7 +40,7 @@
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )

viya/mv_getgroups.sas

@@ -52,7 +52,7 @@
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )

viya/mv_getjobcode.sas

@@ -49,7 +49,6 @@
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )

viya/mv_getjoblog.sas

@@ -49,7 +49,7 @@
 
       %mv_getjoblog(uri=&uri,outref=mylog)
 
-  This macro is used by the mv_jobwaitfor macro, which is generally a more
+  This macro is used by the mv_jobwaitfor.sas macro, which is generally a more
   convenient way to wait for the job to finish before fetching the log.
 
 
@@ -93,7 +93,7 @@
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
@@ -245,7 +245,7 @@ data _null_;
   end;
   input;
   put _infile_;
-  %if &mdebug=0 %then %do;
+  %if &mdebug=1 %then %do;
     putlog _infile_;
   %end;
   if last then do;

viya/mv_getjobstate.sas

@@ -101,7 +101,7 @@
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )

viya/mv_jobflow.sas

@@ -9,18 +9,19 @@
 
   ## Input table (minimum variables needed)
 
-  @li FLOW_ID - Numeric value, provides sequential ordering capability
-  @li _CONTEXTNAME - Dictates which context should be used to run the job. If
-    blank, will default to `SAS Job Execution compute context`.
   @li _PROGRAM - Provides the path to the job itself
+  @li FLOW_ID - Numeric value, provides sequential ordering capability. Is
+    optional, will default to 0 if not provided.
+  @li _CONTEXTNAME - Dictates which context should be used to run the job. If
+    blank (or not provided), will default to `SAS Job Execution compute context`.
 
   Any additional variables provided in this table are converted into macro
   variables and passed into the relevant job.
 
-  | FLOW_ID| _CONTEXTNAME   |_PROGRAM|
+  |_PROGRAM| FLOW_ID (optional)| _CONTEXTNAME (optional) |
   |---|---|---|
-  |0|SAS Job Execution compute context|/Public/jobs/somejob1|
-  |0|SAS Job Execution compute context|/Public/jobs/somejob2|
+  |/Public/jobs/somejob1|0|SAS Job Execution compute context|
+  |/Public/jobs/somejob2|0|SAS Job Execution compute context|
 
   ## Output table (minimum variables produced)
 
@@ -33,6 +34,9 @@
 
   ![https://i.imgur.com/nZE9PvT.png](https://i.imgur.com/nZE9PvT.png)
 
+  To avoid hammering the box with many hits in rapid succession, a one
+  second pause is made between every request.
+
 
   ## Example
 
@@ -81,7 +85,16 @@
 
   Trigger the flow
 
-      %mv_jobflow(inds=work.inputjobs,outds=work.results,maxconcurrency=4)
+      %mv_jobflow(inds=work.inputjobs
+        ,maxconcurrency=4
+        ,outds=work.results
+        ,outref=myjoblog
+      )
+
+      data _null_;
+        infile myjoblog;
+        input; put _infile_;
+      run;
 
 
   @param [in] access_token_var= The global macro variable to contain the access token
@@ -93,7 +106,9 @@
       @li sas_services - will use oauth_bearer=sas_services
   @param [in] inds= The input dataset containing a list of jobs and parameters
   @param [in] maxconcurrency= The max number of parallel jobs to run.  Default=8.
+  @param [in] mdebug= set to 1 to enable DEBUG messages
   @param [out] outds= The output dataset containing the results
+  @param [out] outref= The output fileref to which to append the log file(s).
 
   @version VIYA V.03.05
   @author Allan Bowe, source: https://github.com/sasjs/core
@@ -113,6 +128,8 @@
     ,maxconcurrency=8
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
+    ,outref=0
+    ,mdebug=0
   );
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
@@ -135,16 +152,29 @@
   ,mac=&sysmacroname
   ,msg=%str(Input dataset was not provided)
 )
-%mp_abort(iftrue=(%mf_existVarList(&inds,_CONTEXTNAME FLOW_ID _PROGRAM)=0)
+%mp_abort(iftrue=(%mf_existVarList(&inds,_PROGRAM)=0)
   ,mac=&sysmacroname
-  ,msg=%str(The following columns must exist on input dataset &inds:
-    _CONTEXTNAME FLOW_ID _PROGRAM)
+  ,msg=%str(The _PROGRAM column must exist on input dataset &inds)
 )
 %mp_abort(iftrue=(&maxconcurrency<1)
   ,mac=&sysmacroname
   ,msg=%str(The maxconcurrency variable should be a positive integer)
 )
 
+/* set defaults if not provided */
+%if %mf_existVarList(&inds,_CONTEXTNAME FLOW_ID)=0 %then %do;
+  data &inds;
+    %if %mf_existvarList(&inds,_CONTEXTNAME)=0 %then %do;
+      length _CONTEXTNAME $128;
+      retain _CONTEXTNAME "SAS Job Execution compute context";
+    %end;
+    %if %mf_existvarList(&inds,FLOW_ID)=0 %then %do;
+      retain FLOW_ID 0;
+    %end;
+    set &inds;
+  run;
+%end;
+
 %local missings;
 proc sql noprint;
 select count(*) into: missings
@@ -222,8 +252,8 @@ data;run;%let jdswaitfor=&syslast;
       jparams='jparams'!!left(symget('jid'));
       call symputx(jparams,substr(_infile_,3,length(_infile_)-4));
     run;
-    %local joburi&jid;
-    %let joburi&jid=0; /* used in next loop */
+    %local jobuid&jid;
+    %let jobuid&jid=0; /* used in next loop */
   %end;
   %local concurrency completed;
   %let concurrency=0;
@@ -234,8 +264,21 @@ data;run;%let jdswaitfor=&syslast;
       * now we can execute the jobs up to the maxconcurrency setting
       */
     %if "&&job&jid" ne "0" %then %do; /* this var is zero if job finished */
-      %if "&&joburi&jid"="0" and &concurrency<&maxconcurrency %then %do;
-        /* job has not been triggered and we have free slots */
+
+      /* check to see if the job finished in the previous round */
+      %if %sysfunc(exist(&outds))=1 %then %do;
+        %local jobcheck;  %let jobcheck=0;
+        proc sql noprint;
+        select count(*) into: jobcheck
+          from &outds where uuid="&&jobuid&jid";
+        %if &jobcheck>0 %then %do;
+          %put &&job&jid in flow &fid with uid &&jobuid&jid completed!;
+          %let job&jid=0;
+        %end;
+      %end;
+
+      /* check if job was triggered and if so, if we have enough slots to run */
+      %if "&&jobuid&jid"="0" and &concurrency<&maxconcurrency %then %do;
         %local jobname jobpath;
         %let jobname=%scan(&&job&jid,-1,/);
         %let jobpath=%substr(&&job&jid,1,%length(&&job&jid)-%length(&jobname)-1);
@@ -249,27 +292,22 @@ data;run;%let jdswaitfor=&syslast;
           format jobparams $32767.;
           set &jdsapp(where=(method='GET' and rel='state'));
           jobparams=symget("jparams&jid");
-          call symputx("joburi&jid",uri,'l');
+          /* uri here has the /state suffix */
+          uuid=scan(uri,-2,'/');
+          call symputx("jobuid&jid",uuid,'l');
         run;
         proc append base=&jdsrunning data=&jdsapp;
         run;
         %let concurrency=%eval(&concurrency+1);
-      %end;
-      %else %if %sysfunc(exist(&outds))=1 %then %do;
-        /* check to see if the job has finished as was previously executed */
-        %local jobcheck;  %let jobcheck=0;
-        proc sql noprint;
-        select count(*) into: jobcheck
-          from &outds where uri="&&joburi&jid";
-        %if &jobcheck>0 %then %do;
-          %put &&job&jid in flow &fid with uri &&joburi&jid completed!;
-          %let job&jid=0;
-        %end;
+        /* sleep one second after every request to smooth the impact */
+        data _null_;
+          call sleep(1,1);
+        run;
       %end;
     %end;
     %if &jid=&jcnt %then %do;
       /* we are at the end of the loop - time to see which jobs have finished */
-      %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor)
+      %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref)
       %local done;
       %let done=%mf_nobs(&jdswaitfor);
       %if &done>0 %then %do;
@@ -278,13 +316,14 @@ data;run;%let jdswaitfor=&syslast;
         data &jdsapp;
           set &jdswaitfor;
           flow_id=&&flow&fid;
+          uuid=scan(uri,-1,'/');
         run;
         proc append base=&outds data=&jdsapp;
         run;
       %end;
       proc sql;
       delete from &jdsrunning
-        where uri in (select uri from &outds
+        where uuid in (select uuid from &outds
           where state in ('canceled','completed','failed')
         );
 
@@ -298,5 +337,8 @@ data;run;%let jdswaitfor=&syslast;
   /* back up and execute the next flow */
 %end;
 
+%if &mdebug=1 %then %do;
+  %put _local_;
+%end;
 
 %mend;