fix: refreshed the testing toolkit, added debug options, updated documentation, included one new test (mv_getjobcode.sas)

feat: new (generic) mp_assert macro, and new feature (type filter) fo…

.gitignore

@@ -1,6 +1,7 @@
 node_modules
 .DS_Store
 sasjsbuild/
+sasjsresults/
 
 # avoid filenames with spaces being committed to source control
 **\ **

base/mf_existvarlist.sas

@@ -4,7 +4,7 @@
   @details Returns 0 if ANY of the variables do not exist, or 1 if they ALL do.
   Usage:
 
-        %put %mf_existVarList(sashelp.class, age sex name dummyvar)
+      %put %mf_existVarList(sashelp.class, age sex name dummyvar);
 
   <h4> SAS Macros </h4>
   @li mf_abort.sas
@@ -29,7 +29,7 @@
   %let dsid=%sysfunc(open(&libds,is));
 
   %if &dsid=0 %then %do;
-    %put WARNING:  unable to open &libds in mf_existvarlist (&dsid);
+    %put %str(WARN)ING:  unable to open &libds in mf_existvarlist (&dsid);
   %end;
 
   %if %sysfunc(attrn(&dsid,NVARS))=0 %then %do;

base/mf_getattrc.sas

@@ -23,7 +23,7 @@
   %local dsid rc;
   %let dsid=%sysfunc(open(&libds,is));
   %if &dsid = 0 %then %do;
-    %put WARNING: Cannot open %trim(&libds), system message below;
+    %put %str(WARN)ING: Cannot open %trim(&libds), system message below;
     %put %sysfunc(sysmsg());
     -1
   %end;

base/mf_getvarformat.sas

@@ -23,9 +23,9 @@
       8.
       NOTE: Variable renegade does not exist in test
 
-  @param libds Two part dataset (or view) reference.
+  @param [in] libds Two part dataset (or view) reference.
-  @param var Variable name for which a format should be returned
+  @param [in] var Variable name for which a format should be returned
-  @param force Set to 1 to supply a default if the variable has no format
+  @param [in] force=(0) Set to 1 to supply a default if the variable has no format
   @returns outputs format
 
   @author Allan Bowe
@@ -60,7 +60,7 @@
     %let vlen = %sysfunc(varlen(&dsid, &vnum));
     %let vtype = %sysfunc(vartype(&dsid, &vnum.));
     %if &vtype=C %then %let vformat=$&vlen..;
-    %else %let vformat=8.;
+    %else %let vformat=best.;
   %end;
 
 
@@ -68,4 +68,4 @@
   %let rc = %sysfunc(close(&dsid));
   /* Return variable format */
   &vformat
-%mend;
+%mend mf_getVarFormat;

base/mf_getvarlist.sas

@@ -21,6 +21,10 @@
   @param [in] dlm= ( ) Provide a delimiter (eg comma or space) to separate the
     variables
   @param [in] quote= (none) use either DOUBLE or SINGLE to quote the results
+  @param [in] typefilter= (A) Filter for certain types of column.  Valid values:
+    @li A Return All columns
+    @li C Return Character columns
+    @li N Return Numeric columns
 
   @version 9.2
   @author Allan Bowe
@@ -30,9 +34,10 @@
 %macro mf_getvarlist(libds
       ,dlm=%str( )
       ,quote=no
+      ,typefilter=A
 )/*/STORE SOURCE*/;
   /* declare local vars */
-  %local outvar dsid nvars x rc dlm q var;
+  %local outvar dsid nvars x rc dlm q var vtype;
 
   /* credit Rowland Hale  - byte34 is double quote, 39 is single quote */
   %if %upcase(&quote)=DOUBLE %then %let q=%qsysfunc(byte(34));
@@ -40,23 +45,24 @@
   /* open dataset in macro */
   %let dsid=%sysfunc(open(&libds));
 
-
   %if &dsid %then %do;
     %let nvars=%sysfunc(attrn(&dsid,NVARS));
     %if &nvars>0 %then %do;
-      /* add first dataset variable to global macro variable */
+      /* add variables with supplied delimeter */
-      %let outvar=&q.%sysfunc(varname(&dsid,1))&q.;
-      /* add remaining variables with supplied delimeter */
       %do x=1 %to &nvars;
+        /* get variable type */
+        %let vtype=%sysfunc(vartype(&dsid,&x));
+        %if &vtype=&typefilter or &typefilter=A %then %do;
           %let var=&q.%sysfunc(varname(&dsid,&x))&q.;
           %if &var=&q&q %then %do;
             %put &sysmacroname: Empty column found in &libds!;
             %let var=&q. &q.;
           %end;
-        %if &x=1 %then %let outvar=&var;
+          %if %quote(&outvar)=%quote() %then %let outvar=&var;
           %else %let outvar=&outvar.&dlm.&var.;
         %end;
       %end;
+    %end;
     %let rc=%sysfunc(close(&dsid));
   %end;
   %else %do;
@@ -64,4 +70,4 @@
     %let rc=%sysfunc(close(&dsid));
   %end;
   &outvar
-%mend;
+%mend mf_getvarlist;

base/mf_getvartype.sas

@@ -45,4 +45,4 @@ Usage:
   %let rc = %sysfunc(close(&dsid));
   /* Return variable type */
   &vtype
-%mend;
+%mend mf_getvartype;

base/mf_uid.sas

@@ -18,4 +18,4 @@
 
   &today._&now._&sysjobid._%sysevalf(%sysfunc(ranuni(0))*999,CEIL)
 
-%mend;
+%mend mf_uid;

base/mf_wordsinstr1butnotstr2.sas

@@ -30,7 +30,7 @@
 
 %local count_base count_extr i i2 extr_word base_word match outvar;
 %if %length(&str1)=0 or %length(&str2)=0 %then %do;
-  %put WARNING: empty string provided!;
+  %put %str(WARN)ING: empty string provided!;
   %put base string (str1)= &str1;
   %put compare string (str2) = &str2;
   %return;

base/mp_assert.sas

@@ -0,0 +1,56 @@
+/**
+  @file
+  @brief Generic assertion
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      %mp_assert(iftrue=(1=1),
+        desc=Obviously true
+      )
+
+      %mp_assert(iftrue=(1=0),
+        desc=Will fail
+      )
+
+  @param [in] iftrue= (1=1) A condition where, if true, the test is a PASS.
+  Else, the test is a fail.
+
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Column &inds contained ALL columns|
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assert(iftrue=(1=1),
+  desc=0,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  data ;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_comments="&sysmacroname: Test result of "!!symget('iftrue');
+  %if %eval(%unquote(&iftrue)) %then %do;
+    test_result='PASS';
+  %end;
+  %else %do;
+    test_result='FAIL';
+  %end;
+  run;
+
+  %local ds ;
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+
+%mend mp_assert;

base/mp_assertcols.sas

@@ -0,0 +1,145 @@
+/**
+  @file
+  @brief Asserts the existence (or not) of columns
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      %mp_assertcols(sashelp.class,
+        cols=name age sex,
+        test=ALL,
+        desc=check all columns exist
+      )
+
+      %mp_assertcols(sashelp.class,
+        cols=a b c,
+        test=NONE
+      )
+
+      %mp_assertcols(sashelp.class,
+        cols=age depth,
+        test=ANY
+      )
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_existvarlist.sas
+  @li mf_getvarlist.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_abort.sas
+
+
+  @param [in] inds The input library.dataset to test for values
+  @param [in] cols= The list of columns to check for
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [in] test= (ALL) The test to apply.  Valid values are:
+    @li ALL - Test is a PASS if ALL columns exist in &inds
+    @li ANY - Test is a PASS if ANY of the columns exist in &inds
+    @li NONE - Test is a PASS if NONE of the columns exist in &inds
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Column &inds contained ALL columns|
+
+
+  <h4> Related Macros </h4>
+  @li mp_assertdsobs.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertcols(inds,
+  cols=0,
+  test=ALL,
+  desc=0,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  %mp_abort(iftrue= (&syscc ne 0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc - on macro entry)
+  )
+
+  %local lib ds ;
+  %let lib=%scan(&inds,1,%str(.));
+  %let ds=%scan(&inds,2,%str(.));
+  %let cols=%upcase(&cols);
+
+  %mp_abort(iftrue= (%mf_existds(&lib..&ds)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(&lib..&ds not found!)
+  )
+
+  %mp_abort(iftrue= (&cols=0)
+    ,mac=&sysmacroname
+    ,msg=%str(No cols provided)
+  )
+
+
+  %let test=%upcase(&test);
+
+  %if &test ne ANY and &test ne ALL and &test ne NONE %then %do;
+    %mp_abort(
+      mac=&sysmacroname,
+      msg=%str(Invalid test - &test)
+    )
+  %end;
+
+  /**
+    * now do the actual test!
+    */
+  %local result;
+  %if %mf_existVarList(&inds,&cols)=1 %then %let result=ALL;
+  %else %do;
+    %local targetcols compare;
+    %let targetcols=%upcase(%mf_getvarlist(&inds));
+    %let compare=%mf_wordsinstr1butnotstr2(
+        Str1=&cols,
+        Str2=&targetcols
+      );
+    %if %cmpres(&compare)=%cmpres(&cols) %then %let result=NONE;
+    %else %let result=SOME;
+  %end;
+
+  data;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    if test_description='0'
+    then test_description="Testing &inds for existence of &test of: &cols";
+
+    test_result='FAIL';
+    test_comments="&sysmacroname: &inds has &result columns ";
+  %if &test=ALL %then %do;
+    %if &result=ALL %then %do;
+      test_result='PASS';
+    %end;
+  %end;
+  %else %if &test=ANY %then %do;
+    %if &result=SOME %then %do;
+      test_result='PASS';
+    %end;
+  %end;
+  %else %if &test=NONE %then %do;
+    %if &result=NONE %then %do;
+      test_result='PASS';
+    %end;
+  %end;
+  %else %do;
+    test_comments="&sysmacroname: Unsatisfied test condition - &test";
+  %end;
+  run;
+
+  %local ds;
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+
+%mend;

base/mp_ds2cards.sas

@@ -43,7 +43,7 @@
 %local i setds nvars;
 
 %if not %sysfunc(exist(&base_ds)) %then %do;
-  %put WARNING:  &base_ds does not exist;
+  %put %str(WARN)ING:  &base_ds does not exist;
   %return;
 %end;
 
@@ -61,7 +61,7 @@ select count(*) into: nvars from dictionary.columns
   where libname="%scan(%upcase(&base_ds),1)"
     and memname="%scan(%upcase(&base_ds),2)";
 %if &nvars=0 %then %do;
-  %put WARNING:  Dataset &base_ds has no variables!  It will not be converted.;
+  %put %str(WARN)ING: Dataset &base_ds has no variables, will not be converted.;
   %return;
 %end;
 

base/mp_ds2csv.sas

@@ -19,7 +19,7 @@
 )/*/STORE SOURCE*/;
 
 %if not %sysfunc(exist(&ds)) %then %do;
-  %put WARNING:  &ds does not exist;
+  %put %str(WARN)ING:  &ds does not exist;
   %return;
 %end;
 

base/mp_filtercheck.sas

@@ -14,7 +14,7 @@
 
   The input table should have the following format:
 
-  |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
+  |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$4000|
   |---|---|---|---|---|---|
   |AND|AND|1|AGE|=|12|
   |AND|AND|1|SEX|<=|'M'|
@@ -27,7 +27,7 @@
   @li SUBGROUP_LOGIC - only AND/OR
   @li SUBGROUP_ID - only integers
   @li VARIABLE_NM - must be in the target table
-  @li OPERATOR_NM - only =/>/</<=/>=/BETWEEN/IN/NOT IN/NOT EQUAL/CONTAINS
+  @li OPERATOR_NM - only =/>/</<=/>=/BETWEEN/IN/NOT IN/NE/CONTAINS
   @li RAW_VALUE - no unquoted values except integers, commas and spaces.
 
   @returns The &outds table containing any bad rows, plus a REASON_CD column.
@@ -42,11 +42,16 @@
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas
+  @li mf_getuniquefileref.sas
   @li mf_getvarlist.sas
+  @li mf_getvartype.sas
   @li mf_nobs.sas
+  @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
 
   <h4> Related Macros </h4>
   @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
 
   @version 9.3
   @author Allan Bowe
@@ -61,13 +66,29 @@
   ,msg=%str(syscc=&syscc - on macro entry)
 )
 
+/* Validate input column */
+%local vtype;
+%let vtype=%mf_getvartype(&inds,RAW_VALUE);
+%mp_abort(iftrue=(&abort=YES and &vtype ne C),
+  mac=&sysmacroname,
+  msg=%str(%str(ERR)OR: RAW_VALUE must be character)
+)
+%if &vtype ne C %then %do;
+  %put &sysmacroname: RAW_VALUE must be character;
+  %let syscc=42;
+  %return;
+%end;
+
+
 /**
   * Sanitise the values based on valid value lists, then strip out
   * quotes, commas, periods and spaces.
   * Only numeric values should remain
   */
-
+%local reason_cd;
 data &outds;
+  /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
+    OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
   length reason_cd $32;
 
@@ -137,21 +158,31 @@ data &outds;
 
 run;
 
-%if %mf_nobs(&outds)>0 %then %do;
-  %if &abort=YES %then %do;
 data _null_;
   set &outds;
   call symputx('REASON_CD',reason_cd,'l');
   stop;
 run;
-    %mp_abort(
+
+%mp_abort(iftrue=(&abort=YES and %mf_nobs(&outds)>0),
   mac=&sysmacroname,
-      msg=%str(Filter issues in &inds, first was &reason_cd, details in &outds)
+  msg=%str(Filter issues in &inds, reason: &reason_cd, details in &outds)
 )
-  %end;
+
+%if %mf_nobs(&outds)>0 %then %do;
   %let syscc=1008;
+  %return;
 %end;
 
+/**
+  * syntax checking passed but it does not mean the filter is valid
+  * for that we can run a proc sql validate query
+  */
+%local fref1;
+%let fref1=%mf_getuniquefileref();
+%mp_filtergenerate(&inds,outref=&fref1)
 
+/* this macro will also set syscc to 1008 if any issues found */
+%mp_filtervalidate(&fref1,&targetds,outds=&outds,abort=&abort)
 
-%mend;
+%mend mp_filtercheck;

base/mp_filtergenerate.sas

@@ -5,7 +5,7 @@
   This feature is used to create dynamic dropdowns in [Data Controller for SAS&reg](
   https://datacontroller.io). The input table should be in the format below:
 
-  |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
+  |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$4000|
   |---|---|---|---|---|---|
   |AND|AND|1|AGE|=|12|
   |AND|AND|1|SEX|<=|'M'|
@@ -21,7 +21,7 @@
       data work.filtertable;
         infile datalines4 dsd;
         input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-          OPERATOR_NM:$10. RAW_VALUE:$32767.;
+          OPERATOR_NM:$10. RAW_VALUE:$4000.;
       datalines4;
       AND,AND,1,AGE,=,12
       AND,AND,1,SEX,<=,"'M'"
@@ -56,6 +56,7 @@
 
   <h4> Related Macros </h4>
   @li mp_filtercheck.sas
+  @li mp_filtervalidate.sas
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas
@@ -87,13 +88,14 @@ filename &outref temp;
     file &outref lrecl=32800;
     set &inds end=last;
     by SUBGROUP_ID;
-    if _n_=1 then put '(';
+    if _n_=1 then put '((';
     else if first.SUBGROUP_ID then put +1 GROUP_LOGIC '(';
     else put +2 SUBGROUP_LOGIC;
 
     put +4 VARIABLE_NM OPERATOR_NM RAW_VALUE;
 
     if last.SUBGROUP_ID then put ')'@;
+    if last then put ')';
   run;
 %end;
 

base/mp_filtervalidate.sas

@@ -0,0 +1,104 @@
+/**
+  @file
+  @brief Checks a generated filter query for validity
+  @details Runs a generated filter in proc sql with the validate option.
+  Used in mp_filtercheck.sas in an fcmp container.
+
+  Built to support dynamic filtering in
+  [Data Controller for SAS®](https://datacontroller.io).
+
+  Usage:
+
+      data work.filtertable;
+        infile datalines4 dsd;
+        input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+          OPERATOR_NM:$10. RAW_VALUE:$4000.;
+      datalines4;
+      AND,AND,1,AGE,=,12
+      AND,AND,1,SEX,<=,"'M'"
+      AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
+      AND,OR,2,Weight,>=,7
+      ;;;;
+      run;
+
+      %mp_filtergenerate(work.filtertable,outref=myfilter)
+
+      %mp_filtervalidate(myfilter,sashelp.class)
+
+
+  @returns The SYSCC value will be 1008 if there are validation issues.
+
+  @param [in] inref The input fileref to validate (generated by
+    mp_filtergenerate.sas)
+  @param [in] targetds The target dataset against which to verify the query
+  @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
+  @param [out] outds= (work.mp_filtervalidate) Output dataset containing the
+  error / warning message, if one exists.  If this table contains any rows,
+  there are problems!
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filtercheck.sas
+  @li mp_filtergenerate.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mp_filtervalidate(inref,targetds,abort=YES,outds=work.mp_filtervalidate);
+
+%mp_abort(iftrue= (&syscc ne 0 or &syserr ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(syscc=&syscc / syserr=&syserr - on macro entry)
+)
+
+%local fref1;
+%let fref1=%mf_getuniquefileref();
+
+data _null_;
+  file &fref1;
+  infile &inref end=eof;
+  if _n_=1 then do;
+    put "proc sql;";
+    put "validate select * from &targetds";
+    put "where " ;
+  end;
+  input;
+  put _infile_;
+  putlog _infile_;
+  if eof then put ";quit;";
+run;
+
+%inc &fref1;
+
+data &outds;
+  if &sqlrc or &syscc or &syserr then do;
+    REASON_CD=coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
+    output;
+  end;
+  else stop;
+run;
+
+filename &fref1 clear;
+
+%if %mf_nobs(&outds)>0 %then %do;
+  %if &abort=YES %then %do;
+    data _null_;
+      set &outds;
+      call symputx('REASON_CD',reason_cd,'l');
+      stop;
+    run;
+    %mp_abort(
+      mac=&sysmacroname,
+      msg=%str(Filter issues in &inref: %quote(&reason_cd))
+    )
+  %end;
+  %let syscc=1008;
+%end;
+
+%mend;

base/mp_hashdataset.sas

@@ -56,7 +56,7 @@
       retain &prevkeyvar;
       set &libds end=&lastvar;
       /* hash should include previous row */
-      if _n_>1 then &keyvar=put(md5(&prevkeyvar
+      &keyvar=put(md5(&prevkeyvar
       /* loop every column, hashing every individual value */
     %do i=1 %to %sysfunc(countw(&varlist));
       %let var=%scan(&varlist,&i,%str( ));

base/mp_streamfile.sas

@@ -36,8 +36,20 @@
 %let contentype=%upcase(&contenttype);
 %local platform; %let platform=%mf_getplatform();
 
+
+/**
+  * check engine type to avoid the below err message:
+  * > Function is only valid for filerefs using the CACHE access method.
+  */
+%local streamweb;
+%let streamweb=0;
+data _null_;
+  set sashelp.vextfl(where=(upcase(fileref)="_WEBOUT"));
+  if xengine='STREAM' then call symputx('streamweb',1,'l');
+run;
+
 %if &contentype=ZIP %then %do;
-  %if &platform=SASMETA %then %do;
+  %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/zip');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -51,7 +63,7 @@
 %end;
 %else %if &contentype=EXCEL %then %do;
   /* suitable for XLS format */
-  %if &platform=SASMETA %then %do;
+  %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/vnd.ms-excel');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -64,7 +76,7 @@
   %end;
 %end;
 %else %if &contentype=XLSX %then %do;
-  %if &platform=SASMETA %then %do;
+  %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
       rc=stpsrv_header('Content-type',
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
@@ -79,7 +91,7 @@
   %end;
 %end;
 %else %if &contentype=TEXT %then %do;
-  %if &platform=SASMETA %then %do;
+  %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/text');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -92,7 +104,7 @@
   %end;
 %end;
 %else %if &contentype=CSV %then %do;
-  %if &platform=SASMETA %then %do;
+  %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/csv');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");

base/mp_testservice.sas

@@ -0,0 +1,259 @@
+/**
+  @file mp_testservice.sas
+  @brief Will execute a test against a SASjs web service on SAS 9 or Viya
+  @details Prepares the input files and retrieves the resulting datasets from
+  the response JSON.
+
+      %mp_testjob(
+        duration=60*5
+      )
+
+  Note - the _webout fileref should NOT be assigned prior to running this macro.
+
+  @param [in] program The _PROGRAM endpoint to test
+  @param [in] inputfiles=(0) A list of space seperated fileref:filename pairs as
+    follows:
+        inputfiles=inref:filename inref2:filename2
+  @param [in] inputparams=(0) A dataset containing name/value pairs in the
+    following format:
+    |name:$32|value:$1000|
+    |---|---|
+    |stpmacname|some value|
+    |mustbevalidname|can be anything, oops, %abort!!|
+
+  @param [in] debug= (log) Provide the _debug value
+  @param [in] viyaresult=(WEBOUT_JSON) The Viya result type to return.  For
+    more info, see mv_getjobresult.sas
+  @param [out] outlib= (0) Output libref to contain the final tables.  Set to
+    0 if the service output is not in JSON format.
+  @param [out] outref= (0) Output fileref to create, to contain the full _webout
+    response.
+
+  <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+  @li mv_getjobresult.sas
+  @li mv_jobflow.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mp_testservice(program,
+  inputfiles=0,
+  inputparams=0,
+  debug=log,
+  outlib=0,
+  outref=0,
+  viyaresult=WEBOUT_JSON
+)/*/STORE SOURCE*/;
+%local mdebug;
+%if &debug ne 0 %then %do;
+  %let mdebug=1;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let mdebug=0;
+
+/* sanitise inputparams */
+%local pcnt;
+%let pcnt=0;
+%if &inputparams ne 0 %then %do;
+  data _null_;
+    set &inputparams;
+    if not nvalid(name,'v7') then putlog (_all_)(=);
+    else if name in (
+      'program','inputfiles','inputparams','debug','outlib','outref'
+    ) then putlog (_all_)(=);
+    else do;
+      x+1;
+      call symputx(name,quote(cats(value)),'l');
+      call symputx('pval'!!left(x),name,'l');
+      call symputx('pcnt',x,'l');
+    end;
+  run;
+  %mp_abort(iftrue= (%mf_nobs(&inputparams) ne &pcnt)
+    ,mac=&sysmacroname
+    ,msg=%str(Invalid values in &inputparams)
+  )
+%end;
+
+
+%local fref1 webref;
+%let fref1=%mf_getuniquefileref();
+%let webref=%mf_getuniquefileref();
+
+%local platform;
+%let platform=%mf_getplatform();
+%if &platform=SASMETA %then %do;
+
+  /* parse the input files */
+  %local webcount i var;
+  %if %quote(&inputfiles) ne 0 %then %do;
+    %let webcount=%sysfunc(countw(&inputfiles));
+    %put &=webcount;
+    %do i=1 %to &webcount;
+      %let var=%scan(&inputfiles,&i,%str( ));
+      %local webfref&i webname&i;
+      %let webref&i=%scan(&var,1,%str(:));
+      %let webname&i=%scan(&var,2,%str(:));
+      %put webref&i=&&webref&i;
+      %put webname&i=&&webname&i;
+    %end;
+  %end;
+  %else %let webcount=0;
+
+  proc stp program="&program";
+    inputparam _program="&program"
+  %do i=1 %to &webcount;
+    %if &webcount=1 %then %do;
+      _webin_fileref="&&webref&i"
+      _webin_name="&&webname&i"
+    %end;
+    %else %do;
+      _webin_fileref&i="&&webref&i"
+      _webin_name&i="&&webname&i"
+    %end;
+  %end;
+    _webin_file_count="&webcount"
+    _debug="&debug"
+  %do i=1 %to &pcnt;
+    /* resolve name only, proc stp fetches value */
+    &&pval&i=&&&&&&pval&i
+  %end;
+    ;
+  %do i=1 %to &webcount;
+    inputfile &&webref&i;
+  %end;
+    outputfile _webout=&webref;
+  run;
+
+  data _null_;
+    infile &webref;
+    file &fref1;
+    input;
+    length line $10000;
+    if index(_infile_,'>>weboutBEGIN<<') then do;
+        line=tranwrd(_infile_,'>>weboutBEGIN<<','');
+        put line;
+    end;
+    else if index(_infile_,'>>weboutEND<<') then do;
+        line=tranwrd(_infile_,'>>weboutEND<<','');
+        put line;
+        stop;
+    end;
+    else put _infile_;
+  run;
+  data _null_;
+    infile &fref1;
+    input;
+    put _infile_;
+  run;
+  %if &outlib ne 0 %then %do;
+    libname &outlib json (&fref1);
+  %end;
+  %if &outref ne 0 %then %do;
+    filename &outref temp;
+    %mp_binarycopy(inref=&webref,outref=&outref)
+  %end;
+
+%end;
+%else %if &platform=SASVIYA %then %do;
+
+  /* prepare inputparams */
+  %local ds1;
+  %let ds1=%mf_getuniquename();
+  %if "&inputparams" ne "0" %then %do;
+    proc transpose data=&inputparams out=&ds1;
+      id name;
+      var value;
+    run;
+  %end;
+  %else %do;
+    data &ds1;run;
+  %end;
+
+  /* parse the input files - convert to sasjs params */
+  %local webcount i var sasjs_tables;
+  %if %quote(&inputfiles) ne 0 %then %do;
+    %let webcount=%sysfunc(countw(&inputfiles));
+    %put &=webcount;
+    %do i=1 %to &webcount;
+      %let var=%scan(&inputfiles,&i,%str( ));
+      %local webfref&i webname&i sasjs&i.data;
+      %let webref&i=%scan(&var,1,%str(:));
+      %let webname&i=%scan(&var,2,%str(:));
+      %put webref&i=&&webref&i;
+      %put webname&i=&&webname&i;
+
+      %let sasjs_tables=&sasjs_tables &&webname&i;
+      data _null_;
+        infile &&webref&i lrecl=32767;
+        input;
+        if _n_=1 then call symputx("sasjs&i.data",_infile_);
+        else call symputx(
+          "sasjs&i.data",cats(symget("sasjs&i.data"),'0D0A'x,_infile_)
+        );
+        putlog "&sysmacroname infile: " _infile_;
+      run;
+      data &ds1;
+        set &ds1;
+        length sasjs&i.data $32767 sasjs_tables $1000;
+        sasjs&i.data=symget("sasjs&i.data");
+        sasjs_tables=symget("sasjs_tables");
+      run;
+    %end;
+  %end;
+  %else %let webcount=0;
+
+  data &ds1;
+    retain _program "&program";
+    set &ds1;
+    putlog "&sysmacroname inputparams:";
+    putlog (_all_)(=);
+  run;
+
+  %mv_jobflow(inds=&ds1
+    ,maxconcurrency=1
+    ,outds=work.results
+    ,outref=&fref1
+    ,mdebug=&mdebug
+  )
+  /* show the log */
+  data _null_;
+    infile &fref1;
+    input;
+    putlog _infile_;
+  run;
+  /* get the uri to fetch results */
+  data _null_;
+    set work.results;
+    call symputx('uri',uri);
+    putlog "&sysmacroname: fetching results for " uri;
+  run;
+  /* fetch results from webout.json */
+  %mv_getjobresult(uri=&uri,
+    result=&viyaresult,
+    outref=&outref,
+    outlib=&outlib,
+    mdebug=&mdebug
+  )
+
+%end;
+%else %do;
+  %put %str(ERR)OR: Unrecognised platform:  &platform;
+%end;
+
+%if &mdebug=0 %then %do;
+  filename &webref clear;
+%end;
+%else %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+
+%mend mp_testservice;

base/mp_validatecol.sas

@@ -0,0 +1,66 @@
+/**
+  @file
+  @brief Used to validate variables in a dataset
+  @details Useful when sanitising inputs, to ensure that they arrive with a
+  certain pattern.
+  Usage:
+
+      data test;
+        infile datalines4 dsd;
+        input;
+        libds=_infile_;
+        %mp_validatecol(libds,LIBDS,is_libds)
+      datalines4;
+      some.libname
+      !lib.blah
+      %abort
+      definite.ok
+      not.ok!
+      nineletrs._
+      ;;;;
+      run;
+
+  @param [in] incol The column to be validated
+  @param [in] rule The rule to apply.  Current rules:
+    @li ISNUM - checks if the variable is numeric
+    @li LIBDS - matches LIBREF.DATASET format
+  @param [out] outcol The variable to create, with the results of the match
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+
+  @version 9.3
+**/
+
+%macro mp_validatecol(incol,rule,outcol);
+
+/* tempcol is given a unique name with every invocation */
+%local tempcol;
+%let tempcol=%mf_getuniquename();
+
+%if &rule=ISNUM %then %do;
+  /*
+    credit SØREN LASSEN
+    https://sasmacro.blogspot.com/2009/06/welcome-isnum-macro.html
+  */
+  &tempcol=input(&incol,?? best32.);
+  if missing(&tempcol) then &outcol=0;
+  else &outcol=1;
+  drop &tempcol;
+%end;
+%else %if &rule=LIBDS %then %do;
+  /* match libref.dataset */
+  if _n_=1 then do;
+    retain &tempcol;
+    &tempcol=prxparse('/^[_a-z]\w{0,7}\.[_a-z]\w{0,31}$/i');
+    if missing(&tempcol) then do;
+      putlog "%str(ERR)OR: Invalid expression for LIBDS";
+      stop;
+    end;
+    drop &tempcol;
+  end;
+  if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
+  else &outcol=0;
+%end;
+
+%mend mp_validatecol;

meta/mm_adduser2group.sas

@@ -70,7 +70,7 @@ run;
 %end;
 
 %if &syscc ge 4 %then %do;
-  %put WARNING:  SYSCC=&syscc, exiting &sysmacroname;
+  %put %str(WARN)ING:  SYSCC=&syscc, exiting &sysmacroname;
   %return;
 %end;
 

meta/mm_assigndirectlib.sas

@@ -332,7 +332,7 @@ run;
   run;
 
   %if %length(&open_passthrough)>0 %then %do;
-    %put WARNING:  Passthrough option for postgres not yet supported;
+    %put %str(WARN)ING: Passthrough option for postgres not yet supported;
     %return;
   %end;
   %else %do;
@@ -455,8 +455,8 @@ run;
   %return;
 %end;
 %else %do;
-  %put WARNING: Engine &engine is currently unsupported;
+  %put %str(WARN)ING: Engine &engine is currently unsupported;
-  %put WARNING- Please contact your support team.;
+  %put %str(WARN)ING- Please contact your support team.;
   %return;
 %end;
 

meta/mm_createlibrary.sas

@@ -88,7 +88,7 @@ data _null_;
   putlog (_all_)(=);
 run;
 %if &checktype = SASLibrary %then %do;
-  %put WARNING: Library (&liburi) already exists with libname (&libname)  ;
+  %put %str(WARN)ING: Library (&liburi) already exists with libname (&libname);
   %return;
 %end;
 
@@ -103,7 +103,7 @@ data _null_;
   putlog (_all_)(=);
 run;
 %if &checktype = SASLibrary %then %do;
-  %put WARNING: Library (&liburi) already exists with libref (&libref)  ;
+  %put %str(WARN)ING: Library (&liburi) already exists with libref (&libref)  ;
   %return;
 %end;
 
@@ -123,7 +123,7 @@ data _null_;
   call symputx('treeuri',uri,'l');
 run;
 %if &foldertype ne Tree %then %do;
-  %put WARNING: Tree &tree does not exist!;
+  %put %str(WARN)ING: Tree &tree does not exist!;
   %return;
 %end;
 
@@ -231,7 +231,7 @@ filename &frefout temp;
     *  check SAS version
     */
   %if %sysevalf(&sysver lt 9.3) %then %do;
-    %put WARNING: Version 9.3 or later required;
+    %put %str(WARN)ING: Version 9.3 or later required;
     %return;
   %end;
 

meta/mm_createstp.sas

@@ -118,7 +118,7 @@ data _null_;
   call symputx('treeuri',uri,'l');
 run;
 %if &foldertype ne Tree %then %do;
-  %put WARNING: Tree &tree does not exist!;
+  %put %str(WARN)ING: Tree &tree does not exist!;
   %return;
 %end;
 
@@ -133,7 +133,7 @@ data _null_;
   call symputx('stpuri',uri,'l');
 run;
 %if &cmtype = ClassifierMap %then %do;
-  %put WARNING: Stored Process &stpname already exists in &tree!;
+  %put %str(WARN)ING: Stored Process &stpname already exists in &tree!;
   %return;
 %end;
 
@@ -141,14 +141,14 @@ run;
   * Check that the physical file exists
   */
 %if %sysfunc(fileexist(&directory/&filename)) ne 1 %then %do;
-  %put WARNING: FILE *&directory/&filename* NOT FOUND!;
+  %put %str(WARN)ING: FILE *&directory/&filename* NOT FOUND!;
   %return;
 %end;
 
 %if &stptype=1 %then %do;
   /* type 1 STP - where code is stored on filesystem */
   %if %sysevalf(&sysver lt 9.2) %then %do;
-    %put WARNING: Version 9.2 or later required;
+    %put %str(WARN)ING: Version 9.2 or later required;
     %return;
   %end;
 
@@ -162,7 +162,7 @@ run;
   %if &checkdirtype ne Directory %then %do;
     %mm_getdirectories(path=&directory,outds=&outds ,mDebug=&mDebug)
     %if %mf_nobs(&outds)=0 or %sysfunc(exist(&outds))=0 %then %do;
-      %put WARNING: The directory object does not exist for &directory;
+      %put %str(WARN)ING: The directory object does not exist for &directory;
       %return;
     %end;
   %end;
@@ -180,12 +180,12 @@ run;
     length id $20 type $256;
     __rc=metadata_resolve("&treeuri",type,id);
     if type ne 'Tree' then do;
-      putlog "WARNING:  Invalid tree URI: &treeuri";
+      putlog "%str(WARN)ING:  Invalid tree URI: &treeuri";
       stopme=1;
     end;
     __rc=metadata_resolve(directoryuri,type,id);
     if type ne 'Directory' then do;
-      putlog 'WARNING:  Invalid directory URI: ' directoryuri;
+      putlog "%str(WARN)ING:  Invalid directory URI: " directoryuri;
       stopme=1;
     end;
 
@@ -194,7 +194,7 @@ run;
     if type ne 'LogicalServer' then do;
       __rc=metadata_getnobj("omsobj:LogicalServer?@Name='&server'",1,serveruri);
       if serveruri='' then do;
-        putlog "WARNING:  Invalid server: &server";
+        putlog "%str(WARN)ING:  Invalid server: &server";
         stopme=1;
       end;
     end;
@@ -217,7 +217,7 @@ run;
     rc6 = METADATA_SETATTR(prompturi, 'GroupInfo',groupinfo);
 
     if sum(of rc1-rc6) ne 0 then do;
-      putlog 'WARNING: Issue creating prompt.';
+      putlog "%str(WARN)ING: Issue creating prompt.";
       if prompturi ne . then do;
         putlog '  Removing orphan: ' prompturi;
         rc = METADATA_DELOBJ(prompturi);
@@ -232,7 +232,7 @@ run;
     rc9=METADATA_SETATTR(fileuri, 'IsARelativeName','1');
     rc10=METADATA_SETASSN(fileuri, 'Directories','MODIFY',directoryuri);
     if sum(of rc7-rc10) ne 0 then do;
-      putlog 'WARNING: Issue creating file.';
+      putlog "%str(WARN)ING: Issue creating file.";
       if fileuri ne . then do;
         putlog '  Removing orphans:' prompturi fileuri;
         rc = METADATA_DELOBJ(prompturi);
@@ -251,7 +251,7 @@ run;
       !!"<OutputParameters/></StoredProcess>";
     rc14= METADATA_SETATTR(texturi, 'StoredText',storedtext);
     if sum(of rc11-rc14) ne 0 then do;
-      putlog 'WARNING: Issue creating TextStore.';
+      putlog "%str(WARN)ING: Issue creating TextStore.";
       if texturi ne . then do;
         putlog '  Removing orphans: ' prompturi fileuri texturi;
         rc = METADATA_DELOBJ(prompturi);
@@ -299,7 +299,7 @@ run;
 %else %if &stptype=2 %then %do;
   /* type 2 stp - code is stored in metadata */
   %if %sysevalf(&sysver lt 9.3) %then %do;
-    %put WARNING: SAS version 9.3 or later required to create type2 STPs;
+    %put %str(WARN)ING: SAS version 9.3 or later required to create type2 STPs;
     %return;
   %end;
   /* check we have the correct ServerContext */
@@ -311,7 +311,7 @@ run;
     call symputx('serveruri',serveruri);
   run;
   %if &serveruri=NOTFOUND %then %do;
-    %put WARNING: ServerContext *&server* not found!;
+    %put %str(WARN)ING: ServerContext *&server* not found!;
     %return;
   %end;
 
@@ -382,7 +382,7 @@ run;
 
 %end;
 %else %do;
-  %put WARNING:  STPTYPE=*&stptype* not recognised!;
+  %put %str(WARN)ING:  STPTYPE=*&stptype* not recognised!;
 %end;
 
 %mend;

meta/mm_deletedocument.sas

@@ -32,7 +32,7 @@ data _null_;
   call symputx('stpuri',uri,'l');
 run;
 %if &type ne Document %then %do;
-  %put WARNING: No Document found at ⌖
+  %put %str(WARN)ING: No Document found at ⌖
   %return;
 %end;
 

meta/mm_getdocument.sas

@@ -125,7 +125,7 @@ data _null_;
           when ('
') rec='0D'x;
           when ('$' ) rec='$'  ;
           when ('	') rec='09'x;
-          otherwise putlog "WARNING: missing value for " entity=;
+          otherwise putlog "%str(WARN)ING: missing value for " entity=;
         end;
         rc =fput(fileid, substr(rec,1,1));
         rc =fwrite(fileid);

meta/mm_getstpcode.sas

@@ -1,20 +1,26 @@
 /**
   @file
-  @brief Writes the code of an to an external file, or the log if none provided
+  @brief Writes the code of an STP to an external file
-  @details Get the
+  @details Fetches the SAS code from a Stored Process where the code is stored
+  in metadata.
 
-  usage:
+  Usage:
 
       %mm_getstpcode(tree=/some/meta/path
         ,name=someSTP
         ,outloc=/some/unquoted/filename.ext
       )
 
-  @param tree= The metadata path of the Stored Process (can also contain name)
+  @param [in] tree= The metadata path of the Stored Process (can also contain
-  @param name= Stored Process name.  Leave blank if included above.
+    name)
-  @param outloc= full and unquoted path to the desired text file.  This will be
+  @param [in] name= Stored Process name.  Leave blank if included above.
-    overwritten if it already exists.  If not provided, the code will be written
+  @param [out] outloc= (0) full and unquoted path to the desired text file.
-    to the log.
+    This will be overwritten if it already exists.
+  @param [out] outref= (0) Fileref to which to write the code.
+  @param [out] showlog=(NO) Set to YES to print log to the window
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
 
   @author Allan Bowe
 
@@ -23,8 +29,10 @@
 %macro mm_getstpcode(
     tree=/User Folders/sasdemo/somestp
     ,name=
-    ,outloc=
+    ,outloc=0
+    ,outref=0
     ,mDebug=1
+    ,showlog=NO
     );
 
 %local mD;
@@ -92,14 +100,18 @@ data _null_;
   stop;
 
 %local outeng;
-%if %length(&outloc)=0 %then %let outeng=TEMP;
+%if "&outloc"="0" %then %let outeng=TEMP;
 %else %let outeng="&outloc";
+%local fref;
+%if &outref=0 %then %let fref=%mf_getuniquefileref();
+%else %let fref=&outref;
+
 /* read the content, byte by byte, resolving escaped chars */
-filename __outdoc &outeng lrecl=100000;
+filename &fref &outeng lrecl=100000;
 data _null_;
   length filein 8 fileid 8;
   filein = fopen("__getdoc","I",1,"B");
-  fileid = fopen("__outdoc","O",1,"B");
+  fileid = fopen("&fref","O",1,"B");
   rec = "20"x;
   length entity $6;
   do while(fread(filein)=0);
@@ -140,9 +152,9 @@ data _null_;
   rc=fclose(fileid);
 run;
 
-%if &outeng=TEMP %then %do;
+%if &showlog=YES %then %do;
   data _null_;
-    infile __outdoc lrecl=32767 end=last;
+    infile &fref lrecl=32767 end=last;
     input;
     if _n_=1 then putlog '>>stpcodeBEGIN<<';
     putlog _infile_;
@@ -151,6 +163,8 @@ run;
 %end;
 
 filename __getdoc clear;
-filename __outdoc clear;
+%if &outref=0 %then %do;
+  filename &fref clear;
+%end;
 
-%mend;
+%mend mm_getstpcode;

meta/mm_getwebappsrvprops.sas

@@ -99,7 +99,7 @@ run;
           when ('
') rec='0D'x;
           when ('$' ) rec='$'  ;
           when ('	') rec='09'x;
-          otherwise putlog "WARNING: missing value for " entity=;
+          otherwise putlog "%str(WARN)ING: missing value for " entity=;
         end;
         rc =fput(fileid, substr(rec,1,1));
         rc =fwrite(fileid);

meta/mm_updateappextension.sas

@@ -70,7 +70,7 @@ data _null_;
 run;
 
 %if &appuri=stopifempty %then %do;
-  %put WARNING:  &app.(Application) not found!;
+  %put %str(WARN)ING:  &app.(Application) not found!;
   %return;
 %end;
 

meta/mm_updatedocument.sas

@@ -58,12 +58,12 @@ data _null_;
 run;
 
 %if &tsuri=stopifempty %then %do;
-  %put WARNING:  &path/&name.(Document) not found!;
+  %put %str(WARN)ING:  &path/&name.(Document) not found!;
   %return;
 %end;
 
 %if %length(&text)<2 %then %do;
-  %put WARNING:  No text supplied!!;
+  %put %str(WARN)ING:  No text supplied!!;
   %return;
 %end;
 

meta/mm_updatestpservertype.sas

@@ -34,7 +34,7 @@ data _null_;
   call symputx('stpuri',uri,'l');
 run;
 %if &cmtype ne ClassifierMap %then %do;
-  %put WARNING: No Stored Process found at ⌖
+  %put %str(WARN)ING: No Stored Process found at ⌖
   %return;
 %end;
 

meta/mm_updatestpsourcecode.sas

@@ -68,12 +68,12 @@ data _null_;
 run;
 
 %if &tsuri=stopifempty %then %do;
-  %put WARNING:  &stp.(StoredProcess) not found!;
+  %put %str(WARN)ING:  &stp.(StoredProcess) not found!;
   %return;
 %end;
 
 %if %length(&stpcode)<2 %then %do;
-  %put WARNING:  No SAS code supplied!!;
+  %put %str(WARN)ING:  No SAS code supplied!!;
   %return;
 %end;
 

sasjs/sasjsconfig.json

@@ -16,9 +16,6 @@
       "readMe": "../../README.md"
     }
   },
-  "serviceConfig": {
-    "initProgram": "tests/testinit.sas"
-  },
   "defaultTarget": "viya",
   "targets": [
     {
@@ -26,19 +23,26 @@
       "serverUrl": "https://sas.analytium.co.uk",
       "serverType": "SASVIYA",
       "appLoc": "/Public/temp/macrocore",
-      "serviceConfig": {
-        "serviceFolders": [
-          "tests/base",
-          "tests/viya"
-        ],
-        "macroVars": {
-          "mcTestAppLoc": "/Public/temp/macrocore"
-        }
-      },
       "deployConfig": {
         "deployServicePack": true
       },
-      "contextName": "SAS Job Execution compute context"
+      "macroFolders": [
+        "base",
+        "meta",
+        "metax",
+        "viya",
+        "lua",
+        "tests/base",
+        "tests/viya"
+      ],
+      "contextName": "SAS Job Execution compute context",
+      "testConfig": {
+        "initProgram": "tests/testinit.sas",
+        "termProgram": "tests/testterm.sas",
+        "macroVars": {
+          "mcTestAppLoc": "/Public/temp/macrocore"
+        }
+      }
     }
   ]
 }

tests/base/mf_getvarlist.test.sas

@@ -0,0 +1,60 @@
+/**
+  @file
+  @brief Testing mf_getvarlist macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getvarlist.sas
+
+**/
+
+
+%let test1=%mf_getvarlist(sashelp.class);
+%let test2=%mf_getvarlist(sashelp.class,dlm=X);
+%let test3=%mf_getvarlist(sashelp.class,dlm=%str(,),quote=double);
+%let test4=%mf_getvarlist(sashelp.class,typefilter=N);
+%let test5=%mf_getvarlist(sashelp.class,typefilter=C);
+
+data work.test_results;
+  length test_description $256 test_result $4 test_comments base result  $256;
+  test_description="Basic test";
+  base=symget('test1');
+  result='Name Sex Age Height Weight';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  test_description="DLM test";
+  base=symget('test2');
+  result='NameXSexXAgeXHeightXWeight';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  test_description="DLM + quote test";
+  base=symget('test3');
+  result='"Name","Sex","Age","Height","Weight"';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  test_description="Numeric Filter";
+  base=symget('test4');
+  result='Age Height Weight';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  test_description="Char Filter";
+  base=symget('test5');
+  result='Name Sex';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  drop base result;
+run;

tests/base/mp_assertcolvals.test.sas

@@ -30,7 +30,3 @@ run;
   test=ALLVALS
 )
 
-
-%webout(OPEN)
-%webout(OBJ, TEST_RESULTS)
-%webout(CLOSE)

tests/base/mp_filtercheck.test.sas

@@ -5,6 +5,7 @@
   <h4> SAS Macros </h4>
   @li mp_filtercheck.sas
   @li mp_assertdsobs.sas
+  @li mp_assert.sas
 
 **/
 
@@ -13,7 +14,7 @@
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
 AND,AND,1,AGE,=,12
 AND,AND,1,SEX,<=,"'M'"
@@ -39,7 +40,7 @@ run;
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
 AND,AND,1,invalid,=,12
 AND,AND,1,SEX,<=,"'M'"
@@ -63,7 +64,7 @@ run;
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
 AND,OR,2,Name,NOT IN,"(''''Jane','Alfred')"
 ;;;;
@@ -85,7 +86,7 @@ run;
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
 AND,AND,1,%abort,=,12
 AND,OR,2,Weight,>=,7
@@ -108,7 +109,7 @@ run;
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
 AND,AND,1,age,=,;;%abort
 ;;;;
@@ -125,8 +126,23 @@ run;
   outds=work.test_results
 )
 
-
+/* Supply variables with incorrect types */
-
+data work.inds;
-%webout(OPEN)
+  infile datalines4 dsd;
-%webout(OBJ, TEST_RESULTS)
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-%webout(CLOSE)
+    OPERATOR_NM:$10. RAW_VALUE:8;
+datalines4;
+AND,AND,1,age,=,0
+;;;;
+run;
+%let syscc=0;
+%mp_filtercheck(work.inds,
+  targetds=sashelp.class,
+  outds=work.badrecords,
+  abort=NO
+)
+%mp_assert(iftrue=(&syscc=42),
+  desc=Throw error if RAW_VALUE is incorrect,
+  outds=work.test_results
+)
+%let syscc=0;

tests/base/mp_filtergenerate.test.sas

@@ -15,7 +15,7 @@ options source2;
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
 AND,AND,1,AGE,>,5
 AND,AND,1,SEX,NE,"'M'"
@@ -39,7 +39,7 @@ run;
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
 ;;;;
 run;
@@ -59,7 +59,7 @@ run;
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
 AND,OR,2,Name,IN,"('Jane','Janet')"
 ;;;;
@@ -80,7 +80,7 @@ run;
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
 OR,OR,2,Name,IN,"('Jane','Janet')"
 OR,OR,3,Name,IN,"('James')"
@@ -102,7 +102,7 @@ run;
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:$32767.;
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
 AND,OR,2,Name,IN,"('Jane','Janet')"
 AND,OR,3,Name,IN,"('James')"
@@ -120,7 +120,3 @@ run;
   outds=work.test_results
 )
 
-
-%webout(OPEN)
-%webout(OBJ, TEST_RESULTS)
-%webout(CLOSE)

tests/base/mp_filtervalidate.test.sas

@@ -0,0 +1,68 @@
+/**
+  @file
+  @brief Testing mp_filtervalidate macro
+
+  <h4> SAS Macros </h4>
+  @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
+  @li mp_assertdsobs.sas
+
+**/
+
+
+/* valid filter */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,AGE,>,5
+AND,AND,1,SEX,NE,"'M'"
+AND,OR,2,Name,NOT IN,"('Jane','Janet')"
+AND,OR,2,Weight,>=,84.6
+;;;;
+run;
+%mp_filtergenerate(work.inds,outref=myfilter)
+%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%mp_assertdsobs(work.results,
+  desc=Valid filter,
+  test=EMPTY,
+  outds=work.test_results
+)
+
+/* empty filter (return all records) */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+;;;;
+run;
+%mp_filtergenerate(work.inds,outref=myfilter)
+%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%mp_assertdsobs(work.results,
+  desc=Valid filter,
+  test=EMPTY,
+  outds=work.test_results
+)
+
+
+
+/* invalid filter*/
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,SEX,NE,2
+;;;;
+run;
+%mp_filtergenerate(work.inds,outref=myfilter)
+%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%let syscc=0;
+%mp_assertdsobs(work.results,
+  desc=Valid filter,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+

tests/base/mp_validatecol.test.sas

@@ -0,0 +1,62 @@
+/**
+  @file
+  @brief Testing mp_validatecol.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assertdsobs.sas
+  @li mp_validatecol.sas
+
+**/
+
+
+/**
+  * Test 1 - LIBDS
+  */
+data test1;
+  infile datalines4 dsd;
+  input;
+  libds=_infile_;
+  %mp_validatecol(libds,LIBDS,is_libds)
+  if is_libds=1;
+datalines4;
+some.libname
+!lib.blah
+%abort
+definite.ok
+not.ok!
+nineletrs._
+;;;;
+run;
+%mp_assertdsobs(work.test1,
+  desc=Testing LIBDS,
+  test=EQUALS 2,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - ISNUM
+  */
+data test2;
+  infile datalines4 dsd;
+  input;
+  infile=_infile_;
+  %mp_validatecol(infile,ISNUM,is_numeric)
+  if is_numeric=1;
+datalines4;
+1
+0001
+1e6
+-44
+above are good
+the rest are bad
+%abort
+1&somethingverybad.
+&
++-1
+;;;;
+run;
+%mp_assertdsobs(work.test2,
+  desc=Test2 - ISNUM,
+  test=EQUALS 4,
+  outds=work.test_results
+)

tests/testinit.sas

@@ -5,4 +5,4 @@
 **/
 
 /* location in metadata or SAS Drive for temporary files */
-%let mcTestAppLoc=/Public/temp/test;
+%let mcTestAppLoc=/Public/temp/macrocore;

tests/testterm.sas

@@ -0,0 +1,9 @@
+/**
+  @file
+  @brief term file for tests
+
+**/
+
+%webout(OPEN)
+%webout(OBJ, TEST_RESULTS)
+%webout(CLOSE)

tests/viya/mv_createwebservice.test.sas

@@ -4,6 +4,7 @@
 
   <h4> SAS Macros </h4>
   @li mv_createwebservice.sas
+  @li mv_getjobcode.sas
 
 **/
 
@@ -17,8 +18,28 @@ data _null_;
   file testref;
   put '01'x;
 run;
+%put TEST1: creating web service;
 %mv_createwebservice(
-  path=&mcTestAppLoc/tests/macros,
+  path=&mcTestAppLoc/temp/macros,
-  code=testref,
+  name=mv_createwebservice,
-  name=mv_createwebservice
+  code=testref
 )
+%put TEST1: fetching web service code;
+%mv_getjobcode(
+  path=&mcTestAppLoc/temp/macros,
+  name=mv_createwebservice,
+  outref=compare
+)
+%put TEST1: checking web service code;
+data work.test_results;
+  length test_description $256 test_result $4 test_comments $256;
+  infile compare end=eof;
+  input;
+  if eof then do;
+    if _infile_='01'x then test_result='PASS';
+    else test_result='FAIL';
+    test_description="Creating web service with invisible character";
+    output;
+    stop;
+  end;
+run;

tests/viya/mv_getjobcode.test.sas

@@ -0,0 +1,49 @@
+/**
+  @file
+  @brief Testing mv_getjobcode macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mv_createjob.sas
+  @li mv_getjobcode.sas
+
+**/
+
+/**
+  * Test Case 1
+  */
+
+/* write some code to a job */
+%let incode=%str(data test; set sashelp.class;run;);
+filename testref temp;
+data _null_;
+  file testref;
+  put "&incode";
+run;
+%mv_createjob(
+  code=testref,
+  path=&mcTestAppLoc/services/temp,
+  name=some_job
+)
+
+/* now get the code back */
+%mv_getjobcode(
+  path=&mcTestAppLoc/services/temp,
+  name=some_job,
+  outref=mycode
+)
+
+%let diditexist=NO;
+data work.test1;
+  infile mycode;
+  input;
+  putlog _infile_;
+  line=_infile_;
+  check=symget('incode');
+  if _infile_=symget('incode') then call symputx('diditexist','YES');
+run;
+
+%mp_assert(
+  iftrue=(&diditexist=NO),
+  desc=Check if the code that was sent was successfully retrieved
+)

tests/viya/mv_getjobresult.test.sas

@@ -0,0 +1,74 @@
+/**
+  @file
+  @brief Testing mv_createwebservice macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assertdsobs.sas
+  @li mv_createwebservice.sas
+  @li mv_getjobresult.sas
+  @li mv_jobflow.sas
+
+**/
+
+/**
+  * Test Case 1
+  */
+
+/* create a service */
+filename testref temp;
+data _null_;
+  file testref;
+  put 'data test; set sashelp.class;run;';
+  put '%webout(OPEN)';
+  put '%webout(OBJ,test)';
+  put '%webout(CLOSE)';
+run;
+%mv_createwebservice(
+  path=&mcTestAppLoc/services/temp,
+  code=testref,
+  name=testsvc
+)
+
+/* trigger and wait for it to finish */
+data work.inputjobs;
+  _program="&mcTestAppLoc/services/temp/testsvc";
+run;
+%mv_jobflow(inds=work.inputjobs
+  ,maxconcurrency=4
+  ,outds=work.results
+  ,outref=myjoblog
+)
+/* stream the log */
+data _null_;
+  infile myjoblog;
+  input;
+  put _infile_;
+run;
+
+/* fetch the uri */
+data _null_;
+  set work.results;
+  call symputx('uri',uri);
+  put (_all_)(=);
+run;
+
+/* now get the results */
+%mv_getjobresult(uri=&uri
+  ,result=WEBOUT_JSON
+  ,outref=myweb
+  ,outlib=myweblib
+)
+data _null_;
+  infile myweb;
+  input;
+  putlog _infile_;
+run;
+data work.out;
+  set myweblib.test;
+  put (_all_)(=);
+run;
+%mp_assertdsobs(work.out,
+  desc=Test1 - 19 obs from sashelp.class in service result,
+  test=EQUALS 19,
+  outds=work.test_results
+)

viya/mv_createwebservice.sas

@@ -39,23 +39,26 @@
   @li mf_isblank.sas
   @li mv_deletejes.sas
 
-  @param path= The full path (on SAS Drive) where the service will be created
+  @param [in] path= The full path (on SAS Drive) where the service will be
-  @param name= The name of the service
+    created
-  @param desc= The description of the service
+  @param [in] name= The name of the service
-  @param precode= Space separated list of filerefs, pointing to the code that
+  @param [in] desc= The description of the service
-    needs to be attached to the beginning of the service
+  @param [in] precode= Space separated list of filerefs, pointing to the code
-  @param code= Fileref(s) of the actual code to be added
+    that needs to be attached to the beginning of the service
-  @param access_token_var= The global macro variable to contain the access token
+  @param [in] code= Fileref(s) of the actual code to be added
-  @param grant_type= valid values are "password" or "authorization_code"
+  @param [in] access_token_var= The global macro variable to contain the access
+    token
+  @param [in] grant_type= valid values are "password" or "authorization_code"
     (unquoted). The default is authorization_code.
-  @param replace= select NO to avoid replacing any existing service in that
+  @param [in] replace=(YES) Select NO to avoid replacing any existing service in
-    location
+    that location
-  @param adapter= the macro uses the sasjs adapter by default.  To use another
+  @param [in] adapter= the macro uses the sasjs adapter by default.  To use
-    adapter, add a (different) fileref here.
+    another adapter, add a (different) fileref here.
-  @param contextname= Choose a specific context on which to run the Job.  Leave
+  @param [in] contextname= Choose a specific context on which to run the Job.  Leave
     blank to use the default context.  From Viya 3.5 it is possible to configure
     a shared context - see
 https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5pyhn1wg3ja0drdl6h.htm&docsetVersion=3.5&locale=en
+  @param [in] mdebug=(0) set to 1 to enable DEBUG messages
 
   @version VIYA V.03.04
   @author Allan Bowe, source: https://github.com/sasjs/core
@@ -71,9 +74,17 @@ https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5p
     ,grant_type=sas_services
     ,replace=YES
     ,adapter=sasjs
-    ,debug=0
+    ,mdebug=0
     ,contextname=
+    ,debug=0 /* @TODO - Deprecate */
   );
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -126,7 +137,7 @@ proc http method='GET' out=&fname1 &oauth_bearer
   headers "Authorization"="Bearer &&&access_token_var";
 %end;
 run;
-%if &debug %then %do;
+%if &mdebug=1 %then %do;
   data _null_;
     infile &fname1;
     input;
@@ -165,7 +176,7 @@ proc http method='GET'
   %end;
             'Accept'='application/vnd.sas.collection+json'
             'Accept-Language'='string';
-%if &debug=1 %then %do;
+%if &mdebug=1 %then %do;
   debug level = 3;
 %end;
 run;
@@ -220,9 +231,9 @@ run;
   * These put statements are auto generated - to change the macro, change the
   * source (mv_webout) and run `build.py`
   */
-filename sasjs temp lrecl=3000;
+filename &adapter temp lrecl=3000;
 data _null_;
-  file sasjs;
+  file &adapter;
   put "/* Created on %sysfunc(datetime(),datetime19.) by &sysuserid */";
 /* WEBOUT BEGIN */
   put ' ';
@@ -561,11 +572,12 @@ data _null_;
 run;
 
 /* insert the code, escaping double quotes and carriage returns */
+%&dbg.put &sysmacroname: Creating final input file;
 %local x fref freflist;
 %let freflist= &adapter &precode &code ;
 %do x=1 %to %sysfunc(countw(&freflist));
   %let fref=%scan(&freflist,&x);
-  %put &sysmacroname: adding &fref;
+  %&dbg.put &sysmacroname: adding &fref fileref;
   data _null_;
     length filein 8 fileid 8;
     filein = fopen("&fref","I",1,"B");
@@ -617,7 +629,12 @@ data _null_;
   put '"}';
 run;
 
-/* now we can create the job!! */
+%if &mdebug=1 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then %do;
+  %put &sysmacroname: input about to be POSTed;
+  data _null_;infile &fname3;input;putlog _infile_;run;
+%end;
+
+%&dbg.put &sysmacroname: Creating the actual service!;
 %local fname4;
 %let fname4=%mf_getuniquefileref();
 proc http method='POST'
@@ -630,22 +647,18 @@ proc http method='POST'
             "Authorization"="Bearer &&&access_token_var"
   %end;
             "Accept"="application/vnd.sas.job.definition+json";
-%if &debug=1 %then %do;
+%if &mdebug=1 %then %do;
     debug level = 3;
 %end;
 run;
-/*data _null_;infile &fname4;input;putlog _infile_;run;*/
+%if &mdebug=1 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then %do;
+  %put &sysmacroname: output from POSTing job definition;
+  data _null_;infile &fname4;input;putlog _infile_;run;
+%end;
 %mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 201)
   ,mac=&sysmacroname
   ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
 )
-/* clear refs */
-filename &fname1 clear;
-filename &fname2 clear;
-filename &fname3 clear;
-filename &fname4 clear;
-filename &adapter clear;
-libname &libref1 clear;
 
 /* get the url so we can give a helpful log message */
 %local url;
@@ -660,6 +673,19 @@ data _null_;
   call symputx('url',url);
 run;
 
+%if &mdebug=1 %then %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%else %do;
+  /* clear refs */
+  filename &fname1 clear;
+  filename &fname2 clear;
+  filename &fname3 clear;
+  filename &fname4 clear;
+  filename &adapter clear;
+  libname &libref1 clear;
+%end;
 
 %put &sysmacroname: Job &name successfully created in &path;
 %put &sysmacroname:;
@@ -669,4 +695,4 @@ run;
 %put &sysmacroname:;
 %put &sysmacroname:;
 
-%mend;
+%mend mv_createwebservice;

viya/mv_getjobcode.sas

@@ -10,17 +10,20 @@
         ,outfile=/tmp/some_job.sas
       )
 
-  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] access_token_var= The global macro variable to contain the access
+    token
   @param [in] grant_type= valid values:
-      * password
+    @li password
-      * authorization_code
+    @liauthorization_code
-      * detect - will check if access_token exists, if not will use sas_services if
+    @li detect - will check if access_token exists, if not will use sas_services
-        a SASStudioV session else authorization_code.  Default option.
+      if a SASStudioV session else authorization_code.  Default option.
-      * sas_services - will use oauth_bearer=sas_services
+    @li  sas_services - will use oauth_bearer=sas_services
   @param [in] path= The SAS Drive path of the job
   @param [in] name= The name of the job
-  @param [out] outref= A fileref to which to write the source code
+  @param [in] mdebug=(0) set to 1 to enable DEBUG messages
-  @param [out] outfile= A file to which to write the source code
+  @param [out] outref=(0) A fileref to which to write the source code (will be
+    created with a TEMP engine)
+  @param [out] outfile=(0) A file to which to write the source code
 
   @version VIYA V.03.04
   @author Allan Bowe, source: https://github.com/sasjs/core
@@ -39,7 +42,15 @@
     ,contextName=SAS Job Execution compute context
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
+    ,mdebug=0
   );
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -133,18 +144,30 @@ data _null_;
 run;
 %inc "&fpath3..lua";
 /* export to desired destination */
+%if "&outref"="0" %then %do;
   data _null_;
-  %if &outref=0 %then %do;
     file "&outfile" lrecl=32767;
 %end;
 %else %do;
+  filename &outref temp;
+  data _null_;
     file &outref;
 %end;
   infile &fname2;
   input;
   put _infile_;
+  &dbg. putlog _infile_;
 run;
+
+%if &mdebug=1 %then %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%else %do;
+  /* clear refs */
   filename &fname1 clear;
   filename &fname2 clear;
   filename &fname3 clear;
-%mend;
+%end;
+
+%mend mv_getjobcode;

viya/mv_getjoblog.sas

@@ -54,13 +54,14 @@
   convenient way to wait for the job to finish before fetching the log.
 
 
-  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] access_token_var= The global macro variable to contain the access
+    token
   @param [in] mdebug= set to 1 to enable DEBUG messages
   @param [in] grant_type= valid values:
     @li password
     @li authorization_code
-    @li detect - will check if access_token exists, if not will use sas_services if
+    @li detect - will check if access_token exists, if not will use sas_services
-        a SASStudioV session else authorization_code.  Default option.
+      if a SASStudioV session else authorization_code.  Default option.
     @li sas_services - will use oauth_bearer=sas_services.
   @param [in] uri= The uri of the running job for which to fetch the status,
     in the format `/jobExecution/jobs/$UUID/state` (unquoted).
@@ -85,6 +86,13 @@
     ,grant_type=sas_services
     ,mdebug=0
   );
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -260,9 +268,10 @@ run;
   filename &fname3 clear;
 %end;
 %else %do;
+  %put &sysmacroname exit vars:;
   %put _local_;
 %end;
-%mend;
+%mend mv_getjoblog;
 
 
 

viya/mv_getjobresult.sas

@@ -0,0 +1,232 @@
+/**
+  @file
+  @brief Extract the result from a completed SAS Viya Job
+  @details Extracts result from a Viya job and writes it out to a fileref
+  and/or a JSON-engine library.
+
+  To query the job, you need the URI.  Sample code for achieving this
+  is provided below.
+
+  ## Example
+
+  First, compile the macros:
+
+      filename mc url
+        "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+  Next, create a job (in this case, a web service):
+
+      filename ft15f001 temp;
+      parmcards4;
+        data test;
+          rand=ranuni(0)*1000;
+          do x=1 to rand;
+            y=rand*4;
+            output;
+          end;
+        run;
+        proc sort data=&syslast
+          by descending y;
+        run;
+        %webout(OPEN)
+        %webout(OBJ, test)
+        %webout(CLOSE)
+      ;;;;
+      %mv_createwebservice(path=/Public/temp,name=demo)
+
+  Execute it:
+
+      %mv_jobexecute(path=/Public/temp
+        ,name=demo
+        ,outds=work.info
+      )
+
+  Wait for it to finish, and grab the uri:
+
+      data _null_;
+        set work.info;
+        if method='GET' and rel='self';
+        call symputx('uri',uri);
+      run;
+
+  Finally, fetch the result (In this case, WEBOUT):
+
+      %mv_getjobresult(uri=&uri,result=WEBOUT_JSON,outref=myweb,outlib=myweblib)
+
+
+  @param [in] access_token_var= The global macro variable containing the access
+    token
+  @param [in] mdebug= set to 1 to enable DEBUG messages
+  @param [in] grant_type= valid values:
+    @li password
+    @li authorization_code
+    @li detect - will check if access_token exists, if not will use sas_services
+        if a SASStudioV session else authorization_code.  Default option.
+    @li sas_services - will use oauth_bearer=sas_services.
+  @param [in] uri= The uri of the running job for which to fetch the status,
+    in the format `/jobExecution/jobs/$UUID` (unquoted).
+
+  @param [out] result= (WEBOUT_JSON) The result type to capture.  Resolves
+  to "_[column name]" from the results table when parsed with the JSON libname
+  engine.  Example values:
+    @li WEBOUT_JSON
+    @li WEBOUT_TXT
+
+  @param [out] outref= (0) The output fileref to which to write the results
+  @param [out] outlib= (0) The output library to which to assign the results
+    (assumes the data is in JSON format)
+
+
+  @version VIYA V.03.05
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+  @li mf_getplatform.sas
+  @li mf_existfileref.sas
+
+**/
+
+%macro mv_getjobresult(uri=0
+    ,contextName=SAS Job Execution compute context
+    ,access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+    ,mdebug=0
+    ,result=WEBOUT_JSON
+    ,outref=0
+    ,outlib=0
+  );
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+
+
+/* validation in datastep for better character safety */
+%local errmsg errflg;
+data _null_;
+  uri=symget('uri');
+  if length(uri)<12 then do;
+    call symputx('errflg',1);
+    call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
+  end;
+  if scan(uri,-1)='state' or scan(uri,1) ne 'jobExecution' then do;
+    call symputx('errflg',1);
+    call symputx('errmsg',
+      "URI should be in format /jobExecution/jobs/$$$$UUID$$$$"
+      !!" but is actually like: &uri",'l');
+  end;
+run;
+
+%mp_abort(iftrue=(&errflg=1)
+  ,mac=&sysmacroname
+  ,msg=%str(&errmsg)
+)
+
+%if &outref ne 0 and %mf_existfileref(&outref) ne 1 %then %do;
+  filename &outref temp;
+%end;
+
+options noquotelenmax;
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+/* fetch job info */
+%local fname1;
+%let fname1=%mf_getuniquefileref();
+proc http method='GET' out=&fname1 &oauth_bearer
+  url="&base_uri&uri";
+  headers "Accept"="application/json"
+  %if &grant_type=authorization_code %then %do;
+          "Authorization"="Bearer &&&access_token_var"
+  %end;
+  ;
+run;
+%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+%do;
+  data _null_;infile &fname1;input;putlog _infile_;run;
+  %mp_abort(mac=&sysmacroname
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+%end;
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fname1 lrecl=32767;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+/* extract results link */
+%local lib1 resuri;
+%let lib1=%mf_getuniquelibref();
+libname &lib1 JSON fileref=&fname1;
+data _null_;
+  set &lib1..results;
+  call symputx('resuri',_&result,'l');
+  &dbg putlog "&sysmacroname results: " (_all_)(=);
+run;
+%mp_abort(iftrue=("&resuri"=".")
+  ,mac=&sysmacroname
+  ,msg=%str(Variable _&result did not exist in the response json)
+)
+
+/* extract results */
+%local fname2;
+%let fname2=%mf_getuniquefileref();
+proc http method='GET' out=&fname2 &oauth_bearer
+  url="&base_uri&resuri/content?limit=10000";
+  headers "Accept"="application/json"
+  %if &grant_type=authorization_code %then %do;
+          "Authorization"="Bearer &&&access_token_var"
+  %end;
+  ;
+run;
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fname2 lrecl=32767;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%if &outref ne 0 %then %do;
+  filename &outref temp;
+  %mp_binarycopy(inref=&fname2,outref=&outref)
+%end;
+%if &outlib ne 0 %then %do;
+  libname &outlib JSON fileref=&fname2;
+%end;
+
+%if &mdebug=0 %then %do;
+  filename &fname1 clear;
+  filename &fname2 clear;
+  libname &lib1 clear;
+%end;
+%else %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+
+%mend mv_getjobresult;

viya/mv_jobexecute.sas

@@ -23,24 +23,25 @@
         ,paramstring=%str("macvarname":"macvarvalue","answer":42)
       )
 
-  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] access_token_var= The global macro variable to contain the access
+    token
   @param [in] grant_type= valid values:
-
+    @li password
-    * password
+    @li authorization_code
-    * authorization_code
+    @li detect - will check if access_token exists, if not will use sas_services
-    * detect - will check if access_token exists, if not will use sas_services if
+      if a SASStudioV session else authorization_code.  Default option.
-      a SASStudioV session else authorization_code.  Default option.
+    @li sas_services - will use oauth_bearer=sas_services
-    * sas_services - will use oauth_bearer=sas_services
 
   @param [in] path= The SAS Drive path to the job being executed
   @param [in] name= The name of the job to execute
-  @param [in] paramstring= A JSON fragment with name:value pairs, eg: `"name":"value"`
+  @param [in] paramstring= A JSON fragment with name:value pairs, eg:
-  or "name":"value","name2":42`.  This will need to be wrapped in `%str()`.
+    `"name":"value"` or "name":"value","name2":42`.  This will need to be
+    wrapped in `%str()`.
 
   @param [in] contextName= Context name with which to run the job.
     Default = `SAS Job Execution compute context`
-
+  @param [in] mdebug= set to 1 to enable DEBUG messages
-  @param [out] outds= The output dataset containing links (Default=work.mv_jobexecute)
+  @param [out] outds= (work.mv_jobexecute) The output dataset containing links
 
 
   @version VIYA V.03.04
@@ -62,7 +63,15 @@
     ,grant_type=sas_services
     ,paramstring=0
     ,outds=work.mv_jobexecute
+    ,mdebug=0
   );
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -164,9 +173,14 @@ data &outds;
   _program="&path/&name";
 run;
 
+%if &mdebug=1 %then %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%else %do;
   /* clear refs */
   filename &fname0 clear;
   filename &fname1 clear;
   libname &libref;
-
+%end;
-%mend;
+%mend mv_jobexecute;

viya/mv_jobflow.sas

@@ -136,6 +136,13 @@
     ,raise_err=0
     ,mdebug=0
   );
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -293,6 +300,7 @@ data;run;%let jdswaitfor=&syslast;
           ,name=&jobname
           ,paramstring=%superq(jparams&jid)
           ,outds=&jdsapp
+          ,mdebug=&mdebug
         )
         data &jdsapp;
           format jobparams $32767.;
@@ -313,8 +321,13 @@ data;run;%let jdswaitfor=&syslast;
     %end;
     %if &jid=&jcnt %then %do;
       /* we are at the end of the loop - time to see which jobs have finished */
-      %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref
+      %mv_jobwaitfor(ANY
-                    ,raise_err=&raise_err)
+        ,inds=&jdsrunning
+        ,outds=&jdswaitfor
+        ,outref=&outref
+        ,raise_err=&raise_err
+        ,mdebug=&mdebug
+      )
       %local done;
       %let done=%mf_nobs(&jdswaitfor);
       %if &done>0 %then %do;
@@ -346,7 +359,8 @@ data;run;%let jdswaitfor=&syslast;
 %end;
 
 %if &mdebug=1 %then %do;
+  %put &sysmacroname exit vars:;
   %put _local_;
 %end;
 
-%mend;
+%mend mv_jobflow;

viya/mv_jobwaitfor.sas

@@ -1,6 +1,6 @@
 /**
   @file
-  @brief Takes a dataset of running jobs and waits for ANY or ALL of them to complete
+  @brief Takes a table of running jobs and waits for ANY/ALL of them to complete
   @details Will poll `/jobs/{jobId}/state` at set intervals until ANY or ALL
   jobs are completed.  Completion is determined by reference to the returned
   _state_, as per the following table:
@@ -55,13 +55,14 @@
 
       %mv_deletejes(path=/Public/temp,name=demo)
 
-  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] access_token_var= The global macro variable to contain the access
+    token
   @param [in] grant_type= valid values:
 
       - password
       - authorization_code
-      - detect - will check if access_token exists, if not will use sas_services if
+      - detect - will check if access_token exists, if not will use sas_services
-        a SASStudioV session else authorization_code.  Default option.
+        if a SASStudioV session else authorization_code.  Default option.
       - sas_services - will use oauth_bearer=sas_services
 
   @param [in] action=Either ALL (to wait for every job) or ANY (if one job
@@ -72,9 +73,11 @@
     should be in a `_program` variable.
   @param [in] raise_err=0 Set to 1 to raise SYSCC when a job does not complete
               succcessfully
+  @param [in] mdebug= set to 1 to enable DEBUG messages
   @param [out] outds= The output dataset containing the list of states by job
     (default=work.mv_jobexecute)
-  @param [out] outref= A fileref to which the spawned job logs should be appended.
+  @param [out] outref= A fileref to which the spawned job logs should be
+    appended.
 
   @version VIYA V.03.04
   @author Allan Bowe, source: https://github.com/sasjs/core
@@ -97,7 +100,15 @@
     ,outds=work.mv_jobwaitfor
     ,outref=0
     ,raise_err=0
+    ,mdebug=0
   );
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -155,7 +166,8 @@ run;
 %let fname0=%mf_getuniquefileref();
 
 data &outds;
-  format _program uri $128. state $32. stateDetails $32. timestamp datetime19. jobparams $32767.;
+  format _program uri $128. state $32. stateDetails $32. timestamp datetime19.
+    jobparams $32767.;
   stop;
 run;
 
@@ -168,8 +180,8 @@ run;
               "Authorization"="Bearer &&&access_token_var"
       %end;  ;
     run;
-    %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+    %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201
-    %do;
+    %then %do;
       data _null_;infile &fname0;input;putlog _infile_;run;
       %mp_abort(mac=&sysmacroname
         ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
@@ -205,7 +217,7 @@ run;
       %let joburi&i=0; /* do not re-check */
       /* fetch log */
       %if %str(&outref) ne 0 %then %do;
-        %mv_getjoblog(uri=&plainuri,outref=&outref)
+        %mv_getjoblog(uri=&plainuri,outref=&outref,mdebug=&mdebug)
       %end;
     %end;
     %else %if &status=idle or &status=pending or &status=running %then %do;
@@ -220,10 +232,11 @@ run;
     %end;
 
     %if (&raise_err) %then %do;
-      %if (&status = canceled or &status = failed or %length(&stateDetails)>0) %then %do;
+      %if (&status = canceled or &status = failed or %length(&stateDetails)>0)
+      %then %do;
         %if ("&stateDetails" = "%str(war)ning") %then %let SYSCC=4;
         %else %let SYSCC=5;
-        %put %str(ERR)OR: Job &&jobname&i. did not complete successfully. &stateDetails;
+        %put %str(ERR)OR: Job &&jobname&i. did not complete. &stateDetails;
         %return;
       %end;
     %end;
@@ -238,7 +251,12 @@ run;
   %end;
 %end;
 
+%if &mdebug=1 %then %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%else %do;
   /* clear refs */
   filename &fname0 clear;
-
+%end;
-%mend;
+%mend mv_jobwaitfor;