Merge branch 'main' of github.com:sasjs/core

fix: Non-stp propagation of syscc

.gitignore

@@ -1,9 +1,13 @@
 node_modules
 .DS_Store
 sasjsbuild/
+sasjsresults/
 
 # avoid filenames with spaces being committed to source control
 **\ **
 
 # ignore the mc_* files - containing macros for individual libraries
-mc_*
+mc_*
+
+# ignore .env files as they can contain sasjs access tokens
+*.env*

README.md

@@ -116,19 +116,19 @@ The **Macro Core** documentation is created using [doxygen](http://www.doxygen.n
 All macros must be commented in the doxygen format, to enable the [online documentation](https://core.sasjs.io).
 
 ### Dependencies
-SAS code can contain one of two types of dependency - SAS Macros, and SAS Programs.  When compiling projects using the [SASjs CLI](https://cli.sasjs.io) the doxygen header is scanned for `  @li` items under the following headers:
+SAS code can contain one of two types of dependency - SAS Macros, and SAS Includes.  When compiling projects using the [SASjs CLI](https://cli.sasjs.io) the doxygen header is scanned for `  @li` items under the following headers:
 
 ```sas
   <h4> SAS Macros </h4>
   @li mf_nobs.sas
   @li mm_assignlib.sas
 
-  <h4> SAS Programs </h4>
+  <h4> SAS Includes </h4>
   @li somefile.ddl SOMEFREF
   @li someprogram.sas FREFTWO
 ```
 
-The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Programs) when creating SAS Jobs and Services.
+The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Includes) when creating SAS Jobs and Services.
 
 When contributing to this library, it is therefore important to ensure that all dependencies are listed in the header in this format.
 

base/mf_abort.sas

@@ -1,23 +1,11 @@
 /**
   @file
-  @brief abort gracefully according to context
-  @details Do not use directly!  See bottom of explanation for details.
+  @brief to be deprecated
+  @details We will deprecate this macro in 2022
 
-  Configures an abort mechanism according to site specific policies or the
-    particulars of an environment.  For instance, can stream custom
-    results back to the client in an STP Web App context, or completely stop
-    in the case of a batch run.
+  As you can see, it's not a macro function.
 
-  For the sharp eyed readers - this is no longer a macro function!! It became
-  a macro procedure during a project and now it's kinda stuck that way until
-  that project is updated (if it's ever updated).  In the meantime we created
-  `mp_abort` which is just a wrapper for this one, and so we recomend you use
-  that for forwards compatibility reasons.
-
-  @param mac= to contain the name of the calling macro
-  @param type= deprecated.  Not used.
-  @param msg= message to be returned
-  @param iftrue= supply a condition under which the macro should be executed.
+  Use mp_abort.sas instead.
 
   @version 9.2
   @author Allan Bowe

base/mf_existvarlist.sas

@@ -2,9 +2,9 @@
   @file
   @brief Checks if a set of variables ALL exist in a data set.
   @details Returns 0 if ANY of the variables do not exist, or 1 if they ALL do.
-    Usage:
+  Usage:
 
-        %put %mf_existVarList(sashelp.class, age sex name dummyvar)
+      %put %mf_existVarList(sashelp.class, age sex name dummyvar);
 
   <h4> SAS Macros </h4>
   @li mf_abort.sas
@@ -29,7 +29,7 @@
   %let dsid=%sysfunc(open(&libds,is));
 
   %if &dsid=0 %then %do;
-    %put WARNING:  unable to open &libds in mf_existvarlist (&dsid);
+    %put %str(WARN)ING:  unable to open &libds in mf_existvarlist (&dsid);
   %end;
 
   %if %sysfunc(attrn(&dsid,NVARS))=0 %then %do;

base/mf_getattrc.sas

@@ -23,7 +23,7 @@
   %local dsid rc;
   %let dsid=%sysfunc(open(&libds,is));
   %if &dsid = 0 %then %do;
-    %put WARNING: Cannot open %trim(&libds), system message below;
+    %put %str(WARN)ING: Cannot open %trim(&libds), system message below;
     %put %sysfunc(sysmsg());
     -1
   %end;

base/mf_getengine.sas

@@ -22,6 +22,9 @@
   @version 9.2
   @author Allan Bowe
 
+  <h4> Related Macros </h4>
+  @li mf_getxengine.sas
+
 **/
 /** @cond */
 

base/mf_getuniquename.sas

@@ -18,5 +18,5 @@
 
 
 %macro mf_getuniquename(prefix=MC);
-  &prefix.%substr(%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32-%length(&prefix))
-%mend;
+&prefix.%substr(%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32-%length(&prefix))
+%mend mf_getuniquename;

base/mf_getvarformat.sas

@@ -23,9 +23,9 @@
       8.
       NOTE: Variable renegade does not exist in test
 
-  @param libds Two part dataset (or view) reference.
-  @param var Variable name for which a format should be returned
-  @param force Set to 1 to supply a default if the variable has no format
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] var Variable name for which a format should be returned
+  @param [in] force=(0) Set to 1 to supply a default if the variable has no format
   @returns outputs format
 
   @author Allan Bowe
@@ -60,7 +60,7 @@
     %let vlen = %sysfunc(varlen(&dsid, &vnum));
     %let vtype = %sysfunc(vartype(&dsid, &vnum.));
     %if &vtype=C %then %let vformat=$&vlen..;
-    %else %let vformat=8.;
+    %else %let vformat=best.;
   %end;
 
 
@@ -68,4 +68,4 @@
   %let rc = %sysfunc(close(&dsid));
   /* Return variable format */
   &vformat
-%mend;
+%mend mf_getVarFormat;

base/mf_getvarlist.sas

@@ -10,14 +10,21 @@
   returns:
   > List of Variables=Name Sex Age Height Weight
 
+  For a seperated list of column values:
+
         %put %mf_getvarlist(sashelp.class,dlm=%str(,),quote=double);
 
   returns:
   > "Name","Sex","Age","Height","Weight"
 
-  @param libds Two part dataset (or view) reference.
-  @param dlm= provide a delimiter (eg comma or space) to separate the vars
-  @param quote= use either DOUBLE or SINGLE to quote the results
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] dlm= ( ) Provide a delimiter (eg comma or space) to separate the
+    variables
+  @param [in] quote= (none) use either DOUBLE or SINGLE to quote the results
+  @param [in] typefilter= (A) Filter for certain types of column.  Valid values:
+    @li A Return All columns
+    @li C Return Character columns
+    @li N Return Numeric columns
 
   @version 9.2
   @author Allan Bowe
@@ -27,9 +34,10 @@
 %macro mf_getvarlist(libds
       ,dlm=%str( )
       ,quote=no
+      ,typefilter=A
 )/*/STORE SOURCE*/;
   /* declare local vars */
-  %local outvar dsid nvars x rc dlm q var;
+  %local outvar dsid nvars x rc dlm q var vtype;
 
   /* credit Rowland Hale  - byte34 is double quote, 39 is single quote */
   %if %upcase(&quote)=DOUBLE %then %let q=%qsysfunc(byte(34));
@@ -37,21 +45,22 @@
   /* open dataset in macro */
   %let dsid=%sysfunc(open(&libds));
 
-
   %if &dsid %then %do;
     %let nvars=%sysfunc(attrn(&dsid,NVARS));
     %if &nvars>0 %then %do;
-      /* add first dataset variable to global macro variable */
-      %let outvar=&q.%sysfunc(varname(&dsid,1))&q.;
-      /* add remaining variables with supplied delimeter */
+      /* add variables with supplied delimeter */
       %do x=1 %to &nvars;
-        %let var=&q.%sysfunc(varname(&dsid,&x))&q.;
-        %if &var=&q&q %then %do;
-          %put &sysmacroname: Empty column found in &libds!;
-          %let var=&q. &q.;
+        /* get variable type */
+        %let vtype=%sysfunc(vartype(&dsid,&x));
+        %if &vtype=&typefilter or &typefilter=A %then %do;
+          %let var=&q.%sysfunc(varname(&dsid,&x))&q.;
+          %if &var=&q&q %then %do;
+            %put &sysmacroname: Empty column found in &libds!;
+            %let var=&q. &q.;
+          %end;
+          %if %quote(&outvar)=%quote() %then %let outvar=&var;
+          %else %let outvar=&outvar.&dlm.&var.;
         %end;
-        %if &x=1 %then %let outvar=&var;
-        %else %let outvar=&outvar.&dlm.&var.;
       %end;
     %end;
     %let rc=%sysfunc(close(&dsid));
@@ -61,4 +70,4 @@
     %let rc=%sysfunc(close(&dsid));
   %end;
   &outvar
-%mend;
+%mend mf_getvarlist;

base/mf_getvartype.sas

@@ -45,4 +45,4 @@ Usage:
   %let rc = %sysfunc(close(&dsid));
   /* Return variable type */
   &vtype
-%mend;
+%mend mf_getvartype;

base/mf_getxengine.sas

@@ -0,0 +1,43 @@
+/**
+  @file
+  @brief Returns the engine type of a SAS fileref
+  @details Queries sashelp.vextfl to get the xengine value.
+  Usage:
+
+      filename feng temp;
+      %put %mf_getxengine(feng);
+
+  returns:
+  > TEMP
+
+  @param fref The fileref to check
+
+  @returns The XENGINE value in sashelp.vextfl or 0 if not found.
+
+  @version 9.2
+  @author Allan Bowe
+
+  <h4> Related Macros </h4>
+  @li mf_getengine.sas
+
+**/
+
+%macro mf_getxengine(fref
+)/*/STORE SOURCE*/;
+  %local dsid engnum rc engine;
+
+  %let dsid=%sysfunc(
+    open(sashelp.vextfl(where=(fileref="%upcase(&fref)")),i)
+  );
+  %if (&dsid ^= 0) %then %do;
+    %let engnum=%sysfunc(varnum(&dsid,XENGINE));
+    %let rc=%sysfunc(fetch(&dsid));
+    %let engine=%sysfunc(getvarc(&dsid,&engnum));
+    %* put &fref. ENGINE is &engine.;
+    %let rc= %sysfunc(close(&dsid));
+  %end;
+  %else %let engine=0;
+
+  &engine
+
+%mend;

base/mf_uid.sas

@@ -18,4 +18,4 @@
 
   &today._&now._&sysjobid._%sysevalf(%sysfunc(ranuni(0))*999,CEIL)
 
-%mend;
+%mend mf_uid;

base/mf_wordsinstr1butnotstr2.sas

@@ -30,7 +30,7 @@
 
 %local count_base count_extr i i2 extr_word base_word match outvar;
 %if %length(&str1)=0 or %length(&str2)=0 %then %do;
-  %put WARNING: empty string provided!;
+  %put %str(WARN)ING: empty string provided!;
   %put base string (str1)= &str1;
   %put compare string (str2) = &str2;
   %return;

base/mp_abort.sas

@@ -128,12 +128,13 @@
       if debug ge '"131"' then put '>>weboutEND<<';
     run;
 
-    %let syscc=0;
     %if %symexist(_metaport) %then %do;
       data _null_;
-        if symexist('sysprocessmode')
-          then if symget("sysprocessmode")="SAS Stored Process Server"
-            then rc=stpsrvset('program error', 0);
+        if symexist('sysprocessmode') then
+          if symget("sysprocessmode")="SAS Stored Process Server" then do;
+            rc=stpsrvset('program error', 0);
+            call symputx("syscc",0,"g");
+          end;
       run;
     %end;
     /**

base/mp_assert.sas

@@ -0,0 +1,56 @@
+/**
+  @file
+  @brief Generic assertion
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      %mp_assert(iftrue=(1=1),
+        desc=Obviously true
+      )
+
+      %mp_assert(iftrue=(1=0),
+        desc=Will fail
+      )
+
+  @param [in] iftrue= (1=1) A condition where, if true, the test is a PASS.
+  Else, the test is a fail.
+
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Column &inds contained ALL columns|
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assert(iftrue=(1=1),
+  desc=0,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  data ;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_comments="&sysmacroname: Test result of "!!symget('iftrue');
+  %if %eval(%unquote(&iftrue)) %then %do;
+    test_result='PASS';
+  %end;
+  %else %do;
+    test_result='FAIL';
+  %end;
+  run;
+
+  %local ds ;
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+
+%mend mp_assert;

base/mp_assertcols.sas

@@ -0,0 +1,145 @@
+/**
+  @file
+  @brief Asserts the existence (or not) of columns
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      %mp_assertcols(sashelp.class,
+        cols=name age sex,
+        test=ALL,
+        desc=check all columns exist
+      )
+
+      %mp_assertcols(sashelp.class,
+        cols=a b c,
+        test=NONE
+      )
+
+      %mp_assertcols(sashelp.class,
+        cols=age depth,
+        test=ANY
+      )
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_existvarlist.sas
+  @li mf_getvarlist.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_abort.sas
+
+
+  @param [in] inds The input library.dataset to test for values
+  @param [in] cols= The list of columns to check for
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [in] test= (ALL) The test to apply.  Valid values are:
+    @li ALL - Test is a PASS if ALL columns exist in &inds
+    @li ANY - Test is a PASS if ANY of the columns exist in &inds
+    @li NONE - Test is a PASS if NONE of the columns exist in &inds
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Column &inds contained ALL columns|
+
+
+  <h4> Related Macros </h4>
+  @li mp_assertdsobs.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertcols(inds,
+  cols=0,
+  test=ALL,
+  desc=0,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  %mp_abort(iftrue= (&syscc ne 0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc - on macro entry)
+  )
+
+  %local lib ds ;
+  %let lib=%scan(&inds,1,%str(.));
+  %let ds=%scan(&inds,2,%str(.));
+  %let cols=%upcase(&cols);
+
+  %mp_abort(iftrue= (%mf_existds(&lib..&ds)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(&lib..&ds not found!)
+  )
+
+  %mp_abort(iftrue= (&cols=0)
+    ,mac=&sysmacroname
+    ,msg=%str(No cols provided)
+  )
+
+
+  %let test=%upcase(&test);
+
+  %if &test ne ANY and &test ne ALL and &test ne NONE %then %do;
+    %mp_abort(
+      mac=&sysmacroname,
+      msg=%str(Invalid test - &test)
+    )
+  %end;
+
+  /**
+    * now do the actual test!
+    */
+  %local result;
+  %if %mf_existVarList(&inds,&cols)=1 %then %let result=ALL;
+  %else %do;
+    %local targetcols compare;
+    %let targetcols=%upcase(%mf_getvarlist(&inds));
+    %let compare=%mf_wordsinstr1butnotstr2(
+        Str1=&cols,
+        Str2=&targetcols
+      );
+    %if %cmpres(&compare)=%cmpres(&cols) %then %let result=NONE;
+    %else %let result=SOME;
+  %end;
+
+  data;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    if test_description='0'
+    then test_description="Testing &inds for existence of &test of: &cols";
+
+    test_result='FAIL';
+    test_comments="&sysmacroname: &inds has &result columns ";
+  %if &test=ALL %then %do;
+    %if &result=ALL %then %do;
+      test_result='PASS';
+    %end;
+  %end;
+  %else %if &test=ANY %then %do;
+    %if &result=SOME %then %do;
+      test_result='PASS';
+    %end;
+  %end;
+  %else %if &test=NONE %then %do;
+    %if &result=NONE %then %do;
+      test_result='PASS';
+    %end;
+  %end;
+  %else %do;
+    test_comments="&sysmacroname: Unsatisfied test condition - &test";
+  %end;
+  run;
+
+  %local ds;
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+
+%mend;

base/mp_assertcolvals.sas

@@ -0,0 +1,147 @@
+/**
+  @file
+  @brief Asserts the values in a column
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      data work.checkds;
+        do checkval='Jane','James','Jill';
+          output;
+        end;
+      run;
+      %mp_assertcolvals(sashelp.class.name,
+        checkvals=work.checkds.checkval,
+        desc=At least one value has a match,
+        test=ANYVAL
+      )
+
+      data work.check;
+        do val='M','F';
+          output;
+        end;
+      run;
+      %mp_assertcolvals(sashelp.class.sex,
+        checkvals=work.check.val,
+        desc=All values have a match,
+        test=ALLVALS
+      )
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+
+  @param [in] indscol The input library.dataset.column to test for values
+  @param [in] checkvals= A library.dataset.column value containing a UNIQUE
+    list of values to be compared against the source (indscol).
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [in] test= (ALLVALS) The test to apply.  Valid values are:
+    @li ALLVALS - Test is a PASS if ALL values have a match in checkvals
+    @li ANYVAL - Test is a PASS if at least 1 value has a match in checkvals
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Column &indscol contained ALL target vals|
+
+
+  <h4> Related Macros </h4>
+  @li mp_assertdsobs.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertcolvals(indscol,
+  checkvals=0,
+  test=ALLVALS,
+  desc=mp_assertcolvals - no desc provided,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  %mp_abort(iftrue= (&syscc ne 0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc - on macro entry)
+  )
+
+  %local lib ds col clib cds ccol nobs;
+  %let lib=%scan(&indscol,1,%str(.));
+  %let ds=%scan(&indscol,2,%str(.));
+  %let col=%scan(&indscol,3,%str(.));
+  %mp_abort(iftrue= (%mf_existds(&lib..&ds)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(&lib..&ds not found!)
+  )
+
+  %mp_abort(iftrue= (&checkvals=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Set CHECKVALS to a library.dataset.column containing check vals)
+  )
+  %let clib=%scan(&checkvals,1,%str(.));
+  %let cds=%scan(&checkvals,2,%str(.));
+  %let ccol=%scan(&checkvals,3,%str(.));
+  %mp_abort(iftrue= (%mf_existds(&clib..&cds)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(&clib..&cds not found!)
+  )
+  %let nobs=%mf_nobs(&clib..&cds);
+  %mp_abort(iftrue= (&nobs=0)
+    ,mac=&sysmacroname
+    ,msg=%str(&clib..&cds is empty!)
+  )
+
+  %let test=%upcase(&test);
+
+  %if &test ne ALLVALS and &test ne ANYVAL %then %do;
+    %mp_abort(
+      mac=&sysmacroname,
+      msg=%str(Invalid test - &test)
+    )
+  %end;
+
+  %local result orig;
+  %let result=-1;
+  %let orig=-1;
+  proc sql noprint;
+  select count(*) into: result
+    from &lib..&ds
+    where &col not in (
+      select &ccol from &clib..&cds
+    );
+  select count(*) into: orig from &lib..&ds;
+  quit;
+
+  %mp_abort(iftrue= (&syscc ne 0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc after macro query)
+  )
+
+  data;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_result='FAIL';
+    test_comments="&sysmacroname: &lib..&ds..&col has &result values "
+      !!"not in &clib..&cds..&ccol ";
+  %if &test=ANYVAL %then %do;
+    if &result < &orig then test_result='PASS';
+  %end;
+  %else %if &test=ALLVALS %then %do;
+    if &result=0 then test_result='PASS';
+  %end;
+  %else %do;
+    test_comments="&sysmacroname: Unsatisfied test condition - &test";
+  %end;
+  run;
+
+  %local ds;
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+
+%mend;

base/mp_assertdsobs.sas

@@ -0,0 +1,89 @@
+/**
+  @file
+  @brief Asserts the number of observations in a dataset
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      %mp_assertdsobs(sashelp.class) %* tests if any observations are present;
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+
+  @param [in] inds input dataset to test for presence of observations
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [in] test= (HASOBS) The test to apply.  Valid values are:
+    @li HASOBS - Test is a PASS if the input dataset has any observations
+    @li EMPTY - Test is a PASS if input dataset is empty
+    @li EQUALS [integer] - Test passes if obs count matches the provided integer
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Dataset &inds has XX obs|
+
+  <h4> Related Macros </h4>
+  @li mp_assertcolvals.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertdsobs(inds,
+  test=HASOBS,
+  desc=Testing observations,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  %local nobs;
+  %let nobs=%mf_nobs(&inds);
+  %let test=%upcase(&test);
+
+  %if %substr(&test.xxxxx,1,6)=EQUALS %then %do;
+    %let val=%scan(&test,2,%str( ));
+    %mp_abort(iftrue= (%DATATYP(&val)=CHAR)
+      ,mac=&sysmacroname
+      ,msg=%str(Invalid test - &test, expected EQUALS [integer])
+    )
+    %let test=EQUALS;
+  %end;
+  %else %if &test ne HASOBS and &test ne EMPTY %then %do;
+    %mp_abort(
+      mac=&sysmacroname,
+      msg=%str(Invalid test - &test)
+    )
+  %end;
+
+  data;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_result='FAIL';
+    test_comments="&sysmacroname: Dataset &inds has &nobs observations";
+  %if &test=HASOBS %then %do;
+    if &nobs>0 then test_result='PASS';
+  %end;
+  %else %if &test=EMPTY %then %do;
+    if &nobs=0 then test_result='PASS';
+  %end;
+  %else %if &test=EQUALS %then %do;
+    if &nobs=&val then test_result='PASS';
+  %end;
+  %else %do;
+    test_comments="&sysmacroname: Unsatisfied test condition - &test";
+  %end;
+  run;
+
+  %local ds;
+  %let ds=&syslast;
+
+  proc append base=&outds data=&ds;
+  run;
+
+  proc sql;
+  drop table &ds;
+
+%mend;

base/mp_ds2cards.sas

@@ -43,7 +43,7 @@
 %local i setds nvars;
 
 %if not %sysfunc(exist(&base_ds)) %then %do;
-  %put WARNING:  &base_ds does not exist;
+  %put %str(WARN)ING:  &base_ds does not exist;
   %return;
 %end;
 
@@ -61,7 +61,7 @@ select count(*) into: nvars from dictionary.columns
   where libname="%scan(%upcase(&base_ds),1)"
     and memname="%scan(%upcase(&base_ds),2)";
 %if &nvars=0 %then %do;
-  %put WARNING:  Dataset &base_ds has no variables!  It will not be converted.;
+  %put %str(WARN)ING: Dataset &base_ds has no variables, will not be converted.;
   %return;
 %end;
 

base/mp_ds2csv.sas

@@ -19,7 +19,7 @@
 )/*/STORE SOURCE*/;
 
 %if not %sysfunc(exist(&ds)) %then %do;
-  %put WARNING:  &ds does not exist;
+  %put %str(WARN)ING:  &ds does not exist;
   %return;
 %end;
 

base/mp_ds2fmtds.sas

@@ -0,0 +1,98 @@
+/**
+  @file
+  @brief Converts every value in a dataset to it's formatted value
+  @details Converts every value to it's formatted value.  All variables will
+  become character, and will be in the same order.
+
+  Usage:
+
+      %mp_ds2fmtds(sashelp.cars,work.cars)
+
+  @param [in] libds The library.dataset to be converted
+  @param [out] outds The dataset to create.
+
+  <h4> Related Macros <h4>
+  @li mp_jsonout.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_ds2fmtds(libds, outds
+)/*/STORE SOURCE*/;
+
+/* validations */
+%if not %sysfunc(exist(&libds)) %then %do;
+  %put %str(WARN)ING:  &libds does not exist;
+  %return;
+%end;
+%if %index(&libds,.)=0 %then %let libds=WORK.&libds;
+
+/* grab metadata */
+proc contents noprint data=&libds
+  out=_data_(keep=name type length format formatl formatd varnum);
+run;
+proc sort;
+  by varnum;
+run;
+
+/* prepare formats and varnames */
+data _null_;
+  set &syslast end=last;
+  name=upcase(name);
+  /* fix formats */
+  if type=2 or type=6 then do;
+    length fmt $49.;
+    if format='' then fmt=cats('$',length,'.');
+    else if formatl=0 then fmt=cats(format,'.');
+    else fmt=cats(format,formatl,'.');
+    newlen=max(formatl,length);
+  end;
+  else do;
+    if format='' then fmt='best.';
+    else if formatl=0 then fmt=cats(format,'.');
+    else if formatd=0 then fmt=cats(format,formatl,'.');
+    else fmt=cats(format,formatl,'.',formatd);
+    /* needs to be wide, for datetimes etc */
+    newlen=max(length,formatl,24);
+  end;
+  /* 32 char unique name */
+  newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
+
+  call symputx(cats('name',_n_),name,'l');
+  call symputx(cats('newname',_n_),newname,'l');
+  call symputx(cats('len',_n_),newlen,'l');
+  call symputx(cats('fmt',_n_),fmt,'l');
+  call symputx(cats('type',_n_),type,'l');
+  if last then call symputx('nobs',_n_,'l');
+run;
+
+/* clean up */
+proc sql;
+drop table &syslast;
+
+%if &nobs=0 %then %do;
+  %put Dataset &libds has no columns!
+  data &outds;
+    set &libds;
+  run;
+  %return;
+%end;
+
+data &outds;
+  /* rename on entry */
+  set &libds(rename=(
+%local i;
+%do i=1 %to &nobs;
+  &&name&i=&&newname&i
+%end;
+  ));
+%do i=1 %to &nobs;
+  length &&name&i $&&len&i;
+  &&name&i=left(put(&&newname&i,&&fmt&i));
+  drop &&newname&i;
+%end;
+  if _error_ then call symputx('syscc',1012);
+run;
+
+%mend mp_ds2fmtds;

base/mp_filtercheck.sas

@@ -0,0 +1,192 @@
+/**
+  @file
+  @brief Checks an input filter table for validity
+  @details Performs checks on the input table to ensure it arrives in the
+  correct format.  This is necessary to prevent code injection.  Will update
+  SYSCC to 1008 if bad records are found, and call mp_abort.sas for a
+  graceful service exit (configurable).
+
+  Used for dynamic filtering in [Data Controller for SAS®](https://datacontroller.io).
+
+  Usage:
+
+      %mp_filtercheck(work.filter,targetds=sashelp.class,outds=work.badrecords)
+
+  The input table should have the following format:
+
+  |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$4000|
+  |---|---|---|---|---|---|
+  |AND|AND|1|AGE|=|12|
+  |AND|AND|1|SEX|<=|'M'|
+  |AND|OR|2|Name|NOT IN|('Jane','Alfred')|
+  |AND|OR|2|Weight|>=|7|
+
+  Rules applied:
+
+  @li GROUP_LOGIC - only AND/OR
+  @li SUBGROUP_LOGIC - only AND/OR
+  @li SUBGROUP_ID - only integers
+  @li VARIABLE_NM - must be in the target table
+  @li OPERATOR_NM - only =/>/</<=/>=/BETWEEN/IN/NOT IN/NE/CONTAINS
+  @li RAW_VALUE - no unquoted values except integers, commas and spaces.
+
+  @returns The &outds table containing any bad rows, plus a REASON_CD column.
+
+  @param [in] inds The table to be checked, with the format above
+  @param [in] targetds= The target dataset against which to verify VARIABLE_NM
+  @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
+  @param [out] outds= The output table, which is a copy of the &inds. table
+  plus a REASON_CD column, containing only bad records.  If bad records found,
+  the SYSCC value will be set to 1008 (general data problem).  Downstream
+  processes should check this table (and return code) before continuing.
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
+  @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+  @todo Support date / hex / name literals and exponents in RAW_VALUE field
+**/
+
+%macro mp_filtercheck(inds,targetds=,outds=work.badrecords,abort=YES);
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(syscc=&syscc - on macro entry)
+)
+
+/* Validate input column */
+%local vtype;
+%let vtype=%mf_getvartype(&inds,RAW_VALUE);
+%mp_abort(iftrue=(&abort=YES and &vtype ne C),
+  mac=&sysmacroname,
+  msg=%str(%str(ERR)OR: RAW_VALUE must be character)
+)
+%if &vtype ne C %then %do;
+  %put &sysmacroname: RAW_VALUE must be character;
+  %let syscc=42;
+  %return;
+%end;
+
+
+/**
+  * Sanitise the values based on valid value lists, then strip out
+  * quotes, commas, periods and spaces.
+  * Only numeric values should remain
+  */
+%local reason_cd;
+data &outds;
+  /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
+    OPERATOR_NM $10 RAW_VALUE $4000;*/
+  set &inds;
+  length reason_cd $32;
+
+  /* closed list checks */
+  if GROUP_LOGIC not in ('AND','OR') then do;
+    REASON_CD='GROUP_LOGIC should be either AND or OR';
+    putlog REASON_CD= GROUP_LOGIC=;
+    output;
+  end;
+  if SUBGROUP_LOGIC not in ('AND','OR') then do;
+    REASON_CD='SUBGROUP_LOGIC should be either AND or OR';
+    putlog REASON_CD= SUBGROUP_LOGIC=;
+    output;
+  end;
+  if mod(SUBGROUP_ID,1) ne 0 then do;
+    REASON_CD='SUBGROUP_ID should be integer';
+    putlog REASON_CD= SUBGROUP_ID=;
+    output;
+  end;
+  if upcase(VARIABLE_NM) not in
+  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
+  then do;
+    REASON_CD="VARIABLE_NM not in &targetds";
+    putlog REASON_CD= VARIABLE_NM=;
+    output;
+  end;
+  if OPERATOR_NM not in
+  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
+  then do;
+    REASON_CD='Invalid OPERATOR_NM';
+    putlog REASON_CD= OPERATOR_NM=;
+    output;
+  end;
+
+  /* special logic */
+  if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ','');
+  else if OPERATOR_NM in ('IN','NOT IN') then do;
+    if substr(raw_value,1,1) ne '('
+    or substr(cats(reverse(raw_value)),1,1) ne ')'
+    then do;
+      REASON_CD='Missing brackets in RAW_VALUE';
+      putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+      output;
+    end;
+    else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
+  end;
+  else raw_value1=raw_value;
+
+  /* remove nested literals eg '' */
+  raw_value1=tranwrd(raw_value1,"''",'');
+
+  /* now match string literals (always single quotes) */
+  raw_value2=raw_value1;
+  regex = prxparse("s/(\').*?(\')//");
+  call prxchange(regex,-1,raw_value2);
+
+  /* remove commas and periods*/
+  raw_value3=compress(raw_value2,',.');
+
+  /* output records that contain values other than digits and spaces */
+  if notdigit(compress(raw_value3,' '))>0 then do;
+    putlog raw_value3= $hex32.;
+    REASON_CD='Invalid RAW_VALUE';
+    putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
+    output;
+  end;
+
+run;
+
+%local nobs;
+%let nobs=0;
+data _null_;
+  set &outds end=last;
+  putlog (_all_)(=);
+  if last then do;
+    call symputx('REASON_CD',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
+  end;
+run;
+
+%mp_abort(iftrue=(&abort=YES and &nobs>0),
+  mac=&sysmacroname,
+  msg=%str(&nobs filter issues in &inds, reason: &reason_cd, details in &outds)
+)
+
+%if &nobs>0 %then %do;
+  %let syscc=1008;
+  %return;
+%end;
+
+/**
+  * syntax checking passed but it does not mean the filter is valid
+  * for that we can run a proc sql validate query
+  */
+%local fref1;
+%let fref1=%mf_getuniquefileref();
+%mp_filtergenerate(&inds,outref=&fref1)
+
+/* this macro will also set syscc to 1008 if any issues found */
+%mp_filtervalidate(&fref1,&targetds,outds=&outds,abort=&abort)
+
+%mend mp_filtercheck;

base/mp_filtergenerate.sas

@@ -0,0 +1,102 @@
+/**
+  @file
+  @brief Generates a filter clause from an input table, to a fileref
+  @details Uses the input table to generate an output filter clause.
+  This feature is used to create dynamic dropdowns in [Data Controller for SAS&reg](
+  https://datacontroller.io). The input table should be in the format below:
+
+  |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$4000|
+  |---|---|---|---|---|---|
+  |AND|AND|1|AGE|=|12|
+  |AND|AND|1|SEX|<=|'M'|
+  |AND|OR|2|Name|NOT IN|('Jane','Alfred')|
+  |AND|OR|2|Weight|>=|7|
+
+  Note - if the above table is received from an external client, the values
+  should first be validated using the mp_filtercheck.sas macro to avoid risk
+  of SQL injection.
+
+  To generate the filter, run the following code:
+
+      data work.filtertable;
+        infile datalines4 dsd;
+        input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+          OPERATOR_NM:$10. RAW_VALUE:$4000.;
+      datalines4;
+      AND,AND,1,AGE,=,12
+      AND,AND,1,SEX,<=,"'M'"
+      AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
+      AND,OR,2,Weight,>=,7
+      ;;;;
+      run;
+
+      %mp_filtergenerate(work.filtertable,outref=myfilter)
+
+      data _null_;
+        infile myfilter;
+        input;
+        put _infile_;
+      run;
+
+  Will write the following query to the log:
+
+  > (
+  >     AGE = 12
+  >   AND
+  >     SEX <= 'M'
+  > ) AND (
+  >     Name NOT IN ('Jane','Alfred')
+  >   OR
+  >     Weight >= 7
+  > )
+
+  @param [in] inds The input table with query values
+  @param [out] outref= The output fileref to contain the filter clause.  Will
+    be created (or replaced).
+
+  <h4> Related Macros </h4>
+  @li mp_filtercheck.sas
+  @li mp_filtervalidate.sas
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_nobs.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mp_filtergenerate(inds,outref=filter);
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(syscc=&syscc - on macro entry)
+)
+
+filename &outref temp;
+
+%if %mf_nobs(&inds)=0 %then %do;
+  /* ensure we have a default filter */
+  data _null_;
+    file &outref;
+    put '1=1';
+  run;
+%end;
+%else %do;
+  data _null_;
+    file &outref lrecl=32800;
+    set &inds end=last;
+    by SUBGROUP_ID;
+    if _n_=1 then put '((';
+    else if first.SUBGROUP_ID then put +1 GROUP_LOGIC '(';
+    else put +2 SUBGROUP_LOGIC;
+
+    put +4 VARIABLE_NM OPERATOR_NM RAW_VALUE;
+
+    if last.SUBGROUP_ID then put ')'@;
+    if last then put ')';
+  run;
+%end;
+
+%mend;

base/mp_filtervalidate.sas

@@ -0,0 +1,104 @@
+/**
+  @file
+  @brief Checks a generated filter query for validity
+  @details Runs a generated filter in proc sql with the validate option.
+  Used in mp_filtercheck.sas in an fcmp container.
+
+  Built to support dynamic filtering in
+  [Data Controller for SAS®](https://datacontroller.io).
+
+  Usage:
+
+      data work.filtertable;
+        infile datalines4 dsd;
+        input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+          OPERATOR_NM:$10. RAW_VALUE:$4000.;
+      datalines4;
+      AND,AND,1,AGE,=,12
+      AND,AND,1,SEX,<=,"'M'"
+      AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
+      AND,OR,2,Weight,>=,7
+      ;;;;
+      run;
+
+      %mp_filtergenerate(work.filtertable,outref=myfilter)
+
+      %mp_filtervalidate(myfilter,sashelp.class)
+
+
+  @returns The SYSCC value will be 1008 if there are validation issues.
+
+  @param [in] inref The input fileref to validate (generated by
+    mp_filtergenerate.sas)
+  @param [in] targetds The target dataset against which to verify the query
+  @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
+  @param [out] outds= (work.mp_filtervalidate) Output dataset containing the
+  error / warning message, if one exists.  If this table contains any rows,
+  there are problems!
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filtercheck.sas
+  @li mp_filtergenerate.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mp_filtervalidate(inref,targetds,abort=YES,outds=work.mp_filtervalidate);
+
+%mp_abort(iftrue= (&syscc ne 0 or &syserr ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(syscc=&syscc / syserr=&syserr - on macro entry)
+)
+
+%local fref1;
+%let fref1=%mf_getuniquefileref();
+
+data _null_;
+  file &fref1;
+  infile &inref end=eof;
+  if _n_=1 then do;
+    put "proc sql;";
+    put "validate select * from &targetds";
+    put "where " ;
+  end;
+  input;
+  put _infile_;
+  putlog _infile_;
+  if eof then put ";quit;";
+run;
+
+%inc &fref1;
+
+data &outds;
+  if &sqlrc or &syscc or &syserr then do;
+    REASON_CD=coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
+    output;
+  end;
+  else stop;
+run;
+
+filename &fref1 clear;
+
+%if %mf_nobs(&outds)>0 %then %do;
+  %if &abort=YES %then %do;
+    data _null_;
+      set &outds;
+      call symputx('REASON_CD',reason_cd,'l');
+      stop;
+    run;
+    %mp_abort(
+      mac=&sysmacroname,
+      msg=%str(Filter issues in &inref: %quote(&reason_cd))
+    )
+  %end;
+  %let syscc=1008;
+%end;
+
+%mend;

base/mp_hashdataset.sas

@@ -0,0 +1,75 @@
+/**
+  @file
+  @brief Returns a unique hash for a dataset
+  @details Ignores metadata attributes, used only to hash values. Compared
+  datasets must be in the same order.
+
+      %mp_hashdataset(sashelp.class,outds=myhash)
+
+      data _null_;
+        set work.myhash;
+        put hashkey=;
+      run;
+
+  ![sas md5 hash dataset log results](https://i.imgur.com/MqF98vk.png)
+
+  <h4> SAS Macros </h4>
+  @li mf_getattrn.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
+
+  @param [in] libds dataset to hash
+  @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
+  will contain one column (hashkey) with one observation (a hex32.
+  representation of the input hash)
+  |hashkey:$32.|
+  |---|
+  |28ABC74ABFC45F50794237BA5566E6CA|
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_hashdataset(
+  libds,
+  outds=
+)/*/STORE SOURCE*/;
+  %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
+    %put %str(WARN)ING: Dataset &libds is empty;, or is not a dataset;
+  %end;
+  %else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
+    %put %str(ERR)OR: Dataset &libds is not a dataset;
+  %end;
+  %else %do;
+    %local keyvar /* roll up the md5 */
+      prevkeyvar /* retain prev record md5 */
+      lastvar /* last var in input ds */
+      varlist var i;
+    /* avoid naming conflict for hash key vars */
+    %let keyvar=%mf_getuniquename();
+    %let prevkeyvar=%mf_getuniquename();
+    %let lastvar=%mf_getuniquename();
+    %let varlist=%mf_getvarlist(&libds);
+    data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
+      length &prevkeyvar &keyvar $32;
+      retain &prevkeyvar;
+      set &libds end=&lastvar;
+      /* hash should include previous row */
+      &keyvar=put(md5(&prevkeyvar
+      /* loop every column, hashing every individual value */
+    %do i=1 %to %sysfunc(countw(&varlist));
+      %let var=%scan(&varlist,&i,%str( ));
+      %if %mf_getvartype(&libds,&var)=C %then %do;
+          !!put(md5(trim(&var)),$hex32.)
+      %end;
+      %else %do;
+          !!put(md5(trim(put(&var*1,binary64.))),$hex32.)
+      %end;
+    %end;
+      ),$hex32.);
+      &prevkeyvar=&keyvar;
+      if &lastvar then output;
+    run;
+  %end;
+%mend;

base/mp_jsonout.sas

@@ -48,6 +48,8 @@
   @param dbg= DEPRECATED - was used to conditionally add PRETTY to
     proc json but this can cause line truncation in large files.
 
+  <h4> Related Macros <h4>
+  @li mp_ds2fmtds.sas
 
   @version 9.2
   @author Allan Bowe
@@ -55,10 +57,11 @@
 
 **/
 
-%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=PROCJSON,dbg=0
+%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0
 )/*/STORE SOURCE*/;
 %put output location=&jref;
 %if &action=OPEN %then %do;
+  OPTIONS NOBOMFILE;
   data _null_;file &jref encoding='utf-8';
     put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
   run;
@@ -86,6 +89,64 @@
       %put &sysmacroname:  &ds NOT FOUND!!!;
       %return;
     %end;
+    %if &fmt=Y %then %do;
+      %put converting every variable to a formatted variable;
+      /* see mp_ds2fmtds.sas for source */
+      proc contents noprint data=&ds
+        out=_data_(keep=name type length format formatl formatd varnum);
+      run;
+      proc sort;
+        by varnum;
+      run;
+      %local fmtds;
+      %let fmtds=%scan(&syslast,2,.);
+      /* prepare formats and varnames */
+      data _null_;
+        set &fmtds end=last;
+        name=upcase(name);
+        /* fix formats */
+        if type=2 or type=6 then do;
+          length fmt $49.;
+          if format='' then fmt=cats('$',length,'.');
+          else if formatl=0 then fmt=cats(format,'.');
+          else fmt=cats(format,formatl,'.');
+          newlen=max(formatl,length);
+        end;
+        else do;
+          if format='' then fmt='best.';
+          else if formatl=0 then fmt=cats(format,'.');
+          else if formatd=0 then fmt=cats(format,formatl,'.');
+          else fmt=cats(format,formatl,'.',formatd);
+          /* needs to be wide, for datetimes etc */
+          newlen=max(length,formatl,24);
+        end;
+        /* 32 char unique name */
+        newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
+
+        call symputx(cats('name',_n_),name,'l');
+        call symputx(cats('newname',_n_),newname,'l');
+        call symputx(cats('len',_n_),newlen,'l');
+        call symputx(cats('fmt',_n_),fmt,'l');
+        call symputx(cats('type',_n_),type,'l');
+        if last then call symputx('nobs',_n_,'l');
+      run;
+      data &fmtds;
+        /* rename on entry */
+        set &ds(rename=(
+      %local i;
+      %do i=1 %to &nobs;
+        &&name&i=&&newname&i
+      %end;
+        ));
+      %do i=1 %to &nobs;
+        length &&name&i $&&len&i;
+        &&name&i=left(put(&&newname&i,&&fmt&i));
+        drop &&newname&i;
+      %end;
+        if _error_ then call symputx('syscc',1012);
+      run;
+      %let ds=&fmtds;
+    %end; /* &fmt=Y */
     data _null_;file &jref mod ;
       put "["; call symputx('cols',0,'l');
     proc sort
@@ -169,4 +230,4 @@
     put "}";
   run;
 %end;
-%mend;
+%mend mp_jsonout;

base/mp_streamfile.sas

@@ -36,8 +36,20 @@
 %let contentype=%upcase(&contenttype);
 %local platform; %let platform=%mf_getplatform();
 
+
+/**
+  * check engine type to avoid the below err message:
+  * > Function is only valid for filerefs using the CACHE access method.
+  */
+%local streamweb;
+%let streamweb=0;
+data _null_;
+  set sashelp.vextfl(where=(upcase(fileref)="_WEBOUT"));
+  if xengine='STREAM' then call symputx('streamweb',1,'l');
+run;
+
 %if &contentype=ZIP %then %do;
-  %if &platform=SASMETA %then %do;
+  %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/zip');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -51,7 +63,7 @@
 %end;
 %else %if &contentype=EXCEL %then %do;
   /* suitable for XLS format */
-  %if &platform=SASMETA %then %do;
+  %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/vnd.ms-excel');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -64,7 +76,7 @@
   %end;
 %end;
 %else %if &contentype=XLSX %then %do;
-  %if &platform=SASMETA %then %do;
+  %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
       rc=stpsrv_header('Content-type',
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
@@ -79,7 +91,7 @@
   %end;
 %end;
 %else %if &contentype=TEXT %then %do;
-  %if &platform=SASMETA %then %do;
+  %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/text');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -92,7 +104,7 @@
   %end;
 %end;
 %else %if &contentype=CSV %then %do;
-  %if &platform=SASMETA %then %do;
+  %if &platform=SASMETA and &streamweb=1 %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/csv');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -122,4 +134,4 @@
   %mp_binarycopy(inloc="&inloc",outref=_webout)
 %end;
 
-%mend;
+%mend;

base/mp_testservice.sas

@@ -0,0 +1,259 @@
+/**
+  @file mp_testservice.sas
+  @brief Will execute a test against a SASjs web service on SAS 9 or Viya
+  @details Prepares the input files and retrieves the resulting datasets from
+  the response JSON.
+
+      %mp_testjob(
+        duration=60*5
+      )
+
+  Note - the _webout fileref should NOT be assigned prior to running this macro.
+
+  @param [in] program The _PROGRAM endpoint to test
+  @param [in] inputfiles=(0) A list of space seperated fileref:filename pairs as
+    follows:
+        inputfiles=inref:filename inref2:filename2
+  @param [in] inputparams=(0) A dataset containing name/value pairs in the
+    following format:
+    |name:$32|value:$1000|
+    |---|---|
+    |stpmacname|some value|
+    |mustbevalidname|can be anything, oops, %abort!!|
+
+  @param [in] debug= (log) Provide the _debug value
+  @param [in] viyaresult=(WEBOUT_JSON) The Viya result type to return.  For
+    more info, see mv_getjobresult.sas
+  @param [out] outlib= (0) Output libref to contain the final tables.  Set to
+    0 if the service output is not in JSON format.
+  @param [out] outref= (0) Output fileref to create, to contain the full _webout
+    response.
+
+  <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+  @li mv_getjobresult.sas
+  @li mv_jobflow.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mp_testservice(program,
+  inputfiles=0,
+  inputparams=0,
+  debug=log,
+  outlib=0,
+  outref=0,
+  viyaresult=WEBOUT_JSON
+)/*/STORE SOURCE*/;
+%local mdebug;
+%if &debug ne 0 %then %do;
+  %let mdebug=1;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let mdebug=0;
+
+/* sanitise inputparams */
+%local pcnt;
+%let pcnt=0;
+%if &inputparams ne 0 %then %do;
+  data _null_;
+    set &inputparams;
+    if not nvalid(name,'v7') then putlog (_all_)(=);
+    else if name in (
+      'program','inputfiles','inputparams','debug','outlib','outref'
+    ) then putlog (_all_)(=);
+    else do;
+      x+1;
+      call symputx(name,quote(cats(value)),'l');
+      call symputx('pval'!!left(x),name,'l');
+      call symputx('pcnt',x,'l');
+    end;
+  run;
+  %mp_abort(iftrue= (%mf_nobs(&inputparams) ne &pcnt)
+    ,mac=&sysmacroname
+    ,msg=%str(Invalid values in &inputparams)
+  )
+%end;
+
+
+%local fref1 webref;
+%let fref1=%mf_getuniquefileref();
+%let webref=%mf_getuniquefileref();
+
+%local platform;
+%let platform=%mf_getplatform();
+%if &platform=SASMETA %then %do;
+
+  /* parse the input files */
+  %local webcount i var;
+  %if %quote(&inputfiles) ne 0 %then %do;
+    %let webcount=%sysfunc(countw(&inputfiles));
+    %put &=webcount;
+    %do i=1 %to &webcount;
+      %let var=%scan(&inputfiles,&i,%str( ));
+      %local webfref&i webname&i;
+      %let webref&i=%scan(&var,1,%str(:));
+      %let webname&i=%scan(&var,2,%str(:));
+      %put webref&i=&&webref&i;
+      %put webname&i=&&webname&i;
+    %end;
+  %end;
+  %else %let webcount=0;
+
+  proc stp program="&program";
+    inputparam _program="&program"
+  %do i=1 %to &webcount;
+    %if &webcount=1 %then %do;
+      _webin_fileref="&&webref&i"
+      _webin_name="&&webname&i"
+    %end;
+    %else %do;
+      _webin_fileref&i="&&webref&i"
+      _webin_name&i="&&webname&i"
+    %end;
+  %end;
+    _webin_file_count="&webcount"
+    _debug="&debug"
+  %do i=1 %to &pcnt;
+    /* resolve name only, proc stp fetches value */
+    &&pval&i=&&&&&&pval&i
+  %end;
+    ;
+  %do i=1 %to &webcount;
+    inputfile &&webref&i;
+  %end;
+    outputfile _webout=&webref;
+  run;
+
+  data _null_;
+    infile &webref;
+    file &fref1;
+    input;
+    length line $10000;
+    if index(_infile_,'>>weboutBEGIN<<') then do;
+        line=tranwrd(_infile_,'>>weboutBEGIN<<','');
+        put line;
+    end;
+    else if index(_infile_,'>>weboutEND<<') then do;
+        line=tranwrd(_infile_,'>>weboutEND<<','');
+        put line;
+        stop;
+    end;
+    else put _infile_;
+  run;
+  data _null_;
+    infile &fref1;
+    input;
+    put _infile_;
+  run;
+  %if &outlib ne 0 %then %do;
+    libname &outlib json (&fref1);
+  %end;
+  %if &outref ne 0 %then %do;
+    filename &outref temp;
+    %mp_binarycopy(inref=&webref,outref=&outref)
+  %end;
+
+%end;
+%else %if &platform=SASVIYA %then %do;
+
+  /* prepare inputparams */
+  %local ds1;
+  %let ds1=%mf_getuniquename();
+  %if "&inputparams" ne "0" %then %do;
+    proc transpose data=&inputparams out=&ds1;
+      id name;
+      var value;
+    run;
+  %end;
+  %else %do;
+    data &ds1;run;
+  %end;
+
+  /* parse the input files - convert to sasjs params */
+  %local webcount i var sasjs_tables;
+  %if %quote(&inputfiles) ne 0 %then %do;
+    %let webcount=%sysfunc(countw(&inputfiles));
+    %put &=webcount;
+    %do i=1 %to &webcount;
+      %let var=%scan(&inputfiles,&i,%str( ));
+      %local webfref&i webname&i sasjs&i.data;
+      %let webref&i=%scan(&var,1,%str(:));
+      %let webname&i=%scan(&var,2,%str(:));
+      %put webref&i=&&webref&i;
+      %put webname&i=&&webname&i;
+
+      %let sasjs_tables=&sasjs_tables &&webname&i;
+      data _null_;
+        infile &&webref&i lrecl=32767;
+        input;
+        if _n_=1 then call symputx("sasjs&i.data",_infile_);
+        else call symputx(
+          "sasjs&i.data",cats(symget("sasjs&i.data"),'0D0A'x,_infile_)
+        );
+        putlog "&sysmacroname infile: " _infile_;
+      run;
+      data &ds1;
+        set &ds1;
+        length sasjs&i.data $32767 sasjs_tables $1000;
+        sasjs&i.data=symget("sasjs&i.data");
+        sasjs_tables=symget("sasjs_tables");
+      run;
+    %end;
+  %end;
+  %else %let webcount=0;
+
+  data &ds1;
+    retain _program "&program";
+    set &ds1;
+    putlog "&sysmacroname inputparams:";
+    putlog (_all_)(=);
+  run;
+
+  %mv_jobflow(inds=&ds1
+    ,maxconcurrency=1
+    ,outds=work.results
+    ,outref=&fref1
+    ,mdebug=&mdebug
+  )
+  /* show the log */
+  data _null_;
+    infile &fref1;
+    input;
+    putlog _infile_;
+  run;
+  /* get the uri to fetch results */
+  data _null_;
+    set work.results;
+    call symputx('uri',uri);
+    putlog "&sysmacroname: fetching results for " uri;
+  run;
+  /* fetch results from webout.json */
+  %mv_getjobresult(uri=&uri,
+    result=&viyaresult,
+    outref=&outref,
+    outlib=&outlib,
+    mdebug=&mdebug
+  )
+
+%end;
+%else %do;
+  %put %str(ERR)OR: Unrecognised platform:  &platform;
+%end;
+
+%if &mdebug=0 %then %do;
+  filename &webref clear;
+%end;
+%else %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+
+%mend mp_testservice;

base/mp_validatecol.sas

@@ -0,0 +1,66 @@
+/**
+  @file
+  @brief Used to validate variables in a dataset
+  @details Useful when sanitising inputs, to ensure that they arrive with a
+  certain pattern.
+  Usage:
+
+      data test;
+        infile datalines4 dsd;
+        input;
+        libds=_infile_;
+        %mp_validatecol(libds,LIBDS,is_libds)
+      datalines4;
+      some.libname
+      !lib.blah
+      %abort
+      definite.ok
+      not.ok!
+      nineletrs._
+      ;;;;
+      run;
+
+  @param [in] incol The column to be validated
+  @param [in] rule The rule to apply.  Current rules:
+    @li ISNUM - checks if the variable is numeric
+    @li LIBDS - matches LIBREF.DATASET format
+  @param [out] outcol The variable to create, with the results of the match
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+
+  @version 9.3
+**/
+
+%macro mp_validatecol(incol,rule,outcol);
+
+/* tempcol is given a unique name with every invocation */
+%local tempcol;
+%let tempcol=%mf_getuniquename();
+
+%if &rule=ISNUM %then %do;
+  /*
+    credit SØREN LASSEN
+    https://sasmacro.blogspot.com/2009/06/welcome-isnum-macro.html
+  */
+  &tempcol=input(&incol,?? best32.);
+  if missing(&tempcol) then &outcol=0;
+  else &outcol=1;
+  drop &tempcol;
+%end;
+%else %if &rule=LIBDS %then %do;
+  /* match libref.dataset */
+  if _n_=1 then do;
+    retain &tempcol;
+    &tempcol=prxparse('/^[_a-z]\w{0,7}\.[_a-z]\w{0,31}$/i');
+    if missing(&tempcol) then do;
+      putlog "%str(ERR)OR: Invalid expression for LIBDS";
+      stop;
+    end;
+    drop &tempcol;
+  end;
+  if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
+  else &outcol=0;
+%end;
+
+%mend mp_validatecol;

base/mp_zip.sas

@@ -2,11 +2,11 @@
   @file
   @brief Creates a zip file
   @details For DIRECTORY usage, will ignore subfolders. For DATASET usage,
-    provide a column that contains the full file path to each file to be zipped.
+  provide a column that contains the full file path to each file to be zipped.
 
-    %mp_zip(in=myzips,type=directory,outname=myDir)
-    %mp_zip(in=/my/file/path.txt,type=FILE,outname=myFile)
-    %mp_zip(in=SOMEDS,incol=FPATH,type=DATASET,outname=myFile)
+      %mp_zip(in=myzips,type=directory,outname=myDir)
+      %mp_zip(in=/my/file/path.txt,type=FILE,outname=myFile)
+      %mp_zip(in=SOMEDS,incol=FPATH,type=DATASET,outname=myFile)
 
   If you are sending zipped output to the _webout destination as part of an STP
   be sure that _debug is not set (else the SPWA will send non zipped content

meta/mm_adduser2group.sas

@@ -70,7 +70,7 @@ run;
 %end;
 
 %if &syscc ge 4 %then %do;
-  %put WARNING:  SYSCC=&syscc, exiting &sysmacroname;
+  %put %str(WARN)ING:  SYSCC=&syscc, exiting &sysmacroname;
   %return;
 %end;
 

meta/mm_assigndirectlib.sas

@@ -332,7 +332,7 @@ run;
   run;
 
   %if %length(&open_passthrough)>0 %then %do;
-    %put WARNING:  Passthrough option for postgres not yet supported;
+    %put %str(WARN)ING: Passthrough option for postgres not yet supported;
     %return;
   %end;
   %else %do;
@@ -455,8 +455,8 @@ run;
   %return;
 %end;
 %else %do;
-  %put WARNING: Engine &engine is currently unsupported;
-  %put WARNING- Please contact your support team.;
+  %put %str(WARN)ING: Engine &engine is currently unsupported;
+  %put %str(WARN)ING- Please contact your support team.;
   %return;
 %end;
 

meta/mm_createlibrary.sas

@@ -88,7 +88,7 @@ data _null_;
   putlog (_all_)(=);
 run;
 %if &checktype = SASLibrary %then %do;
-  %put WARNING: Library (&liburi) already exists with libname (&libname)  ;
+  %put %str(WARN)ING: Library (&liburi) already exists with libname (&libname);
   %return;
 %end;
 
@@ -103,7 +103,7 @@ data _null_;
   putlog (_all_)(=);
 run;
 %if &checktype = SASLibrary %then %do;
-  %put WARNING: Library (&liburi) already exists with libref (&libref)  ;
+  %put %str(WARN)ING: Library (&liburi) already exists with libref (&libref)  ;
   %return;
 %end;
 
@@ -123,7 +123,7 @@ data _null_;
   call symputx('treeuri',uri,'l');
 run;
 %if &foldertype ne Tree %then %do;
-  %put WARNING: Tree &tree does not exist!;
+  %put %str(WARN)ING: Tree &tree does not exist!;
   %return;
 %end;
 
@@ -231,7 +231,7 @@ filename &frefout temp;
     *  check SAS version
     */
   %if %sysevalf(&sysver lt 9.3) %then %do;
-    %put WARNING: Version 9.3 or later required;
+    %put %str(WARN)ING: Version 9.3 or later required;
     %return;
   %end;
 

meta/mm_createstp.sas

@@ -118,7 +118,7 @@ data _null_;
   call symputx('treeuri',uri,'l');
 run;
 %if &foldertype ne Tree %then %do;
-  %put WARNING: Tree &tree does not exist!;
+  %put %str(WARN)ING: Tree &tree does not exist!;
   %return;
 %end;
 
@@ -133,7 +133,7 @@ data _null_;
   call symputx('stpuri',uri,'l');
 run;
 %if &cmtype = ClassifierMap %then %do;
-  %put WARNING: Stored Process &stpname already exists in &tree!;
+  %put %str(WARN)ING: Stored Process &stpname already exists in &tree!;
   %return;
 %end;
 
@@ -141,14 +141,14 @@ run;
   * Check that the physical file exists
   */
 %if %sysfunc(fileexist(&directory/&filename)) ne 1 %then %do;
-  %put WARNING: FILE *&directory/&filename* NOT FOUND!;
+  %put %str(WARN)ING: FILE *&directory/&filename* NOT FOUND!;
   %return;
 %end;
 
 %if &stptype=1 %then %do;
   /* type 1 STP - where code is stored on filesystem */
   %if %sysevalf(&sysver lt 9.2) %then %do;
-    %put WARNING: Version 9.2 or later required;
+    %put %str(WARN)ING: Version 9.2 or later required;
     %return;
   %end;
 
@@ -162,7 +162,7 @@ run;
   %if &checkdirtype ne Directory %then %do;
     %mm_getdirectories(path=&directory,outds=&outds ,mDebug=&mDebug)
     %if %mf_nobs(&outds)=0 or %sysfunc(exist(&outds))=0 %then %do;
-      %put WARNING: The directory object does not exist for &directory;
+      %put %str(WARN)ING: The directory object does not exist for &directory;
       %return;
     %end;
   %end;
@@ -180,12 +180,12 @@ run;
     length id $20 type $256;
     __rc=metadata_resolve("&treeuri",type,id);
     if type ne 'Tree' then do;
-      putlog "WARNING:  Invalid tree URI: &treeuri";
+      putlog "%str(WARN)ING:  Invalid tree URI: &treeuri";
       stopme=1;
     end;
     __rc=metadata_resolve(directoryuri,type,id);
     if type ne 'Directory' then do;
-      putlog 'WARNING:  Invalid directory URI: ' directoryuri;
+      putlog "%str(WARN)ING:  Invalid directory URI: " directoryuri;
       stopme=1;
     end;
 
@@ -194,7 +194,7 @@ run;
     if type ne 'LogicalServer' then do;
       __rc=metadata_getnobj("omsobj:LogicalServer?@Name='&server'",1,serveruri);
       if serveruri='' then do;
-        putlog "WARNING:  Invalid server: &server";
+        putlog "%str(WARN)ING:  Invalid server: &server";
         stopme=1;
       end;
     end;
@@ -217,7 +217,7 @@ run;
     rc6 = METADATA_SETATTR(prompturi, 'GroupInfo',groupinfo);
 
     if sum(of rc1-rc6) ne 0 then do;
-      putlog 'WARNING: Issue creating prompt.';
+      putlog "%str(WARN)ING: Issue creating prompt.";
       if prompturi ne . then do;
         putlog '  Removing orphan: ' prompturi;
         rc = METADATA_DELOBJ(prompturi);
@@ -232,7 +232,7 @@ run;
     rc9=METADATA_SETATTR(fileuri, 'IsARelativeName','1');
     rc10=METADATA_SETASSN(fileuri, 'Directories','MODIFY',directoryuri);
     if sum(of rc7-rc10) ne 0 then do;
-      putlog 'WARNING: Issue creating file.';
+      putlog "%str(WARN)ING: Issue creating file.";
       if fileuri ne . then do;
         putlog '  Removing orphans:' prompturi fileuri;
         rc = METADATA_DELOBJ(prompturi);
@@ -251,7 +251,7 @@ run;
       !!"<OutputParameters/></StoredProcess>";
     rc14= METADATA_SETATTR(texturi, 'StoredText',storedtext);
     if sum(of rc11-rc14) ne 0 then do;
-      putlog 'WARNING: Issue creating TextStore.';
+      putlog "%str(WARN)ING: Issue creating TextStore.";
       if texturi ne . then do;
         putlog '  Removing orphans: ' prompturi fileuri texturi;
         rc = METADATA_DELOBJ(prompturi);
@@ -299,7 +299,7 @@ run;
 %else %if &stptype=2 %then %do;
   /* type 2 stp - code is stored in metadata */
   %if %sysevalf(&sysver lt 9.3) %then %do;
-    %put WARNING: SAS version 9.3 or later required to create type2 STPs;
+    %put %str(WARN)ING: SAS version 9.3 or later required to create type2 STPs;
     %return;
   %end;
   /* check we have the correct ServerContext */
@@ -311,7 +311,7 @@ run;
     call symputx('serveruri',serveruri);
   run;
   %if &serveruri=NOTFOUND %then %do;
-    %put WARNING: ServerContext *&server* not found!;
+    %put %str(WARN)ING: ServerContext *&server* not found!;
     %return;
   %end;
 
@@ -382,7 +382,7 @@ run;
 
 %end;
 %else %do;
-  %put WARNING:  STPTYPE=*&stptype* not recognised!;
+  %put %str(WARN)ING:  STPTYPE=*&stptype* not recognised!;
 %end;
 
 %mend;

meta/mm_createwebservice.sas

@@ -86,10 +86,11 @@ data _null_;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
   put ' ';
-  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=PROCJSON,dbg=0 ';
+  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0 ';
   put ')/*/STORE SOURCE*/; ';
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
+  put '  OPTIONS NOBOMFILE; ';
   put '  data _null_;file &jref encoding=''utf-8''; ';
   put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
   put '  run; ';
@@ -117,6 +118,64 @@ data _null_;
   put '      %put &sysmacroname:  &ds NOT FOUND!!!; ';
   put '      %return; ';
   put '    %end; ';
+  put '    %if &fmt=Y %then %do; ';
+  put '      %put converting every variable to a formatted variable; ';
+  put '      /* see mp_ds2fmtds.sas for source */ ';
+  put '      proc contents noprint data=&ds ';
+  put '        out=_data_(keep=name type length format formatl formatd varnum); ';
+  put '      run; ';
+  put '      proc sort; ';
+  put '        by varnum; ';
+  put '      run; ';
+  put '      %local fmtds; ';
+  put '      %let fmtds=%scan(&syslast,2,.); ';
+  put '      /* prepare formats and varnames */ ';
+  put '      data _null_; ';
+  put '        set &fmtds end=last; ';
+  put '        name=upcase(name); ';
+  put '        /* fix formats */ ';
+  put '        if type=2 or type=6 then do; ';
+  put '          length fmt $49.; ';
+  put '          if format='''' then fmt=cats(''$'',length,''.''); ';
+  put '          else if formatl=0 then fmt=cats(format,''.''); ';
+  put '          else fmt=cats(format,formatl,''.''); ';
+  put '          newlen=max(formatl,length); ';
+  put '        end; ';
+  put '        else do; ';
+  put '          if format='''' then fmt=''best.''; ';
+  put '          else if formatl=0 then fmt=cats(format,''.''); ';
+  put '          else if formatd=0 then fmt=cats(format,formatl,''.''); ';
+  put '          else fmt=cats(format,formatl,''.'',formatd); ';
+  put '          /* needs to be wide, for datetimes etc */ ';
+  put '          newlen=max(length,formatl,24); ';
+  put '        end; ';
+  put '        /* 32 char unique name */ ';
+  put '        newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put ' ';
+  put '        call symputx(cats(''name'',_n_),name,''l''); ';
+  put '        call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '        call symputx(cats(''len'',_n_),newlen,''l''); ';
+  put '        call symputx(cats(''fmt'',_n_),fmt,''l''); ';
+  put '        call symputx(cats(''type'',_n_),type,''l''); ';
+  put '        if last then call symputx(''nobs'',_n_,''l''); ';
+  put '      run; ';
+  put '      data &fmtds; ';
+  put '        /* rename on entry */ ';
+  put '        set &ds(rename=( ';
+  put '      %local i; ';
+  put '      %do i=1 %to &nobs; ';
+  put '        &&name&i=&&newname&i ';
+  put '      %end; ';
+  put '        )); ';
+  put '      %do i=1 %to &nobs; ';
+  put '        length &&name&i $&&len&i; ';
+  put '        &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '        drop &&newname&i; ';
+  put '      %end; ';
+  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '      run; ';
+  put '      %let ds=&fmtds; ';
+  put '    %end; /* &fmt=Y */ ';
   put '    data _null_;file &jref mod ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
@@ -200,7 +259,7 @@ data _null_;
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
-  put '%mend; ';
+  put '%mend mp_jsonout; ';
   put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
@@ -237,8 +296,16 @@ data _null_;
   put '%else %if &action=OPEN %then %do; ';
   put '  /* fix encoding */ ';
   put '  OPTIONS NOBOMFILE; ';
+  put ' ';
+  put '  /** ';
+  put '    * check engine type to avoid the below err message: ';
+  put '    * > Function is only valid for filerefs using the CACHE access method. ';
+  put '    */ ';
   put '  data _null_; ';
-  put '    rc = stpsrv_header(''Content-type'',"text/html; encoding=utf-8"); ';
+  put '    set sashelp.vextfl(where=(fileref="_WEBOUT")); ';
+  put '    if xengine=''STREAM'' then do; ';
+  put '      rc=stpsrv_header(''Content-type'',"text/html; encoding=utf-8"); ';
+  put '    end; ';
   put '  run; ';
   put ' ';
   put '  /* setup json */ ';
@@ -252,16 +319,9 @@ data _null_;
   put '%end; ';
   put ' ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
-  put '  %if &sysver=9.4 %then %do; ';
-  put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
-  put '      ,engine=PROCJSON,dbg=%str(&_debug) ';
-  put '    ) ';
-  put '  %end; ';
-  put '  %else %do; ';
-  put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
-  put '      ,engine=DATASTEP,dbg=%str(&_debug) ';
-  put '    ) ';
-  put '  %end; ';
+  put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
+  put '    ,engine=DATASTEP,dbg=%str(&_debug) ';
+  put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
   put '  %if %str(&_debug) ge 131 %then %do; ';

meta/mm_deletedocument.sas

@@ -32,7 +32,7 @@ data _null_;
   call symputx('stpuri',uri,'l');
 run;
 %if &type ne Document %then %do;
-  %put WARNING: No Document found at ⌖
+  %put %str(WARN)ING: No Document found at ⌖
   %return;
 %end;
 

meta/mm_getdocument.sas

@@ -125,7 +125,7 @@ data _null_;
           when ('
') rec='0D'x;
           when ('$' ) rec='$'  ;
           when ('	') rec='09'x;
-          otherwise putlog "WARNING: missing value for " entity=;
+          otherwise putlog "%str(WARN)ING: missing value for " entity=;
         end;
         rc =fput(fileid, substr(rec,1,1));
         rc =fwrite(fileid);

meta/mm_getstpcode.sas

@@ -1,20 +1,26 @@
 /**
   @file
-  @brief Writes the code of an to an external file, or the log if none provided
-  @details Get the
+  @brief Writes the code of an STP to an external file
+  @details Fetches the SAS code from a Stored Process where the code is stored
+  in metadata.
 
-  usage:
+  Usage:
 
       %mm_getstpcode(tree=/some/meta/path
         ,name=someSTP
         ,outloc=/some/unquoted/filename.ext
       )
 
-  @param tree= The metadata path of the Stored Process (can also contain name)
-  @param name= Stored Process name.  Leave blank if included above.
-  @param outloc= full and unquoted path to the desired text file.  This will be
-    overwritten if it already exists.  If not provided, the code will be written
-    to the log.
+  @param [in] tree= The metadata path of the Stored Process (can also contain
+    name)
+  @param [in] name= Stored Process name.  Leave blank if included above.
+  @param [out] outloc= (0) full and unquoted path to the desired text file.
+    This will be overwritten if it already exists.
+  @param [out] outref= (0) Fileref to which to write the code.
+  @param [out] showlog=(NO) Set to YES to print log to the window
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
 
   @author Allan Bowe
 
@@ -23,8 +29,10 @@
 %macro mm_getstpcode(
     tree=/User Folders/sasdemo/somestp
     ,name=
-    ,outloc=
+    ,outloc=0
+    ,outref=0
     ,mDebug=1
+    ,showlog=NO
     );
 
 %local mD;
@@ -92,14 +100,18 @@ data _null_;
   stop;
 
 %local outeng;
-%if %length(&outloc)=0 %then %let outeng=TEMP;
+%if "&outloc"="0" %then %let outeng=TEMP;
 %else %let outeng="&outloc";
+%local fref;
+%if &outref=0 %then %let fref=%mf_getuniquefileref();
+%else %let fref=&outref;
+
 /* read the content, byte by byte, resolving escaped chars */
-filename __outdoc &outeng lrecl=100000;
+filename &fref &outeng lrecl=100000;
 data _null_;
   length filein 8 fileid 8;
   filein = fopen("__getdoc","I",1,"B");
-  fileid = fopen("__outdoc","O",1,"B");
+  fileid = fopen("&fref","O",1,"B");
   rec = "20"x;
   length entity $6;
   do while(fread(filein)=0);
@@ -140,9 +152,9 @@ data _null_;
   rc=fclose(fileid);
 run;
 
-%if &outeng=TEMP %then %do;
+%if &showlog=YES %then %do;
   data _null_;
-    infile __outdoc lrecl=32767 end=last;
+    infile &fref lrecl=32767 end=last;
     input;
     if _n_=1 then putlog '>>stpcodeBEGIN<<';
     putlog _infile_;
@@ -151,6 +163,8 @@ run;
 %end;
 
 filename __getdoc clear;
-filename __outdoc clear;
+%if &outref=0 %then %do;
+  filename &fref clear;
+%end;
 
-%mend;
+%mend mm_getstpcode;

meta/mm_getwebappsrvprops.sas

@@ -99,7 +99,7 @@ run;
           when ('
') rec='0D'x;
           when ('$' ) rec='$'  ;
           when ('	') rec='09'x;
-          otherwise putlog "WARNING: missing value for " entity=;
+          otherwise putlog "%str(WARN)ING: missing value for " entity=;
         end;
         rc =fput(fileid, substr(rec,1,1));
         rc =fwrite(fileid);

meta/mm_updateappextension.sas

@@ -70,7 +70,7 @@ data _null_;
 run;
 
 %if &appuri=stopifempty %then %do;
-  %put WARNING:  &app.(Application) not found!;
+  %put %str(WARN)ING:  &app.(Application) not found!;
   %return;
 %end;
 

meta/mm_updatedocument.sas

@@ -58,12 +58,12 @@ data _null_;
 run;
 
 %if &tsuri=stopifempty %then %do;
-  %put WARNING:  &path/&name.(Document) not found!;
+  %put %str(WARN)ING:  &path/&name.(Document) not found!;
   %return;
 %end;
 
 %if %length(&text)<2 %then %do;
-  %put WARNING:  No text supplied!!;
+  %put %str(WARN)ING:  No text supplied!!;
   %return;
 %end;
 

meta/mm_updatestpservertype.sas

@@ -34,7 +34,7 @@ data _null_;
   call symputx('stpuri',uri,'l');
 run;
 %if &cmtype ne ClassifierMap %then %do;
-  %put WARNING: No Stored Process found at ⌖
+  %put %str(WARN)ING: No Stored Process found at ⌖
   %return;
 %end;
 

meta/mm_updatestpsourcecode.sas

@@ -68,12 +68,12 @@ data _null_;
 run;
 
 %if &tsuri=stopifempty %then %do;
-  %put WARNING:  &stp.(StoredProcess) not found!;
+  %put %str(WARN)ING:  &stp.(StoredProcess) not found!;
   %return;
 %end;
 
 %if %length(&stpcode)<2 %then %do;
-  %put WARNING:  No SAS code supplied!!;
+  %put %str(WARN)ING:  No SAS code supplied!!;
   %return;
 %end;
 

meta/mm_webout.sas

@@ -68,8 +68,16 @@
 %else %if &action=OPEN %then %do;
   /* fix encoding */
   OPTIONS NOBOMFILE;
+
+  /**
+    * check engine type to avoid the below err message:
+    * > Function is only valid for filerefs using the CACHE access method.
+    */
   data _null_;
-    rc = stpsrv_header('Content-type',"text/html; encoding=utf-8");
+    set sashelp.vextfl(where=(fileref="_WEBOUT"));
+    if xengine='STREAM' then do;
+      rc=stpsrv_header('Content-type',"text/html; encoding=utf-8");
+    end;
   run;
 
   /* setup json */
@@ -83,16 +91,9 @@
 %end;
 
 %else %if &action=ARR or &action=OBJ %then %do;
-  %if &sysver=9.4 %then %do;
-    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt
-      ,engine=PROCJSON,dbg=%str(&_debug)
-    )
-  %end;
-  %else %do;
-    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt
-      ,engine=DATASTEP,dbg=%str(&_debug)
-    )
-  %end;
+  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt
+    ,engine=DATASTEP,dbg=%str(&_debug)
+  )
 %end;
 %else %if &action=CLOSE %then %do;
   %if %str(&_debug) ge 131 %then %do;

sasjs/sasjsconfig.json

@@ -1,7 +1,13 @@
 {
   "$schema": "https://cli.sasjs.io/sasjsconfig-schema.json",
-  "macroFolders": ["base", "meta", "metax", "viya", "lua"],
-  "docConfig":{
+  "macroFolders": [
+    "base",
+    "meta",
+    "metax",
+    "viya",
+    "lua"
+  ],
+  "docConfig": {
     "displayMacroCore": false,
     "enableLineage": false,
     "doxyContent": {
@@ -9,5 +15,34 @@
       "logo": "Macro_core_website_1.png",
       "readMe": "../../README.md"
     }
-  }
-}
+  },
+  "defaultTarget": "viya",
+  "targets": [
+    {
+      "name": "viya",
+      "serverUrl": "https://sas.analytium.co.uk",
+      "serverType": "SASVIYA",
+      "appLoc": "/Public/temp/macrocore",
+      "deployConfig": {
+        "deployServicePack": true
+      },
+      "macroFolders": [
+        "base",
+        "meta",
+        "metax",
+        "viya",
+        "lua",
+        "tests/base",
+        "tests/viya"
+      ],
+      "contextName": "SAS Job Execution compute context",
+      "testConfig": {
+        "initProgram": "tests/testinit.sas",
+        "termProgram": "tests/testterm.sas",
+        "macroVars": {
+          "mcTestAppLoc": "/Public/temp/macrocore"
+        }
+      }
+    }
+  ]
+}

tests/base/mf_getvarlist.test.sas

@@ -0,0 +1,60 @@
+/**
+  @file
+  @brief Testing mf_getvarlist macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getvarlist.sas
+
+**/
+
+
+%let test1=%mf_getvarlist(sashelp.class);
+%let test2=%mf_getvarlist(sashelp.class,dlm=X);
+%let test3=%mf_getvarlist(sashelp.class,dlm=%str(,),quote=double);
+%let test4=%mf_getvarlist(sashelp.class,typefilter=N);
+%let test5=%mf_getvarlist(sashelp.class,typefilter=C);
+
+data work.test_results;
+  length test_description $256 test_result $4 test_comments base result  $256;
+  test_description="Basic test";
+  base=symget('test1');
+  result='Name Sex Age Height Weight';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  test_description="DLM test";
+  base=symget('test2');
+  result='NameXSexXAgeXHeightXWeight';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  test_description="DLM + quote test";
+  base=symget('test3');
+  result='"Name","Sex","Age","Height","Weight"';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  test_description="Numeric Filter";
+  base=symget('test4');
+  result='Age Height Weight';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  test_description="Char Filter";
+  base=symget('test5');
+  result='Name Sex';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  drop base result;
+run;

tests/base/mp_assertcolvals.test.sas

@@ -0,0 +1,32 @@
+/**
+  @file
+  @brief Testing mp_assertcolvals macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assertcolvals.sas
+
+**/
+
+
+data work.checkds;
+  do checkval='Jane','James','Jill';
+    output;
+  end;
+run;
+%mp_assertcolvals(sashelp.class.name,
+  checkvals=work.checkds.checkval,
+  desc=At least one value has a match,
+  test=ANYVAL
+)
+
+data work.check;
+  do val='M','F';
+    output;
+  end;
+run;
+%mp_assertcolvals(sashelp.class.sex,
+  checkvals=work.check.val,
+  desc=All values have a match,
+  test=ALLVALS
+)
+

tests/base/mp_ds2fmtds.test.sas

@@ -0,0 +1,28 @@
+/**
+  @file
+  @brief Testing mp_ds2fmtds.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2fmtds.sas
+  @li mp_assert.sas
+
+**/
+
+proc sql;
+create table test as select * from dictionary.tables where libname='SASHELP';
+
+filename inc temp;
+data _null_;
+  set work.test;
+  file inc;
+  line=cats('%mp_ds2fmtds(sashelp.',memname,',',memname,')');
+  put line;
+run;
+
+%inc inc;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking tables were created successfully,
+  outds=work.test_results
+)

tests/base/mp_filtercheck.test.sas

@@ -0,0 +1,148 @@
+/**
+  @file
+  @brief Testing mp_filtercheck macro
+
+  <h4> SAS Macros </h4>
+  @li mp_filtercheck.sas
+  @li mp_assertdsobs.sas
+  @li mp_assert.sas
+
+**/
+
+
+/* valid filter */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,AGE,=,12
+AND,AND,1,SEX,<=,"'M'"
+AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
+AND,OR,2,Weight,>=,77.7
+AND,OR,2,Weight,NE,77.7
+;;;;
+run;
+
+%mp_filtercheck(work.inds,
+  targetds=sashelp.class,
+  outds=work.badrecords,
+  abort=NO
+)
+%let syscc=0;
+%mp_assertdsobs(work.badrecords,
+  desc=Valid filter query,
+  test=EMPTY,
+  outds=work.test_results
+)
+
+/* invalid column */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,invalid,=,12
+AND,AND,1,SEX,<=,"'M'"
+AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
+AND,OR,2,Weight,>=,7
+;;;;
+run;
+%mp_filtercheck(work.inds,
+  targetds=sashelp.class,
+  outds=work.badrecords,
+  abort=NO
+)
+%let syscc=0;
+%mp_assertdsobs(work.badrecords,
+  desc=Invalid column name,
+  test=HASOBS,
+  outds=work.test_results
+)
+
+/* invalid raw value */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,OR,2,Name,NOT IN,"(''''Jane','Alfred')"
+;;;;
+run;
+
+%mp_filtercheck(work.inds,
+  targetds=sashelp.class,
+  outds=work.badrecords,
+  abort=NO
+)
+%let syscc=0;
+%mp_assertdsobs(work.badrecords,
+  desc=Invalid raw value,
+  test=HASOBS,
+  outds=work.test_results
+)
+
+/* Code injection - column name */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,%abort,=,12
+AND,OR,2,Weight,>=,7
+;;;;
+run;
+
+%mp_filtercheck(work.inds,
+  targetds=sashelp.class,
+  outds=work.badrecords,
+  abort=NO
+)
+%let syscc=0;
+%mp_assertdsobs(work.badrecords,
+  desc=Code injection - column name,
+  test=HASOBS,
+  outds=work.test_results
+)
+
+/* Code injection - raw values*/
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,age,=,;;%abort
+;;;;
+run;
+%mp_filtercheck(work.inds,
+  targetds=sashelp.class,
+  outds=work.badrecords,
+  abort=NO
+)
+%let syscc=0;
+%mp_assertdsobs(work.badrecords,
+  desc=Code injection - raw value abort,
+  test=HASOBS,
+  outds=work.test_results
+)
+
+/* Supply variables with incorrect types */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:8;
+datalines4;
+AND,AND,1,age,=,0
+;;;;
+run;
+%let syscc=0;
+%mp_filtercheck(work.inds,
+  targetds=sashelp.class,
+  outds=work.badrecords,
+  abort=NO
+)
+%mp_assert(iftrue=(&syscc=42),
+  desc=Throw error if RAW_VALUE is incorrect,
+  outds=work.test_results
+)
+%let syscc=0;

tests/base/mp_filtergenerate.test.sas

@@ -0,0 +1,122 @@
+/**
+  @file
+  @brief Testing mp_filtergenerate macro
+
+  <h4> SAS Macros </h4>
+  @li mp_filtergenerate.sas
+  @li mp_filtercheck.sas
+  @li mp_assertdsobs.sas
+
+**/
+
+options source2;
+
+/* valid filter */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,AGE,>,5
+AND,AND,1,SEX,NE,"'M'"
+AND,OR,2,Name,NOT IN,"('Jane','Janet')"
+AND,OR,2,Weight,>=,84.6
+;;;;
+run;
+%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtergenerate(work.inds,outref=myfilter)
+data work.test;
+  set sashelp.class;
+  where %inc myfilter;;
+run;
+%mp_assertdsobs(work.test,
+  desc=Valid filter,
+  test=EQUALS 8,
+  outds=work.test_results
+)
+
+/* empty filter (return all records) */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+;;;;
+run;
+%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtergenerate(work.inds,outref=myfilter)
+data work.test;
+  set sashelp.class;
+  where %inc myfilter;;
+run;
+%mp_assertdsobs(work.test,
+  desc=Empty filter (return all records) ,
+  test=EQUALS 19,
+  outds=work.test_results
+)
+
+/* single line filter */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,OR,2,Name,IN,"('Jane','Janet')"
+;;;;
+run;
+%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtergenerate(work.inds,outref=myfilter)
+data work.test;
+  set sashelp.class;
+  where %inc myfilter;;
+run;
+%mp_assertdsobs(work.test,
+  desc=Single line filter ,
+  test=EQUALS 2,
+  outds=work.test_results
+)
+
+/* single line 2 group filter */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+OR,OR,2,Name,IN,"('Jane','Janet')"
+OR,OR,3,Name,IN,"('James')"
+;;;;
+run;
+%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtergenerate(work.inds,outref=myfilter)
+data work.test;
+  set sashelp.class;
+  where %inc myfilter;;
+run;
+%mp_assertdsobs(work.test,
+  desc=Single line 2 group filter ,
+  test=EQUALS 3,
+  outds=work.test_results
+)
+
+/* filter with nothing returned */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,OR,2,Name,IN,"('Jane','Janet')"
+AND,OR,3,Name,IN,"('James')"
+;;;;
+run;
+%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtergenerate(work.inds,outref=myfilter)
+data work.test;
+  set sashelp.class;
+  where %inc myfilter;;
+run;
+%mp_assertdsobs(work.test,
+  desc=Filter with nothing returned,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+

tests/base/mp_filtervalidate.test.sas

@@ -0,0 +1,68 @@
+/**
+  @file
+  @brief Testing mp_filtervalidate macro
+
+  <h4> SAS Macros </h4>
+  @li mp_filtergenerate.sas
+  @li mp_filtervalidate.sas
+  @li mp_assertdsobs.sas
+
+**/
+
+
+/* valid filter */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,AGE,>,5
+AND,AND,1,SEX,NE,"'M'"
+AND,OR,2,Name,NOT IN,"('Jane','Janet')"
+AND,OR,2,Weight,>=,84.6
+;;;;
+run;
+%mp_filtergenerate(work.inds,outref=myfilter)
+%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%mp_assertdsobs(work.results,
+  desc=Valid filter,
+  test=EMPTY,
+  outds=work.test_results
+)
+
+/* empty filter (return all records) */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+;;;;
+run;
+%mp_filtergenerate(work.inds,outref=myfilter)
+%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%mp_assertdsobs(work.results,
+  desc=Valid filter,
+  test=EMPTY,
+  outds=work.test_results
+)
+
+
+
+/* invalid filter*/
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,SEX,NE,2
+;;;;
+run;
+%mp_filtergenerate(work.inds,outref=myfilter)
+%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%let syscc=0;
+%mp_assertdsobs(work.results,
+  desc=Valid filter,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+

tests/base/mp_jsonout.test.sas

@@ -0,0 +1,42 @@
+/**
+  @file
+  @brief Testing mp_jsonout.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_jsonout.sas
+  @li mp_assert.sas
+
+**/
+
+filename webref temp;
+
+data demo;
+  dtval=date();
+  format dtval date9.;
+  compare=put(date(),date9.);
+  call symputx('compare',compare);
+run;
+
+%mp_jsonout(OPEN,jref=webref)
+%mp_jsonout(OBJ,demo,jref=webref,fmt=Y)
+%mp_jsonout(CLOSE,jref=webref)
+
+data _null_;
+  infile webref;
+  input;
+  putlog _infile_;
+run;
+
+libname web JSON fileref=webref;
+%let dtval=0;
+data work.test;
+  set web.demo;
+  call symputx('dtval',dtval);
+run;
+
+
+%mp_assert(
+  iftrue=(&dtval=&compare),
+  desc=Checking tables were created successfully,
+  outds=work.test_results
+)

tests/base/mp_validatecol.test.sas

@@ -0,0 +1,62 @@
+/**
+  @file
+  @brief Testing mp_validatecol.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assertdsobs.sas
+  @li mp_validatecol.sas
+
+**/
+
+
+/**
+  * Test 1 - LIBDS
+  */
+data test1;
+  infile datalines4 dsd;
+  input;
+  libds=_infile_;
+  %mp_validatecol(libds,LIBDS,is_libds)
+  if is_libds=1;
+datalines4;
+some.libname
+!lib.blah
+%abort
+definite.ok
+not.ok!
+nineletrs._
+;;;;
+run;
+%mp_assertdsobs(work.test1,
+  desc=Testing LIBDS,
+  test=EQUALS 2,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - ISNUM
+  */
+data test2;
+  infile datalines4 dsd;
+  input;
+  infile=_infile_;
+  %mp_validatecol(infile,ISNUM,is_numeric)
+  if is_numeric=1;
+datalines4;
+1
+0001
+1e6
+-44
+above are good
+the rest are bad
+%abort
+1&somethingverybad.
+&
++-1
+;;;;
+run;
+%mp_assertdsobs(work.test2,
+  desc=Test2 - ISNUM,
+  test=EQUALS 4,
+  outds=work.test_results
+)

tests/testinit.sas

@@ -0,0 +1,8 @@
+/**
+  @file
+  @brief init file for tests
+
+**/
+
+/* location in metadata or SAS Drive for temporary files */
+%let mcTestAppLoc=/Public/temp/macrocore;

tests/testterm.sas

@@ -0,0 +1,9 @@
+/**
+  @file
+  @brief term file for tests
+
+**/
+
+%webout(OPEN)
+%webout(OBJ, TEST_RESULTS)
+%webout(CLOSE)

tests/viya/mv_createwebservice.test.sas

@@ -0,0 +1,45 @@
+/**
+  @file
+  @brief Testing mv_createwebservice macro
+
+  <h4> SAS Macros </h4>
+  @li mv_createwebservice.sas
+  @li mv_getjobcode.sas
+
+**/
+
+/**
+  * Test Case 1
+  * Send special char in a service
+  */
+
+filename testref temp;
+data _null_;
+  file testref;
+  put '01'x;
+run;
+%put TEST1: creating web service;
+%mv_createwebservice(
+  path=&mcTestAppLoc/temp/macros,
+  name=mv_createwebservice,
+  code=testref
+)
+%put TEST1: fetching web service code;
+%mv_getjobcode(
+  path=&mcTestAppLoc/temp/macros,
+  name=mv_createwebservice,
+  outref=compare
+)
+%put TEST1: checking web service code;
+data work.test_results;
+  length test_description $256 test_result $4 test_comments $256;
+  infile compare end=eof;
+  input;
+  if eof then do;
+    if _infile_='01'x then test_result='PASS';
+    else test_result='FAIL';
+    test_description="Creating web service with invisible character";
+    output;
+    stop;
+  end;
+run;

tests/viya/mv_getjobcode.test.sas

@@ -0,0 +1,49 @@
+/**
+  @file
+  @brief Testing mv_getjobcode macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mv_createjob.sas
+  @li mv_getjobcode.sas
+
+**/
+
+/**
+  * Test Case 1
+  */
+
+/* write some code to a job */
+%let incode=%str(data test; set sashelp.class;run;);
+filename testref temp;
+data _null_;
+  file testref;
+  put "&incode";
+run;
+%mv_createjob(
+  code=testref,
+  path=&mcTestAppLoc/services/temp,
+  name=some_job
+)
+
+/* now get the code back */
+%mv_getjobcode(
+  path=&mcTestAppLoc/services/temp,
+  name=some_job,
+  outref=mycode
+)
+
+%let diditexist=NO;
+data work.test1;
+  infile mycode;
+  input;
+  putlog _infile_;
+  line=_infile_;
+  check=symget('incode');
+  if _infile_=symget('incode') then call symputx('diditexist','YES');
+run;
+
+%mp_assert(
+  iftrue=(&diditexist=NO),
+  desc=Check if the code that was sent was successfully retrieved
+)

tests/viya/mv_getjobresult.test.sas

@@ -0,0 +1,74 @@
+/**
+  @file
+  @brief Testing mv_createwebservice macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assertdsobs.sas
+  @li mv_createwebservice.sas
+  @li mv_getjobresult.sas
+  @li mv_jobflow.sas
+
+**/
+
+/**
+  * Test Case 1
+  */
+
+/* create a service */
+filename testref temp;
+data _null_;
+  file testref;
+  put 'data test; set sashelp.class;run;';
+  put '%webout(OPEN)';
+  put '%webout(OBJ,test)';
+  put '%webout(CLOSE)';
+run;
+%mv_createwebservice(
+  path=&mcTestAppLoc/services/temp,
+  code=testref,
+  name=testsvc
+)
+
+/* trigger and wait for it to finish */
+data work.inputjobs;
+  _program="&mcTestAppLoc/services/temp/testsvc";
+run;
+%mv_jobflow(inds=work.inputjobs
+  ,maxconcurrency=4
+  ,outds=work.results
+  ,outref=myjoblog
+)
+/* stream the log */
+data _null_;
+  infile myjoblog;
+  input;
+  put _infile_;
+run;
+
+/* fetch the uri */
+data _null_;
+  set work.results;
+  call symputx('uri',uri);
+  put (_all_)(=);
+run;
+
+/* now get the results */
+%mv_getjobresult(uri=&uri
+  ,result=WEBOUT_JSON
+  ,outref=myweb
+  ,outlib=myweblib
+)
+data _null_;
+  infile myweb;
+  input;
+  putlog _infile_;
+run;
+data work.out;
+  set myweblib.test;
+  put (_all_)(=);
+run;
+%mp_assertdsobs(work.out,
+  desc=Test1 - 19 obs from sashelp.class in service result,
+  test=EQUALS 19,
+  outds=work.test_results
+)

viya/mv_createfolder.sas

@@ -9,8 +9,8 @@
 
   @param path= The full path of the folder to be created
   @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values are "password" or "authorization_code" (unquoted).
-    The default is authorization_code.
+  @param grant_type= (authorization_code) Valid values are "password" or
+    "authorization_code" (unquoted).
 
 
   @version VIYA V.03.04
@@ -39,7 +39,6 @@
     %let &access_token_var=;
 %end;
 
-%put &sysmacroname: grant_type=&grant_type;
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
@@ -85,12 +84,15 @@ options noquotelenmax;
   %local libref1;
   %let libref1=%mf_getuniquelibref();
   libname &libref1 JSON fileref=&fname1;
-  %mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 404)
+  %mp_abort(
+    iftrue=(
+      &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 404
+    )
     ,mac=&sysmacroname
     ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
   )
   %if &SYS_PROCHTTP_STATUS_CODE=200 %then %do;
-    %put &sysmacroname &newpath exists so grab the follow on link ;
+    %*put &sysmacroname &newpath exists so grab the follow on link ;
     data _null_;
       set &libref1..links;
       if rel='createChild' then

viya/mv_createjob.sas

@@ -72,7 +72,6 @@ https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5p
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
 
 /* initial validation checking */
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password

viya/mv_createwebservice.sas

@@ -39,22 +39,26 @@
   @li mf_isblank.sas
   @li mv_deletejes.sas
 
-  @param path= The full path (on SAS Drive) where the service will be created
-  @param name= The name of the service
-  @param desc= The description of the service
-  @param precode= Space separated list of filerefs, pointing to the code that
-    needs to be attached to the beginning of the service
-  @param code= Fileref(s) of the actual code to be added
-  @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values are "password" or "authorization_code" (unquoted).
-    The default is authorization_code.
-  @param replace= select NO to avoid replacing any existing service in that location
-  @param adapter= the macro uses the sasjs adapter by default.  To use another
-    adapter, add a (different) fileref here.
-  @param contextname= Choose a specific context on which to run the Job.  Leave
+  @param [in] path= The full path (on SAS Drive) where the service will be
+    created
+  @param [in] name= The name of the service
+  @param [in] desc= The description of the service
+  @param [in] precode= Space separated list of filerefs, pointing to the code
+    that needs to be attached to the beginning of the service
+  @param [in] code= Fileref(s) of the actual code to be added
+  @param [in] access_token_var= The global macro variable to contain the access
+    token
+  @param [in] grant_type= valid values are "password" or "authorization_code"
+    (unquoted). The default is authorization_code.
+  @param [in] replace=(YES) Select NO to avoid replacing any existing service in
+    that location
+  @param [in] adapter= the macro uses the sasjs adapter by default.  To use
+    another adapter, add a (different) fileref here.
+  @param [in] contextname= Choose a specific context on which to run the Job.  Leave
     blank to use the default context.  From Viya 3.5 it is possible to configure
     a shared context - see
 https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5pyhn1wg3ja0drdl6h.htm&docsetVersion=3.5&locale=en
+  @param [in] mdebug=(0) set to 1 to enable DEBUG messages
 
   @version VIYA V.03.04
   @author Allan Bowe, source: https://github.com/sasjs/core
@@ -70,9 +74,17 @@ https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5p
     ,grant_type=sas_services
     ,replace=YES
     ,adapter=sasjs
-    ,debug=0
+    ,mdebug=0
     ,contextname=
+    ,debug=0 /* @TODO - Deprecate */
   );
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -82,7 +94,6 @@ https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5p
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
 
 /* initial validation checking */
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
@@ -126,7 +137,7 @@ proc http method='GET' out=&fname1 &oauth_bearer
   headers "Authorization"="Bearer &&&access_token_var";
 %end;
 run;
-%if &debug %then %do;
+%if &mdebug=1 %then %do;
   data _null_;
     infile &fname1;
     input;
@@ -145,7 +156,8 @@ libname &libref1 JSON fileref=&fname1;
 
 data _null_;
   set &libref1..links;
-  if rel='members' then call symputx('membercheck',quote("&base_uri"!!trim(href)),'l');
+  if rel='members' then
+    call symputx('membercheck',quote("&base_uri"!!trim(href)),'l');
   else if rel='self' then call symputx('parentFolderUri',href,'l');
 run;
 data _null_;
@@ -164,7 +176,7 @@ proc http method='GET'
   %end;
             'Accept'='application/vnd.sas.collection+json'
             'Accept-Language'='string';
-%if &debug=1 %then %do;
+%if &mdebug=1 %then %do;
   debug level = 3;
 %end;
 run;
@@ -219,16 +231,17 @@ run;
   * These put statements are auto generated - to change the macro, change the
   * source (mv_webout) and run `build.py`
   */
-filename sasjs temp lrecl=3000;
+filename &adapter temp lrecl=3000;
 data _null_;
-  file sasjs;
+  file &adapter;
   put "/* Created on %sysfunc(datetime(),datetime19.) by &sysuserid */";
 /* WEBOUT BEGIN */
   put ' ';
-  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=PROCJSON,dbg=0 ';
+  put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y,engine=DATASTEP,dbg=0 ';
   put ')/*/STORE SOURCE*/; ';
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
+  put '  OPTIONS NOBOMFILE; ';
   put '  data _null_;file &jref encoding=''utf-8''; ';
   put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
   put '  run; ';
@@ -256,6 +269,64 @@ data _null_;
   put '      %put &sysmacroname:  &ds NOT FOUND!!!; ';
   put '      %return; ';
   put '    %end; ';
+  put '    %if &fmt=Y %then %do; ';
+  put '      %put converting every variable to a formatted variable; ';
+  put '      /* see mp_ds2fmtds.sas for source */ ';
+  put '      proc contents noprint data=&ds ';
+  put '        out=_data_(keep=name type length format formatl formatd varnum); ';
+  put '      run; ';
+  put '      proc sort; ';
+  put '        by varnum; ';
+  put '      run; ';
+  put '      %local fmtds; ';
+  put '      %let fmtds=%scan(&syslast,2,.); ';
+  put '      /* prepare formats and varnames */ ';
+  put '      data _null_; ';
+  put '        set &fmtds end=last; ';
+  put '        name=upcase(name); ';
+  put '        /* fix formats */ ';
+  put '        if type=2 or type=6 then do; ';
+  put '          length fmt $49.; ';
+  put '          if format='''' then fmt=cats(''$'',length,''.''); ';
+  put '          else if formatl=0 then fmt=cats(format,''.''); ';
+  put '          else fmt=cats(format,formatl,''.''); ';
+  put '          newlen=max(formatl,length); ';
+  put '        end; ';
+  put '        else do; ';
+  put '          if format='''' then fmt=''best.''; ';
+  put '          else if formatl=0 then fmt=cats(format,''.''); ';
+  put '          else if formatd=0 then fmt=cats(format,formatl,''.''); ';
+  put '          else fmt=cats(format,formatl,''.'',formatd); ';
+  put '          /* needs to be wide, for datetimes etc */ ';
+  put '          newlen=max(length,formatl,24); ';
+  put '        end; ';
+  put '        /* 32 char unique name */ ';
+  put '        newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
+  put ' ';
+  put '        call symputx(cats(''name'',_n_),name,''l''); ';
+  put '        call symputx(cats(''newname'',_n_),newname,''l''); ';
+  put '        call symputx(cats(''len'',_n_),newlen,''l''); ';
+  put '        call symputx(cats(''fmt'',_n_),fmt,''l''); ';
+  put '        call symputx(cats(''type'',_n_),type,''l''); ';
+  put '        if last then call symputx(''nobs'',_n_,''l''); ';
+  put '      run; ';
+  put '      data &fmtds; ';
+  put '        /* rename on entry */ ';
+  put '        set &ds(rename=( ';
+  put '      %local i; ';
+  put '      %do i=1 %to &nobs; ';
+  put '        &&name&i=&&newname&i ';
+  put '      %end; ';
+  put '        )); ';
+  put '      %do i=1 %to &nobs; ';
+  put '        length &&name&i $&&len&i; ';
+  put '        &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '        drop &&newname&i; ';
+  put '      %end; ';
+  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '      run; ';
+  put '      %let ds=&fmtds; ';
+  put '    %end; /* &fmt=Y */ ';
   put '    data _null_;file &jref mod ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
@@ -339,7 +410,7 @@ data _null_;
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
-  put '%mend; ';
+  put '%mend mp_jsonout; ';
   put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y); ';
   put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
   put '  sasjs_tables SYS_JES_JOB_URI; ';
@@ -560,11 +631,12 @@ data _null_;
 run;
 
 /* insert the code, escaping double quotes and carriage returns */
+%&dbg.put &sysmacroname: Creating final input file;
 %local x fref freflist;
 %let freflist= &adapter &precode &code ;
 %do x=1 %to %sysfunc(countw(&freflist));
   %let fref=%scan(&freflist,&x);
-  %put &sysmacroname: adding &fref;
+  %&dbg.put &sysmacroname: adding &fref fileref;
   data _null_;
     length filein 8 fileid 8;
     filein = fopen("&fref","I",1,"B");
@@ -592,6 +664,14 @@ run;
         rc =fput(fileid,'\');rc =fwrite(fileid);
         rc =fput(fileid,'\');rc =fwrite(fileid);
       end;
+      else if rec='01'x then do; /* Unprintable */
+        rc =fput(fileid,'\');rc =fwrite(fileid);
+        rc =fput(fileid,'u');rc =fwrite(fileid);
+        rc =fput(fileid,'0');rc =fwrite(fileid);
+        rc =fput(fileid,'0');rc =fwrite(fileid);
+        rc =fput(fileid,'0');rc =fwrite(fileid);
+        rc =fput(fileid,'1');rc =fwrite(fileid);
+      end;
       else do;
         rc =fput(fileid,rec);
         rc =fwrite(fileid);
@@ -608,7 +688,12 @@ data _null_;
   put '"}';
 run;
 
-/* now we can create the job!! */
+%if &mdebug=1 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then %do;
+  %put &sysmacroname: input about to be POSTed;
+  data _null_;infile &fname3;input;putlog _infile_;run;
+%end;
+
+%&dbg.put &sysmacroname: Creating the actual service!;
 %local fname4;
 %let fname4=%mf_getuniquefileref();
 proc http method='POST'
@@ -621,22 +706,18 @@ proc http method='POST'
             "Authorization"="Bearer &&&access_token_var"
   %end;
             "Accept"="application/vnd.sas.job.definition+json";
-%if &debug=1 %then %do;
+%if &mdebug=1 %then %do;
     debug level = 3;
 %end;
 run;
-/*data _null_;infile &fname4;input;putlog _infile_;run;*/
+%if &mdebug=1 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then %do;
+  %put &sysmacroname: output from POSTing job definition;
+  data _null_;infile &fname4;input;putlog _infile_;run;
+%end;
 %mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 201)
   ,mac=&sysmacroname
   ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
 )
-/* clear refs */
-filename &fname1 clear;
-filename &fname2 clear;
-filename &fname3 clear;
-filename &fname4 clear;
-filename &adapter clear;
-libname &libref1 clear;
 
 /* get the url so we can give a helpful log message */
 %local url;
@@ -651,6 +732,19 @@ data _null_;
   call symputx('url',url);
 run;
 
+%if &mdebug=1 %then %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%else %do;
+  /* clear refs */
+  filename &fname1 clear;
+  filename &fname2 clear;
+  filename &fname3 clear;
+  filename &fname4 clear;
+  filename &adapter clear;
+  libname &libref1 clear;
+%end;
 
 %put &sysmacroname: Job &name successfully created in &path;
 %put &sysmacroname:;
@@ -660,4 +754,4 @@ run;
 %put &sysmacroname:;
 %put &sysmacroname:;
 
-%mend;
+%mend mv_createwebservice;

viya/mv_deletejes.sas

@@ -44,7 +44,7 @@
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )

viya/mv_deleteviyafolder.sas

@@ -39,7 +39,7 @@
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )

viya/mv_getgroupmembers.sas

@@ -54,7 +54,7 @@
     %let oauth_bearer=oauth_bearer=sas_services;
     %let &access_token_var=;
 %end;
-%put &sysmacroname: grant_type=&grant_type;
+
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )

viya/mv_getjobcode.sas

@@ -10,17 +10,20 @@
         ,outfile=/tmp/some_job.sas
       )
 
-  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] access_token_var= The global macro variable to contain the access
+    token
   @param [in] grant_type= valid values:
-      * password
-      * authorization_code
-      * detect - will check if access_token exists, if not will use sas_services if
-        a SASStudioV session else authorization_code.  Default option.
-      * sas_services - will use oauth_bearer=sas_services
+    @li password
+    @liauthorization_code
+    @li detect - will check if access_token exists, if not will use sas_services
+      if a SASStudioV session else authorization_code.  Default option.
+    @li  sas_services - will use oauth_bearer=sas_services
   @param [in] path= The SAS Drive path of the job
   @param [in] name= The name of the job
-  @param [out] outref= A fileref to which to write the source code
-  @param [out] outfile= A file to which to write the source code
+  @param [in] mdebug=(0) set to 1 to enable DEBUG messages
+  @param [out] outref=(0) A fileref to which to write the source code (will be
+    created with a TEMP engine)
+  @param [out] outfile=(0) A file to which to write the source code
 
   @version VIYA V.03.04
   @author Allan Bowe, source: https://github.com/sasjs/core
@@ -39,7 +42,15 @@
     ,contextName=SAS Job Execution compute context
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
+    ,mdebug=0
   );
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -133,18 +144,30 @@ data _null_;
 run;
 %inc "&fpath3..lua";
 /* export to desired destination */
-data _null_;
-  %if &outref=0 %then %do;
+%if "&outref"="0" %then %do;
+  data _null_;
     file "&outfile" lrecl=32767;
-  %end;
-  %else %do;
+%end;
+%else %do;
+  filename &outref temp;
+  data _null_;
     file &outref;
-  %end;
+%end;
   infile &fname2;
   input;
   put _infile_;
+  &dbg. putlog _infile_;
 run;
-filename &fname1 clear;
-filename &fname2 clear;
-filename &fname3 clear;
-%mend;
+
+%if &mdebug=1 %then %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%else %do;
+  /* clear refs */
+  filename &fname1 clear;
+  filename &fname2 clear;
+  filename &fname3 clear;
+%end;
+
+%mend mv_getjobcode;

viya/mv_getjoblog.sas

@@ -54,13 +54,14 @@
   convenient way to wait for the job to finish before fetching the log.
 
 
-  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] access_token_var= The global macro variable to contain the access
+    token
   @param [in] mdebug= set to 1 to enable DEBUG messages
   @param [in] grant_type= valid values:
     @li password
     @li authorization_code
-    @li detect - will check if access_token exists, if not will use sas_services if
-        a SASStudioV session else authorization_code.  Default option.
+    @li detect - will check if access_token exists, if not will use sas_services
+      if a SASStudioV session else authorization_code.  Default option.
     @li sas_services - will use oauth_bearer=sas_services.
   @param [in] uri= The uri of the running job for which to fetch the status,
     in the format `/jobExecution/jobs/$UUID/state` (unquoted).
@@ -85,6 +86,13 @@
     ,grant_type=sas_services
     ,mdebug=0
   );
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -260,9 +268,10 @@ run;
   filename &fname3 clear;
 %end;
 %else %do;
+  %put &sysmacroname exit vars:;
   %put _local_;
 %end;
-%mend;
+%mend mv_getjoblog;
 
 
 

viya/mv_getjobresult.sas

@@ -0,0 +1,232 @@
+/**
+  @file
+  @brief Extract the result from a completed SAS Viya Job
+  @details Extracts result from a Viya job and writes it out to a fileref
+  and/or a JSON-engine library.
+
+  To query the job, you need the URI.  Sample code for achieving this
+  is provided below.
+
+  ## Example
+
+  First, compile the macros:
+
+      filename mc url
+        "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+  Next, create a job (in this case, a web service):
+
+      filename ft15f001 temp;
+      parmcards4;
+        data test;
+          rand=ranuni(0)*1000;
+          do x=1 to rand;
+            y=rand*4;
+            output;
+          end;
+        run;
+        proc sort data=&syslast
+          by descending y;
+        run;
+        %webout(OPEN)
+        %webout(OBJ, test)
+        %webout(CLOSE)
+      ;;;;
+      %mv_createwebservice(path=/Public/temp,name=demo)
+
+  Execute it:
+
+      %mv_jobexecute(path=/Public/temp
+        ,name=demo
+        ,outds=work.info
+      )
+
+  Wait for it to finish, and grab the uri:
+
+      data _null_;
+        set work.info;
+        if method='GET' and rel='self';
+        call symputx('uri',uri);
+      run;
+
+  Finally, fetch the result (In this case, WEBOUT):
+
+      %mv_getjobresult(uri=&uri,result=WEBOUT_JSON,outref=myweb,outlib=myweblib)
+
+
+  @param [in] access_token_var= The global macro variable containing the access
+    token
+  @param [in] mdebug= set to 1 to enable DEBUG messages
+  @param [in] grant_type= valid values:
+    @li password
+    @li authorization_code
+    @li detect - will check if access_token exists, if not will use sas_services
+        if a SASStudioV session else authorization_code.  Default option.
+    @li sas_services - will use oauth_bearer=sas_services.
+  @param [in] uri= The uri of the running job for which to fetch the status,
+    in the format `/jobExecution/jobs/$UUID` (unquoted).
+
+  @param [out] result= (WEBOUT_JSON) The result type to capture.  Resolves
+  to "_[column name]" from the results table when parsed with the JSON libname
+  engine.  Example values:
+    @li WEBOUT_JSON
+    @li WEBOUT_TXT
+
+  @param [out] outref= (0) The output fileref to which to write the results
+  @param [out] outlib= (0) The output library to which to assign the results
+    (assumes the data is in JSON format)
+
+
+  @version VIYA V.03.05
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+  @li mf_getplatform.sas
+  @li mf_existfileref.sas
+
+**/
+
+%macro mv_getjobresult(uri=0
+    ,contextName=SAS Job Execution compute context
+    ,access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+    ,mdebug=0
+    ,result=WEBOUT_JSON
+    ,outref=0
+    ,outlib=0
+  );
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+
+
+/* validation in datastep for better character safety */
+%local errmsg errflg;
+data _null_;
+  uri=symget('uri');
+  if length(uri)<12 then do;
+    call symputx('errflg',1);
+    call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
+  end;
+  if scan(uri,-1)='state' or scan(uri,1) ne 'jobExecution' then do;
+    call symputx('errflg',1);
+    call symputx('errmsg',
+      "URI should be in format /jobExecution/jobs/$$$$UUID$$$$"
+      !!" but is actually like: &uri",'l');
+  end;
+run;
+
+%mp_abort(iftrue=(&errflg=1)
+  ,mac=&sysmacroname
+  ,msg=%str(&errmsg)
+)
+
+%if &outref ne 0 and %mf_existfileref(&outref) ne 1 %then %do;
+  filename &outref temp;
+%end;
+
+options noquotelenmax;
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+/* fetch job info */
+%local fname1;
+%let fname1=%mf_getuniquefileref();
+proc http method='GET' out=&fname1 &oauth_bearer
+  url="&base_uri&uri";
+  headers "Accept"="application/json"
+  %if &grant_type=authorization_code %then %do;
+          "Authorization"="Bearer &&&access_token_var"
+  %end;
+  ;
+run;
+%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+%do;
+  data _null_;infile &fname1;input;putlog _infile_;run;
+  %mp_abort(mac=&sysmacroname
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+%end;
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fname1 lrecl=32767;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+/* extract results link */
+%local lib1 resuri;
+%let lib1=%mf_getuniquelibref();
+libname &lib1 JSON fileref=&fname1;
+data _null_;
+  set &lib1..results;
+  call symputx('resuri',_&result,'l');
+  &dbg putlog "&sysmacroname results: " (_all_)(=);
+run;
+%mp_abort(iftrue=("&resuri"=".")
+  ,mac=&sysmacroname
+  ,msg=%str(Variable _&result did not exist in the response json)
+)
+
+/* extract results */
+%local fname2;
+%let fname2=%mf_getuniquefileref();
+proc http method='GET' out=&fname2 &oauth_bearer
+  url="&base_uri&resuri/content?limit=10000";
+  headers "Accept"="application/json"
+  %if &grant_type=authorization_code %then %do;
+          "Authorization"="Bearer &&&access_token_var"
+  %end;
+  ;
+run;
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fname2 lrecl=32767;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%if &outref ne 0 %then %do;
+  filename &outref temp;
+  %mp_binarycopy(inref=&fname2,outref=&outref)
+%end;
+%if &outlib ne 0 %then %do;
+  libname &outlib JSON fileref=&fname2;
+%end;
+
+%if &mdebug=0 %then %do;
+  filename &fname1 clear;
+  filename &fname2 clear;
+  libname &lib1 clear;
+%end;
+%else %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+
+%mend mv_getjobresult;

viya/mv_jobexecute.sas

@@ -23,24 +23,25 @@
         ,paramstring=%str("macvarname":"macvarvalue","answer":42)
       )
 
-  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] access_token_var= The global macro variable to contain the access
+    token
   @param [in] grant_type= valid values:
-
-    * password
-    * authorization_code
-    * detect - will check if access_token exists, if not will use sas_services if
-      a SASStudioV session else authorization_code.  Default option.
-    * sas_services - will use oauth_bearer=sas_services
+    @li password
+    @li authorization_code
+    @li detect - will check if access_token exists, if not will use sas_services
+      if a SASStudioV session else authorization_code.  Default option.
+    @li sas_services - will use oauth_bearer=sas_services
 
   @param [in] path= The SAS Drive path to the job being executed
   @param [in] name= The name of the job to execute
-  @param [in] paramstring= A JSON fragment with name:value pairs, eg: `"name":"value"`
-  or "name":"value","name2":42`.  This will need to be wrapped in `%str()`.
+  @param [in] paramstring= A JSON fragment with name:value pairs, eg:
+    `"name":"value"` or "name":"value","name2":42`.  This will need to be
+    wrapped in `%str()`.
 
   @param [in] contextName= Context name with which to run the job.
     Default = `SAS Job Execution compute context`
-
-  @param [out] outds= The output dataset containing links (Default=work.mv_jobexecute)
+  @param [in] mdebug= set to 1 to enable DEBUG messages
+  @param [out] outds= (work.mv_jobexecute) The output dataset containing links
 
 
   @version VIYA V.03.04
@@ -62,7 +63,15 @@
     ,grant_type=sas_services
     ,paramstring=0
     ,outds=work.mv_jobexecute
+    ,mdebug=0
   );
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -164,9 +173,14 @@ data &outds;
   _program="&path/&name";
 run;
 
-/* clear refs */
-filename &fname0 clear;
-filename &fname1 clear;
-libname &libref;
-
-%mend;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%else %do;
+  /* clear refs */
+  filename &fname0 clear;
+  filename &fname1 clear;
+  libname &libref;
+%end;
+%mend mv_jobexecute;

viya/mv_jobflow.sas

@@ -136,6 +136,13 @@
     ,raise_err=0
     ,mdebug=0
   );
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -293,6 +300,7 @@ data;run;%let jdswaitfor=&syslast;
           ,name=&jobname
           ,paramstring=%superq(jparams&jid)
           ,outds=&jdsapp
+          ,mdebug=&mdebug
         )
         data &jdsapp;
           format jobparams $32767.;
@@ -313,8 +321,13 @@ data;run;%let jdswaitfor=&syslast;
     %end;
     %if &jid=&jcnt %then %do;
       /* we are at the end of the loop - time to see which jobs have finished */
-      %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref
-                    ,raise_err=&raise_err)
+      %mv_jobwaitfor(ANY
+        ,inds=&jdsrunning
+        ,outds=&jdswaitfor
+        ,outref=&outref
+        ,raise_err=&raise_err
+        ,mdebug=&mdebug
+      )
       %local done;
       %let done=%mf_nobs(&jdswaitfor);
       %if &done>0 %then %do;
@@ -346,7 +359,8 @@ data;run;%let jdswaitfor=&syslast;
 %end;
 
 %if &mdebug=1 %then %do;
+  %put &sysmacroname exit vars:;
   %put _local_;
 %end;
 
-%mend;
+%mend mv_jobflow;

viya/mv_jobwaitfor.sas

@@ -1,6 +1,6 @@
 /**
   @file
-  @brief Takes a dataset of running jobs and waits for ANY or ALL of them to complete
+  @brief Takes a table of running jobs and waits for ANY/ALL of them to complete
   @details Will poll `/jobs/{jobId}/state` at set intervals until ANY or ALL
   jobs are completed.  Completion is determined by reference to the returned
   _state_, as per the following table:
@@ -55,13 +55,14 @@
 
       %mv_deletejes(path=/Public/temp,name=demo)
 
-  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] access_token_var= The global macro variable to contain the access
+    token
   @param [in] grant_type= valid values:
 
       - password
       - authorization_code
-      - detect - will check if access_token exists, if not will use sas_services if
-        a SASStudioV session else authorization_code.  Default option.
+      - detect - will check if access_token exists, if not will use sas_services
+        if a SASStudioV session else authorization_code.  Default option.
       - sas_services - will use oauth_bearer=sas_services
 
   @param [in] action=Either ALL (to wait for every job) or ANY (if one job
@@ -72,9 +73,11 @@
     should be in a `_program` variable.
   @param [in] raise_err=0 Set to 1 to raise SYSCC when a job does not complete
               succcessfully
+  @param [in] mdebug= set to 1 to enable DEBUG messages
   @param [out] outds= The output dataset containing the list of states by job
     (default=work.mv_jobexecute)
-  @param [out] outref= A fileref to which the spawned job logs should be appended.
+  @param [out] outref= A fileref to which the spawned job logs should be
+    appended.
 
   @version VIYA V.03.04
   @author Allan Bowe, source: https://github.com/sasjs/core
@@ -97,7 +100,15 @@
     ,outds=work.mv_jobwaitfor
     ,outref=0
     ,raise_err=0
+    ,mdebug=0
   );
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -155,7 +166,8 @@ run;
 %let fname0=%mf_getuniquefileref();
 
 data &outds;
-  format _program uri $128. state $32. stateDetails $32. timestamp datetime19. jobparams $32767.;
+  format _program uri $128. state $32. stateDetails $32. timestamp datetime19.
+    jobparams $32767.;
   stop;
 run;
 
@@ -168,8 +180,8 @@ run;
               "Authorization"="Bearer &&&access_token_var"
       %end;  ;
     run;
-    %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
-    %do;
+    %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201
+    %then %do;
       data _null_;infile &fname0;input;putlog _infile_;run;
       %mp_abort(mac=&sysmacroname
         ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
@@ -205,7 +217,7 @@ run;
       %let joburi&i=0; /* do not re-check */
       /* fetch log */
       %if %str(&outref) ne 0 %then %do;
-        %mv_getjoblog(uri=&plainuri,outref=&outref)
+        %mv_getjoblog(uri=&plainuri,outref=&outref,mdebug=&mdebug)
       %end;
     %end;
     %else %if &status=idle or &status=pending or &status=running %then %do;
@@ -220,10 +232,11 @@ run;
     %end;
 
     %if (&raise_err) %then %do;
-      %if (&status = canceled or &status = failed or %length(&stateDetails)>0) %then %do;
+      %if (&status = canceled or &status = failed or %length(&stateDetails)>0)
+      %then %do;
         %if ("&stateDetails" = "%str(war)ning") %then %let SYSCC=4;
         %else %let SYSCC=5;
-        %put %str(ERR)OR: Job &&jobname&i. did not complete successfully. &stateDetails;
+        %put %str(ERR)OR: Job &&jobname&i. did not complete. &stateDetails;
         %return;
       %end;
     %end;
@@ -238,7 +251,12 @@ run;
   %end;
 %end;
 
-/* clear refs */
-filename &fname0 clear;
-
-%mend;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%else %do;
+  /* clear refs */
+  filename &fname0 clear;
+%end;
+%mend mv_jobwaitfor;