Merge pull request #19 from sasjs/mp_assert

feat: new (generic) mp_assert macro, and new feature (type filter) fo…

all.sas

@@ -1051,6 +1051,10 @@ options noquotelenmax;
   @param [in] dlm= ( ) Provide a delimiter (eg comma or space) to separate the
     variables
   @param [in] quote= (none) use either DOUBLE or SINGLE to quote the results
+  @param [in] typefilter= (A) Filter for certain types of column.  Valid values:
+    @li A Return All columns
+    @li C Return Character columns
+    @li N Return Numeric columns
 
   @version 9.2
   @author Allan Bowe
@@ -1060,9 +1064,10 @@ options noquotelenmax;
 %macro mf_getvarlist(libds
       ,dlm=%str( )
       ,quote=no
+      ,typefilter=A
 )/*/STORE SOURCE*/;
   /* declare local vars */
-  %local outvar dsid nvars x rc dlm q var;
+  %local outvar dsid nvars x rc dlm q var vtype;
 
   /* credit Rowland Hale  - byte34 is double quote, 39 is single quote */
   %if %upcase(&quote)=DOUBLE %then %let q=%qsysfunc(byte(34));
@@ -1070,21 +1075,22 @@ options noquotelenmax;
   /* open dataset in macro */
   %let dsid=%sysfunc(open(&libds));
 
-
   %if &dsid %then %do;
     %let nvars=%sysfunc(attrn(&dsid,NVARS));
     %if &nvars>0 %then %do;
-      /* add first dataset variable to global macro variable */
+      /* add variables with supplied delimeter */
-      %let outvar=&q.%sysfunc(varname(&dsid,1))&q.;
-      /* add remaining variables with supplied delimeter */
       %do x=1 %to &nvars;
-        %let var=&q.%sysfunc(varname(&dsid,&x))&q.;
+        /* get variable type */
-        %if &var=&q&q %then %do;
+        %let vtype=%sysfunc(vartype(&dsid,&x));
-          %put &sysmacroname: Empty column found in &libds!;
+        %if &vtype=&typefilter or &typefilter=A %then %do;
-          %let var=&q. &q.;
+          %let var=&q.%sysfunc(varname(&dsid,&x))&q.;
+          %if &var=&q&q %then %do;
+            %put &sysmacroname: Empty column found in &libds!;
+            %let var=&q. &q.;
+          %end;
+          %if %quote(&outvar)=%quote() %then %let outvar=&var;
+          %else %let outvar=&outvar.&dlm.&var.;
         %end;
-        %if &x=1 %then %let outvar=&var;
-        %else %let outvar=&outvar.&dlm.&var.;
       %end;
     %end;
     %let rc=%sysfunc(close(&dsid));
@@ -1094,7 +1100,7 @@ options noquotelenmax;
     %let rc=%sysfunc(close(&dsid));
   %end;
   &outvar
-%mend;/**
+%mend mf_getvarlist;/**
   @file
   @brief Returns the position of a variable in dataset (varnum attribute).
   @details Uses varnum function to determine position.
@@ -1194,7 +1200,7 @@ Usage:
   %let rc = %sysfunc(close(&dsid));
   /* Return variable type */
   &vtype
-%mend;/**
+%mend mf_getvartype;/**
   @file
   @brief Returns the engine type of a SAS fileref
   @details Queries sashelp.vextfl to get the xengine value.
@@ -1505,7 +1511,7 @@ Usage:
 
   &today._&now._&sysjobid._%sysevalf(%sysfunc(ranuni(0))*999,CEIL)
 
-%mend;/**
+%mend mf_uid;/**
   @file
   @brief Checks if a set of macro variables exist / contain values.
   @details Writes ERROR to log if abortType is SOFT, else will call %mf_abort.
@@ -1782,6 +1788,61 @@ Usage:
 %mend;
 
 /** @endcond *//**
+  @file
+  @brief Generic assertion
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      %mp_assert(iftrue=(1=1),
+        desc=Obviously true
+      )
+
+      %mp_assert(iftrue=(1=0),
+        desc=Will fail
+      )
+
+  @param [in] iftrue= (1=1) A condition where, if true, the test is a PASS.
+  Else, the test is a fail.
+
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Column &inds contained ALL columns|
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assert(iftrue=(1=1),
+  desc=0,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  data ;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_comments="&sysmacroname: Test result of "!!symget('iftrue');
+  %if %eval(%unquote(&iftrue)) %then %do;
+    test_result='PASS';
+  %end;
+  %else %do;
+    test_result='FAIL';
+  %end;
+  run;
+
+  %local ds ;
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+
+%mend mp_assert;/**
   @file
   @brief Asserts the existence (or not) of columns
   @details Useful in the context of writing sasjs tests.  The results of the
@@ -3238,6 +3299,7 @@ run;
   @li mp_abort.sas
   @li mf_getuniquefileref.sas
   @li mf_getvarlist.sas
+  @li mf_getvartype.sas
   @li mf_nobs.sas
   @li mp_filtergenerate.sas
   @li mp_filtervalidate.sas
@@ -3259,13 +3321,29 @@ run;
   ,msg=%str(syscc=&syscc - on macro entry)
 )
 
+/* Validate input column */
+%local vtype;
+%let vtype=%mf_getvartype(&inds,RAW_VALUE);
+%mp_abort(iftrue=(&abort=YES and &vtype ne C),
+  mac=&sysmacroname,
+  msg=%str(%str(ERR)OR: RAW_VALUE must be character)
+)
+%if &vtype ne C %then %do;
+  %put &sysmacroname: RAW_VALUE must be character;
+  %let syscc=42;
+  %return;
+%end;
+
+
 /**
   * Sanitise the values based on valid value lists, then strip out
   * quotes, commas, periods and spaces.
   * Only numeric values should remain
   */
-
+%local reason_cd;
 data &outds;
+  /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
+    OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
   length reason_cd $32;
 
@@ -3341,9 +3419,9 @@ data _null_;
   stop;
 run;
 
-%mp_abort(iftrue=(&abort=YES),
+%mp_abort(iftrue=(&abort=YES and %mf_nobs(&outds)>0),
   mac=&sysmacroname,
-  msg=%str(Filter issues in &inds, first was &reason_cd, details in &outds)
+  msg=%str(Filter issues in &inds, reason: &reason_cd, details in &outds)
 )
 
 %if %mf_nobs(&outds)>0 %then %do;
@@ -3362,7 +3440,7 @@ run;
 /* this macro will also set syscc to 1008 if any issues found */
 %mp_filtervalidate(&fref1,&targetds,outds=&outds,abort=&abort)
 
-%mend;
+%mend mp_filtercheck;
 /**
   @file
   @brief Generates a filter clause from an input table, to a fileref
@@ -4775,7 +4853,7 @@ create table &outds (rename=(
       retain &prevkeyvar;
       set &libds end=&lastvar;
       /* hash should include previous row */
-      if _n_>1 then &keyvar=put(md5(&prevkeyvar
+      &keyvar=put(md5(&prevkeyvar
       /* loop every column, hashing every individual value */
     %do i=1 %to %sysfunc(countw(&varlist));
       %let var=%scan(&varlist,&i,%str( ));
@@ -5991,9 +6069,16 @@ libname &lib clear;
   Note - the _webout fileref should NOT be assigned prior to running this macro.
 
   @param [in] program The _PROGRAM endpoint to test
-  @param [in] inputfiles= A list of space seperated fileref:filename pairs as
+  @param [in] inputfiles=(0) A list of space seperated fileref:filename pairs as
-  follows:
+    follows:
-      inputfiles=inref:filename inref2:filename2
+        inputfiles=inref:filename inref2:filename2
+  @param [in] inputparams=(0) A dataset containing name/value pairs in the
+    following format:
+    |name:$32|value:$1000|
+    |---|---|
+    |stpmacname|some value|
+    |mustbevalidname|can be anything, oops, %abort!!|
+
   @param [in] debug= (log) Provide the _debug value
   @param [out] outlib= (0) Output libref to contain the final tables.  Set to
     0 if the service output is not in JSON format.
@@ -6001,8 +6086,13 @@ libname &lib clear;
     response.
 
   <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
   @li mf_getuniquefileref.sas
   @li mf_getuniquename.sas
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+  @li mv_getjobresult.sas
+  @li mv_jobflow.sas
 
   @version 9.4
   @author Allan Bowe
@@ -6010,30 +6100,62 @@ libname &lib clear;
 **/
 
 %macro mp_testservice(program,
-  inputfiles=,
+  inputfiles=0,
+  inputparams=0,
   debug=log,
   outlib=0,
   outref=0
 )/*/STORE SOURCE*/;
 
-/* parse the input files */
+/* sanitise inputparams */
-%local webcount i var;
+%local pcnt;
-%let webcount=%sysfunc(countw(&inputfiles));
+%let pcnt=0;
-%do i=1 %to &webcount;
+%if &inputparams ne 0 %then %do;
-  %let var=%scan(&inputfiles,&i,%str( ));
+  data _null_;
-  %local webfref&i webname&i;
+    set &inputparams;
-  %let webref&i=%scan(&var,1,%str(:));
+    if not nvalid(name,'v7') then putlog (_all_)(=);
-  %let webname&i=%scan(&var,2,%str(:));
+    else if name in (
+      'program','inputfiles','inputparams','debug','outlib','outref'
+    ) then putlog (_all_)(=);
+    else do;
+      x+1;
+      call symputx(name,quote(cats(value)),'l');
+      call symputx('pval'!!left(x),name,'l');
+      call symputx('pcnt',x,'l');
+    end;
+  run;
+  %mp_abort(iftrue= (%mf_nobs(&inputparams) ne &pcnt)
+    ,mac=&sysmacroname
+    ,msg=%str(Invalid values in &inputparams)
+  )
 %end;
 
-%local fref1 ;
+/* parse the input files */
+%local webcount i var;
+%if %quote(&inputfiles) ne 0 %then %do;
+  %let webcount=%sysfunc(countw(&inputfiles));
+  %put &=webcount;
+  %do i=1 %to &webcount;
+    %let var=%scan(&inputfiles,&i,%str( ));
+    %local webfref&i webname&i;
+    %let webref&i=%scan(&var,1,%str(:));
+    %let webname&i=%scan(&var,2,%str(:));
+    %put webref&i=&&webref&i;
+    %put webname&i=&&webname&i;
+  %end;
+%end;
+%else %let webcount=0;
+
+
+%local fref1 webref;
 %let fref1=%mf_getuniquefileref();
-filename _webout "%sysfunc(pathname(work))/%mf_getuniquename().txt";
+%let webref=%mf_getuniquefileref();
 
 %local platform;
 %let platform=%mf_getplatform();
 %if &platform=SASMETA %then %do;
-  proc stp program="&program"
+  proc stp program="&program";
+    inputparam _program="&program"
   %do i=1 %to &webcount;
     %if &webcount=1 %then %do;
       _webin_fileref="&&webref&i"
@@ -6045,12 +6167,20 @@ filename _webout "%sysfunc(pathname(work))/%mf_getuniquename().txt";
     %end;
   %end;
     _webin_file_count="&webcount"
-    _debug="&debug";
+    _debug="&debug"
-    outputfile=_webout;
+  %do i=1 %to &pcnt;
+    /* resolve name only, proc stp fetches value */
+    &&pval&i=&&&&&&pval&i
+  %end;
+    ;
+  %do i=1 %to &webcount;
+    inputfile &&webref&i;
+  %end;
+    outputfile _webout=&webref;
   run;
 
   data _null_;
-    infile _webout;
+    infile &webref;
     file &fref1;
     input;
     length line $10000;
@@ -6075,19 +6205,44 @@ filename _webout "%sysfunc(pathname(work))/%mf_getuniquename().txt";
   %end;
   %if &outref ne 0 %then %do;
     filename &outref temp;
-    %mp_binarycopy(inref=_webout,outref=&outref)
+    %mp_binarycopy(inref=&webref,outref=&outref)
   %end;
-  filename _webout clear;
 
 %end;
 %else %if &platform=SASVIYA %then %do;
+  data ;
+    _program="&program";
+  run;
+
+  %mv_jobflow(inds=&syslast
+    ,maxconcurrency=1
+    ,outds=work.results
+    ,outref=&fref1
+  )
+  /* show the log */
+  data _null_;
+    infile &fref1;
+    input;
+    putlog _infile_;
+  run;
+  /* get the uri to fetch results */
+  data _null_;
+    set work.results;
+    call symputx('uri',uri);
+  run;
+  /* fetch results from webout.json */
+  %mv_getjobresult(uri=&uri,
+    result=WEBOUT_JSON,
+    outref=&outref,
+    outlib=&outlib
+  )
 
 %end;
 %else %do;
   %put %str(ERR)OR: Unrecognised platform:  &platform;
 %end;
 
-filename _webout clear;
+filename &webref clear;
 
 %mend;/**
   @file mp_testwritespeedlibrary.sas
@@ -6399,6 +6554,7 @@ alter table &libds modify &var char(&len);
 
   @param [in] incol The column to be validated
   @param [in] rule The rule to apply.  Current rules:
+    @li ISNUM - checks if the variable is numeric
     @li LIBDS - matches LIBREF.DATASET format
   @param [out] outcol The variable to create, with the results of the match
 
@@ -6439,7 +6595,7 @@ alter table &libds modify &var char(&len);
   else &outcol=0;
 %end;
 
-%mend;
+%mend mp_validatecol;
 /**
   @file
   @brief Creates a zip file
@@ -10481,21 +10637,27 @@ filename __mc2 clear;
 
 %mend;/**
   @file
-  @brief Writes the code of an to an external file, or the log if none provided
+  @brief Writes the code of an STP to an external file
-  @details Get the
+  @details Fetches the SAS code from a Stored Process where the code is stored
+  in metadata.
 
-  usage:
+  Usage:
 
       %mm_getstpcode(tree=/some/meta/path
         ,name=someSTP
         ,outloc=/some/unquoted/filename.ext
       )
 
-  @param tree= The metadata path of the Stored Process (can also contain name)
+  @param [in] tree= The metadata path of the Stored Process (can also contain
-  @param name= Stored Process name.  Leave blank if included above.
+    name)
-  @param outloc= full and unquoted path to the desired text file.  This will be
+  @param [in] name= Stored Process name.  Leave blank if included above.
-    overwritten if it already exists.  If not provided, the code will be written
+  @param [out] outloc= (0) full and unquoted path to the desired text file.
-    to the log.
+    This will be overwritten if it already exists.
+  @param [out] outref= (0) Fileref to which to write the code.
+  @param [out] showlog=(NO) Set to YES to print log to the window
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
 
   @author Allan Bowe
 
@@ -10504,8 +10666,10 @@ filename __mc2 clear;
 %macro mm_getstpcode(
     tree=/User Folders/sasdemo/somestp
     ,name=
-    ,outloc=
+    ,outloc=0
+    ,outref=0
     ,mDebug=1
+    ,showlog=NO
     );
 
 %local mD;
@@ -10573,14 +10737,18 @@ data _null_;
   stop;
 
 %local outeng;
-%if %length(&outloc)=0 %then %let outeng=TEMP;
+%if "&outloc"="0" %then %let outeng=TEMP;
 %else %let outeng="&outloc";
+%local fref;
+%if &outref=0 %then %let fref=%mf_getuniquefileref();
+%else %let fref=&outref;
+
 /* read the content, byte by byte, resolving escaped chars */
-filename __outdoc &outeng lrecl=100000;
+filename &fref &outeng lrecl=100000;
 data _null_;
   length filein 8 fileid 8;
   filein = fopen("__getdoc","I",1,"B");
-  fileid = fopen("__outdoc","O",1,"B");
+  fileid = fopen("&fref","O",1,"B");
   rec = "20"x;
   length entity $6;
   do while(fread(filein)=0);
@@ -10621,9 +10789,9 @@ data _null_;
   rc=fclose(fileid);
 run;
 
-%if &outeng=TEMP %then %do;
+%if &showlog=YES %then %do;
   data _null_;
-    infile __outdoc lrecl=32767 end=last;
+    infile &fref lrecl=32767 end=last;
     input;
     if _n_=1 then putlog '>>stpcodeBEGIN<<';
     putlog _infile_;
@@ -10632,9 +10800,11 @@ run;
 %end;
 
 filename __getdoc clear;
-filename __outdoc clear;
+%if &outref=0 %then %do;
+  filename &fref clear;
+%end;
 
-%mend;
+%mend mm_getstpcode;
 /**
   @file
   @brief Returns a dataset with all Stored Processes, or just those in a
@@ -14550,13 +14720,14 @@ filename &fname3 clear;
   convenient way to wait for the job to finish before fetching the log.
 
 
-  @param [in] access_token_var= The global macro variable to contain the access token
+  @param [in] access_token_var= The global macro variable to contain the access
+    token
   @param [in] mdebug= set to 1 to enable DEBUG messages
   @param [in] grant_type= valid values:
     @li password
     @li authorization_code
-    @li detect - will check if access_token exists, if not will use sas_services if
+    @li detect - will check if access_token exists, if not will use sas_services
-        a SASStudioV session else authorization_code.  Default option.
+      if a SASStudioV session else authorization_code.  Default option.
     @li sas_services - will use oauth_bearer=sas_services.
   @param [in] uri= The uri of the running job for which to fetch the status,
     in the format `/jobExecution/jobs/$UUID/state` (unquoted).
@@ -14762,6 +14933,213 @@ run;
 
 
 
+/**
+  @file
+  @brief Extract the result from a completed SAS Viya Job
+  @details Extracts result from a Viya job and writes it out to a fileref
+  and/or a JSON-engine library.
+
+  To query the job, you need the URI.  Sample code for achieving this
+  is provided below.
+
+  ## Example
+
+  First, compile the macros:
+
+      filename mc url
+        "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+  Next, create a job (in this case, a web service):
+
+      filename ft15f001 temp;
+      parmcards4;
+        data test;
+          rand=ranuni(0)*1000;
+          do x=1 to rand;
+            y=rand*4;
+            output;
+          end;
+        run;
+        proc sort data=&syslast
+          by descending y;
+        run;
+        %webout(OPEN)
+        %webout(OBJ, test)
+        %webout(CLOSE)
+      ;;;;
+      %mv_createwebservice(path=/Public/temp,name=demo)
+
+  Execute it:
+
+      %mv_jobexecute(path=/Public/temp
+        ,name=demo
+        ,outds=work.info
+      )
+
+  Wait for it to finish, and grab the uri:
+
+      data _null_;
+        set work.info;
+        if method='GET' and rel='self';
+        call symputx('uri',uri);
+      run;
+
+  Finally, fetch the result (In this case, WEBOUT):
+
+      %mv_getjobresult(uri=&uri,result=WEBOUT_JSON,outref=myweb,outlib=myweblib)
+
+
+  @param [in] access_token_var= The global macro variable containing the access
+    token
+  @param [in] mdebug= set to 1 to enable DEBUG messages
+  @param [in] grant_type= valid values:
+    @li password
+    @li authorization_code
+    @li detect - will check if access_token exists, if not will use sas_services
+        if a SASStudioV session else authorization_code.  Default option.
+    @li sas_services - will use oauth_bearer=sas_services.
+  @param [in] uri= The uri of the running job for which to fetch the status,
+    in the format `/jobExecution/jobs/$UUID` (unquoted).
+
+  @param [out] result= (WEBOUT_JSON) The result type to capture.  Resolves
+  to "_[column name]" from the results table when parsed with the JSON libname
+  engine.
+
+  @param [out] outref= (0) The output fileref to which to write the results
+  @param [out] outlib= (0) The output library to which to assign the results
+    (assumes the data is in JSON format)
+
+
+  @version VIYA V.03.05
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+  @li mf_getplatform.sas
+  @li mf_existfileref.sas
+
+**/
+
+%macro mv_getjobresult(uri=0
+    ,contextName=SAS Job Execution compute context
+    ,access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+    ,mdebug=0
+    ,result=WEBOUT_JSON
+    ,outref=0
+    ,outlib=0
+  );
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+
+
+/* validation in datastep for better character safety */
+%local errmsg errflg;
+data _null_;
+  uri=symget('uri');
+  if length(uri)<12 then do;
+    call symputx('errflg',1);
+    call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
+  end;
+  if scan(uri,-1)='state' or scan(uri,1) ne 'jobExecution' then do;
+    call symputx('errflg',1);
+    call symputx('errmsg',
+      "URI should be in format /jobExecution/jobs/$$$$UUID$$$$"
+      !!" but is actually like: &uri",'l');
+  end;
+run;
+
+%mp_abort(iftrue=(&errflg=1)
+  ,mac=&sysmacroname
+  ,msg=%str(&errmsg)
+)
+
+%if &outref ne 0 and %mf_existfileref(&outref) ne 1 %then %do;
+  filename &outref temp;
+%end;
+
+options noquotelenmax;
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+/* fetch job info */
+%local fname1;
+%let fname1=%mf_getuniquefileref();
+proc http method='GET' out=&fname1 &oauth_bearer
+  url="&base_uri&uri";
+  headers "Accept"="application/json"
+  %if &grant_type=authorization_code %then %do;
+          "Authorization"="Bearer &&&access_token_var"
+  %end;
+  ;
+run;
+%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+%do;
+  data _null_;infile &fname1;input;putlog _infile_;run;
+  %mp_abort(mac=&sysmacroname
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+%end;
+
+/* extract results link */
+%local lib1 resuri;
+%let lib1=%mf_getuniquelibref();
+libname &lib1 JSON fileref=&fname1;
+data _null_;
+  set &lib1..results;
+  call symputx('resuri',_&result,'l');
+  putlog (_all_)(=);
+run;
+%mp_abort(iftrue=("&resuri"=".")
+  ,mac=&sysmacroname
+  ,msg=%str(Variable _&result did not exist in the response json)
+)
+
+/* extract results */
+%local fname2;
+%let fname2=%mf_getuniquefileref();
+proc http method='GET' out=&fname2 &oauth_bearer
+  url="&base_uri&resuri/content?limit=10000";
+  headers "Accept"="application/json"
+  %if &grant_type=authorization_code %then %do;
+          "Authorization"="Bearer &&&access_token_var"
+  %end;
+  ;
+run;
+
+%if &outref ne 0 %then %do;
+  filename &outref temp;
+  %mp_binarycopy(inref=&fname2,outref=&outref)
+%end;
+%if &outlib ne 0 %then %do;
+  libname &outlib JSON fileref=&fname2;
+%end;
+
+%if &mdebug=0 %then %do;
+  filename &fname1 clear;
+  filename &fname2 clear;
+  libname &lib1 clear;
+%end;
+%else %do;
+  %put _local_;
+%end;
+%mend;
 /**
   @file
   @brief Extract the status from a running SAS Viya job

base/mf_getvarlist.sas

@@ -21,6 +21,10 @@
   @param [in] dlm= ( ) Provide a delimiter (eg comma or space) to separate the
     variables
   @param [in] quote= (none) use either DOUBLE or SINGLE to quote the results
+  @param [in] typefilter= (A) Filter for certain types of column.  Valid values:
+    @li A Return All columns
+    @li C Return Character columns
+    @li N Return Numeric columns
 
   @version 9.2
   @author Allan Bowe
@@ -30,9 +34,10 @@
 %macro mf_getvarlist(libds
       ,dlm=%str( )
       ,quote=no
+      ,typefilter=A
 )/*/STORE SOURCE*/;
   /* declare local vars */
-  %local outvar dsid nvars x rc dlm q var;
+  %local outvar dsid nvars x rc dlm q var vtype;
 
   /* credit Rowland Hale  - byte34 is double quote, 39 is single quote */
   %if %upcase(&quote)=DOUBLE %then %let q=%qsysfunc(byte(34));
@@ -40,21 +45,22 @@
   /* open dataset in macro */
   %let dsid=%sysfunc(open(&libds));
 
-
   %if &dsid %then %do;
     %let nvars=%sysfunc(attrn(&dsid,NVARS));
     %if &nvars>0 %then %do;
-      /* add first dataset variable to global macro variable */
+      /* add variables with supplied delimeter */
-      %let outvar=&q.%sysfunc(varname(&dsid,1))&q.;
-      /* add remaining variables with supplied delimeter */
       %do x=1 %to &nvars;
-        %let var=&q.%sysfunc(varname(&dsid,&x))&q.;
+        /* get variable type */
-        %if &var=&q&q %then %do;
+        %let vtype=%sysfunc(vartype(&dsid,&x));
-          %put &sysmacroname: Empty column found in &libds!;
+        %if &vtype=&typefilter or &typefilter=A %then %do;
-          %let var=&q. &q.;
+          %let var=&q.%sysfunc(varname(&dsid,&x))&q.;
+          %if &var=&q&q %then %do;
+            %put &sysmacroname: Empty column found in &libds!;
+            %let var=&q. &q.;
+          %end;
+          %if %quote(&outvar)=%quote() %then %let outvar=&var;
+          %else %let outvar=&outvar.&dlm.&var.;
         %end;
-        %if &x=1 %then %let outvar=&var;
-        %else %let outvar=&outvar.&dlm.&var.;
       %end;
     %end;
     %let rc=%sysfunc(close(&dsid));
@@ -64,4 +70,4 @@
     %let rc=%sysfunc(close(&dsid));
   %end;
   &outvar
-%mend;
+%mend mf_getvarlist;

base/mf_getvartype.sas

@@ -45,4 +45,4 @@ Usage:
   %let rc = %sysfunc(close(&dsid));
   /* Return variable type */
   &vtype
-%mend;
+%mend mf_getvartype;

base/mf_uid.sas

@@ -18,4 +18,4 @@
 
   &today._&now._&sysjobid._%sysevalf(%sysfunc(ranuni(0))*999,CEIL)
 
-%mend;
+%mend mf_uid;

base/mp_assert.sas

@@ -0,0 +1,56 @@
+/**
+  @file
+  @brief Generic assertion
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      %mp_assert(iftrue=(1=1),
+        desc=Obviously true
+      )
+
+      %mp_assert(iftrue=(1=0),
+        desc=Will fail
+      )
+
+  @param [in] iftrue= (1=1) A condition where, if true, the test is a PASS.
+  Else, the test is a fail.
+
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Column &inds contained ALL columns|
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assert(iftrue=(1=1),
+  desc=0,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  data ;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_comments="&sysmacroname: Test result of "!!symget('iftrue');
+  %if %eval(%unquote(&iftrue)) %then %do;
+    test_result='PASS';
+  %end;
+  %else %do;
+    test_result='FAIL';
+  %end;
+  run;
+
+  %local ds ;
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+
+%mend mp_assert;

base/mp_filtercheck.sas

@@ -44,6 +44,7 @@
   @li mp_abort.sas
   @li mf_getuniquefileref.sas
   @li mf_getvarlist.sas
+  @li mf_getvartype.sas
   @li mf_nobs.sas
   @li mp_filtergenerate.sas
   @li mp_filtervalidate.sas
@@ -65,6 +66,20 @@
   ,msg=%str(syscc=&syscc - on macro entry)
 )
 
+/* Validate input column */
+%local vtype;
+%let vtype=%mf_getvartype(&inds,RAW_VALUE);
+%mp_abort(iftrue=(&abort=YES and &vtype ne C),
+  mac=&sysmacroname,
+  msg=%str(%str(ERR)OR: RAW_VALUE must be character)
+)
+%if &vtype ne C %then %do;
+  %put &sysmacroname: RAW_VALUE must be character;
+  %let syscc=42;
+  %return;
+%end;
+
+
 /**
   * Sanitise the values based on valid value lists, then strip out
   * quotes, commas, periods and spaces.
@@ -72,6 +87,8 @@
   */
 %local reason_cd;
 data &outds;
+  /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
+    OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
   length reason_cd $32;
 
@@ -168,4 +185,4 @@ run;
 /* this macro will also set syscc to 1008 if any issues found */
 %mp_filtervalidate(&fref1,&targetds,outds=&outds,abort=&abort)
 
-%mend;
+%mend mp_filtercheck;

base/mp_validatecol.sas

@@ -22,6 +22,7 @@
 
   @param [in] incol The column to be validated
   @param [in] rule The rule to apply.  Current rules:
+    @li ISNUM - checks if the variable is numeric
     @li LIBDS - matches LIBREF.DATASET format
   @param [out] outcol The variable to create, with the results of the match
 
@@ -62,4 +63,4 @@
   else &outcol=0;
 %end;
 
-%mend;
+%mend mp_validatecol;

meta/mm_getstpcode.sas

@@ -1,20 +1,26 @@
 /**
   @file
-  @brief Writes the code of an to an external file, or the log if none provided
+  @brief Writes the code of an STP to an external file
-  @details Get the
+  @details Fetches the SAS code from a Stored Process where the code is stored
+  in metadata.
 
-  usage:
+  Usage:
 
       %mm_getstpcode(tree=/some/meta/path
         ,name=someSTP
         ,outloc=/some/unquoted/filename.ext
       )
 
-  @param tree= The metadata path of the Stored Process (can also contain name)
+  @param [in] tree= The metadata path of the Stored Process (can also contain
-  @param name= Stored Process name.  Leave blank if included above.
+    name)
-  @param outloc= full and unquoted path to the desired text file.  This will be
+  @param [in] name= Stored Process name.  Leave blank if included above.
-    overwritten if it already exists.  If not provided, the code will be written
+  @param [out] outloc= (0) full and unquoted path to the desired text file.
-    to the log.
+    This will be overwritten if it already exists.
+  @param [out] outref= (0) Fileref to which to write the code.
+  @param [out] showlog=(NO) Set to YES to print log to the window
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
 
   @author Allan Bowe
 
@@ -23,8 +29,10 @@
 %macro mm_getstpcode(
     tree=/User Folders/sasdemo/somestp
     ,name=
-    ,outloc=
+    ,outloc=0
+    ,outref=0
     ,mDebug=1
+    ,showlog=NO
     );
 
 %local mD;
@@ -92,14 +100,18 @@ data _null_;
   stop;
 
 %local outeng;
-%if %length(&outloc)=0 %then %let outeng=TEMP;
+%if "&outloc"="0" %then %let outeng=TEMP;
 %else %let outeng="&outloc";
+%local fref;
+%if &outref=0 %then %let fref=%mf_getuniquefileref();
+%else %let fref=&outref;
+
 /* read the content, byte by byte, resolving escaped chars */
-filename __outdoc &outeng lrecl=100000;
+filename &fref &outeng lrecl=100000;
 data _null_;
   length filein 8 fileid 8;
   filein = fopen("__getdoc","I",1,"B");
-  fileid = fopen("__outdoc","O",1,"B");
+  fileid = fopen("&fref","O",1,"B");
   rec = "20"x;
   length entity $6;
   do while(fread(filein)=0);
@@ -140,9 +152,9 @@ data _null_;
   rc=fclose(fileid);
 run;
 
-%if &outeng=TEMP %then %do;
+%if &showlog=YES %then %do;
   data _null_;
-    infile __outdoc lrecl=32767 end=last;
+    infile &fref lrecl=32767 end=last;
     input;
     if _n_=1 then putlog '>>stpcodeBEGIN<<';
     putlog _infile_;
@@ -151,6 +163,8 @@ run;
 %end;
 
 filename __getdoc clear;
-filename __outdoc clear;
+%if &outref=0 %then %do;
+  filename &fref clear;
+%end;
 
-%mend;
+%mend mm_getstpcode;

tests/base/mf_getvarlist.test.sas

@@ -0,0 +1,60 @@
+/**
+  @file
+  @brief Testing mf_getvarlist macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getvarlist.sas
+
+**/
+
+
+%let test1=%mf_getvarlist(sashelp.class);
+%let test2=%mf_getvarlist(sashelp.class,dlm=X);
+%let test3=%mf_getvarlist(sashelp.class,dlm=%str(,),quote=double);
+%let test4=%mf_getvarlist(sashelp.class,typefilter=N);
+%let test5=%mf_getvarlist(sashelp.class,typefilter=C);
+
+data work.test_results;
+  length test_description $256 test_result $4 test_comments base result  $256;
+  test_description="Basic test";
+  base=symget('test1');
+  result='Name Sex Age Height Weight';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  test_description="DLM test";
+  base=symget('test2');
+  result='NameXSexXAgeXHeightXWeight';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  test_description="DLM + quote test";
+  base=symget('test3');
+  result='"Name","Sex","Age","Height","Weight"';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  test_description="Numeric Filter";
+  base=symget('test4');
+  result='Age Height Weight';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  test_description="Char Filter";
+  base=symget('test5');
+  result='Name Sex';
+  if base=result then test_result='PASS';
+  else test_result='FAIL';
+  test_comments="Comparing "!!trim(base)!!' vs '!!trim(result);
+  output;
+
+  drop base result;
+run;

tests/base/mp_filtercheck.test.sas

@@ -5,6 +5,7 @@
   <h4> SAS Macros </h4>
   @li mp_filtercheck.sas
   @li mp_assertdsobs.sas
+  @li mp_assert.sas
 
 **/
 
@@ -125,3 +126,23 @@ run;
   outds=work.test_results
 )
 
+/* Supply variables with incorrect types */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:8;
+datalines4;
+AND,AND,1,age,=,0
+;;;;
+run;
+%let syscc=0;
+%mp_filtercheck(work.inds,
+  targetds=sashelp.class,
+  outds=work.badrecords,
+  abort=NO
+)
+%mp_assert(iftrue=(&syscc=42),
+  desc=Throw error if RAW_VALUE is incorrect,
+  outds=work.test_results
+)
+%let syscc=0;

tests/base/mp_validatecol.test.sas

@@ -17,7 +17,7 @@ data test1;
   input;
   libds=_infile_;
   %mp_validatecol(libds,LIBDS,is_libds)
-  if libds=1;
+  if is_libds=1;
 datalines4;
 some.libname
 !lib.blah