Merge pull request #116 from sasjs/fix/mp_init_options_fix

fix: Aligns the autocorrect option with the intent. Closes #115.

.git-hooks/pre-commit

@@ -1,2 +1,11 @@
-#!/bin/sh
-sasjs lint
+#!/bin/bash
+sasjs lint
+
+# Avoid commits to the master branch
+BRANCH=`git rev-parse --abbrev-ref HEAD`
+
+if [[ "$BRANCH" =~ ^(master|main|develop)$ ]]; then
+  echo "You are on branch $BRANCH. Are you sure you want to commit to this branch?"
+  echo "If so, commit with -n to bypass the pre-commit hook."
+  exit 1
+fi

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2020 (Allan Bowe)
+Copyright 2021 (Allan Bowe)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

README.md

@@ -1,7 +1,7 @@
 # Macro Core
 [![npm package][npm-image]][npm-url]
 [![Github Workflow][githubworkflow-image]][githubworkflow-url]
-[![npm](https://img.shields.io/npm/dt/@sasjs/core)]()
+![npm](https://img.shields.io/npm/dt/@sasjs/core)
 ![Snyk Vulnerabilities for npm package](https://img.shields.io/snyk/vulnerabilities/npm/@sasjs/core)
 [![License](https://img.shields.io/apm/l/atomic-design-ui.svg)](/LICENSE)
 ![GitHub top language](https://img.shields.io/github/languages/top/sasjs/core)
@@ -31,21 +31,21 @@ Documentation: https://core.sasjs.io
 
 ## Components
 
-### **base** library (SAS9/Viya)
+### BASE library (SAS9/Viya)
 
 - OS independent
 - Not metadata aware
 - No X command
 - Prefixes: _mf_, _mp_
 
-#### **fcmp** library (SAS9/Viya)
+#### FCMP library (SAS9/Viya)
 - Function and macro names are identical, except for special cases
 - Prefixes: _mcf_
 
 The fcmp macros are used to generate fcmp functions, and can be used with or
 without the `proc fcmp` wrapper.
 
-### **meta** library (SAS9 only)
+### META library (SAS9 only)
 
 Macros used in SAS EBI, which connect to the metadata server.
 
@@ -54,7 +54,7 @@ Macros used in SAS EBI, which connect to the metadata server.
 - No X command
 - Prefixes: _mm_
 
-### **server** library (@sasjs/server only)
+### SERVER library (@sasjs/server only)
 These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
 
 - OS independent
@@ -62,7 +62,7 @@ These macros are used for building applications using [@sasjs/server](https://se
 - No X command
 - Prefixes: _ms_
 
-### **viya** library (Viya only)
+### VIYA library (Viya only)
 
 Macros used for interfacing with SAS Viya.
 
@@ -70,14 +70,14 @@ Macros used for interfacing with SAS Viya.
 - No X command
 - Prefixes: _mv_, _mvf_
 
-### **metax** library (SAS9 only)
+### METAX library (SAS9 only)
 
 - OS specific
 - Metadata aware
 - X command enabled
 - Prefixes: _mmw_,_mmu_,_mmx_
 
-### **lua** library
+### LUA library
 
 Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
 

all.sas

@@ -79,7 +79,7 @@ options noquotelenmax;
     Run without arguments to see a list of detectable features.
     Note - this list is based on known versions of SAS rather than
     actual feature detection, as that is tricky / impossible to do
-    without generating errors in most cases.
+    without generating errs in most cases.
 
         %put %mf_existfeature(PROCLUA);
 
@@ -362,7 +362,7 @@ https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionex
   @param attr full list in [documentation](
     https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000147794.htm)
   @return output returns result of the attrc value supplied, or -1 and log
-    message if error.
+    message if err.
 
   @version 9.2
   @author Allan Bowe
@@ -395,7 +395,7 @@ https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionex
   @param attr Common values are NLOBS and NVARS, full list in [documentation](
   http://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000212040.htm)
   @return output returns result of the attrn value supplied, or -1 and log
-    message if error.
+    message if err.
 
   @version 9.2
   @author Allan Bowe
@@ -626,12 +626,12 @@ https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionex
   for:
 > "these","words","are","double","quoted"
 
-  @param in_str the unquoted, spaced delimited string to transform
-  @param dlm= the delimeter to be applied to the output (default comma)
-  @param indlm= the delimeter used for the input (default is space)
-  @param quote= the quote mark to apply (S=Single, D=Double). If any other value
-    than uppercase S or D is supplied, then that value will be used as the
-    quoting character.
+  @param [in] in_str The unquoted, spaced delimited string to transform
+  @param [in] dlm= The delimeter to be applied to the output (default comma)
+  @param [in] indlm= (,) The delimeter used for the input (default is space)
+  @param [in] quote= (S) The quote mark to apply (S=Single, D=Double, N=None).
+    If any other value than uppercase S or D is supplied, then that value will
+    be used as the quoting character.
   @return output returns a string with the newly quoted / delimited output.
 
   @version 9.2
@@ -641,9 +641,10 @@ https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionex
 
 %macro mf_getquotedstr(IN_STR,DLM=%str(,),QUOTE=S,indlm=%str( )
 )/*/STORE SOURCE*/;
-  %if &quote=S %then %let quote=%str(%');
-  %else %if &quote=D %then %let quote=%str(%");
-  %else %let quote=%str();
+  /* credit Rowland Hale  - byte34 is double quote, 39 is single quote */
+  %if &quote=S %then %let quote=%qsysfunc(byte(39));
+  %else %if &quote=D %then %let quote=%qsysfunc(byte(34));
+  %else %if &quote=N %then %let quote=;
   %local i item buffer;
   %let i=1;
   %do %while (%qscan(&IN_STR,&i,%str(&indlm)) ne %str() ) ;
@@ -1339,6 +1340,38 @@ Usage:
   &is_directory
 
 %mend mf_isdir;/**
+  @file
+  @brief Returns 1 if the variable contains only digits 0-9, else 0
+  @details Note that numerics containing any punctuation (including decimals
+    or exponents) will be flagged zero.
+
+  If you'd like support for this, then do raise an issue (or even better, a
+  pull request!)
+
+  Usage:
+
+      %put %mf_isint(1) returns 1;
+      %put %mf_isint(1.1) returns 0;
+      %put %mf_isint(%str(1,1)) returns 0;
+
+  @param [in] arg input value to check
+
+  @version 9.2
+**/
+
+%macro mf_isint(arg
+)/*/STORE SOURCE*/;
+  /* remove minus sign if exists */
+
+  %local val;
+  %if "%substr(%str(&arg),1,1)"="-" %then %let val=%substr(%str(&arg),2);
+  %else %let val=&arg;
+
+  /* check remaining chars */
+  %if %sysfunc(findc(%str(&val),,kd)) %then %do;0%end;
+  %else %do;1%end;
+
+%mend mf_isint;/**
   @file
   @brief Returns physical location of various SAS items
   @details Returns location of the PlatformObjectFramework tools
@@ -1420,7 +1453,7 @@ Usage:
     %end;
 
     /*
-      Now create the directory.  Complain loudly of any errors.
+      Now create the directory.  Complain loudly of any errs.
     */
 
     %let dname = %sysfunc(dcreate(&child, &parent));
@@ -1467,7 +1500,7 @@ Usage:
   @param libds library.dataset
 
   @return output returns result of the attrn value supplied, or log message
-    if error.
+    if err.
 
 
   @version 9.2
@@ -2789,6 +2822,8 @@ run;
 
   /* create folders and copy content */
   data _null_;
+    length msg $200;
+    call missing(msg);
     set work.&tempds;
     if _n_ = 1 then dpos+sum(length(directory),2);
     filepath2="&target/"!!substr(filepath,dpos);
@@ -2798,9 +2833,9 @@ run;
       rc1=filename(fref1,filepath,'disk','recfm=n');
       rc2=filename(fref2,filepath2,'disk','recfm=n');
       if fcopy(fref1,fref2) ne 0 then do;
-        sysmsg=sysmsg();
+        msg=sysmsg();
         putlog "%str(ERR)OR: Unable to copy " filepath " to " filepath2;
-        putlog sysmg=;
+        putlog msg=;
       end;
     end;
     rc=filename(fref1);
@@ -3875,17 +3910,23 @@ run;
 
 %mend mp_ds2csv;/**
   @file
-  @brief Converts every value in a dataset to it's formatted value
+  @brief Converts every value in a dataset to formatted value
   @details Converts every value to it's formatted value.  All variables will
-  become character, and will be in the same order.
+  become character, and will be in the same order as the original dataset.
+
+  Lengths will be adjusted according to the format lengths, where applicable.
 
   Usage:
 
       %mp_ds2fmtds(sashelp.cars,work.cars)
+      %mp_ds2fmtds(sashelp.vallopt,vw_vallopt)
 
   @param [in] libds The library.dataset to be converted
   @param [out] outds The dataset to create.
 
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+
   <h4> Related Macros <h4>
   @li mp_jsonout.sas
 
@@ -3897,8 +3938,9 @@ run;
 )/*/STORE SOURCE*/;
 
 /* validations */
-%if not %sysfunc(exist(&libds)) %then %do;
-  %put %str(WARN)ING:  &libds does not exist;
+
+%if not %mf_existds(libds=&libds) %then %do;
+  %put %str(WARN)ING:  &libds does not exist as either a VIEW or DATASET;
   %return;
 %end;
 %if %index(&libds,.)=0 %then %let libds=WORK.&libds;
@@ -4088,6 +4130,7 @@ data _null_;
     if _n_>&maxobs then stop;
   %end;
   length _____str $32767;
+  call missing(_____str);
   format _numeric_ best.;
   format _character_ ;
   %local i comma var vtype vfmt;
@@ -4488,8 +4531,8 @@ filename &outref temp;
   @param [in] targetds The target dataset against which to verify the query
   @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
   @param [out] outds= (work.mp_filtervalidate) Output dataset containing the
-  error / warning message, if one exists.  If this table contains any rows,
-  there are problems!
+    err / warning message, if one exists.  If this table contains any rows,
+    there are problems!
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
@@ -4612,7 +4655,7 @@ data &outds(keep=name type length varnum format label ddtype);
     else if formatd=0 then format=cats(format2,formatl,'.');
     else format=cats(format2,formatl,'.',formatd);
     type='N';
-    if format=:'DATETIME' then ddtype='DATETIME';
+    if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
     else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
       or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
       or format=:'MONYY'
@@ -4673,17 +4716,17 @@ run;
 %let vw=%mf_getuniquename(prefix=mp_getconstraints_vw_);
 data &vw /view=&vw;
   set sashelp.vcncolu;
-  where TABLE_CATALOG="&lib";
+  where table_catalog="&lib";
 
   /* use retain approach to reset the constraint order with each constraint */
   length tmp $1000;
   retain tmp;
   drop tmp;
-  if tmp ne catx('|',libref,table_name,constraint_type,constraint_name) then do;
+  if tmp ne catx('|',table_catalog,table_name,constraint_name) then do;
     constraint_order=1;
   end;
   else constraint_order+1;
-  tmp=catx('|',libref, table_name, constraint_type,constraint_name);
+  tmp=catx('|',table_catalog, table_name,constraint_name);
 run;
 
 /* must use SQL as proc datasets does not support length changes */
@@ -5193,7 +5236,7 @@ run;
     %let curds=%scan(&dsnlist,&x);
     data _null_;
       file &fref mod;
-      length nm lab $1024 typ $20;
+      length lab $1024 typ $20;
       set &colinfo (where=(upcase(memname)="&curds")) end=last;
 
       if _n_=1 then do;
@@ -5548,6 +5591,10 @@ create table &outds (rename=(
 
   Returns:
 
+|libref:$8.|dsn:$32.|memtype:$8.|dbms_memtype:$32.|typemem:$8.|memlabel:$256.|nvar:best.|compress:$8.|pk_fields:$512.|
+|---|---|---|---|---|---|---|---|---|
+|WORK|EXAMPLE|DATA| |DATA| |4|NO|TX_FROM DD_TYPE DD_SOURCE|
+
 
   @param [in] lib The libref to examine
   @param [in] ds= (0) Select the dataset to examine, else use 0 for all tables
@@ -5562,6 +5609,7 @@ create table &outds (rename=(
 
   <h4> Related Macros </h4>
   @li mp_getpk.test.sas
+  @li mp_guesspk.sas
 
   @version 9.3
   @author Macro People Ltd
@@ -5830,34 +5878,39 @@ create table &outds as
 
 %mend mp_gsubfile;
 /**
-  @file mp_guesspk.sas
+  @file
   @brief Guess the primary key of a table
-  @details Tries to guess the primary key of a table based on the following logic:
+  @details Tries to guess the primary key of a table based on the following
+  logic:
 
       * Columns with nulls are ignored
       * Return only column combinations that provide unique results
-      * Start from one column, then move out to include composite keys of 2 to 6 columns
+      * Start from one column, then move out to composite keys of 2 to 6 columns
 
   The library of the target should be assigned before using this macro.
 
   Usage:
 
-      filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      filename mc url
+        "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
       %mp_guesspk(sashelp.class,outds=classpks)
 
   @param baseds The dataset to analyse
   @param outds= The output dataset to contain the possible PKs
-  @param max_guesses= The total number of possible primary keys to generate.  A
-    table is likely to have multiple unlikely PKs, so no need to list them all. Default=3.
-  @param min_rows= The minimum number of rows a table should have in order to try
-    and guess the PK.  Default=5.
+  @param max_guesses= (3) The total number of possible primary keys to generate.
+    A table may have multiple unlikely PKs, so no need to list them all.
+  @param min_rows= (5) The minimum number of rows a table should have in order
+    to try and guess the PK.
 
   <h4> SAS Macros </h4>
   @li mf_getvarlist.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
 
+  <h4> Related Macros </h4>
+  @li mp_getpk.sas
+
   @version 9.3
   @author Allan Bowe
 
@@ -6027,7 +6080,8 @@ create table &outds as
           %let lev4=%scan(&posspks,&l);
           %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then %do;
             /* check for four level uniqueness */
-            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4) out=&tmpds noduprec;
+            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4)
+                out=&tmpds noduprec;
               by _all_;
             run;
             %if %mf_nobs(&tmpds)=&rows %then %do;
@@ -6064,7 +6118,8 @@ create table &outds as
             %let lev5=%scan(&posspks,&m);
             %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then %do;
               /* check for four level uniqueness */
-              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5) out=&tmpds noduprec;
+              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5)
+                  out=&tmpds noduprec;
                 by _all_;
               run;
               %if %mf_nobs(&tmpds)=&rows %then %do;
@@ -6113,7 +6168,8 @@ create table &outds as
                 run;
                 %if %mf_nobs(&tmpds)=&rows %then %do;
                   proc sql;
-                  insert into &outds values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
+                  insert into &outds
+                    values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
                   %if %mf_nobs(&outds) ge &max_guesses %then %do;
                     %put &sysmacroname: Max PKs reached at Level 6 for &baseds;
                     %return;
@@ -6155,6 +6211,7 @@ create table &outds as
 
   @param [in] libds dataset to hash
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
+  @param [in] iftrue= A condition under which the macro should be executed.
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
   will contain one column (hashkey) with one observation (a hex32.
   representation of the input hash)
@@ -6169,10 +6226,14 @@ create table &outds as
 %macro mp_hashdataset(
   libds,
   outds=,
-  salt=
+  salt=,
+  iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
+
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
   %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
-    %put %str(WARN)ING: Dataset &libds is empty;, or is not a dataset;
+    %put %str(WARN)ING: Dataset &libds is empty, or is not a dataset;
   %end;
   %else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
     %put %str(ERR)OR: Dataset &libds is not a dataset;
@@ -6312,6 +6373,70 @@ run;
 filename &tempref clear;
 
 %mend mp_include;/**
+  @file
+  @brief Initialise session with useful settings and variables
+  @details Implements a "strict" set of SAS options for use in defensive
+    programming.  Highly recommended, if you want your code to run on some
+    other machine.
+
+    This macro is recommended to be compiled and invoked in the `initProgram`
+    for SASjs [Jobs](https://cli.sasjs.io/sasjsconfig.html#jobConfig_initProgram
+    ), [Services](
+    https://cli.sasjs.io/sasjsconfig.html#serviceConfig_initProgram) and [Tests]
+    (https://cli.sasjs.io/sasjsconfig.html#testConfig_initProgram).
+
+    For non SASjs projects, you could invoke in the autoexec, or in your own
+    solution initialisation macro.
+
+
+    If you have a good idea for another useful option, setting, or global
+    variable - feel free to [raise an issue](
+    https://github.com/sasjs/core/issues/new)!
+
+    All global variables are prefixed with "SASJS" (unless modified with the
+    prefix parameter).
+
+  @param [in] prefix= (SASJS) The prefix to apply to the global macro variables
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_init(prefix=
+)/*/STORE SOURCE*/;
+
+  %global
+    &prefix._INIT_NUM   /* initialisation time as numeric             */
+    &prefix._INIT_DTTM  /* initialisation time in E8601DT26.6 format */
+  ;
+  %if %eval(&&&prefix._INIT_NUM>0) %then %return;  /* only run once */
+
+  data _null_;
+    dttm=datetime();
+    call symputx("&prefix._init_num",dttm);
+    call symputx("&prefix._init_dttm",put(dttm,E8601DT26.6));
+  run;
+
+  options
+    autocorrect             /* disallow mis-spelled procedure names           */
+    compress=CHAR           /* default is none so ensure we have something!   */
+    datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
+    %str(err)orcheck=STRICT /* catch errs in libname/filename statements      */
+    fmterr                  /* ensure err when a format cannot be found       */
+    mergenoby=%str(ERR)OR   /* throw err when a merge has no BY variables     */
+    missing=.               /* changing this can cause hard to detect errs    */
+    noquotelenmax           /* avoid warnings for long strings                */
+    noreplace               /* avoid overwriting permanent datasets           */
+    ps=max                  /* reduce log size slightly                       */
+    validmemname=COMPATIBLE /* avoid special characters etc in table names    */
+    validvarname=V7         /* avoid special characters etc in variable names */
+    varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
+    varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
+  ;
+
+%mend mp_init;/**
   @file mp_jsonout.sas
   @brief Writes JSON in SASjs format to a fileref
   @details PROC JSON is faster but will produce errs like the ones below if
@@ -6376,7 +6501,7 @@ filename &tempref clear;
 %if &action=OPEN %then %do;
   options nobomfile;
   data _null_;file &jref encoding='utf-8';
-    put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
+    put '{"PROCESSED_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '"';
   run;
 %end;
 %else %if (&action=ARR or &action=OBJ) %then %do;
@@ -6938,7 +7063,7 @@ run;
   %let abortme=1;
 %end;
 
-/* catch errors - mp_abort must be called outside of a logic block */
+/* catch errs - mp_abort must be called outside of a logic block */
 %mp_abort(iftrue=(&abortme=1),
   msg=%superq(msg),
   mac=&sysmacroname
@@ -7049,6 +7174,92 @@ lock &libds clear;
 )
 
 %mend mp_lockfilecheck;/**
+  @file
+  @brief Create sample data based on the structure of an empty table
+  @details Many SAS projects involve sensitive datasets.  One way to _ensure_
+    the data is anonymised, is never to receive it in the first place!  Often
+    consultants are provided with empty tables, and expected to create complex
+    ETL flows.
+
+    This macro can help by taking an empty table, and populating it with data
+    according to the variable types and formats.
+
+    TODO:
+      @li Respect PKs
+      @li Respect NOT NULLs
+      @li Consider dates, datetimes, times, integers etc
+
+  Usage:
+
+      proc sql;
+      create table work.example(
+        TX_FROM float format=datetime19.,
+        DD_TYPE char(16),
+        DD_SOURCE char(2048),
+        DD_SHORTDESC char(256),
+        constraint pk primary key(tx_from, dd_type,dd_source),
+        constraint nnn not null(DD_SHORTDESC)
+      );
+      %mp_makedata(work.example)
+
+  @param [in] libds The empty table in which to create data
+  @param [out] obs= (500) The number of records to create.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_getvarlen.sas
+  @li mf_nobs.sas
+  @li mp_getcols.sas
+  @li mp_getpk.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_makedata(libds
+  ,obs=500
+)/*/STORE SOURCE*/;
+
+%local ds1 c1 n1 i col charvars numvars;
+
+%if %mf_nobs(&libds)>0 %then %do;
+  %put &sysmacroname: &libds has data, it will not be recreated;
+  %return;
+%end;
+
+%local ds1 c1 n1;
+%let ds1=%mf_getuniquename(prefix=mp_makedata);
+%let c1=%mf_getuniquename(prefix=mp_makedatacol);
+%let n1=%mf_getuniquename(prefix=mp_makedatacol);
+data &ds1;
+  if 0 then set &libds;
+  do _n_=1 to &obs;
+    &c1=repeat(uuidgen(),10);
+    &n1=ranuni(1)*5000000;
+    drop &c1 &n1;
+    %let charvars=%mf_getvarlist(&libds,typefilter=C);
+    %do i=1 %to %sysfunc(countw(&charvars));
+      %let col=%scan(&charvars,&i);
+      &col=subpad(&c1,1,%mf_getvarlen(&libds,&col));
+    %end;
+
+    %let numvars=%mf_getvarlist(&libds,typefilter=N);
+    %do i=1 %to %sysfunc(countw(&numvars));
+      %let col=%scan(&numvars,&i);
+      &col=&n1;
+    %end;
+    output;
+  end;
+run;
+
+proc append base=&libds data=&ds1;
+run;
+
+proc sql;
+drop table &ds1;
+
+%mend mp_makedata;/**
   @file
   @brief Create a Markdown Table from a dataset
   @details A markdown table is a simple table representation for use in
@@ -7364,6 +7575,32 @@ insert into &outds select distinct * from &append_ds;
 
 %mend mp_recursivejoin;
 /**
+  @file
+  @brief Reset when an err condition occurs
+  @details When building apps, sometimes an operation must be attempted that
+  can cause an err condition.  There is no try catch in SAS! So the err state
+  must be caught and reset.
+
+  This macro attempts to do that reset.
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_reseterror(
+)/*/STORE SOURCE*/;
+
+options obs=max replace nosyntaxcheck;
+%let syscc=0;
+
+%if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
+  data _null_;
+    rc=stpsrvset('program error', 0);
+  run;
+%end;
+
+%mend mp_reseterror;/**
   @file
   @brief Reset an option to original value
   @details Inspired by the SAS Jedi -
@@ -7558,22 +7795,28 @@ drop table &tempds;
   Usage:
 
       %mp_searchdata(lib=sashelp, string=Jan)
-      %mp_searchdata(lib=sashelp, numval=1)
+      %mp_searchdata(lib=sashelp, ds=bird, numval=1)
+      %mp_searchdata(lib=sashelp, ds=class, string=l,outobs=5)
 
 
   Outputs zero or more tables to an MPSEARCH library with specific records.
 
-  @param lib=  the libref to search (should be already assigned)
-  @param ds= the dataset to search (leave blank to search entire library)
-  @param string= the string value to search
-  @param numval= the numeric value to search (must be exact)
-  @param outloc= the directory in which to create the output datasets with
-    matching rows.  Will default to a subfolder in the WORK library.
-  @param outobs= set to a positive integer to restrict the number of
+  @param [in] lib=  The libref to search (should be already assigned)
+  @param [in] ds= The dataset to search (leave blank to search entire library)
+  @param [in] string= String value to search (case sensitive, can be partial)
+  @param [in] numval= Numeric value to search (must be exact)
+  @param [out] outloc= (0) Optionally specify the directory in which to
+    create the the output datasets with matching rows.  By default it will
+    write them to a temporary subdirectory within the WORK folder.
+  @param [out] outlib= (MPSEARCH) Assign a different libref to the output
+    library containing the matching datasets / records
+  @param [in] outobs= set to a positive integer to restrict the number of
     observations
-  @param filter_text= add a (valid) filter clause to further filter the results
+  @param [in] filter_text= (1=1) Add a (valid) filter clause to further filter
+    the results.
 
   <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
   @li mf_mkdir.sas
@@ -7583,11 +7826,12 @@ drop table &tempds;
   @author Allan Bowe
 **/
 
-%macro mp_searchdata(lib=sashelp
+%macro mp_searchdata(lib=
   ,ds=
   ,string= /* the query will use a contains (?) operator */
   ,numval= /* numeric must match exactly */
-  ,outloc=%sysfunc(pathname(work))/mpsearch
+  ,outloc=0
+  ,outlib=MPSEARCH
   ,outobs=-1
   ,filter_text=%str(1=1)
 )/*/STORE SOURCE*/;
@@ -7604,8 +7848,12 @@ drop table &tempds;
 %if &string = %then %let type=N;
 %else %let type=C;
 
+%if "&outloc"="0" %then %do;
+  %let outloc=%sysfunc(pathname(work))/%mf_getuniquename();
+%end;
+
 %mf_mkdir(&outloc)
-libname mpsearch "&outloc";
+libname &outlib "&outloc";
 
 /* get the list of tables in the library */
 proc sql noprint;
@@ -7617,11 +7865,6 @@ select distinct memname into: table_list separated by ' '
 %end;
   ;
 /* check that we have something to check */
-proc sql
-%if &outobs>-1 %then %do;
-  outobs=&outobs
-%end;
-;
 %if %length(&table_list)=0 %then %put library &lib contains no tables!;
 /* loop through each table */
 %else %do table_num=1 %to %sysfunc(countw(&table_list,%str( )));
@@ -7632,10 +7875,10 @@ proc sql
   %end;
   %else %do;
     %let check_tm=%sysfunc(datetime());
-    /* build sql statement */
-    create table mpsearch.&table as select * from &lib..&table
-      where %unquote(&filter_text) and
-    (0
+    /* prep input */
+    data &outlib..&table;
+      set &lib..&table;
+      where %unquote(&filter_text) and ( 0
     /* loop through columns */
     %do colnum=1 %to %sysfunc(countw(&vars,%str( )));
       %let col=%scan(&vars,&colnum,%str( ));
@@ -7649,15 +7892,20 @@ proc sql
         or ("&col"n = &numval)
       %end;
     %end;
-    );
+      );
+    %if &outobs>-1 %then %do;
+      if _n_ > &outobs then stop;
+    %end;
+    run;
     %put Search query for &table took
       %sysevalf(%sysfunc(datetime())-&check_tm) seconds;
-    %if &sqlrc ne 0 %then %do;
-      %put %str(WAR)NING: SQLRC=&sqlrc when processing &table;
+    %if &syscc ne 0 %then %do;
+      %put %str(ERR)ROR: SYSCC=&syscc when processing &lib..&table;
       %return;
     %end;
-    %if %mf_nobs(mpsearch.&table)=0 %then %do;
-      drop table mpsearch.&table;
+    %if %mf_nobs(&outlib..&table)=0 %then %do;
+      proc sql;
+      drop table &outlib..&table;
     %end;
   %end;
 %end;
@@ -7810,7 +8058,7 @@ run;
 %let tempvw=%mf_getuniquename(prefix=&sysmacroname);
 proc sql;
 create view work.&tempvw as select * from &lib..&ds
-order by %mf_getquotedstr(&sortkey,quote=%str());
+order by %mf_getquotedstr(&sortkey,quote=N);
 
 /* append sorted data */
 proc append base=&lib..&tempds2 data=work.&tempvw;
@@ -10917,7 +11165,7 @@ data _null_;
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
   put '  data _null_;file &jref encoding=''utf-8''; ';
-  put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
+  put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
   put '  run; ';
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
@@ -15999,7 +16247,7 @@ data _null_;
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
   put '  data _null_;file &jref encoding=''utf-8''; ';
-  put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
+  put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
   put '  run; ';
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
@@ -17223,6 +17471,8 @@ options noquotelenmax;
   %local href cnt;
   %let cnt=0;
   data _null_;
+    length rel href $512;
+    call missing(rel,href);
     set &libref1..links;
     if rel='members' then do;
       url=cats("'","&base_uri",href,"?limit=10000'");
@@ -17246,6 +17496,7 @@ options noquotelenmax;
   libname &libref2 JSON fileref=&fname2;
   data &outds;
     length id $36 name $128 uri $64 type $32 description $256;
+    if _n_=1 then call missing (of _all_);
     set &libref2..items;
   run;
   filename &fname2 clear;
@@ -17541,6 +17792,8 @@ data;run;
 %local joburi;
 %let joburi=0;
 data _null_;
+  length name uri $512;
+  call missing(name,uri);
   set &foldermembers;
   if name="&name" and uri=:'/jobDefinitions/definitions'
     then call symputx('joburi',uri);
@@ -18697,6 +18950,8 @@ data;run;
 %local joburi;
 %let joburi=0;
 data _null_;
+  length name uri $512;
+  call missing(name,uri);
   set &foldermembers;
   if name="&name" and uri=:'/jobDefinitions/definitions'
     then call symputx('joburi',uri);
@@ -19330,6 +19585,7 @@ run;
 data &outds;
   format _program uri $128. state $32. stateDetails $32. timestamp datetime19.
     jobparams $32767.;
+  call missing (of _all_);
   stop;
 run;
 

base/mf_existfeature.sas

@@ -5,7 +5,7 @@
     Run without arguments to see a list of detectable features.
     Note - this list is based on known versions of SAS rather than
     actual feature detection, as that is tricky / impossible to do
-    without generating errors in most cases.
+    without generating errs in most cases.
 
         %put %mf_existfeature(PROCLUA);
 

base/mf_getattrc.sas

@@ -10,7 +10,7 @@
   @param attr full list in [documentation](
     https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000147794.htm)
   @return output returns result of the attrc value supplied, or -1 and log
-    message if error.
+    message if err.
 
   @version 9.2
   @author Allan Bowe

base/mf_getattrn.sas

@@ -10,7 +10,7 @@
   @param attr Common values are NLOBS and NVARS, full list in [documentation](
   http://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000212040.htm)
   @return output returns result of the attrn value supplied, or -1 and log
-    message if error.
+    message if err.
 
   @version 9.2
   @author Allan Bowe

base/mf_getquotedstr.sas

@@ -15,12 +15,12 @@
   for:
 > "these","words","are","double","quoted"
 
-  @param in_str the unquoted, spaced delimited string to transform
-  @param dlm= the delimeter to be applied to the output (default comma)
-  @param indlm= the delimeter used for the input (default is space)
-  @param quote= the quote mark to apply (S=Single, D=Double). If any other value
-    than uppercase S or D is supplied, then that value will be used as the
-    quoting character.
+  @param [in] in_str The unquoted, spaced delimited string to transform
+  @param [in] dlm= The delimeter to be applied to the output (default comma)
+  @param [in] indlm= (,) The delimeter used for the input (default is space)
+  @param [in] quote= (S) The quote mark to apply (S=Single, D=Double, N=None).
+    If any other value than uppercase S or D is supplied, then that value will
+    be used as the quoting character.
   @return output returns a string with the newly quoted / delimited output.
 
   @version 9.2
@@ -30,9 +30,10 @@
 
 %macro mf_getquotedstr(IN_STR,DLM=%str(,),QUOTE=S,indlm=%str( )
 )/*/STORE SOURCE*/;
-  %if &quote=S %then %let quote=%str(%');
-  %else %if &quote=D %then %let quote=%str(%");
-  %else %let quote=%str();
+  /* credit Rowland Hale  - byte34 is double quote, 39 is single quote */
+  %if &quote=S %then %let quote=%qsysfunc(byte(39));
+  %else %if &quote=D %then %let quote=%qsysfunc(byte(34));
+  %else %if &quote=N %then %let quote=;
   %local i item buffer;
   %let i=1;
   %do %while (%qscan(&IN_STR,&i,%str(&indlm)) ne %str() ) ;

base/mf_isint.sas

@@ -0,0 +1,33 @@
+/**
+  @file
+  @brief Returns 1 if the variable contains only digits 0-9, else 0
+  @details Note that numerics containing any punctuation (including decimals
+    or exponents) will be flagged zero.
+
+  If you'd like support for this, then do raise an issue (or even better, a
+  pull request!)
+
+  Usage:
+
+      %put %mf_isint(1) returns 1;
+      %put %mf_isint(1.1) returns 0;
+      %put %mf_isint(%str(1,1)) returns 0;
+
+  @param [in] arg input value to check
+
+  @version 9.2
+**/
+
+%macro mf_isint(arg
+)/*/STORE SOURCE*/;
+  /* remove minus sign if exists */
+
+  %local val;
+  %if "%substr(%str(&arg),1,1)"="-" %then %let val=%substr(%str(&arg),2);
+  %else %let val=&arg;
+
+  /* check remaining chars */
+  %if %sysfunc(findc(%str(&val),,kd)) %then %do;0%end;
+  %else %do;1%end;
+
+%mend mf_isint;

base/mf_mkdir.sas

@@ -51,7 +51,7 @@ Usage:
     %end;
 
     /*
-      Now create the directory.  Complain loudly of any errors.
+      Now create the directory.  Complain loudly of any errs.
     */
 
     %let dname = %sysfunc(dcreate(&child, &parent));

base/mf_nobs.sas

@@ -12,7 +12,7 @@
   @param libds library.dataset
 
   @return output returns result of the attrn value supplied, or log message
-    if error.
+    if err.
 
 
   @version 9.2

base/mp_copyfolder.sas

@@ -54,6 +54,8 @@
 
   /* create folders and copy content */
   data _null_;
+    length msg $200;
+    call missing(msg);
     set work.&tempds;
     if _n_ = 1 then dpos+sum(length(directory),2);
     filepath2="&target/"!!substr(filepath,dpos);
@@ -63,9 +65,9 @@
       rc1=filename(fref1,filepath,'disk','recfm=n');
       rc2=filename(fref2,filepath2,'disk','recfm=n');
       if fcopy(fref1,fref2) ne 0 then do;
-        sysmsg=sysmsg();
+        msg=sysmsg();
         putlog "%str(ERR)OR: Unable to copy " filepath " to " filepath2;
-        putlog sysmg=;
+        putlog msg=;
       end;
     end;
     rc=filename(fref1);

base/mp_ds2fmtds.sas

@@ -1,16 +1,22 @@
 /**
   @file
-  @brief Converts every value in a dataset to it's formatted value
+  @brief Converts every value in a dataset to formatted value
   @details Converts every value to it's formatted value.  All variables will
-  become character, and will be in the same order.
+  become character, and will be in the same order as the original dataset.
+
+  Lengths will be adjusted according to the format lengths, where applicable.
 
   Usage:
 
       %mp_ds2fmtds(sashelp.cars,work.cars)
+      %mp_ds2fmtds(sashelp.vallopt,vw_vallopt)
 
   @param [in] libds The library.dataset to be converted
   @param [out] outds The dataset to create.
 
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+
   <h4> Related Macros <h4>
   @li mp_jsonout.sas
 
@@ -22,8 +28,9 @@
 )/*/STORE SOURCE*/;
 
 /* validations */
-%if not %sysfunc(exist(&libds)) %then %do;
-  %put %str(WARN)ING:  &libds does not exist;
+
+%if not %mf_existds(libds=&libds) %then %do;
+  %put %str(WARN)ING:  &libds does not exist as either a VIEW or DATASET;
   %return;
 %end;
 %if %index(&libds,.)=0 %then %let libds=WORK.&libds;

base/mp_ds2inserts.sas

@@ -116,6 +116,7 @@ data _null_;
     if _n_>&maxobs then stop;
   %end;
   length _____str $32767;
+  call missing(_____str);
   format _numeric_ best.;
   format _character_ ;
   %local i comma var vtype vfmt;

base/mp_filtervalidate.sas

@@ -33,8 +33,8 @@
   @param [in] targetds The target dataset against which to verify the query
   @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
   @param [out] outds= (work.mp_filtervalidate) Output dataset containing the
-  error / warning message, if one exists.  If this table contains any rows,
-  there are problems!
+    err / warning message, if one exists.  If this table contains any rows,
+    there are problems!
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas

base/mp_getcols.sas

@@ -51,7 +51,7 @@ data &outds(keep=name type length varnum format label ddtype);
     else if formatd=0 then format=cats(format2,formatl,'.');
     else format=cats(format2,formatl,'.',formatd);
     type='N';
-    if format=:'DATETIME' then ddtype='DATETIME';
+    if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
     else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
       or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
       or format=:'MONYY'

base/mp_getconstraints.sas

@@ -48,17 +48,17 @@
 %let vw=%mf_getuniquename(prefix=mp_getconstraints_vw_);
 data &vw /view=&vw;
   set sashelp.vcncolu;
-  where TABLE_CATALOG="&lib";
+  where table_catalog="&lib";
 
   /* use retain approach to reset the constraint order with each constraint */
   length tmp $1000;
   retain tmp;
   drop tmp;
-  if tmp ne catx('|',libref,table_name,constraint_type,constraint_name) then do;
+  if tmp ne catx('|',table_catalog,table_name,constraint_name) then do;
     constraint_order=1;
   end;
   else constraint_order+1;
-  tmp=catx('|',libref, table_name, constraint_type,constraint_name);
+  tmp=catx('|',table_catalog, table_name,constraint_name);
 run;
 
 /* must use SQL as proc datasets does not support length changes */

base/mp_getddl.sas

@@ -139,7 +139,7 @@ run;
     %let curds=%scan(&dsnlist,&x);
     data _null_;
       file &fref mod;
-      length nm lab $1024 typ $20;
+      length lab $1024 typ $20;
       set &colinfo (where=(upcase(memname)="&curds")) end=last;
 
       if _n_=1 then do;

base/mp_getpk.sas

@@ -23,6 +23,10 @@
 
   Returns:
 
+|libref:$8.|dsn:$32.|memtype:$8.|dbms_memtype:$32.|typemem:$8.|memlabel:$256.|nvar:best.|compress:$8.|pk_fields:$512.|
+|---|---|---|---|---|---|---|---|---|
+|WORK|EXAMPLE|DATA| |DATA| |4|NO|TX_FROM DD_TYPE DD_SOURCE|
+
 
   @param [in] lib The libref to examine
   @param [in] ds= (0) Select the dataset to examine, else use 0 for all tables
@@ -37,6 +41,7 @@
 
   <h4> Related Macros </h4>
   @li mp_getpk.test.sas
+  @li mp_guesspk.sas
 
   @version 9.3
   @author Macro People Ltd

base/mp_guesspk.sas

@@ -1,32 +1,37 @@
 /**
-  @file mp_guesspk.sas
+  @file
   @brief Guess the primary key of a table
-  @details Tries to guess the primary key of a table based on the following logic:
+  @details Tries to guess the primary key of a table based on the following
+  logic:
 
       * Columns with nulls are ignored
       * Return only column combinations that provide unique results
-      * Start from one column, then move out to include composite keys of 2 to 6 columns
+      * Start from one column, then move out to composite keys of 2 to 6 columns
 
   The library of the target should be assigned before using this macro.
 
   Usage:
 
-      filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      filename mc url
+        "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
       %mp_guesspk(sashelp.class,outds=classpks)
 
   @param baseds The dataset to analyse
   @param outds= The output dataset to contain the possible PKs
-  @param max_guesses= The total number of possible primary keys to generate.  A
-    table is likely to have multiple unlikely PKs, so no need to list them all. Default=3.
-  @param min_rows= The minimum number of rows a table should have in order to try
-    and guess the PK.  Default=5.
+  @param max_guesses= (3) The total number of possible primary keys to generate.
+    A table may have multiple unlikely PKs, so no need to list them all.
+  @param min_rows= (5) The minimum number of rows a table should have in order
+    to try and guess the PK.
 
   <h4> SAS Macros </h4>
   @li mf_getvarlist.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
 
+  <h4> Related Macros </h4>
+  @li mp_getpk.sas
+
   @version 9.3
   @author Allan Bowe
 
@@ -196,7 +201,8 @@
           %let lev4=%scan(&posspks,&l);
           %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then %do;
             /* check for four level uniqueness */
-            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4) out=&tmpds noduprec;
+            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4)
+                out=&tmpds noduprec;
               by _all_;
             run;
             %if %mf_nobs(&tmpds)=&rows %then %do;
@@ -233,7 +239,8 @@
             %let lev5=%scan(&posspks,&m);
             %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then %do;
               /* check for four level uniqueness */
-              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5) out=&tmpds noduprec;
+              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5)
+                  out=&tmpds noduprec;
                 by _all_;
               run;
               %if %mf_nobs(&tmpds)=&rows %then %do;
@@ -282,7 +289,8 @@
                 run;
                 %if %mf_nobs(&tmpds)=&rows %then %do;
                   proc sql;
-                  insert into &outds values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
+                  insert into &outds
+                    values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
                   %if %mf_nobs(&outds) ge &max_guesses %then %do;
                     %put &sysmacroname: Max PKs reached at Level 6 for &baseds;
                     %return;

base/mp_hashdataset.sas

@@ -21,6 +21,7 @@
 
   @param [in] libds dataset to hash
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
+  @param [in] iftrue= A condition under which the macro should be executed.
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
   will contain one column (hashkey) with one observation (a hex32.
   representation of the input hash)
@@ -35,10 +36,14 @@
 %macro mp_hashdataset(
   libds,
   outds=,
-  salt=
+  salt=,
+  iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
+
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
   %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
-    %put %str(WARN)ING: Dataset &libds is empty;, or is not a dataset;
+    %put %str(WARN)ING: Dataset &libds is empty, or is not a dataset;
   %end;
   %else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
     %put %str(ERR)OR: Dataset &libds is not a dataset;

base/mp_init.sas

@@ -0,0 +1,65 @@
+/**
+  @file
+  @brief Initialise session with useful settings and variables
+  @details Implements a "strict" set of SAS options for use in defensive
+    programming.  Highly recommended, if you want your code to run on some
+    other machine.
+
+    This macro is recommended to be compiled and invoked in the `initProgram`
+    for SASjs [Jobs](https://cli.sasjs.io/sasjsconfig.html#jobConfig_initProgram
+    ), [Services](
+    https://cli.sasjs.io/sasjsconfig.html#serviceConfig_initProgram) and [Tests]
+    (https://cli.sasjs.io/sasjsconfig.html#testConfig_initProgram).
+
+    For non SASjs projects, you could invoke in the autoexec, or in your own
+    solution initialisation macro.
+
+
+    If you have a good idea for another useful option, setting, or global
+    variable - feel free to [raise an issue](
+    https://github.com/sasjs/core/issues/new)!
+
+    All global variables are prefixed with "SASJS" (unless modified with the
+    prefix parameter).
+
+  @param [in] prefix= (SASJS) The prefix to apply to the global macro variables
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_init(prefix=
+)/*/STORE SOURCE*/;
+
+  %global
+    &prefix._INIT_NUM   /* initialisation time as numeric             */
+    &prefix._INIT_DTTM  /* initialisation time in E8601DT26.6 format */
+  ;
+  %if %eval(&&&prefix._INIT_NUM>0) %then %return;  /* only run once */
+
+  data _null_;
+    dttm=datetime();
+    call symputx("&prefix._init_num",dttm);
+    call symputx("&prefix._init_dttm",put(dttm,E8601DT26.6));
+  run;
+
+  options
+    noautocorrect           /* disallow misspelled procedure names            */
+    compress=CHAR           /* default is none so ensure we have something!   */
+    datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
+    %str(err)orcheck=STRICT /* catch errs in libname/filename statements      */
+    fmterr                  /* ensure err when a format cannot be found       */
+    mergenoby=%str(ERR)OR   /* throw err when a merge has no BY variables     */
+    missing=.               /* changing this can cause hard to detect errs    */
+    noquotelenmax           /* avoid warnings for long strings                */
+    noreplace               /* avoid overwriting permanent datasets           */
+    ps=max                  /* reduce log size slightly                       */
+    validmemname=COMPATIBLE /* avoid special characters etc in table names    */
+    validvarname=V7         /* avoid special characters etc in variable names */
+    varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
+    varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
+  ;
+
+%mend mp_init;

base/mp_jsonout.sas

@@ -63,7 +63,7 @@
 %if &action=OPEN %then %do;
   options nobomfile;
   data _null_;file &jref encoding='utf-8';
-    put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
+    put '{"PROCESSED_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '"';
   run;
 %end;
 %else %if (&action=ARR or &action=OBJ) %then %do;

base/mp_lockanytable.sas

@@ -239,7 +239,7 @@ run;
   %let abortme=1;
 %end;
 
-/* catch errors - mp_abort must be called outside of a logic block */
+/* catch errs - mp_abort must be called outside of a logic block */
 %mp_abort(iftrue=(&abortme=1),
   msg=%superq(msg),
   mac=&sysmacroname

base/mp_makedata.sas

@@ -0,0 +1,87 @@
+/**
+  @file
+  @brief Create sample data based on the structure of an empty table
+  @details Many SAS projects involve sensitive datasets.  One way to _ensure_
+    the data is anonymised, is never to receive it in the first place!  Often
+    consultants are provided with empty tables, and expected to create complex
+    ETL flows.
+
+    This macro can help by taking an empty table, and populating it with data
+    according to the variable types and formats.
+
+    TODO:
+      @li Respect PKs
+      @li Respect NOT NULLs
+      @li Consider dates, datetimes, times, integers etc
+
+  Usage:
+
+      proc sql;
+      create table work.example(
+        TX_FROM float format=datetime19.,
+        DD_TYPE char(16),
+        DD_SOURCE char(2048),
+        DD_SHORTDESC char(256),
+        constraint pk primary key(tx_from, dd_type,dd_source),
+        constraint nnn not null(DD_SHORTDESC)
+      );
+      %mp_makedata(work.example)
+
+  @param [in] libds The empty table in which to create data
+  @param [out] obs= (500) The number of records to create.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_getvarlen.sas
+  @li mf_nobs.sas
+  @li mp_getcols.sas
+  @li mp_getpk.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_makedata(libds
+  ,obs=500
+)/*/STORE SOURCE*/;
+
+%local ds1 c1 n1 i col charvars numvars;
+
+%if %mf_nobs(&libds)>0 %then %do;
+  %put &sysmacroname: &libds has data, it will not be recreated;
+  %return;
+%end;
+
+%local ds1 c1 n1;
+%let ds1=%mf_getuniquename(prefix=mp_makedata);
+%let c1=%mf_getuniquename(prefix=mp_makedatacol);
+%let n1=%mf_getuniquename(prefix=mp_makedatacol);
+data &ds1;
+  if 0 then set &libds;
+  do _n_=1 to &obs;
+    &c1=repeat(uuidgen(),10);
+    &n1=ranuni(1)*5000000;
+    drop &c1 &n1;
+    %let charvars=%mf_getvarlist(&libds,typefilter=C);
+    %do i=1 %to %sysfunc(countw(&charvars));
+      %let col=%scan(&charvars,&i);
+      &col=subpad(&c1,1,%mf_getvarlen(&libds,&col));
+    %end;
+
+    %let numvars=%mf_getvarlist(&libds,typefilter=N);
+    %do i=1 %to %sysfunc(countw(&numvars));
+      %let col=%scan(&numvars,&i);
+      &col=&n1;
+    %end;
+    output;
+  end;
+run;
+
+proc append base=&libds data=&ds1;
+run;
+
+proc sql;
+drop table &ds1;
+
+%mend mp_makedata;

base/mp_reseterror.sas

@@ -0,0 +1,27 @@
+/**
+  @file
+  @brief Reset when an err condition occurs
+  @details When building apps, sometimes an operation must be attempted that
+  can cause an err condition.  There is no try catch in SAS! So the err state
+  must be caught and reset.
+
+  This macro attempts to do that reset.
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_reseterror(
+)/*/STORE SOURCE*/;
+
+options obs=max replace nosyntaxcheck;
+%let syscc=0;
+
+%if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
+  data _null_;
+    rc=stpsrvset('program error', 0);
+  run;
+%end;
+
+%mend mp_reseterror;

base/mp_searchdata.sas

@@ -11,22 +11,28 @@
   Usage:
 
       %mp_searchdata(lib=sashelp, string=Jan)
-      %mp_searchdata(lib=sashelp, numval=1)
+      %mp_searchdata(lib=sashelp, ds=bird, numval=1)
+      %mp_searchdata(lib=sashelp, ds=class, string=l,outobs=5)
 
 
   Outputs zero or more tables to an MPSEARCH library with specific records.
 
-  @param lib=  the libref to search (should be already assigned)
-  @param ds= the dataset to search (leave blank to search entire library)
-  @param string= the string value to search
-  @param numval= the numeric value to search (must be exact)
-  @param outloc= the directory in which to create the output datasets with
-    matching rows.  Will default to a subfolder in the WORK library.
-  @param outobs= set to a positive integer to restrict the number of
+  @param [in] lib=  The libref to search (should be already assigned)
+  @param [in] ds= The dataset to search (leave blank to search entire library)
+  @param [in] string= String value to search (case sensitive, can be partial)
+  @param [in] numval= Numeric value to search (must be exact)
+  @param [out] outloc= (0) Optionally specify the directory in which to
+    create the the output datasets with matching rows.  By default it will
+    write them to a temporary subdirectory within the WORK folder.
+  @param [out] outlib= (MPSEARCH) Assign a different libref to the output
+    library containing the matching datasets / records
+  @param [in] outobs= set to a positive integer to restrict the number of
     observations
-  @param filter_text= add a (valid) filter clause to further filter the results
+  @param [in] filter_text= (1=1) Add a (valid) filter clause to further filter
+    the results.
 
   <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
   @li mf_mkdir.sas
@@ -36,11 +42,12 @@
   @author Allan Bowe
 **/
 
-%macro mp_searchdata(lib=sashelp
+%macro mp_searchdata(lib=
   ,ds=
   ,string= /* the query will use a contains (?) operator */
   ,numval= /* numeric must match exactly */
-  ,outloc=%sysfunc(pathname(work))/mpsearch
+  ,outloc=0
+  ,outlib=MPSEARCH
   ,outobs=-1
   ,filter_text=%str(1=1)
 )/*/STORE SOURCE*/;
@@ -57,8 +64,12 @@
 %if &string = %then %let type=N;
 %else %let type=C;
 
+%if "&outloc"="0" %then %do;
+  %let outloc=%sysfunc(pathname(work))/%mf_getuniquename();
+%end;
+
 %mf_mkdir(&outloc)
-libname mpsearch "&outloc";
+libname &outlib "&outloc";
 
 /* get the list of tables in the library */
 proc sql noprint;
@@ -70,11 +81,6 @@ select distinct memname into: table_list separated by ' '
 %end;
   ;
 /* check that we have something to check */
-proc sql
-%if &outobs>-1 %then %do;
-  outobs=&outobs
-%end;
-;
 %if %length(&table_list)=0 %then %put library &lib contains no tables!;
 /* loop through each table */
 %else %do table_num=1 %to %sysfunc(countw(&table_list,%str( )));
@@ -85,10 +91,10 @@ proc sql
   %end;
   %else %do;
     %let check_tm=%sysfunc(datetime());
-    /* build sql statement */
-    create table mpsearch.&table as select * from &lib..&table
-      where %unquote(&filter_text) and
-    (0
+    /* prep input */
+    data &outlib..&table;
+      set &lib..&table;
+      where %unquote(&filter_text) and ( 0
     /* loop through columns */
     %do colnum=1 %to %sysfunc(countw(&vars,%str( )));
       %let col=%scan(&vars,&colnum,%str( ));
@@ -102,15 +108,20 @@ proc sql
         or ("&col"n = &numval)
       %end;
     %end;
-    );
+      );
+    %if &outobs>-1 %then %do;
+      if _n_ > &outobs then stop;
+    %end;
+    run;
     %put Search query for &table took
       %sysevalf(%sysfunc(datetime())-&check_tm) seconds;
-    %if &sqlrc ne 0 %then %do;
-      %put %str(WAR)NING: SQLRC=&sqlrc when processing &table;
+    %if &syscc ne 0 %then %do;
+      %put %str(ERR)ROR: SYSCC=&syscc when processing &lib..&table;
       %return;
     %end;
-    %if %mf_nobs(mpsearch.&table)=0 %then %do;
-      drop table mpsearch.&table;
+    %if %mf_nobs(&outlib..&table)=0 %then %do;
+      proc sql;
+      drop table &outlib..&table;
     %end;
   %end;
 %end;

base/mp_sortinplace.sas

@@ -89,7 +89,7 @@ run;
 %let tempvw=%mf_getuniquename(prefix=&sysmacroname);
 proc sql;
 create view work.&tempvw as select * from &lib..&ds
-order by %mf_getquotedstr(&sortkey,quote=%str());
+order by %mf_getquotedstr(&sortkey,quote=N);
 
 /* append sorted data */
 proc append base=&lib..&tempds2 data=work.&tempvw;

meta/mm_createwebservice.sas

@@ -95,7 +95,7 @@ data _null_;
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
   put '  data _null_;file &jref encoding=''utf-8''; ';
-  put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
+  put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
   put '  run; ';
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';

tests/crossplatform/mf_isint.test.sas

@@ -0,0 +1,33 @@
+/**
+  @file
+  @brief Testing mf_isint macro
+
+  <h4> SAS Macros </h4>
+  @li mf_isint.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(
+    "%mf_isint(1)"="1"
+  ),
+  desc=Checking basic mf_isint(1),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_isint(1.1)"="0"
+  ),
+  desc=Checking basic mf_isint(1.1),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_isint(-1)"="1"
+  ),
+  desc=Checking mf_isint(-1),
+  outds=work.test_results
+)

tests/crossplatform/mp_ds2fmtds.test.sas

@@ -15,10 +15,12 @@ filename inc temp;
 data _null_;
   set work.test;
   file inc;
-  line=cats('%mp_ds2fmtds(sashelp.',memname,',',memname,')');
+  libds=cats('sashelp.',memname);
+  if exist(libds) then line=cats('%mp_ds2fmtds(',libds,',',memname,')');
   put line;
 run;
 
+options obs=50;
 %inc inc;
 
 %mp_assert(

tests/crossplatform/mp_init.test.sas

@@ -0,0 +1,24 @@
+/**
+  @file
+  @brief Testing mp_gsubfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_init.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * Test 1 - mp_init.sas actually already ran as part of testinit
+  * So lets test to make sure it will not run again
+  */
+
+%let initial_value=&sasjs_init_num;
+
+%mp_init();
+
+%mp_assert(
+  iftrue=("&initial_value"="&sasjs_init_num"),
+  desc=Check that mp_init() did not run twice,
+  outds=work.test_results
+)

tests/crossplatform/mp_lib2inserts.test.sas

@@ -11,9 +11,10 @@
 **/
 
 /* grab 20 datasets from SASHELP */
-%let path=%sysfunc(pathname(work));
+%let work=%sysfunc(pathname(work));
+%let path=&work/new;
 %mf_mkdir(&path)
-libname sashlp "&path";
+libname sashlp "&work";
 proc sql noprint;
 create table members as
   select distinct lowcase(memname) as memname
@@ -31,6 +32,7 @@ run;
 %mp_lib2inserts(sashlp, schema=work, outref=tempref,maxobs=50)
 
 /* check if it actually runs */
+libname sashlp "&path";
 options source2;
 %inc tempref;
 

tests/crossplatform/mp_lockfilecheck.test.sas

@@ -5,6 +5,7 @@
   <h4> SAS Macros </h4>
   @li mp_lockfilecheck.sas
   @li mp_assert.sas
+  @li mp_reseterror.sas
 
 **/
 
@@ -29,6 +30,8 @@ data work.test; a=1;run;
 
 %mp_lockfilecheck(sashelp.class)
 
+%mp_reseterror()
+
 %mp_assert(
   iftrue=(&success=1),
   desc=Checking sashelp table cannot be locked,

tests/crossplatform/mp_reseterror.test.sas

@@ -0,0 +1,23 @@
+/**
+  @file
+  @brief Testing mp_reseterror macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mp_reseterror.sas
+
+**/
+
+
+/* cause an error */
+
+lock sashelp.class;
+
+/* recover ? */
+%mp_reseterror()
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking error condition was fixed,
+  outds=work.test_results
+)

tests/crossplatform/mp_searchdata.test.sas

@@ -0,0 +1,29 @@
+/**
+  @file
+  @brief Testing mp_searchdata.sas
+
+  <h4> SAS Macros </h4>
+  @li mp_searchdata.sas
+  @li mp_assert.sas
+
+
+**/
+
+/** Test 1 - generic useage */
+
+%mp_searchdata(lib=sashelp, ds=class, string=a)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errors in regular usage,
+  outds=work.test_results
+)
+
+/** Test 2 - with obs issue  */
+
+%mp_searchdata(lib=sashelp, ds=class, string=l,outobs=5)
+
+%mp_assert(
+  iftrue=("&SYSWARNINGTEXT" = ""),
+  desc=Ensuring WARN status is clean,
+  outds=work.test_results
+)

tests/testinit.sas

@@ -2,11 +2,17 @@
   @file
   @brief init file for tests
 
+  <h4> SAS Macros </h4>
+  @li mp_init.sas
+
 **/
 
 /* location in metadata or SAS Drive for temporary files */
 %let mcTestAppLoc=/Public/temp/macrocore;
 
+/* set defaults */
+%mp_init()
+
 %macro loglevel();
   %if &_debug=2477 %then %do;
     options mprint;

tests/viyaonly/mv_createwebservice.test.sas

@@ -33,6 +33,7 @@ run;
 %put TEST1: checking web service code;
 data work.test_results;
   length test_description $256 test_result $4 test_comments $256;
+  if _n_=1 then call missing (of _all_);
   infile compare end=eof;
   input;
   if eof then do;

viya/mv_createwebservice.sas

@@ -243,7 +243,7 @@ data _null_;
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
   put '  data _null_;file &jref encoding=''utf-8''; ';
-  put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
+  put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
   put '  run; ';
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';

viya/mv_getfoldermembers.sas

@@ -100,6 +100,8 @@ options noquotelenmax;
   %local href cnt;
   %let cnt=0;
   data _null_;
+    length rel href $512;
+    call missing(rel,href);
     set &libref1..links;
     if rel='members' then do;
       url=cats("'","&base_uri",href,"?limit=10000'");
@@ -123,6 +125,7 @@ options noquotelenmax;
   libname &libref2 JSON fileref=&fname2;
   data &outds;
     length id $36 name $128 uri $64 type $32 description $256;
+    if _n_=1 then call missing (of _all_);
     set &libref2..items;
   run;
   filename &fname2 clear;

viya/mv_getjobcode.sas

@@ -92,6 +92,8 @@ data;run;
 %local joburi;
 %let joburi=0;
 data _null_;
+  length name uri $512;
+  call missing(name,uri);
   set &foldermembers;
   if name="&name" and uri=:'/jobDefinitions/definitions'
     then call symputx('joburi',uri);

viya/mv_jobexecute.sas

@@ -115,6 +115,8 @@ data;run;
 %local joburi;
 %let joburi=0;
 data _null_;
+  length name uri $512;
+  call missing(name,uri);
   set &foldermembers;
   if name="&name" and uri=:'/jobDefinitions/definitions'
     then call symputx('joburi',uri);

viya/mv_jobwaitfor.sas

@@ -168,6 +168,7 @@ run;
 data &outds;
   format _program uri $128. state $32. stateDetails $32. timestamp datetime19.
     jobparams $32767.;
+  call missing (of _all_);
   stop;
 run;