Merge pull request #126 from sasjs/applyformats

Applyformats

.git-hooks/pre-commit

@@ -1,2 +1,11 @@
-#!/bin/sh
-sasjs lint
+#!/bin/bash
+sasjs lint
+
+# Avoid commits to the master branch
+BRANCH=`git rev-parse --abbrev-ref HEAD`
+
+if [[ "$BRANCH" =~ ^(master|main|develop)$ ]]; then
+  echo "You are on branch $BRANCH. Are you sure you want to commit to this branch?"
+  echo "If so, commit with -n to bypass the pre-commit hook."
+  exit 1
+fi

.sasjslint

@@ -6,7 +6,7 @@
     "hasMacroParentheses": true,
     "noNestedMacros": false,
     "noSpacesInFileNames": true,
-    "maxLineLength": 230,
+    "maxLineLength": 300,
     "lowerCaseFileNames": true,
     "noTabIndentation": true,
     "indentationMultiple": 2

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2020 (Allan Bowe)
+Copyright 2021 (Allan Bowe)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

README.md

@@ -1,7 +1,7 @@
 # Macro Core
 [![npm package][npm-image]][npm-url]
 [![Github Workflow][githubworkflow-image]][githubworkflow-url]
-[![npm](https://img.shields.io/npm/dt/@sasjs/core)]()
+![npm](https://img.shields.io/npm/dt/@sasjs/core)
 ![Snyk Vulnerabilities for npm package](https://img.shields.io/snyk/vulnerabilities/npm/@sasjs/core)
 [![License](https://img.shields.io/apm/l/atomic-design-ui.svg)](/LICENSE)
 ![GitHub top language](https://img.shields.io/github/languages/top/sasjs/core)
@@ -31,21 +31,21 @@ Documentation: https://core.sasjs.io
 
 ## Components
 
-### **base** library (SAS9/Viya)
+### BASE library (SAS9/Viya)
 
 - OS independent
 - Not metadata aware
 - No X command
 - Prefixes: _mf_, _mp_
 
-#### **fcmp** library (SAS9/Viya)
+#### FCMP library (SAS9/Viya)
 - Function and macro names are identical, except for special cases
 - Prefixes: _mcf_
 
 The fcmp macros are used to generate fcmp functions, and can be used with or
 without the `proc fcmp` wrapper.
 
-### **meta** library (SAS9 only)
+### META library (SAS9 only)
 
 Macros used in SAS EBI, which connect to the metadata server.
 
@@ -54,7 +54,7 @@ Macros used in SAS EBI, which connect to the metadata server.
 - No X command
 - Prefixes: _mm_
 
-### **server** library (@sasjs/server only)
+### SERVER library (@sasjs/server only)
 These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
 
 - OS independent
@@ -62,7 +62,7 @@ These macros are used for building applications using [@sasjs/server](https://se
 - No X command
 - Prefixes: _ms_
 
-### **viya** library (Viya only)
+### VIYA library (Viya only)
 
 Macros used for interfacing with SAS Viya.
 
@@ -70,14 +70,14 @@ Macros used for interfacing with SAS Viya.
 - No X command
 - Prefixes: _mv_, _mvf_
 
-### **metax** library (SAS9 only)
+### METAX library (SAS9 only)
 
 - OS specific
 - Metadata aware
 - X command enabled
 - Prefixes: _mmw_,_mmu_,_mmx_
 
-### **lua** library
+### LUA library
 
 Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
 

base/mf_dedup.sas

@@ -0,0 +1,54 @@
+/**
+  @file
+  @brief de-duplicates a macro string
+  @details Removes all duplicates from a string of words.  A delimeter can be
+  chosen.  Is inspired heavily by this excellent [macro](
+  https://github.com/scottbass/SAS/blob/master/Macro/dedup_mstring.sas) from
+  [Scott Base](https://www.linkedin.com/in/scottbass).  Case sensitive.
+
+  Usage:
+
+      %let str=One two one two and through and through;
+      %put %mf_dedup(&str);
+      %put %mf_dedup(&str,outdlm=%str(,));
+
+  Which returns:
+
+      > One two one and through
+      > One,two,one,and,through
+
+  @param [in] str String to be deduplicated
+  @param [in] indlm= ( ) Delimeter of the input string
+  @param [out] outdlm= ( ) Delimiter of the output string
+
+  <h4> Related Macros </h4>
+  @li mf_trimstr.sas
+  @li mf_wordsinstr1butnotstr2.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_dedup(str
+  ,indlm=%str( )
+  ,outdlm=%str( )
+)/*/STORE SOURCE*/;
+
+%local num word i pos out;
+
+%* loop over each token, searching the target for that token ;
+%let num=%sysfunc(countc(%superq(str),%str(&indlm)));
+%do i=1 %to %eval(&num+1);
+  %let word=%scan(%superq(str),&i,%str(&indlm));
+  %let pos=%sysfunc(indexw(&out,&word,%str(&outdlm)));
+  %if (&pos eq 0) %then %do;
+    %if (&i gt 1) %then %let out=&out%str(&outdlm);
+    %let out=&out&word;
+  %end;
+%end;
+
+%unquote(&out)
+
+%mend mf_dedup;
+
+

base/mf_existfeature.sas

@@ -5,7 +5,7 @@
     Run without arguments to see a list of detectable features.
     Note - this list is based on known versions of SAS rather than
     actual feature detection, as that is tricky / impossible to do
-    without generating errors in most cases.
+    without generating errs in most cases.
 
         %put %mf_existfeature(PROCLUA);
 

base/mf_getattrc.sas

@@ -10,7 +10,7 @@
   @param attr full list in [documentation](
     https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000147794.htm)
   @return output returns result of the attrc value supplied, or -1 and log
-    message if error.
+    message if err.
 
   @version 9.2
   @author Allan Bowe

base/mf_getattrn.sas

@@ -10,7 +10,7 @@
   @param attr Common values are NLOBS and NVARS, full list in [documentation](
   http://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000212040.htm)
   @return output returns result of the attrn value supplied, or -1 and log
-    message if error.
+    message if err.
 
   @version 9.2
   @author Allan Bowe

base/mf_getengine.sas

@@ -12,9 +12,10 @@
   contributors of Chris Hemedingers blog [post](
   http://blogs.sas.com/content/sasdummy/2013/06/04/find-a-sas-library-engine/)
 
-  @param libref Library reference (also accepts a 2 level libds ref).
+  @param [in] libref Library reference (also accepts a 2 level libds ref).
 
-  @return output returns the library engine for the FIRST library encountered.
+  @return output returns the library engine (uppercase) for the FIRST library
+    encountered.
 
   @warning will only return the FIRST library engine - for concatenated
     libraries, with different engines, inconsistent results may be encountered.
@@ -46,7 +47,7 @@
     %let rc= %sysfunc(close(&dsid));
   %end;
 
-  &engine
+  %upcase(&engine)
 
 %mend mf_getengine;
 

base/mf_getfmtlist.sas

@@ -0,0 +1,61 @@
+/**
+  @file
+  @brief Returns a distinct list of formats from a table
+  @details Reads the dataset header and returns a distinct list of formats
+  applied.
+
+        %put NOTE- %mf_getfmtlist(sashelp.prdsale);
+        %put NOTE- %mf_getfmtlist(sashelp.shoes);
+        %put NOTE- %mf_getfmtlist(sashelp.demographics);
+
+  returns:
+
+  > DOLLAR $CHAR W MONNAME
+  > $CHAR BEST DOLLAR
+  > BEST Z $CHAR COMMA PERCENTN
+
+
+  @param [in] libds Two part library.dataset reference.
+
+  <h4> SAS Macros </h4>
+  @li mf_getfmtname.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_getfmtlist(libds
+)/*/STORE SOURCE*/;
+/* declare local vars */
+%local out dsid nvars x rc fmt;
+
+/* open dataset in macro */
+%let dsid=%sysfunc(open(&libds));
+
+/* continue if dataset exists */
+%if &dsid %then %do;
+  /* loop each variable in the dataset */
+  %let nvars=%sysfunc(attrn(&dsid,NVARS));
+  %do x=1 %to &nvars;
+    /* grab format and check it exists */
+    %let fmt=%sysfunc(varfmt(&dsid,&x));
+    %if %quote(&fmt) ne %quote() %then %let fmt=%mf_getfmtname(&fmt);
+    %else %do;
+      /* assign default format depending on variable type */
+      %if %sysfunc(vartype(&dsid, &x))=C %then %let fmt=$CHAR;
+      %else %let fmt=BEST;
+    %end;
+    /* concatenate unique list of formats */
+    %if %sysfunc(indexw(&out,&fmt,%str( )))=0 %then %let out=&out &fmt;
+  %end;
+  %let rc=%sysfunc(close(&dsid));
+%end;
+%else %do;
+  %put &sysmacroname: Unable to open &libds (rc=&dsid);
+  %put &sysmacroname: SYSMSG= %sysfunc(sysmsg());
+  %let rc=%sysfunc(close(&dsid));
+%end;
+/* send them out without spaces or quote markers */
+%do;%unquote(&out)%end;
+%mend mf_getfmtlist;

base/mf_getfmtname.sas

@@ -0,0 +1,44 @@
+/**
+  @file
+  @brief Extracts a format name from a fully defined format
+  @details Converts formats in like $thi3. and th13.2 $THI and TH.
+  Usage:
+
+      %put %mf_getfmtname(8.);
+      %put %mf_getfmtname($4.);
+      %put %mf_getfmtname(comma14.10);
+
+  Returns:
+
+  > W
+  > $CHAR
+  > COMMA
+
+  Note that system defaults are inferred from the values provided.
+
+  @param [in] fmt The fully defined format. If left blank, nothing is returned.
+
+  @returns The name (without width or decimal) of the format.
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_getfmtname(fmt
+)/*/STORE SOURCE*/ /minoperator mindelimiter=' ';
+
+%local out dsid nvars x rc fmt;
+
+/* extract actual format name from the format definition */
+%let fmt=%scan(&fmt,1,.);
+%do %while(%substr(&fmt,%length(&fmt),1) in 1 2 3 4 5 6 7 8 9 0);
+  %if %length(&fmt)=1 %then %let fmt=W;
+  %else %let fmt=%substr(&fmt,1,%length(&fmt)-1);
+%end;
+
+%if &fmt=$ %then %let fmt=$CHAR;
+
+/* send them out without spaces or quote markers */
+%do;%unquote(%upcase(&fmt))%end;
+%mend mf_getfmtname;

base/mf_getquotedstr.sas

@@ -15,12 +15,12 @@
   for:
 > "these","words","are","double","quoted"
 
-  @param in_str the unquoted, spaced delimited string to transform
-  @param dlm= the delimeter to be applied to the output (default comma)
-  @param indlm= the delimeter used for the input (default is space)
-  @param quote= the quote mark to apply (S=Single, D=Double). If any other value
-    than uppercase S or D is supplied, then that value will be used as the
-    quoting character.
+  @param [in] in_str The unquoted, spaced delimited string to transform
+  @param [in] dlm= (,) The delimeter to be applied to the output (default comma)
+  @param [in] indlm= ( ) The delimeter used for the input (default is space)
+  @param [in] quote= (S) The quote mark to apply (S=Single, D=Double, N=None).
+    If any other value than uppercase S or D is supplied, then that value will
+    be used as the quoting character.
   @return output returns a string with the newly quoted / delimited output.
 
   @version 9.2
@@ -28,11 +28,15 @@
 **/
 
 
-%macro mf_getquotedstr(IN_STR,DLM=%str(,),QUOTE=S,indlm=%str( )
+%macro mf_getquotedstr(IN_STR
+  ,DLM=%str(,)
+  ,QUOTE=S
+  ,indlm=%str( )
 )/*/STORE SOURCE*/;
-  %if &quote=S %then %let quote=%str(%');
-  %else %if &quote=D %then %let quote=%str(%");
-  %else %let quote=%str();
+  /* credit Rowland Hale  - byte34 is double quote, 39 is single quote */
+  %if &quote=S %then %let quote=%qsysfunc(byte(39));
+  %else %if &quote=D %then %let quote=%qsysfunc(byte(34));
+  %else %if &quote=N %then %let quote=;
   %local i item buffer;
   %let i=1;
   %do %while (%qscan(&IN_STR,&i,%str(&indlm)) ne %str() ) ;

base/mf_getvarlist.sas

@@ -70,5 +70,5 @@
     %put &sysmacroname: SYSMSG= %sysfunc(sysmsg());
     %let rc=%sysfunc(close(&dsid));
   %end;
-  &outvar
+  %do;%unquote(&outvar)%end;
 %mend mf_getvarlist;

base/mf_isint.sas

@@ -0,0 +1,33 @@
+/**
+  @file
+  @brief Returns 1 if the variable contains only digits 0-9, else 0
+  @details Note that numerics containing any punctuation (including decimals
+    or exponents) will be flagged zero.
+
+  If you'd like support for this, then do raise an issue (or even better, a
+  pull request!)
+
+  Usage:
+
+      %put %mf_isint(1) returns 1;
+      %put %mf_isint(1.1) returns 0;
+      %put %mf_isint(%str(1,1)) returns 0;
+
+  @param [in] arg input value to check
+
+  @version 9.2
+**/
+
+%macro mf_isint(arg
+)/*/STORE SOURCE*/;
+  /* remove minus sign if exists */
+
+  %local val;
+  %if "%substr(%str(&arg),1,1)"="-" %then %let val=%substr(%str(&arg),2);
+  %else %let val=&arg;
+
+  /* check remaining chars */
+  %if %sysfunc(findc(%str(&val),,kd)) %then %do;0%end;
+  %else %do;1%end;
+
+%mend mf_isint;

base/mf_islibds.sas

@@ -0,0 +1,40 @@
+/**
+  @file
+  @brief Checks whether a string follows correct library.dataset format
+  @details Many macros in the core library accept a library.dataset parameter
+  referred to as 'libds'.  This macro validates the structure of that parameter,
+  eg:
+
+    @li 8 character libref?
+    @li 32 character dataset?
+    @li contains a period?
+
+  It does NOT check whether the dataset exists, or if the library is assigned.
+
+  Usage:
+
+      %put %mf_islibds(work.something)=1;
+      %put %mf_islibds(nolib)=0;
+      %put %mf_islibds(badlibref.ds)=0;
+      %put %mf_islibds(w.t.f)=0;
+
+  @param [in] libds The string to be checked
+
+  @return output Returns 1 if libds is valid, 0 if it is not
+
+  <h4> Related Macros </h4>
+  @li mf_islibds.test.sas
+  @li mp_validatecol.sas
+
+  @version 9.2
+**/
+
+%macro mf_islibds(libds
+)/*/STORE SOURCE*/;
+
+%local regex;
+%let regex=%sysfunc(prxparse(%str(/^[_a-z]\w{0,7}\.[_a-z]\w{0,31}$/i)));
+
+%sysfunc(prxmatch(&regex,&libds))
+
+%mend mf_islibds;

base/mf_mkdir.sas

@@ -51,7 +51,7 @@ Usage:
     %end;
 
     /*
-      Now create the directory.  Complain loudly of any errors.
+      Now create the directory.  Complain loudly of any errs.
     */
 
     %let dname = %sysfunc(dcreate(&child, &parent));

base/mf_nobs.sas

@@ -12,7 +12,7 @@
   @param libds library.dataset
 
   @return output returns result of the attrn value supplied, or log message
-    if error.
+    if err.
 
 
   @version 9.2

base/mf_wordsinstr1andstr2.sas

@@ -0,0 +1,54 @@
+/**
+  @file
+  @brief Returns words that are in both string 1 and string 2
+  @details  Compares two space separated strings and returns the words that are
+  in both.
+  Usage:
+
+      %put %mf_wordsInStr1andStr2(
+        Str1=blah sss blaaah brah bram boo
+        ,Str2=   blah blaaah brah ssss
+      );
+
+  returns:
+  > blah blaaah brah
+
+  @param str1= string containing words to extract
+  @param str2= used to compare with the extract string
+
+  @warning CASE SENSITIVE!
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_wordsInStr1andStr2(
+  Str1= /* string containing words to extract */
+  ,Str2= /* used to compare with the extract string */
+)/*/STORE SOURCE*/;
+
+%local count_base count_extr i i2 extr_word base_word match outvar;
+%if %length(&str1)=0 or %length(&str2)=0 %then %do;
+  %put %str(WARN)ING: empty string provided!;
+  %put base string (str1)= &str1;
+  %put compare string (str2) = &str2;
+  %return;
+%end;
+%let count_base=%sysfunc(countw(&Str2));
+%let count_extr=%sysfunc(countw(&Str1));
+
+%do i=1 %to &count_extr;
+  %let extr_word=%scan(&Str1,&i,%str( ));
+  %let match=0;
+  %do i2=1 %to &count_base;
+    %let base_word=%scan(&Str2,&i2,%str( ));
+    %if &extr_word=&base_word %then %let match=1;
+  %end;
+  %if &match=1 %then %let outvar=&outvar &extr_word;
+%end;
+
+  &outvar
+
+%mend mf_wordsInStr1andStr2;
+

base/mp_applyformats.sas

@@ -0,0 +1,181 @@
+/**
+  @file
+  @brief Apply a set of formats to a table
+  @details Applies a set of formats to one or more SAS datasets.  Can be used
+    to migrate formats from one table to another.  The input table must contain
+    the following columns:
+
+    @li lib - the libref of the table to be updated
+    @li ds - the dataset to be updated
+    @li var - the variable to be updated
+    @li fmt - the format to apply.  Missing or default ($CHAR, 8.) formats are
+      ignored.
+
+  The macro will abort in the following scenarios:
+
+    @li Libref not assigned
+    @li Dataset does not exist
+    @li Input table contains null or invalid values
+
+  Example usage:
+
+      data work.example;
+        set sashelp.prdsale;
+        format _all_ clear;
+      run;
+
+      %mp_getcols(sashelp.prdsale,outds=work.cols)
+
+      data work.cols2;
+        set work.cols;
+        lib='WORK';
+        ds='EXAMPLE';
+        var=name;
+        fmt=format;
+        keep lib ds var fmt;
+      run;
+
+      %mp_applyformats(work.cols2)
+
+  @param [in] inds The input dataset containing the formats to apply (and where
+    to apply them).  Example structure:
+  |LIB:$8.|DS:$32.|VAR:$32.|FMT:$49.|
+  |---|---|---|---|
+  |`WORK `|`EXAMPLE `|`ACTUAL `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`COUNTRY `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`DIVISION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`MONTH `|`MONNAME3. `|
+  |`WORK `|`EXAMPLE `|`PREDICT `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`PRODTYPE `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`PRODUCT `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`QUARTER `|`8. `|
+  |`WORK `|`EXAMPLE `|`REGION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`YEAR `|`8. `|
+
+  @param [out] errds= (0) Provide a libds reference here to export the
+    error messages to a table.  In this case, they will not be printed to the
+    log.
+
+  <h4> SAS Macros </h4>
+  @li mf_getengine.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_validatecol.sas
+
+
+  <h4> Related Macros </h4>
+  @li mp_getformats.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_applyformats(inds,errds=0
+)/*/STORE SOURCE*/;
+%local outds liblist i engine lib msg ;
+
+/**
+  * Validations
+  */
+proc sort data=&inds;
+  by lib ds var fmt;
+run;
+
+%if &errds=0 %then %let outds=%mf_getuniquename(prefix=mp_applyformats);
+%else %let outds=&errds;
+
+data &outds;
+  set &inds;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  length msg $128;
+  by lib ds var fmt;
+  if libref(lib) ne 0 then do;
+    msg=catx(' ','libref',lib,'is not assigned!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  if exist(cats(lib,'.',ds)) ne 1 then do;
+    msg=catx(' ','libds',lib,'.',ds,'does not exist!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  %mp_validatecol(fmt,FORMAT,is_fmt)
+  if is_fmt=0 then do;
+    msg=catx(' ','format',fmt,'on libds',lib,'.',ds,'.',var,'is not valid!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+
+  if first.ds then do;
+    retain dsid;
+    dsid=open(cats(lib,'.',ds));
+    if dsid=0 then do;
+      msg=catx(' ','libds',lib,'.',ds,' could not be opened!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+      return;
+    end;
+    if varnum(dsid,var)<1 then do;
+      msg=catx(' ','Variable',lib,'.',ds,'.',var,' was not found!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+    end;
+  end;
+  if last.ds then rc=close(dsid);
+run;
+
+proc sql noprint;
+select distinct lib into: liblist separated by ' ' from &inds;
+%put &=liblist;
+%do i=1 %to %sysfunc(countw(&liblist));
+  %let lib=%scan(&liblist,1);
+  %let engine=%mf_getengine(&lib);
+  %if &engine ne V9 and &engine ne BASE %then %do;
+    %let msg=&lib has &engine engine - formats cannot be applied;
+    proc sql;
+    insert into &outds set lib="&lib",ds="_all_",var="_all", msg="&msg" ;
+    %if &errds=0 %then %put %str(ERR)OR: &msg;
+  %end;
+%end;
+
+%if %mf_nobs(&outds)>0 %then %return;
+
+/**
+  * Validations complete - now apply the actual formats!
+  */
+%let fref=%mf_getuniquefileref();
+data _null_;
+  set &inds;
+  by lib ds var fmt;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  file &fref;
+  if first.lib then put 'proc datasets nolist lib=' lib ';';
+  if first.ds then put '  modify ' ds ';';
+  put '    format ' var fmt ';';
+  if last.ds then put '  run;';
+  if last.lib then put 'quit;';
+run;
+
+%inc &fref/source2;
+
+%if &errds=0 %then %do;
+  proc sql;
+  drop table &outds;
+%end;
+
+%mend mp_applyformats;

base/mp_copyfolder.sas

@@ -54,6 +54,8 @@
 
   /* create folders and copy content */
   data _null_;
+    length msg $200;
+    call missing(msg);
     set work.&tempds;
     if _n_ = 1 then dpos+sum(length(directory),2);
     filepath2="&target/"!!substr(filepath,dpos);
@@ -63,9 +65,9 @@
       rc1=filename(fref1,filepath,'disk','recfm=n');
       rc2=filename(fref2,filepath2,'disk','recfm=n');
       if fcopy(fref1,fref2) ne 0 then do;
-        sysmsg=sysmsg();
+        msg=sysmsg();
         putlog "%str(ERR)OR: Unable to copy " filepath " to " filepath2;
-        putlog sysmg=;
+        putlog msg=;
       end;
     end;
     rc=filename(fref1);

base/mp_ds2fmtds.sas

@@ -1,16 +1,22 @@
 /**
   @file
-  @brief Converts every value in a dataset to it's formatted value
+  @brief Converts every value in a dataset to formatted value
   @details Converts every value to it's formatted value.  All variables will
-  become character, and will be in the same order.
+  become character, and will be in the same order as the original dataset.
+
+  Lengths will be adjusted according to the format lengths, where applicable.
 
   Usage:
 
       %mp_ds2fmtds(sashelp.cars,work.cars)
+      %mp_ds2fmtds(sashelp.vallopt,vw_vallopt)
 
   @param [in] libds The library.dataset to be converted
   @param [out] outds The dataset to create.
 
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+
   <h4> Related Macros <h4>
   @li mp_jsonout.sas
 
@@ -22,8 +28,9 @@
 )/*/STORE SOURCE*/;
 
 /* validations */
-%if not %sysfunc(exist(&libds)) %then %do;
-  %put %str(WARN)ING:  &libds does not exist;
+
+%if not %mf_existds(libds=&libds) %then %do;
+  %put %str(WARN)ING:  &libds does not exist as either a VIEW or DATASET;
   %return;
 %end;
 %if %index(&libds,.)=0 %then %let libds=WORK.&libds;

base/mp_ds2inserts.sas

@@ -116,6 +116,7 @@ data _null_;
     if _n_>&maxobs then stop;
   %end;
   length _____str $32767;
+  call missing(_____str);
   format _numeric_ best.;
   format _character_ ;
   %local i comma var vtype vfmt;

base/mp_ds2md.sas

@@ -29,44 +29,44 @@
 
   Usage:
 
-      %mp_mdtablewrite(libds=sashelp.class,showlog=YES)
+      %mp_ds2md(sashelp.class)
 
+  @param [in] libds the library / dataset to create or read from.
+  @param [out] outref= (mdtable) Fileref to contain the markdown
+  @param [out] showlog= (YES) Set to NO to avoid printing markdown to the log
 
   <h4> SAS Macros </h4>
   @li mf_getvarlist.sas
   @li mf_getvarformat.sas
 
-  @param [in] libds= the library / dataset to create or read from.
-  @param [out] fref= Fileref to contain the markdown. Default=mdtable.
-  @param [out] showlog= set to YES to show the markdown in the log. Default=NO.
-
   @version 9.3
   @author Allan Bowe
 **/
 
-%macro mp_mdtablewrite(
-  libds=,
-  fref=mdtable,
-  showlog=NO
+%macro mp_ds2md(
+  libds,
+  outref=mdtable,
+  showlog=YES
 )/*/STORE SOURCE*/;
 
 /* check fileref is assigned */
-%if %sysfunc(fileref(&fref)) > 0 %then %do;
-  filename &fref temp;
+%if %sysfunc(fileref(&outref)) > 0 %then %do;
+  filename &outref temp;
 %end;
 
 %local vars;
-%let vars=%mf_getvarlist(&libds);
+%let vars=%upcase(%mf_getvarlist(&libds));
 
 /* create the header row */
 data _null_;
-  file &fref;
+  file &outref;
   length line $32767;
+  call missing(line);
   put '|'
 %local i var fmt;
 %do i=1 %to %sysfunc(countw(&vars));
   %let var=%scan(&vars,&i);
-  %let fmt=%mf_getvarformat(&libds,&var,force=1);
+  %let fmt=%lowcase(%mf_getvarformat(&libds,&var,force=1));
   "&var:&fmt|"
 %end;
   ;
@@ -79,20 +79,20 @@ run;
 
 /* write out the data */
 data _null_;
-  file &fref mod dlm='|' lrecl=32767;
+  file &outref mod dlm='|' lrecl=32767;
   set &libds ;
   length line $32767;
-  line=cats('|',%mf_getvarlist(&libds,dlm=%str(,'|',)),'|');
+  line='|`'!!cats(%mf_getvarlist(&libds,dlm=%str(%)!!' `|`'!!cats%()))!!' `|';
   put line;
 run;
 
 %if %upcase(&showlog)=YES %then %do;
   options ps=max;
   data _null_;
-    infile &fref;
+    infile &outref;
     input;
     putlog _infile_;
   run;
 %end;
 
-%mend mp_mdtablewrite;
+%mend mp_ds2md;

base/mp_filtervalidate.sas

@@ -33,8 +33,8 @@
   @param [in] targetds The target dataset against which to verify the query
   @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
   @param [out] outds= (work.mp_filtervalidate) Output dataset containing the
-  error / warning message, if one exists.  If this table contains any rows,
-  there are problems!
+    err / warning message, if one exists.  If this table contains any rows,
+    there are problems!
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas

base/mp_getcols.sas

@@ -51,7 +51,7 @@ data &outds(keep=name type length varnum format label ddtype);
     else if formatd=0 then format=cats(format2,formatl,'.');
     else format=cats(format2,formatl,'.',formatd);
     type='N';
-    if format=:'DATETIME' then ddtype='DATETIME';
+    if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
     else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
       or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
       or format=:'MONYY'

base/mp_getconstraints.sas

@@ -48,17 +48,17 @@
 %let vw=%mf_getuniquename(prefix=mp_getconstraints_vw_);
 data &vw /view=&vw;
   set sashelp.vcncolu;
-  where TABLE_CATALOG="&lib";
+  where table_catalog="&lib";
 
   /* use retain approach to reset the constraint order with each constraint */
   length tmp $1000;
   retain tmp;
   drop tmp;
-  if tmp ne catx('|',libref,table_name,constraint_type,constraint_name) then do;
+  if tmp ne catx('|',table_catalog,table_name,constraint_name) then do;
     constraint_order=1;
   end;
   else constraint_order+1;
-  tmp=catx('|',libref, table_name, constraint_type,constraint_name);
+  tmp=catx('|',table_catalog, table_name,constraint_name);
 run;
 
 /* must use SQL as proc datasets does not support length changes */

base/mp_getddl.sas

@@ -139,7 +139,7 @@ run;
     %let curds=%scan(&dsnlist,&x);
     data _null_;
       file &fref mod;
-      length nm lab $1024 typ $20;
+      length lab $1024 typ $20;
       set &colinfo (where=(upcase(memname)="&curds")) end=last;
 
       if _n_=1 then do;

base/mp_getformats.sas

@@ -0,0 +1,142 @@
+/**
+  @file
+  @brief Export format definitions
+  @details Formats are exported from the first (if any) catalog entry in the
+  FMTSEARCH path.
+
+  Formats are taken from the library / dataset reference and / or a static
+  format list.
+
+  Example usage:
+
+      %mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable)
+
+  @param [in] lib= (0) The libref for which to return formats.
+  @todo Enable exporting of formats for an entire library
+  @param [in] ds= (0) The dataset from which to obtain format definitions
+  @param [in] fmtlist= (0) A list of additional format names
+  @param [out] outsummary= (work.mp_getformats_summary) Output dataset
+    containing summary definitions - structure taken from dictionary.formats as
+    follows:
+
+  |libname:$8.|memname:$32.|path:$1024.|objname:$32.|fmtname:$32.|fmttype:$1.|source:$1.|minw:best.|mind:best.|maxw:best.|maxd:best.|defw:best.|defd:best.|
+  |---|---|---|---|---|---|---|---|---|---|---|---|---|
+  | | | | |$|F|B|1|0|32767|0|1|0|
+  | | | | |$|I|B|1|0|32767|0|1|0|
+  |` `|` `|/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWIANYDT|$ANYDTIF|I|U|1|0|60|0|19|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWFASCII|$ASCII|F|U|1|0|32767|0|1|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWIASCII|$ASCII|I|U|1|0|32767|0|1|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWFBASE6|$BASE64X|F|U|1|0|32767|0|1|0|
+
+
+  @param [out] outdetail= (0) Provide an output dataset in which to export all
+    the custom format definitions (from proc format CNTLOUT).  Definitions:
+https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#a002473477.htm
+    Sample data:
+
+  |FMTNAME:$32.|START:$16.|END:$16.|LABEL:$256.|MIN:best.|MAX:best.|DEFAULT:best.|LENGTH:best.|FUZZ:best.|PREFIX:$2.|MULT:best.|FILL:$1.|NOEDIT:best.|TYPE:$1.|SEXCL:$1.|EEXCL:$1.|HLO:$13.|DECSEP:$1.|DIG3SEP:$1.|DATATYPE:$8.|LANGUAGE:$8.|
+  |---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
+  |`WHICHPATH `|`0 `|`0 `|`path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|` `|` `|` `|` `|` `|
+  |`WHICHPATH `|`**OTHER** `|`**OTHER** `|`big fat problem if not path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|`O `|` `|` `|` `|` `|
+
+  <h4> SAS Macros </h4>
+  @li mf_dedup.sas
+  @li mf_getfmtlist.sas
+  @li mf_getfmtname.sas
+  @li mf_getquotedstr.sas
+  @li mf_getuniquename.sas
+
+
+  <h4> Related Macros </h4>
+  @li mp_applyformats.sas
+  @li mp_getformats.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_getformats(lib=0
+  ,ds=0
+  ,fmtlist=0
+  ,outsummary=work.mp_getformats_summary
+  ,outdetail=0
+);
+
+%local i fmt allfmts tempds fmtcnt;
+
+%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( )));
+  /* ensure format list contains format _name_ only */
+  %let fmt=%scan(&fmtlist,&i,%str( ));
+  %let fmt=%mf_getfmtname(&fmt);
+  %let allfmts=&allfmts &fmt;
+%end;
+
+%if &ds=0 and &lib ne 0 %then %do;
+  /* grab formats from library */
+  /* to do */
+%end;
+%else %if &ds ne 0 and &lib ne 0 %then %do;
+  /* grab formats from dataset */
+  %let allfmts=%mf_getfmtlist(&lib..&ds) &allfmts;
+%end;
+
+/* ensure list is unique */
+%let allfmts=%mf_dedup(%upcase(&allfmts));
+
+/* create summary table */
+%if %index(&outsummary,.)=0 %then %let outsummary=WORK.&outsummary;
+proc sql;
+create table &outsummary as
+  select * from dictionary.formats
+  where fmtname in (%mf_getquotedstr(&allfmts,quote=D))
+    and fmttype='F';
+
+%if "&outdetail" ne "0" %then %do;
+  /* ensure base table always exists */
+  proc sql;
+  create table &outdetail(
+      FMTNAME char(32)     label='Format name'
+      ,START char(16)     label='Starting value for format'
+      ,END char(16)     label='Ending value for format'
+      ,LABEL char(256)     label='Format value label'
+      ,MIN num length=3     label='Minimum length'
+      ,MAX num length=3     label='Maximum length'
+      ,DEFAULT num length=3     label='Default length'
+      ,LENGTH num length=3     label='Format length'
+      ,FUZZ num     label='Fuzz value'
+      ,PREFIX char(2)     label='Prefix characters'
+      ,MULT num     label='Multiplier'
+      ,FILL char(1)     label='Fill character'
+      ,NOEDIT num length=3     label='Is picture string noedit?'
+      ,TYPE char(1)     label='Type of format'
+      ,SEXCL char(1)     label='Start exclusion'
+      ,EEXCL char(1)     label='End exclusion'
+      ,HLO char(13)     label='Additional information'
+      ,DECSEP char(1)     label='Decimal separator'
+      ,DIG3SEP char(1)     label='Three-digit separator'
+      ,DATATYPE char(8)     label='Date/time/datetime?'
+      ,LANGUAGE char(8)     label='Language for date strings'
+  );
+  /* grab the location of each format */
+  %let fmtcnt=0;
+  data _null_;
+    set &outsummary;
+    if not missing(libname);
+    x+1;
+    call symputx(cats('fmtloc',x),cats(libname,'.',memname),'l');
+    call symputx(cats('fmtname',x),fmtname,'l');
+    call symputx('fmtcnt',x,'l');
+  run;
+  /* export each format and append to the output table */
+  %let tempds=%mf_getuniquename(prefix=mp_getformats);
+  %do i=1 %to &fmtcnt;
+    proc format library=&&fmtloc&i CNTLOUT=&tempds;
+      select &&fmtname&i;
+    run;
+    proc append base=&outdetail data=&tempds;
+    run;
+  %end;
+%end;
+
+%mend mp_getformats;

base/mp_getpk.sas

@@ -23,6 +23,10 @@
 
   Returns:
 
+|libref:$8.|dsn:$32.|memtype:$8.|dbms_memtype:$32.|typemem:$8.|memlabel:$256.|nvar:best.|compress:$8.|pk_fields:$512.|
+|---|---|---|---|---|---|---|---|---|
+|WORK|EXAMPLE|DATA| |DATA| |4|NO|TX_FROM DD_TYPE DD_SOURCE|
+
 
   @param [in] lib The libref to examine
   @param [in] ds= (0) Select the dataset to examine, else use 0 for all tables
@@ -37,6 +41,7 @@
 
   <h4> Related Macros </h4>
   @li mp_getpk.test.sas
+  @li mp_guesspk.sas
 
   @version 9.3
   @author Macro People Ltd

base/mp_guesspk.sas

@@ -1,32 +1,37 @@
 /**
-  @file mp_guesspk.sas
+  @file
   @brief Guess the primary key of a table
-  @details Tries to guess the primary key of a table based on the following logic:
+  @details Tries to guess the primary key of a table based on the following
+  logic:
 
       * Columns with nulls are ignored
       * Return only column combinations that provide unique results
-      * Start from one column, then move out to include composite keys of 2 to 6 columns
+      * Start from one column, then move out to composite keys of 2 to 6 columns
 
   The library of the target should be assigned before using this macro.
 
   Usage:
 
-      filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      filename mc url
+        "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
       %mp_guesspk(sashelp.class,outds=classpks)
 
   @param baseds The dataset to analyse
   @param outds= The output dataset to contain the possible PKs
-  @param max_guesses= The total number of possible primary keys to generate.  A
-    table is likely to have multiple unlikely PKs, so no need to list them all. Default=3.
-  @param min_rows= The minimum number of rows a table should have in order to try
-    and guess the PK.  Default=5.
+  @param max_guesses= (3) The total number of possible primary keys to generate.
+    A table may have multiple unlikely PKs, so no need to list them all.
+  @param min_rows= (5) The minimum number of rows a table should have in order
+    to try and guess the PK.
 
   <h4> SAS Macros </h4>
   @li mf_getvarlist.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
 
+  <h4> Related Macros </h4>
+  @li mp_getpk.sas
+
   @version 9.3
   @author Allan Bowe
 
@@ -196,7 +201,8 @@
           %let lev4=%scan(&posspks,&l);
           %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then %do;
             /* check for four level uniqueness */
-            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4) out=&tmpds noduprec;
+            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4)
+                out=&tmpds noduprec;
               by _all_;
             run;
             %if %mf_nobs(&tmpds)=&rows %then %do;
@@ -233,7 +239,8 @@
             %let lev5=%scan(&posspks,&m);
             %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then %do;
               /* check for four level uniqueness */
-              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5) out=&tmpds noduprec;
+              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5)
+                  out=&tmpds noduprec;
                 by _all_;
               run;
               %if %mf_nobs(&tmpds)=&rows %then %do;
@@ -282,7 +289,8 @@
                 run;
                 %if %mf_nobs(&tmpds)=&rows %then %do;
                   proc sql;
-                  insert into &outds values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
+                  insert into &outds
+                    values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
                   %if %mf_nobs(&outds) ge &max_guesses %then %do;
                     %put &sysmacroname: Max PKs reached at Level 6 for &baseds;
                     %return;

base/mp_hashdataset.sas

@@ -21,6 +21,7 @@
 
   @param [in] libds dataset to hash
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
+  @param [in] iftrue= A condition under which the macro should be executed.
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
   will contain one column (hashkey) with one observation (a hex32.
   representation of the input hash)
@@ -35,10 +36,14 @@
 %macro mp_hashdataset(
   libds,
   outds=,
-  salt=
+  salt=,
+  iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
+
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
   %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
-    %put %str(WARN)ING: Dataset &libds is empty;, or is not a dataset;
+    %put %str(WARN)ING: Dataset &libds is empty, or is not a dataset;
   %end;
   %else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
     %put %str(ERR)OR: Dataset &libds is not a dataset;

base/mp_include.sas

@@ -88,6 +88,7 @@ run;
 /* prepare the errds */
 data &errds;
   length msg mac $1000;
+  call missing(msg,mac);
   iftrue='1=0';
 run;
 

base/mp_init.sas

@@ -0,0 +1,68 @@
+/**
+  @file
+  @brief Initialise session with useful settings and variables
+  @details Implements a "strict" set of SAS options for use in defensive
+    programming.  Highly recommended, if you want your code to run on some
+    other machine.
+
+    This macro is recommended to be compiled and invoked in the `initProgram`
+    for SASjs [Jobs](https://cli.sasjs.io/sasjsconfig.html#jobConfig_initProgram
+    ), [Services](
+    https://cli.sasjs.io/sasjsconfig.html#serviceConfig_initProgram) and [Tests]
+    (https://cli.sasjs.io/sasjsconfig.html#testConfig_initProgram).
+
+    For non SASjs projects, you could invoke in the autoexec, or in your own
+    solution initialisation macro.
+
+
+    If you have a good idea for another useful option, setting, or global
+    variable - feel free to [raise an issue](
+    https://github.com/sasjs/core/issues/new)!
+
+    All global variables are prefixed with "SASJS" (unless modified with the
+    prefix parameter).
+
+  @param [in] prefix= (SASJS) The prefix to apply to the global macro variables
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_init(prefix=SASJS
+)/*/STORE SOURCE*/;
+
+  %global
+    &prefix._INIT_NUM   /* initialisation time as numeric                   */
+    &prefix._INIT_DTTM  /* initialisation time in E8601DT26.6 format        */
+    &prefix.WORK        /* avoid typing %sysfunc(pathname(work)) every time */
+  ;
+  %if %eval(&&&prefix._INIT_NUM>0) %then %return;  /* only run once */
+
+  data _null_;
+    dttm=datetime();
+    call symputx("&prefix._init_num",dttm,'g');
+    call symputx("&prefix._init_dttm",put(dttm,E8601DT26.6),'g');
+    call symputx("&prefix.work",pathname('WORK'),'g');
+  run;
+
+  options
+    noautocorrect           /* disallow misspelled procedure names            */
+    compress=CHAR           /* default is none so ensure we have something!   */
+    datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
+    %str(err)orcheck=STRICT /* catch errs in libname/filename statements      */
+    fmterr                  /* ensure err when a format cannot be found       */
+    mergenoby=%str(ERR)OR   /* throw err when a merge has no BY variables     */
+    missing=.               /* changing this can cause hard to detect errs    */
+    noquotelenmax           /* avoid warnings for long strings                */
+    noreplace               /* avoid overwriting permanent datasets           */
+    ps=max                  /* reduce log size slightly                       */
+    ls=max                  /* reduce log even more and avoid word truncation */
+    validmemname=COMPATIBLE /* avoid special characters etc in table names    */
+    validvarname=V7         /* avoid special characters etc in variable names */
+    varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
+    varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
+  ;
+
+%mend mp_init;

base/mp_jsonout.sas

@@ -62,13 +62,13 @@
 %put output location=&jref;
 %if &action=OPEN %then %do;
   options nobomfile;
-  data _null_;file &jref encoding='utf-8';
-    put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
+  data _null_;file &jref encoding='utf-8' ;
+    put '{"PROCESSED_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '"';
   run;
 %end;
 %else %if (&action=ARR or &action=OBJ) %then %do;
   options validvarname=upcase;
-  data _null_;file &jref mod encoding='utf-8';
+  data _null_;file &jref mod encoding='utf-8' ;
     put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":";
 
   %if &engine=PROCJSON %then %do;
@@ -147,7 +147,7 @@
       run;
       %let ds=&fmtds;
     %end; /* &fmt=Y */
-    data _null_;file &jref mod encoding='utf-8';
+    data _null_;file &jref mod encoding='utf-8' ;
       put "["; call symputx('cols',0,'l');
     proc sort
       data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
@@ -192,7 +192,7 @@
     /* write to temp loc to avoid _webout truncation
       - https://support.sas.com/kb/49/325.html */
     filename _sjs temp lrecl=131068 encoding='utf-8';
-    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod;
+    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod ;
       set &tempds;
       if _n_>1 then put "," @; put
       %if &action=ARR %then "[" ; %else "{" ;
@@ -219,14 +219,14 @@
       rc = fclose(fileid);
     run;
     filename _sjs clear;
-    data _null_; file &jref mod encoding='utf-8';
+    data _null_; file &jref mod encoding='utf-8' ;
       put "]";
     run;
   %end;
 %end;
 
 %else %if &action=CLOSE %then %do;
-  data _null_;file &jref encoding='utf-8' mod;
+  data _null_;file &jref encoding='utf-8' mod ;
     put "}";
   run;
 %end;

base/mp_lockanytable.sas

@@ -180,7 +180,7 @@ run;
     %else %do;
       data _null_;
         putlog 'NOTE-' / 'NOTE-';
-        putlog "NOTE- &sysmacroname: Table &lib..&ds locked at "@
+        putlog "NOTE- &sysmacroname: Table &lib..&ds locked at "@;
         putlog " %sysfunc(datetime(),datetime19.) (iteration &x)"@;
         putlog 'NOTE-' / 'NOTE-';
       run;
@@ -239,7 +239,7 @@ run;
   %let abortme=1;
 %end;
 
-/* catch errors - mp_abort must be called outside of a logic block */
+/* catch errs - mp_abort must be called outside of a logic block */
 %mp_abort(iftrue=(&abortme=1),
   msg=%superq(msg),
   mac=&sysmacroname

base/mp_makedata.sas

@@ -0,0 +1,106 @@
+/**
+  @file
+  @brief Create sample data based on the structure of an empty table
+  @details Many SAS projects involve sensitive datasets.  One way to _ensure_
+    the data is anonymised, is never to receive it in the first place!  Often
+    consultants are provided with empty tables, and expected to create complex
+    ETL flows.
+
+    This macro can help by taking an empty table, and populating it with data
+    according to the variable types and formats.
+
+    TODO:
+      @li Consider dates, datetimes, times, integers etc
+
+  Usage:
+
+      proc sql;
+      create table work.example(
+        TX_FROM float format=datetime19.,
+        DD_TYPE char(16),
+        DD_SOURCE char(2048),
+        DD_SHORTDESC char(256),
+        constraint pk primary key(tx_from, dd_type,dd_source),
+        constraint nnn not null(DD_SHORTDESC)
+      );
+      %mp_makedata(work.example)
+
+  @param [in] libds The empty table (libref.dataset) in which to create data
+  @param [out] obs= (500) The maximum number of records to create.  The table
+    is sorted with nodup on the primary key, so the actual number of records may
+    be lower than this.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_getvarlen.sas
+  @li mf_getvarlist.sas
+  @li mf_islibds.sas
+  @li mf_nobs.sas
+  @li mp_getcols.sas
+  @li mp_getpk.sas
+
+  <h4> Related Macros </h4>
+  @li mp_makedata.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_makedata(libds
+  ,obs=500
+  ,seed=1
+)/*/STORE SOURCE*/;
+
+%local ds1 ds2 lib ds pk_fields i col charvars numvars ispk;
+
+%if %mf_islibds(&libds)=0 %then %do;
+  %put &sysmacroname: Invalid libds (&libds) - should be library.dataset format;
+  %return;
+%end;
+%else %if %mf_nobs(&libds)>0 %then %do;
+  %put &sysmacroname: &libds has data, it will not be recreated;
+  %return;
+%end;
+
+/* set up temporary vars */
+%let ds1=%mf_getuniquename(prefix=mp_makedatads1);
+%let ds2=%mf_getuniquename(prefix=mp_makedatads2);
+%let lib=%scan(&libds,1,.);
+%let ds=%scan(&libds,2,.);
+
+/* grab the primary key vars */
+%mp_getpk(&lib,ds=&ds,outds=&ds1)
+
+proc sql noprint;
+select coalescec(pk_fields,'_all_') into: pk_fields from &ds1;
+
+data &ds2;
+  if 0 then set &libds;
+  do _n_=1 to &obs;
+    %let charvars=%mf_getvarlist(&libds,typefilter=C);
+    %if &charvars ^= %then %do i=1 %to %sysfunc(countw(&charvars));
+      %let col=%scan(&charvars,&i);
+      /* create random value based on observation number and colum length */
+      &col=repeat(put(md5(cats(_n_)),$hex32.),%mf_getvarlen(&libds,&col)/32);
+    %end;
+
+    %let numvars=%mf_getvarlist(&libds,typefilter=N);
+    %if &numvars ^= %then %do i=1 %to %sysfunc(countw(&numvars));
+      %let col=%scan(&numvars,&i);
+      &col=_n_;
+    %end;
+    output;
+  end;
+run;
+proc sort data=&ds2 nodupkey;
+  by &pk_fields;
+run;
+
+proc append base=&libds data=&ds2;
+run;
+
+proc sql;
+drop table &ds1, &ds2;
+
+%mend mp_makedata;

base/mp_reseterror.sas

@@ -0,0 +1,27 @@
+/**
+  @file
+  @brief Reset when an err condition occurs
+  @details When building apps, sometimes an operation must be attempted that
+  can cause an err condition.  There is no try catch in SAS! So the err state
+  must be caught and reset.
+
+  This macro attempts to do that reset.
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_reseterror(
+)/*/STORE SOURCE*/;
+
+options obs=max replace nosyntaxcheck;
+%let syscc=0;
+
+%if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
+  data _null_;
+    rc=stpsrvset('program error', 0);
+  run;
+%end;
+
+%mend mp_reseterror;

base/mp_resetoption.sas

@@ -3,13 +3,15 @@
   @brief Reset an option to original value
   @details Inspired by the SAS Jedi -
 https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options
-    Called as follows:
 
-    options obs=30;
-    %mp_resetoption(OBS)
+  Called as follows:
+
+      options obs=30 ps=max;
+      %mp_resetoption(OBS)
+      %mp_resetoption(PS)
 
 
-  @param option the option to reset
+  @param [in] option the option to reset
 
   @version 9.2
   @author Allan Bowe

base/mp_searchdata.sas

@@ -11,22 +11,28 @@
   Usage:
 
       %mp_searchdata(lib=sashelp, string=Jan)
-      %mp_searchdata(lib=sashelp, numval=1)
+      %mp_searchdata(lib=sashelp, ds=bird, numval=1)
+      %mp_searchdata(lib=sashelp, ds=class, string=l,outobs=5)
 
 
   Outputs zero or more tables to an MPSEARCH library with specific records.
 
-  @param lib=  the libref to search (should be already assigned)
-  @param ds= the dataset to search (leave blank to search entire library)
-  @param string= the string value to search
-  @param numval= the numeric value to search (must be exact)
-  @param outloc= the directory in which to create the output datasets with
-    matching rows.  Will default to a subfolder in the WORK library.
-  @param outobs= set to a positive integer to restrict the number of
+  @param [in] lib=  The libref to search (should be already assigned)
+  @param [in] ds= The dataset to search (leave blank to search entire library)
+  @param [in] string= String value to search (case sensitive, can be partial)
+  @param [in] numval= Numeric value to search (must be exact)
+  @param [out] outloc= (0) Optionally specify the directory in which to
+    create the the output datasets with matching rows.  By default it will
+    write them to a temporary subdirectory within the WORK folder.
+  @param [out] outlib= (MPSEARCH) Assign a different libref to the output
+    library containing the matching datasets / records
+  @param [in] outobs= set to a positive integer to restrict the number of
     observations
-  @param filter_text= add a (valid) filter clause to further filter the results
+  @param [in] filter_text= (1=1) Add a (valid) filter clause to further filter
+    the results.
 
   <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
   @li mf_mkdir.sas
@@ -36,11 +42,12 @@
   @author Allan Bowe
 **/
 
-%macro mp_searchdata(lib=sashelp
+%macro mp_searchdata(lib=
   ,ds=
   ,string= /* the query will use a contains (?) operator */
   ,numval= /* numeric must match exactly */
-  ,outloc=%sysfunc(pathname(work))/mpsearch
+  ,outloc=0
+  ,outlib=MPSEARCH
   ,outobs=-1
   ,filter_text=%str(1=1)
 )/*/STORE SOURCE*/;
@@ -57,8 +64,12 @@
 %if &string = %then %let type=N;
 %else %let type=C;
 
+%if "&outloc"="0" %then %do;
+  %let outloc=%sysfunc(pathname(work))/%mf_getuniquename();
+%end;
+
 %mf_mkdir(&outloc)
-libname mpsearch "&outloc";
+libname &outlib "&outloc";
 
 /* get the list of tables in the library */
 proc sql noprint;
@@ -70,11 +81,6 @@ select distinct memname into: table_list separated by ' '
 %end;
   ;
 /* check that we have something to check */
-proc sql
-%if &outobs>-1 %then %do;
-  outobs=&outobs
-%end;
-;
 %if %length(&table_list)=0 %then %put library &lib contains no tables!;
 /* loop through each table */
 %else %do table_num=1 %to %sysfunc(countw(&table_list,%str( )));
@@ -85,10 +91,10 @@ proc sql
   %end;
   %else %do;
     %let check_tm=%sysfunc(datetime());
-    /* build sql statement */
-    create table mpsearch.&table as select * from &lib..&table
-      where %unquote(&filter_text) and
-    (0
+    /* prep input */
+    data &outlib..&table;
+      set &lib..&table;
+      where %unquote(&filter_text) and ( 0
     /* loop through columns */
     %do colnum=1 %to %sysfunc(countw(&vars,%str( )));
       %let col=%scan(&vars,&colnum,%str( ));
@@ -102,15 +108,20 @@ proc sql
         or ("&col"n = &numval)
       %end;
     %end;
-    );
+      );
+    %if &outobs>-1 %then %do;
+      if _n_ > &outobs then stop;
+    %end;
+    run;
     %put Search query for &table took
       %sysevalf(%sysfunc(datetime())-&check_tm) seconds;
-    %if &sqlrc ne 0 %then %do;
-      %put %str(WAR)NING: SQLRC=&sqlrc when processing &table;
+    %if &syscc ne 0 %then %do;
+      %put %str(ERR)ROR: SYSCC=&syscc when processing &lib..&table;
       %return;
     %end;
-    %if %mf_nobs(mpsearch.&table)=0 %then %do;
-      drop table mpsearch.&table;
+    %if %mf_nobs(&outlib..&table)=0 %then %do;
+      proc sql;
+      drop table &outlib..&table;
     %end;
   %end;
 %end;

base/mp_sortinplace.sas

@@ -5,9 +5,9 @@
   sort it before performing operations such as merges / joins etc.
   That said, there are a few edge cases where it can be desirable:
 
-    @li To improve performance for particular scenarios
     @li To allow adjacent records to be viewed directly in the dataset
-    @li To reduce dataset size (eg when there are deleted records)
+    @li To apply compression, or to remove deleted records
+    @li To improve performance for specific queries
 
   This macro will only work for BASE (V9) engine libraries.  It works by
   creating a copy of the dataset (without data, WITH constraints) in the same
@@ -29,6 +29,7 @@
   @li mf_getengine.sas
   @li mf_getquotedstr.sas
   @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
   @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_getpk.sas
@@ -38,7 +39,6 @@
 
   @version 9.2
   @author Allan Bowe
-  @source https://github.com/sasjs/core
 
 **/
 
@@ -74,9 +74,13 @@
   %return;
 %end;
 
+/* fallback sortkey is all fields */
+%let sortkey=%mf_getvarlist(&libds);
+
+/* overlay actual sort key if it exists */
 data _null_;
   set work.&tempds1;
-  call symputx('sortkey',pk_fields);
+  call symputx('sortkey',coalescec(pk_fields,symget('sortkey')));
 run;
 
 
@@ -89,7 +93,7 @@ run;
 %let tempvw=%mf_getuniquename(prefix=&sysmacroname);
 proc sql;
 create view work.&tempvw as select * from &lib..&ds
-order by %mf_getquotedstr(&sortkey,quote=%str());
+order by %mf_getquotedstr(&sortkey,quote=N);
 
 /* append sorted data */
 proc append base=&lib..&tempds2 data=work.&tempvw;

base/mp_storediffs.sas

@@ -0,0 +1,237 @@
+/**
+  @file
+  @brief Converts deletes/changes/appends into a single audit table.
+  @details When tracking changes to data over time, it can be helpful to have
+    a single base table to track ALL modifications - enabling audit trail,
+    data recovery, and change re-application.  This macro is one of many
+    data management utilities used in [Data Controller for SAS](
+    https:datacontroller.io) - a comprehensive data ingestion solution, which
+    works on any SAS platform (Viya, SAS 9, Foundation) and is free for up to 5
+    users.
+
+    NOTE - this macro does not validate the inputs. It is assumed that the
+    datasets containing the new / changed / deleted rows are CORRECT, contain
+    no additional (or missing columns), and that the originals dataset contains
+    all relevant base records (and no additionals).
+
+    Usage:
+
+        data work.orig work.deleted work.changed work.appended;
+          set sashelp.class;
+          if _n_=1 then do;
+            output work.orig work.deleted;
+          end;
+          else if _n_=2 then do;
+            output work.orig;
+            age=99;
+            output work.changed;
+          end;
+          else do;
+            name='Newbie';
+            output work.appended;
+            stop;
+          end;
+        run;
+
+        %mp_storediffs(sashelp.class,work.orig,NAME
+          ,delds=work.deleted
+          ,modds=work.changed
+          ,appds=work.appended
+          ,outds=work.final
+          ,mdebug=1
+        )
+
+  @param [in] libds Target table against which the changes were applied
+  @param [in] origds Dataset with original (unchanged) records.  Can be empty if
+    only appending.
+  @param [in] key Space seperated list of key variables
+  @param [in] delds= (0) Dataset with deleted records
+  @param [in] appds= (0) Dataset with appended records
+  @param [in] modds= (0) Dataset with modified records
+  @param [out] outds= (work.mp_storediffs) Output table containing stored data.
+    Has the following format:
+
+        proc sql;
+        create table &outds(
+          load_ref char(36) label='unique load reference',
+          processed_dttm num format=E8601DT26.6 label='Processed at timestamp',
+          libref char(8) label='Library Reference (8 chars)',
+          dsn char(32) label='Dataset Name (32 chars)',
+          key_hash char(32) label=
+            'MD5 Hash of primary key values (pipe seperated)',
+          move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified',
+          is_pk num label='Is Primary Key Field? (1/0)',
+          is_diff num label=
+            'Did value change? (1/0/-1).  Always -1 for appends and deletes.',
+          tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
+          tgtvar_nm char(32) label='Target variable name (32 chars)',
+          oldval_num num format=best32. label='Old (numeric) value',
+          newval_num num format=best32. label='New (numeric) value',
+          oldval_char char(32765) label='Old (character) value',
+          newval_char char(32765) label='New (character) value',
+          constraint pk_mpe_audit
+            primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
+        );
+
+    @param [in] processed_dttm= (0) Provide a datetime constant in relation to
+      the actual load time.  If not provided, current timestamp is used.
+    @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
+    @param [out] loadref= (0) Provide a unique key to reference the load,
+      otherwise a UUID will be generated.
+
+  <h4> SAS Macros </h4>
+  @li mf_getquotedstr.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mp_storediffs(libds
+  ,origds
+  ,key
+  ,delds=0
+  ,appds=0
+  ,modds=0
+  ,outds=work.mp_storediffs
+  ,loadref=0
+  ,processed_dttm=0
+  ,mdebug=0
+)/*/STORE SOURCE*/;
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+/* set up unique and temporary vars */
+%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist;
+%let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_ds1));
+%let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_ds2));
+%let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_ds3));
+%let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_ds4));
+%let hashkey=%upcase(%mf_getuniquename(prefix=mpsd_hashkey));
+%let inds_auto=%upcase(%mf_getuniquename(prefix=mpsd_inds_auto));
+%let inds_keep=%upcase(%mf_getuniquename(prefix=mpsd_inds_keep));
+
+%let dslist=&origds;
+%if &delds ne 0 %then %do;
+  %let delds=%upcase(&delds);
+  %if %scan(&delds,-1,.)=&delds %then %let delds=WORK.&delds;
+  %let dslist=&dslist &delds;
+%end;
+%if &appds ne 0 %then %do;
+  %let appds=%upcase(&appds);
+  %if %scan(&appds,-1,.)=&appds %then %let appds=WORK.&appds;
+  %let dslist=&dslist &appds;
+%end;
+%if &modds ne 0 %then %do;
+  %let modds=%upcase(&modds);
+  %if %scan(&modds,-1,.)=&modds %then %let modds=WORK.&modds;
+  %let dslist=&dslist &modds;
+%end;
+
+%let origds=%upcase(&origds);
+%if %scan(&origds,-1,.)=&origds %then %let origds=WORK.&origds;
+
+%let key=%upcase(&key);
+
+/* hash the key and append all the tables (marking the source) */
+data &ds1;
+  set &dslist indsname=&inds_auto;
+  &hashkey=put(md5(catx('|',%mf_getquotedstr(&key,quote=N))),$hex32.);
+  &inds_keep=&inds_auto;
+proc sort;
+  by &inds_keep &hashkey;
+run;
+
+/* transpose numeric & char vars */
+proc transpose data=&ds1
+    out=&ds2(rename=(&hashkey=key_hash _name_=tgtvar_nm col1=newval_num));
+  by &inds_keep &hashkey;
+  var _numeric_;
+run;
+proc transpose data=&ds1
+    out=&ds3(
+      rename=(&hashkey=key_hash _name_=tgtvar_nm col1=newval_char)
+      where=(tgtvar_nm not in ("&hashkey","&inds_keep"))
+    );
+  by &inds_keep &hashkey;
+  var _character_;
+run;
+data &ds4;
+  length &inds_keep $41 tgtvar_nm $32;
+  set &ds2 &ds3 indsname=&inds_auto;
+
+  tgtvar_nm=upcase(tgtvar_nm);
+  if tgtvar_nm in (%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE)));
+
+  if &inds_auto="&ds2" then tgtvar_type='N';
+  else if &inds_auto="&ds3" then tgtvar_type='C';
+  else do;
+    putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
+    call symputx('syscc',98);
+  end;
+
+  if &inds_keep="&appds" then move_type='A';
+  else if &inds_keep="&delds" then move_type='D';
+  else if &inds_keep="&modds" then move_type='M';
+  else if &inds_keep="&origds" then move_type='O';
+  else do;
+    putlog "%str(ERR)OR: unidentified movetype input!" &inds_keep;
+    call symputx('syscc',99);
+  end;
+  tgtvar_nm=upcase(tgtvar_nm);
+  if tgtvar_nm in (%mf_getquotedstr(&key)) then is_pk=1;
+  else is_pk=0;
+  drop &inds_keep;
+run;
+
+%if "&loadref"="0" %then %let loadref=%sysfunc(uuidgen());
+%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime());
+%let libds=%upcase(&libds);
+
+/* join orig vals for modified & deleted */
+proc sql;
+create table &outds as
+  select "&loadref" as load_ref length=36
+    ,&processed_dttm as processed_dttm format=E8601DT26.6
+    ,"%scan(&libds,1,.)" as libref length=8
+    ,"%scan(&libds,2,.)" as dsn length=32
+    ,b.key_hash length=32
+    ,b.move_type length=1
+    ,b.tgtvar_nm length=32
+    ,b.is_pk
+    ,case when b.move_type ne 'M' then -1
+      when a.newval_num=b.newval_num and a.newval_char=b.newval_char then 0
+      else 1
+      end as is_diff
+    ,b.tgtvar_type length=1
+    ,case when b.move_type='D' then b.newval_num
+      else a.newval_num
+      end as oldval_num format=best32.
+    ,case when b.move_type='D' then .
+      else b.newval_num
+      end as newval_num format=best32.
+    ,case when b.move_type='D' then b.newval_char
+      else a.newval_char
+      end as oldval_char length=32765
+    ,case when b.move_type='D' then ''
+      else b.newval_char
+      end as newval_char length=32765
+  from &ds4(where=(move_type='O')) as a
+  right join &ds4(where=(move_type ne 'O')) as b
+  on a.tgtvar_nm=b.tgtvar_nm
+  and a.key_hash=b.key_hash
+  order by move_type, key_hash,is_pk desc, tgtvar_nm;
+
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &ds1, &ds2, &ds3, &ds4;
+%end;
+
+%mend mp_storediffs;
+/** @endcond */

base/mp_validatecol.sas

@@ -20,10 +20,13 @@
       ;;;;
       run;
 
+  Tip - when contributing, use https://regex101.com to test the regex validity!
+
   @param [in] incol The column to be validated
   @param [in] rule The rule to apply.  Current rules:
     @li ISNUM - checks if the variable is numeric
     @li LIBDS - matches LIBREF.DATASET format
+    @li FORMAT - checks if the provided format is syntactically valid
   @param [out] outcol The variable to create, with the results of the match
 
   <h4> SAS Macros </h4>
@@ -62,5 +65,19 @@
   if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
   else &outcol=0;
 %end;
+%else %if &rule=FORMAT %then %do;
+  /* match valid format - regex could probably be improved */
+  if _n_=1 then do;
+    retain &tempcol;
+    &tempcol=prxparse('/^[_a-z\$]\w{0,31}\.[0-9]*$/i');
+    if missing(&tempcol) then do;
+      putlog "%str(ERR)OR: Invalid expression for FORMAT";
+      stop;
+    end;
+    drop &tempcol;
+  end;
+  if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
+  else &outcol=0;
+%end;
 
 %mend mp_validatecol;

meta/mm_assigndirectlib.sas

@@ -295,7 +295,7 @@ run;
       prop='Connection.DBMS.Property.SERVER.Name.xmlKey.txt';
       rc=metadata_getprop(uri,prop,server,"");
     end;
-    if server^='' then server='server='!!server;
+    if server^='' then server='server='!!quote(cats(server));
     call symputx('server',server,'l');
 
     /* get SCHEMA value */
@@ -441,11 +441,11 @@ run;
   run;
 
   %put NOTE: Executing the following:/; %put NOTE-;
-  %put NOTE- libname &libref TERADATA server=&path schema=&schema ;
+  %put NOTE- libname &libref TERADATA server="&path" schema=&schema ;
   %put NOTe-   authdomain=&authdomain;
   %put NOTE-;
 
-  libname &libref TERADATA server=&path schema=&schema authdomain=&authdomain;
+  libname &libref TERADATA server="&path" schema=&schema authdomain=&authdomain;
 %end;
 %else %if &engine= %then %do;
   %put NOTE: Libref &libref is not registered in metadata;

meta/mm_createwebservice.sas

@@ -94,13 +94,13 @@ data _null_;
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
-  put '  data _null_;file &jref encoding=''utf-8''; ';
-  put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
+  put '  data _null_;file &jref encoding=''utf-8'' ; ';
+  put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
   put '  run; ';
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
   put '  options validvarname=upcase; ';
-  put '  data _null_;file &jref mod encoding=''utf-8''; ';
+  put '  data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
@@ -179,7 +179,7 @@ data _null_;
   put '      run; ';
   put '      %let ds=&fmtds; ';
   put '    %end; /* &fmt=Y */ ';
-  put '    data _null_;file &jref mod encoding=''utf-8''; ';
+  put '    data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
   put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
@@ -224,7 +224,7 @@ data _null_;
   put '    /* write to temp loc to avoid _webout truncation ';
   put '      - https://support.sas.com/kb/49/325.html */ ';
   put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
-  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod; ';
+  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
   put '      %if &action=ARR %then "[" ; %else "{" ; ';
@@ -251,14 +251,14 @@ data _null_;
   put '      rc = fclose(fileid); ';
   put '    run; ';
   put '    filename _sjs clear; ';
-  put '    data _null_; file &jref mod encoding=''utf-8''; ';
+  put '    data _null_; file &jref mod encoding=''utf-8'' ; ';
   put '      put "]"; ';
   put '    run; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8'' mod; ';
+  put '  data _null_;file &jref encoding=''utf-8'' mod ; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';

meta/mm_getdetails.sas

@@ -45,7 +45,7 @@ run;
 data &outattrs;
   keep type name value;
   length type $4 name $256 value $32767;
-  rc1=1;n1=1;type='Prop';
+  rc1=1;n1=1;type='Prop';name='';value='';
   do while(rc1>0);
     rc1=metadata_getnprp("&uri",n1,name,value);
     if rc1>0 then output;

meta/mm_getstps.sas

@@ -63,14 +63,13 @@ run;
 %if %length(&tree)>0 %then %do;
   /* get tree info */
   %mm_gettree(tree=&tree,inds=&outds, outds=&outds, mDebug=&mDebug)
-    %if %mf_nobs(&outds)=0 %then %do;
+  %if %mf_nobs(&outds)=0 %then %do;
     %put NOTE:  Tree &tree did not exist!!;
     %return;
   %end;
 %end;
 
 
-
 data &outds ;
   set &outds(rename=(treeuri=treeuri_compare));
   length treeuri query stpuri $256;

meta/mm_gettables.sas

@@ -39,7 +39,7 @@
 data &outds;
   length uri serveruri conn_uri domainuri libname ServerContext AuthDomain
     path_schema usingpkguri type tableuri $256 id $17
-    libdesc $200 libref engine $8 IsDBMSLibname $1
+    libdesc $200 libref engine $8 IsDBMSLibname IsPreassigned $1
     tablename $50 /* metadata table names can be longer than $32 */
     ;
   keep libname libdesc libref engine ServerContext path_schema AuthDomain

sasjs/sasjsconfig.json

@@ -32,7 +32,9 @@
       "name": "viya",
       "serverUrl": "https://sas.analytium.co.uk",
       "serverType": "SASVIYA",
-      "allowInsecureRequests": false,
+      "httpsAgentOptions": {
+        "allowInsecureRequests": false
+      },
       "appLoc": "/Public/temp/macrocore",
       "macroFolders": [
         "tests/viyaonly"

server/ms_webout.sas

@@ -79,7 +79,7 @@
   OPTIONS NOBOMFILE;
 
   /* setup json */
-  data _null_;file &fref encoding='utf-8';
+  data _null_;file &fref encoding='utf-8' termstr=lf;
   %if %str(&_debug) ge 131 %then %do;
     put '>>weboutBEGIN<<';
   %end;
@@ -108,14 +108,14 @@
       i+1;
       call symputx('wt'!!left(i),name,'l');
       call symputx('wtcnt',i,'l');
-    data _null_; file &fref mod encoding='utf-8';
+    data _null_; file &fref mod encoding='utf-8' termstr=lf;
       put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
       proc contents noprint data=&wt
         out=_data_ (keep=name type length format:);
       run;%let tempds=%scan(&syslast,2,.);
-      data _null_; file &fref mod encoding='utf-8';
+      data _null_; file &fref mod encoding='utf-8' termstr=lf;
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
         nvars=attrn(dsid,'NVARS');
@@ -126,15 +126,15 @@
         put ',"nvars":' nvars;
       %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
       %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
-      data _null_; file &fref mod encoding='utf-8';
+      data _null_; file &fref mod encoding='utf-8' termstr=lf;
         put "}";
     %end;
-    data _null_; file &fref mod encoding='utf-8';
+    data _null_; file &fref mod encoding='utf-8' termstr=lf termstr=lf;
       put "}";
     run;
   %end;
   /* close off json */
-  data _null_;file &fref mod encoding='utf-8';
+  data _null_;file &fref mod encoding='utf-8' termstr=lf;
     _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ",""SYSUSERID"" : ""&sysuserid"" ";
     put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
@@ -147,7 +147,9 @@
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
     put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
-    put ",""SYSPROCESSNAME"" : ""&SYSPROCESSNAME"" ";
+    length SYSPROCESSNAME $512;
+    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
+    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
@@ -156,7 +158,8 @@
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
     put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
-    autoexec=quote(trim(getoption('autoexec')));
+    length autoexec $512;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
     put ',"AUTOEXEC" : ' autoexec;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
     memsize=quote(cats(memsize));

tests/crossplatform/mf_dedup.test.sas

@@ -0,0 +1,23 @@
+/**
+  @file
+  @brief Testing mf_dedup macro
+
+  <h4> SAS Macros </h4>
+  @li mf_dedup.sas
+  @li mp_assert.sas
+
+**/
+
+%let str=One two one two and through and through;
+
+%mp_assert(
+  iftrue=("%mf_dedup(&str)"="One two one and through"),
+  desc=Basic test,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=("%mf_dedup(&str,outdlm=%str(,))"="One,two,one,and,through"),
+  desc=Outdlm test,
+  outds=work.test_results
+)

tests/crossplatform/mf_getfmtlist.test.sas

@@ -0,0 +1,33 @@
+/**
+  @file
+  @brief Testing mf_getfmtlist macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getfmtlist.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(
+    "%mf_getfmtlist(sashelp.prdsale)"="DOLLAR $CHAR W MONNAME"
+  ),
+  desc=Checking basic numeric,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getfmtlist(sashelp.shoes)"="$CHAR BEST DOLLAR"
+  ),
+  desc=Checking basic char,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getfmtlist(sashelp.demographics)"="BEST Z $CHAR COMMA PERCENTN"
+  ),
+  desc=Checking longer numeric,
+  outds=work.test_results
+)

tests/crossplatform/mf_getfmtname.test.sas

@@ -0,0 +1,33 @@
+/**
+  @file
+  @brief Testing mf_getfmtname macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getfmtname.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(
+    "%mf_getfmtname(8.)"="W"
+  ),
+  desc=Checking basic numeric,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getfmtname($4.)"="$CHAR"
+  ),
+  desc=Checking basic char,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getfmtname(comma14.10)"="COMMA"
+  ),
+  desc=Checking longer numeric,
+  outds=work.test_results
+)

tests/crossplatform/mf_isint.test.sas

@@ -0,0 +1,33 @@
+/**
+  @file
+  @brief Testing mf_isint macro
+
+  <h4> SAS Macros </h4>
+  @li mf_isint.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(
+    "%mf_isint(1)"="1"
+  ),
+  desc=Checking basic mf_isint(1),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_isint(1.1)"="0"
+  ),
+  desc=Checking basic mf_isint(1.1),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_isint(-1)"="1"
+  ),
+  desc=Checking mf_isint(-1),
+  outds=work.test_results
+)

tests/crossplatform/mf_islibds.test.sas

@@ -0,0 +1,46 @@
+/**
+  @file
+  @brief Testing mf_islibds macro
+
+      %put %mf_islibds(work.something)=1;
+      %put %mf_islibds(nolib)=0;
+      %put %mf_islibds(badlibref.ds)=0;
+      %put %mf_islibds(w.t.f)=0;
+
+  <h4> SAS Macros </h4>
+  @li mf_islibds.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(
+    %mf_islibds(work.something)=1
+  ),
+  desc=%str(Checking mf_islibds(work.something)=1),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    %mf_islibds(nolib)=0
+  ),
+  desc=%str(Checking mf_islibds(nolib)=0),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    %mf_islibds(badlibref.ds)=0
+  ),
+  desc=%str(Checking mf_islibds(badlibref.ds)=0),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    %mf_islibds(w.t.f)=0
+  ),
+  desc=%str(Checking mf_islibds(w.t.f)=0),
+  outds=work.test_results
+)

tests/crossplatform/mf_wordsinstr1andstr2.test.sas

@@ -0,0 +1,20 @@
+/**
+  @file
+  @brief Testing mf_wordsinstr1andstr2 macro
+
+  <h4> SAS Macros </h4>
+  @li mf_wordsinstr1andstr2.sas
+  @li mp_assert.sas
+
+**/
+
+%let x=%mf_wordsinstr1andstr2(str1=xx DOLLAR x $CHAR xxx W MONNAME
+  ,str2=DOLLAR $CHAR W MONNAME xxxxxx
+);
+%mp_assert(
+  iftrue=(
+    "&x"="DOLLAR $CHAR W MONNAME"
+  ),
+  desc=Checking basic string,
+  outds=work.test_results
+)

tests/crossplatform/mf_wordsinstr1butnotstr2.test.sas

@@ -0,0 +1,20 @@
+/**
+  @file
+  @brief Testing mf_wordsinstr1butnotstr2 macro
+
+  <h4> SAS Macros </h4>
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_assert.sas
+
+**/
+
+%let x=%mf_wordsinstr1butnotstr2(str1=xx DOLLAR x $CHAR xxx W MONNAME
+  ,str2=ff xx x xxx xxxxxx
+);
+%mp_assert(
+  iftrue=(
+    "&x"="DOLLAR $CHAR W MONNAME"
+  ),
+  desc=Checking basic string,
+  outds=work.test_results
+)

tests/crossplatform/mp_applyformats.test.sas

@@ -0,0 +1,45 @@
+/**
+  @file
+  @brief Testing mp_applyformats.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getvarformat.sas
+  @li mp_applyformats.sas
+  @li mp_assert.sas
+  @li mp_getcols.sas
+
+**/
+
+/**
+  * Test 1 Base case
+  */
+
+data work.example;
+  set sashelp.prdsale;
+  format _all_;
+run;
+%let origfmt=%mf_getvarformat(work.example,month);
+
+%mp_getcols(sashelp.prdsale,outds=work.cols)
+
+data work.cols2;
+  set work.cols;
+  lib='WORK';
+  ds='EXAMPLE';
+  var=name;
+  fmt=format;
+  keep lib ds var fmt;
+run;
+
+%mp_applyformats(work.cols2)
+
+%mp_assert(
+  iftrue=("&orig_fmt"=""),
+  desc=Check that formats were cleared,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("%mf_getvarformat(work.example,month)"="MONNAME3."),
+  desc=Check that formats were applied,
+  outds=work.test_results
+)

tests/crossplatform/mp_ds2fmtds.test.sas

@@ -15,10 +15,12 @@ filename inc temp;
 data _null_;
   set work.test;
   file inc;
-  line=cats('%mp_ds2fmtds(sashelp.',memname,',',memname,')');
+  libds=cats('sashelp.',memname);
+  if exist(libds) then line=cats('%mp_ds2fmtds(',libds,',',memname,')');
   put line;
 run;
 
+options obs=50;
 %inc inc;
 
 %mp_assert(

tests/crossplatform/mp_ds2md.test.sas

@@ -0,0 +1,34 @@
+/**
+  @file
+  @brief Testing mp_ds2md.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2md.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_ds2md(sashelp.class,outref=md)
+
+data _null_;
+  infile md;
+  input;
+  call symputx(cats('test',_n_),_infile_);
+  if _n_=4 then stop;
+run;
+
+%mp_assert(
+  iftrue=("&test1"="|NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.|"),
+  desc=Checking header row,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test2"="|---|---|---|---|---|"),
+  desc=Checking divider row,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test3"="|`Alfred `|`M `|`14 `|`69 `|`112.5 `|"),
+  desc=Checking data row,
+  outds=work.test_results
+)

tests/crossplatform/mp_getformats.test.sas

@@ -0,0 +1,76 @@
+/**
+  @file
+  @brief Testing mp_getformats.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mp_getformats.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * Test - setup
+  */
+
+%mf_mkdir(&sasjswork/path1)
+%mf_mkdir(&sasjswork/path2)
+
+libname path1 "&sasjswork/path1";
+libname path2 "&sasjswork/path2";
+
+
+PROC FORMAT library=path1;
+  value whichpath 0 = 'path1' other='big fat problem if not path1';
+PROC FORMAT library=path2;
+  value whichpath 0 = 'path2' other='big fat problem if not path2';
+RUN;
+
+
+/** run with path1 path2 FMTSEARCH */
+options insert=(fmtsearch=(path1 path2));
+data _null_;
+  test=0;
+  call symputx('test1',put(test,whichpath.));
+run;
+%mp_assert(
+  iftrue=("&test1"="path1"),
+  desc=Check correct format is applied,
+  outds=work.test_results
+)
+%mp_getformats(fmtlist=WHICHPATH,outsummary=sum,outdetail=detail1)
+%let tst1=0;
+data _null_;
+  set detail1;
+  if fmtname='WHICHPATH' and start='**OTHER**' then call symputx('tst1',label);
+  putlog (_all_)(=);
+run;
+%mp_assert(
+  iftrue=("&tst1"="big fat problem if not path1"),
+  desc=Check correct detail results are applied,
+  outds=work.test_results
+)
+
+/** run with path2 path1 FMTSEARCH */
+options insert=(fmtsearch=(path2 path1));
+data _null_;
+  test=0;
+  call symputx('test2',put(test,whichpath.));
+run;
+%mp_assert(
+  iftrue=("&test2"="path2"),
+  desc=Check correct format is applied,
+  outds=work.test_results
+)
+%mp_getformats(fmtlist=WHICHPATH,outsummary=sum,outdetail=detail2)
+%let tst2=0;
+data _null_;
+  set detail2;
+  if fmtname='WHICHPATH' and start='**OTHER**' then call symputx('tst2',label);
+  putlog (_all_)(=);
+run;
+%mp_assert(
+  iftrue=("&tst2"="big fat problem if not path2"),
+  desc=Check correct detail results are applied,
+  outds=work.test_results
+)

tests/crossplatform/mp_init.test.sas

@@ -0,0 +1,24 @@
+/**
+  @file
+  @brief Testing mp_gsubfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_init.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * Test 1 - mp_init.sas actually already ran as part of testinit
+  * So lets test to make sure it will not run again
+  */
+
+%let initial_value=&sasjs_init_num;
+
+%mp_init()
+
+%mp_assert(
+  iftrue=("&initial_value"="&sasjs_init_num"),
+  desc=Check that mp_init() did not run twice,
+  outds=work.test_results
+)

tests/crossplatform/mp_lib2inserts.test.sas

@@ -11,9 +11,10 @@
 **/
 
 /* grab 20 datasets from SASHELP */
-%let path=%sysfunc(pathname(work));
+%let work=%sysfunc(pathname(work));
+%let path=&work/new;
 %mf_mkdir(&path)
-libname sashlp "&path";
+libname sashlp "&work";
 proc sql noprint;
 create table members as
   select distinct lowcase(memname) as memname
@@ -31,6 +32,7 @@ run;
 %mp_lib2inserts(sashlp, schema=work, outref=tempref,maxobs=50)
 
 /* check if it actually runs */
+libname sashlp "&path";
 options source2;
 %inc tempref;
 

tests/crossplatform/mp_lockfilecheck.test.sas

@@ -5,6 +5,7 @@
   <h4> SAS Macros </h4>
   @li mp_lockfilecheck.sas
   @li mp_assert.sas
+  @li mp_reseterror.sas
 
 **/
 
@@ -29,6 +30,8 @@ data work.test; a=1;run;
 
 %mp_lockfilecheck(sashelp.class)
 
+%mp_reseterror()
+
 %mp_assert(
   iftrue=(&success=1),
   desc=Checking sashelp table cannot be locked,

tests/crossplatform/mp_makedata.test.sas

@@ -0,0 +1,59 @@
+/**
+  @file
+  @brief Testing mp_makedata.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_makedata.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * Test 1 - Regular makedata call
+  */
+
+proc sql;
+create table work.example(
+  TX_FROM float format=datetime19.,
+  DD_TYPE char(16),
+  DD_SOURCE char(2048),
+  DD_SHORTDESC char(256),
+  constraint pk primary key(tx_from, dd_type,dd_source),
+  constraint nnn not null(DD_SHORTDESC)
+);
+%mp_makedata(work.example,obs=500)
+
+%mp_assert(
+  iftrue=("%mf_nobs(work.example)"="500"),
+  desc=Check that 500 rows were created,
+  outds=work.test_results
+)
+
+data _null_;
+  set work.example;
+  call symputx('lenvar',length(dd_source));
+  stop;
+run;
+%mp_assert(
+  iftrue=("&lenvar"="2048"),
+  desc=Check that entire length of variable is populated,
+  outds=work.test_results
+)
+
+
+proc sql;
+create table work.example2(
+  TX_FROM float format=datetime19.,
+  DD_TYPE char(16),
+  DD_SOURCE char(2048),
+  DD_SHORTDESC char(256),
+  some_num num
+);
+%mp_makedata(work.example2)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Ensure tables without keys still generate,
+  outds=work.test_results
+)

tests/crossplatform/mp_reseterror.test.sas

@@ -0,0 +1,23 @@
+/**
+  @file
+  @brief Testing mp_reseterror macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mp_reseterror.sas
+
+**/
+
+
+/* cause an error */
+
+lock sashelp.class;
+
+/* recover ? */
+%mp_reseterror()
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking error condition was fixed,
+  outds=work.test_results
+)

tests/crossplatform/mp_searchdata.test.sas

@@ -0,0 +1,29 @@
+/**
+  @file
+  @brief Testing mp_searchdata.sas
+
+  <h4> SAS Macros </h4>
+  @li mp_searchdata.sas
+  @li mp_assert.sas
+
+
+**/
+
+/** Test 1 - generic useage */
+
+%mp_searchdata(lib=sashelp, ds=class, string=a)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errors in regular usage,
+  outds=work.test_results
+)
+
+/** Test 2 - with obs issue  */
+
+%mp_searchdata(lib=sashelp, ds=class, string=l,outobs=5)
+
+%mp_assert(
+  iftrue=("&SYSWARNINGTEXT" = ""),
+  desc=Ensuring WARN status is clean,
+  outds=work.test_results
+)

tests/crossplatform/mp_sortinplace.test.sas

@@ -39,4 +39,31 @@ run;
   ),
   desc=Check if sort was appplied,
   outds=work.test_results
+)
+
+/** Test 2 - table without PK  */
+proc sql;
+create table work.example2 as
+  select * from sashelp.classfit;
+%mp_sortinplace(work.example2)
+%mp_assert(
+  iftrue=(
+    %str(&syscc)=%str(0)
+  ),
+  desc=Ensure no errors when no key exists,
+  outds=work.test_results
+)
+
+%let test2=0;
+data _null_;
+  set work.example2;
+  call symputx('test2',name);
+  stop;
+run;
+%mp_assert(
+  iftrue=(
+    %str(&test2)=%str(Alfred)
+  ),
+  desc=Check if sort was appplied when no index exists,
+  outds=work.test_results
 )

tests/crossplatform/mp_storediffs.test.sas

@@ -0,0 +1,102 @@
+/**
+  @file
+  @brief Testing mp_storediffs macro
+
+  <h4> SAS Macros </h4>
+  @li mp_storediffs.sas
+  @li mp_assert.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
+
+**/
+
+/* make some data */
+
+data work.orig work.deleted work.changed work.appended;
+  set sashelp.class;
+  if _n_=1 then do;
+    output work.orig work.deleted;
+  end;
+  else if _n_=2 then do;
+    output work.orig;
+    age=99;
+    output work.changed;
+  end;
+  else do;
+    name='Newbie';
+    output work.appended;
+    stop;
+  end;
+run;
+
+%mp_storediffs(sashelp.class,work.orig,NAME
+  ,delds=work.deleted
+  ,modds=work.changed
+  ,appds=work.appended
+  ,outds=work.final
+  ,mdebug=1
+)
+
+%mp_assert(
+  iftrue=(
+    %str(&syscc)=%str(0)
+  ),
+  desc=ensure no errors,
+  outds=work.test_results
+)
+
+%mp_assertdsobs(work.final,
+  desc=Has 15 records,
+  test=EQUALS 15,
+  outds=work.test_results
+)
+
+data work.check;
+  length val $10;
+  do val='C','N';
+    output;
+  end;
+run;
+%mp_assertcolvals(work.final.tgtvar_type,
+  checkvals=work.check.val,
+  desc=All values have a match,
+  test=ALLVALS
+)
+
+/* Test for when there are no actual changes */
+data work.orig work.deleted work.changed work.appended;
+  set sashelp.class;
+  output work.orig;
+run;
+%mp_storediffs(sashelp.class,work.orig,NAME
+  ,delds=work.deleted
+  ,modds=work.changed
+  ,appds=work.appended
+  ,outds=work.final2
+  ,mdebug=1
+)
+%mp_assertdsobs(work.final2,
+  desc=No changes produces 0 records,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+/* Test for deletes only */
+data work.orig work.deleted work.changed work.appended;
+  set sashelp.class;
+  output work.orig;
+  if _n_>5 then output work.deleted;
+run;
+
+%mp_storediffs(sashelp.class,work.orig,NAME
+  ,delds=work.deleted
+  ,modds=work.changed
+  ,appds=work.appended
+  ,outds=work.final3
+  ,mdebug=1
+)
+%mp_assertdsobs(work.final3,
+  desc=Delete has 70 records,
+  test=EQUALS 70,
+  outds=work.test_results
+)

tests/crossplatform/mp_validatecol.test.sas

@@ -59,4 +59,37 @@ run;
   desc=Test2 - ISNUM,
   test=EQUALS 4,
   outds=work.test_results
+)
+
+/**
+  * Test 3 - FORMAT
+  */
+data test3;
+  infile datalines4 dsd;
+  input;
+  infile=_infile_;
+  %mp_validatecol(infile,FORMAT,is_format)
+  if is_format=1;
+datalines4;
+$.
+$format.
+$format12.2
+somenum.
+somenum12.4
+above are good
+the rest are bad
+%abort
+1&somethingverybad.
+&
++-1
+.
+a.A
+$format12.1b
+$format12.1b1
+;;;;
+run;
+%mp_assertdsobs(work.test3,
+  desc=Test3 - ISFORMAT,
+  test=EQUALS 5,
+  outds=work.test_results
 )

tests/testinit.sas

@@ -2,11 +2,17 @@
   @file
   @brief init file for tests
 
+  <h4> SAS Macros </h4>
+  @li mp_init.sas
+
 **/
 
 /* location in metadata or SAS Drive for temporary files */
 %let mcTestAppLoc=/Public/temp/macrocore;
 
+/* set defaults */
+%mp_init()
+
 %macro loglevel();
   %if &_debug=2477 %then %do;
     options mprint;

tests/viyaonly/mv_createwebservice.test.sas

@@ -33,6 +33,7 @@ run;
 %put TEST1: checking web service code;
 data work.test_results;
   length test_description $256 test_result $4 test_comments $256;
+  if _n_=1 then call missing (of _all_);
   infile compare end=eof;
   input;
   if eof then do;

viya/mv_createwebservice.sas

@@ -242,13 +242,13 @@ data _null_;
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
-  put '  data _null_;file &jref encoding=''utf-8''; ';
-  put '    put ''{"START_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''"''; ';
+  put '  data _null_;file &jref encoding=''utf-8'' ; ';
+  put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
   put '  run; ';
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
   put '  options validvarname=upcase; ';
-  put '  data _null_;file &jref mod encoding=''utf-8''; ';
+  put '  data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
@@ -327,7 +327,7 @@ data _null_;
   put '      run; ';
   put '      %let ds=&fmtds; ';
   put '    %end; /* &fmt=Y */ ';
-  put '    data _null_;file &jref mod encoding=''utf-8''; ';
+  put '    data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
   put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
@@ -372,7 +372,7 @@ data _null_;
   put '    /* write to temp loc to avoid _webout truncation ';
   put '      - https://support.sas.com/kb/49/325.html */ ';
   put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
-  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod; ';
+  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
   put '      %if &action=ARR %then "[" ; %else "{" ; ';
@@ -399,14 +399,14 @@ data _null_;
   put '      rc = fclose(fileid); ';
   put '    run; ';
   put '    filename _sjs clear; ';
-  put '    data _null_; file &jref mod encoding=''utf-8''; ';
+  put '    data _null_; file &jref mod encoding=''utf-8'' ; ';
   put '      put "]"; ';
   put '    run; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8'' mod; ';
+  put '  data _null_;file &jref encoding=''utf-8'' mod ; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';

viya/mv_getfoldermembers.sas

@@ -100,6 +100,8 @@ options noquotelenmax;
   %local href cnt;
   %let cnt=0;
   data _null_;
+    length rel href $512;
+    call missing(rel,href);
     set &libref1..links;
     if rel='members' then do;
       url=cats("'","&base_uri",href,"?limit=10000'");
@@ -123,6 +125,7 @@ options noquotelenmax;
   libname &libref2 JSON fileref=&fname2;
   data &outds;
     length id $36 name $128 uri $64 type $32 description $256;
+    if _n_=1 then call missing (of _all_);
     set &libref2..items;
   run;
   filename &fname2 clear;

viya/mv_getjobcode.sas

@@ -92,6 +92,8 @@ data;run;
 %local joburi;
 %let joburi=0;
 data _null_;
+  length name uri $512;
+  call missing(name,uri);
   set &foldermembers;
   if name="&name" and uri=:'/jobDefinitions/definitions'
     then call symputx('joburi',uri);

viya/mv_jobexecute.sas

@@ -115,6 +115,8 @@ data;run;
 %local joburi;
 %let joburi=0;
 data _null_;
+  length name uri $512;
+  call missing(name,uri);
   set &foldermembers;
   if name="&name" and uri=:'/jobDefinitions/definitions'
     then call symputx('joburi',uri);

viya/mv_jobwaitfor.sas

@@ -168,6 +168,7 @@ run;
 data &outds;
   format _program uri $128. state $32. stateDetails $32. timestamp datetime19.
     jobparams $32767.;
+  call missing (of _all_);
   stop;
 run;