Merge pull request #122 from sasjs/serverfix

fix: making ms_webout work with SAS on Windows Desktop

.sasjslint

@@ -6,7 +6,7 @@
     "hasMacroParentheses": true,
     "noNestedMacros": false,
     "noSpacesInFileNames": true,
-    "maxLineLength": 230,
+    "maxLineLength": 300,
     "lowerCaseFileNames": true,
     "noTabIndentation": true,
     "indentationMultiple": 2

all.sas

@@ -43,6 +43,60 @@ options noquotelenmax;
 %mend mf_abort;
 
 /** @endcond *//**
+  @file
+  @brief de-duplicates a macro string
+  @details Removes all duplicates from a string of words.  A delimeter can be
+  chosen.  Is inspired heavily by this excellent [macro](
+  https://github.com/scottbass/SAS/blob/master/Macro/dedup_mstring.sas) from
+  [Scott Base](https://www.linkedin.com/in/scottbass).  Case sensitive.
+
+  Usage:
+
+      %let str=One two one two and through and through;
+      %put %mf_dedup(&str);
+      %put %mf_dedup(&str,outdlm=%str(,));
+
+  Which returns:
+
+      > One two one and through
+      > One,two,one,and,through
+
+  @param [in] str String to be deduplicated
+  @param [in] indlm= ( ) Delimeter of the input string
+  @param [out] outdlm= ( ) Delimiter of the output string
+
+  <h4> Related Macros </h4>
+  @li mf_trimstr.sas
+  @li mf_wordsinstr1butnotstr2.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_dedup(str
+  ,indlm=%str( )
+  ,outdlm=%str( )
+)/*/STORE SOURCE*/;
+
+%local num word i pos out;
+
+%* loop over each token, searching the target for that token ;
+%let num=%sysfunc(countc(%superq(str),%str(&indlm)));
+%do i=1 %to %eval(&num+1);
+  %let word=%scan(%superq(str),&i,%str(&indlm));
+  %let pos=%sysfunc(indexw(&out,&word,%str(&outdlm)));
+  %if (&pos eq 0) %then %do;
+    %if (&i gt 1) %then %let out=&out%str(&outdlm);
+    %let out=&out&word;
+  %end;
+%end;
+
+%unquote(&out)
+
+%mend mf_dedup;
+
+
+/**
   @file mf_existds.sas
   @brief Checks whether a dataset OR a view exists.
   @details Can be used in open code, eg as follows:
@@ -515,6 +569,109 @@ https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionex
   %end;
 
 %mend mf_getfilesize ;/**
+  @file
+  @brief Returns a distinct list of formats from a table
+  @details Reads the dataset header and returns a distinct list of formats
+  applied.
+
+        %put NOTE- %mf_getfmtlist(sashelp.prdsale);
+        %put NOTE- %mf_getfmtlist(sashelp.shoes);
+        %put NOTE- %mf_getfmtlist(sashelp.demographics);
+
+  returns:
+
+  > DOLLAR $CHAR W MONNAME
+  > $CHAR BEST DOLLAR
+  > BEST Z $CHAR COMMA PERCENTN
+
+
+  @param [in] libds Two part library.dataset reference.
+
+  <h4> SAS Macros </h4>
+  @li mf_getfmtname.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_getfmtlist(libds
+)/*/STORE SOURCE*/;
+/* declare local vars */
+%local out dsid nvars x rc fmt;
+
+/* open dataset in macro */
+%let dsid=%sysfunc(open(&libds));
+
+/* continue if dataset exists */
+%if &dsid %then %do;
+  /* loop each variable in the dataset */
+  %let nvars=%sysfunc(attrn(&dsid,NVARS));
+  %do x=1 %to &nvars;
+    /* grab format and check it exists */
+    %let fmt=%sysfunc(varfmt(&dsid,&x));
+    %if %quote(&fmt) ne %quote() %then %let fmt=%mf_getfmtname(&fmt);
+    %else %do;
+      /* assign default format depending on variable type */
+      %if %sysfunc(vartype(&dsid, &x))=C %then %let fmt=$CHAR;
+      %else %let fmt=BEST;
+    %end;
+    /* concatenate unique list of formats */
+    %if %sysfunc(indexw(&out,&fmt,%str( )))=0 %then %let out=&out &fmt;
+  %end;
+  %let rc=%sysfunc(close(&dsid));
+%end;
+%else %do;
+  %put &sysmacroname: Unable to open &libds (rc=&dsid);
+  %put &sysmacroname: SYSMSG= %sysfunc(sysmsg());
+  %let rc=%sysfunc(close(&dsid));
+%end;
+/* send them out without spaces or quote markers */
+%do;%unquote(&out)%end;
+%mend mf_getfmtlist;/**
+  @file
+  @brief Extracts a format name from a fully defined format
+  @details Converts formats in like $thi3. and th13.2 $THI and TH.
+  Usage:
+
+      %put %mf_getfmtname(8.);
+      %put %mf_getfmtname($4.);
+      %put %mf_getfmtname(comma14.10);
+
+  Returns:
+
+  > W
+  > $CHAR
+  > COMMA
+
+  Note that system defaults are inferred from the values provided.
+
+  @param [in] fmt The fully defined format. If left blank, nothing is returned.
+
+  @returns The name (without width or decimal) of the format.
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_getfmtname(fmt
+)/*/STORE SOURCE*/ /minoperator mindelimiter=' ';
+
+%local out dsid nvars x rc fmt;
+
+/* extract actual format name from the format definition */
+%let fmt=%scan(&fmt,1,.);
+%do %while(%substr(&fmt,%length(&fmt),1) in 1 2 3 4 5 6 7 8 9 0);
+  %if %length(&fmt)=1 %then %let fmt=W;
+  %else %let fmt=%substr(&fmt,1,%length(&fmt)-1);
+%end;
+
+%if &fmt=$ %then %let fmt=$CHAR;
+
+/* send them out without spaces or quote markers */
+%do;%unquote(%upcase(&fmt))%end;
+%mend mf_getfmtname;/**
   @file
   @brief retrieves a key value pair from a control dataset
   @details By default, control dataset is work.mp_setkeyvalue.  Usage:
@@ -627,8 +784,8 @@ https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionex
 > "these","words","are","double","quoted"
 
   @param [in] in_str The unquoted, spaced delimited string to transform
-  @param [in] dlm= The delimeter to be applied to the output (default comma)
+  @param [in] dlm= (,) The delimeter to be applied to the output (default comma)
-  @param [in] indlm= (,) The delimeter used for the input (default is space)
+  @param [in] indlm= ( ) The delimeter used for the input (default is space)
   @param [in] quote= (S) The quote mark to apply (S=Single, D=Double, N=None).
     If any other value than uppercase S or D is supplied, then that value will
     be used as the quoting character.
@@ -639,7 +796,10 @@ https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionex
 **/
 
 
-%macro mf_getquotedstr(IN_STR,DLM=%str(,),QUOTE=S,indlm=%str( )
+%macro mf_getquotedstr(IN_STR
+  ,DLM=%str(,)
+  ,QUOTE=S
+  ,indlm=%str( )
 )/*/STORE SOURCE*/;
   /* credit Rowland Hale  - byte34 is double quote, 39 is single quote */
   %if &quote=S %then %let quote=%qsysfunc(byte(39));
@@ -4192,6 +4352,103 @@ data _null_;
 run;
 
 %mend mp_ds2inserts;/**
+  @file
+  @brief Create a Markdown Table from a dataset
+  @details A markdown table is a simple table representation for use in
+  documents written in markdown format.
+
+  An online generator is available here:
+  https://www.tablesgenerator.com/markdown_tables
+
+  This structure is also used by the Macro Core library for documenting input/
+  output datasets, as well as the sasjs/cli tool for documenting inputs/outputs
+  for web services.
+
+  We take the standard definition one step further by embedding the informat
+  in the table header row, like so:
+
+      |var1:$32|var2:best.|var3:date9.|
+      |---|---|---|
+      |some text|42|01JAN1960|
+      |blah|1|31DEC1999|
+
+  Which resolves to:
+
+  |var1:$32|var2:best.|var3:date9.|
+  |---|---|---|
+  |some text|42|01JAN1960|
+  |blah|1|31DEC1999|
+
+
+  Usage:
+
+      %mp_ds2md(sashelp.class)
+
+  @param [in] libds the library / dataset to create or read from.
+  @param [out] outref= (mdtable) Fileref to contain the markdown
+  @param [out] showlog= (YES) Set to NO to avoid printing markdown to the log
+
+  <h4> SAS Macros </h4>
+  @li mf_getvarlist.sas
+  @li mf_getvarformat.sas
+
+  @version 9.3
+  @author Allan Bowe
+**/
+
+%macro mp_ds2md(
+  libds,
+  outref=mdtable,
+  showlog=YES
+)/*/STORE SOURCE*/;
+
+/* check fileref is assigned */
+%if %sysfunc(fileref(&outref)) > 0 %then %do;
+  filename &outref temp;
+%end;
+
+%local vars;
+%let vars=%upcase(%mf_getvarlist(&libds));
+
+/* create the header row */
+data _null_;
+  file &outref;
+  length line $32767;
+  call missing(line);
+  put '|'
+%local i var fmt;
+%do i=1 %to %sysfunc(countw(&vars));
+  %let var=%scan(&vars,&i);
+  %let fmt=%lowcase(%mf_getvarformat(&libds,&var,force=1));
+  "&var:&fmt|"
+%end;
+  ;
+  put '|'
+%do i=1 %to %sysfunc(countw(&vars));
+  "---|"
+%end;
+  ;
+run;
+
+/* write out the data */
+data _null_;
+  file &outref mod dlm='|' lrecl=32767;
+  set &libds ;
+  length line $32767;
+  line='|`'!!cats(%mf_getvarlist(&libds,dlm=%str(%)!!' `|`'!!cats%()))!!' `|';
+  put line;
+run;
+
+%if %upcase(&showlog)=YES %then %do;
+  options ps=max;
+  data _null_;
+    infile &outref;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_ds2md;/**
   @file
   @brief Checks an input filter table for validity
   @details Performs checks on the input table to ensure it arrives in the
@@ -5495,6 +5752,146 @@ run;
 %end;
 
 %mend mp_getddl;/**
+  @file
+  @brief Export format definitions
+  @details Formats are exported from the first (if any) catalog entry in the
+  FMTSEARCH path.
+
+  Formats are taken from the library / dataset reference and / or a static
+  format list.
+
+  Example usage:
+
+      %mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable)
+
+  @param [in] lib= (0) The libref for which to return formats.
+  @todo Enable exporting of formats for an entire library
+  @param [in] ds= (0) The dataset from which to obtain format definitions
+  @param [in] fmtlist= (0) A list of additional format names
+  @param [out] outsummary= (work.mp_getformats_summary) Output dataset
+    containing summary definitions - structure taken from dictionary.formats as
+    follows:
+
+  |libname:$8.|memname:$32.|path:$1024.|objname:$32.|fmtname:$32.|fmttype:$1.|source:$1.|minw:best.|mind:best.|maxw:best.|maxd:best.|defw:best.|defd:best.|
+  |---|---|---|---|---|---|---|---|---|---|---|---|---|
+  | | | | |$|F|B|1|0|32767|0|1|0|
+  | | | | |$|I|B|1|0|32767|0|1|0|
+  |` `|` `|/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWIANYDT|$ANYDTIF|I|U|1|0|60|0|19|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWFASCII|$ASCII|F|U|1|0|32767|0|1|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWIASCII|$ASCII|I|U|1|0|32767|0|1|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWFBASE6|$BASE64X|F|U|1|0|32767|0|1|0|
+
+
+  @param [out] outdetail= (0) Provide an output dataset in which to export all
+    the custom format definitions (from proc format CNTLOUT).  Definitions:
+https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#a002473477.htm
+    Sample data:
+
+  |FMTNAME:$32.|START:$16.|END:$16.|LABEL:$256.|MIN:best.|MAX:best.|DEFAULT:best.|LENGTH:best.|FUZZ:best.|PREFIX:$2.|MULT:best.|FILL:$1.|NOEDIT:best.|TYPE:$1.|SEXCL:$1.|EEXCL:$1.|HLO:$13.|DECSEP:$1.|DIG3SEP:$1.|DATATYPE:$8.|LANGUAGE:$8.|
+  |---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
+  |`WHICHPATH `|`0 `|`0 `|`path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|` `|` `|` `|` `|` `|
+  |`WHICHPATH `|`**OTHER** `|`**OTHER** `|`big fat problem if not path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|`O `|` `|` `|` `|` `|
+
+  <h4> SAS Macros </h4>
+  @li mf_dedup.sas
+  @li mf_getfmtlist.sas
+  @li mf_getfmtname.sas
+  @li mf_getquotedstr.sas
+  @li mf_getuniquename.sas
+
+
+  <h4> Related Macros </h4>
+  @li mp_getformats.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_getformats(lib=0
+  ,ds=0
+  ,fmtlist=0
+  ,outsummary=work.mp_getformats_summary
+  ,outdetail=0
+);
+
+%local i fmt allfmts tempds fmtcnt;
+
+%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( )));
+  /* ensure format list contains format _name_ only */
+  %let fmt=%scan(&fmtlist,&i,%str( ));
+  %let fmt=%mf_getfmtname(&fmt);
+  %let allfmts=&allfmts &fmt;
+%end;
+
+%if &ds=0 and &lib ne 0 %then %do;
+  /* grab formats from library */
+  /* to do */
+%end;
+%else %if &ds ne 0 and &lib ne 0 %then %do;
+  /* grab formats from dataset */
+  %let allfmts=%mf_getfmtlist(&lib..&ds) &allfmts;
+%end;
+
+/* ensure list is unique */
+%let allfmts=%mf_dedup(%upcase(&allfmts));
+
+/* create summary table */
+%if %index(&outsummary,.)=0 %then %let outsummary=WORK.&outsummary;
+proc sql;
+create table &outsummary as
+  select * from dictionary.formats
+  where fmtname in (%mf_getquotedstr(&allfmts,quote=D))
+    and fmttype='F';
+
+%if "&outdetail" ne "0" %then %do;
+  /* ensure base table always exists */
+  proc sql;
+  create table &outdetail(
+      FMTNAME char(32)     label='Format name'
+      ,START char(16)     label='Starting value for format'
+      ,END char(16)     label='Ending value for format'
+      ,LABEL char(256)     label='Format value label'
+      ,MIN num length=3     label='Minimum length'
+      ,MAX num length=3     label='Maximum length'
+      ,DEFAULT num length=3     label='Default length'
+      ,LENGTH num length=3     label='Format length'
+      ,FUZZ num     label='Fuzz value'
+      ,PREFIX char(2)     label='Prefix characters'
+      ,MULT num     label='Multiplier'
+      ,FILL char(1)     label='Fill character'
+      ,NOEDIT num length=3     label='Is picture string noedit?'
+      ,TYPE char(1)     label='Type of format'
+      ,SEXCL char(1)     label='Start exclusion'
+      ,EEXCL char(1)     label='End exclusion'
+      ,HLO char(13)     label='Additional information'
+      ,DECSEP char(1)     label='Decimal separator'
+      ,DIG3SEP char(1)     label='Three-digit separator'
+      ,DATATYPE char(8)     label='Date/time/datetime?'
+      ,LANGUAGE char(8)     label='Language for date strings'
+  );
+  /* grab the location of each format */
+  %let fmtcnt=0;
+  data _null_;
+    set &outsummary;
+    if not missing(libname);
+    x+1;
+    call symputx(cats('fmtloc',x),cats(libname,'.',memname),'l');
+    call symputx(cats('fmtname',x),fmtname,'l');
+    call symputx('fmtcnt',x,'l');
+  run;
+  /* export each format and append to the output table */
+  %let tempds=%mf_getuniquename(prefix=mp_getformats);
+  %do i=1 %to &fmtcnt;
+    proc format library=&&fmtloc&i CNTLOUT=&tempds;
+      select &&fmtname&i;
+    run;
+    proc append base=&outdetail data=&tempds;
+    run;
+  %end;
+%end;
+
+%mend mp_getformats;/**
   @file mp_getmaxvarlengths.sas
   @brief Scans a dataset to find the max length of the variable values
   @details
@@ -6405,23 +6802,25 @@ filename &tempref clear;
 
 **/
 
-%macro mp_init(prefix=
+%macro mp_init(prefix=SASJS
 )/*/STORE SOURCE*/;
 
   %global
-    &prefix._INIT_NUM   /* initialisation time as numeric             */
+    &prefix._INIT_NUM   /* initialisation time as numeric                   */
-    &prefix._INIT_DTTM  /* initialisation time in E8601DT26.6 format */
+    &prefix._INIT_DTTM  /* initialisation time in E8601DT26.6 format        */
+    &prefix.WORK        /* avoid typing %sysfunc(pathname(work)) every time */
   ;
   %if %eval(&&&prefix._INIT_NUM>0) %then %return;  /* only run once */
 
   data _null_;
     dttm=datetime();
-    call symputx("&prefix._init_num",dttm);
+    call symputx("&prefix._init_num",dttm,'g');
-    call symputx("&prefix._init_dttm",put(dttm,E8601DT26.6));
+    call symputx("&prefix._init_dttm",put(dttm,E8601DT26.6),'g');
+    call symputx("&prefix.work",pathname('WORK'),'g');
   run;
 
   options
-    autocorrect             /* disallow mis-spelled procedure names           */
+    noautocorrect           /* disallow misspelled procedure names            */
     compress=CHAR           /* default is none so ensure we have something!   */
     datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
     %str(err)orcheck=STRICT /* catch errs in libname/filename statements      */
@@ -6431,6 +6830,7 @@ filename &tempref clear;
     noquotelenmax           /* avoid warnings for long strings                */
     noreplace               /* avoid overwriting permanent datasets           */
     ps=max                  /* reduce log size slightly                       */
+    ls=max                  /* reduce log even more and avoid word truncation */
     validmemname=COMPATIBLE /* avoid special characters etc in table names    */
     validvarname=V7         /* avoid special characters etc in variable names */
     varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
@@ -7240,13 +7640,13 @@ data &ds1;
     &n1=ranuni(1)*5000000;
     drop &c1 &n1;
     %let charvars=%mf_getvarlist(&libds,typefilter=C);
-    %do i=1 %to %sysfunc(countw(&charvars));
+    %if &charvars ^= %then %do i=1 %to %sysfunc(countw(&charvars));
       %let col=%scan(&charvars,&i);
       &col=subpad(&c1,1,%mf_getvarlen(&libds,&col));
     %end;
 
     %let numvars=%mf_getvarlist(&libds,typefilter=N);
-    %do i=1 %to %sysfunc(countw(&numvars));
+    %if &numvars ^= %then %do i=1 %to %sysfunc(countw(&numvars));
       %let col=%scan(&numvars,&i);
       &col=&n1;
     %end;
@@ -7261,103 +7661,6 @@ proc sql;
 drop table &ds1;
 
 %mend mp_makedata;/**
-  @file
-  @brief Create a Markdown Table from a dataset
-  @details A markdown table is a simple table representation for use in
-  documents written in markdown format.
-
-  An online generator is available here:
-  https://www.tablesgenerator.com/markdown_tables
-
-  This structure is also used by the Macro Core library for documenting input/
-  output datasets, as well as the sasjs/cli tool for documenting inputs/outputs
-  for web services.
-
-  We take the standard definition one step further by embedding the informat
-  in the table header row, like so:
-
-      |var1:$32|var2:best.|var3:date9.|
-      |---|---|---|
-      |some text|42|01JAN1960|
-      |blah|1|31DEC1999|
-
-  Which resolves to:
-
-  |var1:$32|var2:best.|var3:date9.|
-  |---|---|---|
-  |some text|42|01JAN1960|
-  |blah|1|31DEC1999|
-
-
-  Usage:
-
-      %mp_mdtablewrite(libds=sashelp.class,showlog=YES)
-
-
-  <h4> SAS Macros </h4>
-  @li mf_getvarlist.sas
-  @li mf_getvarformat.sas
-
-  @param [in] libds= the library / dataset to create or read from.
-  @param [out] fref= Fileref to contain the markdown. Default=mdtable.
-  @param [out] showlog= set to YES to show the markdown in the log. Default=NO.
-
-  @version 9.3
-  @author Allan Bowe
-**/
-
-%macro mp_mdtablewrite(
-  libds=,
-  fref=mdtable,
-  showlog=NO
-)/*/STORE SOURCE*/;
-
-/* check fileref is assigned */
-%if %sysfunc(fileref(&fref)) > 0 %then %do;
-  filename &fref temp;
-%end;
-
-%local vars;
-%let vars=%mf_getvarlist(&libds);
-
-/* create the header row */
-data _null_;
-  file &fref;
-  length line $32767;
-  put '|'
-%local i var fmt;
-%do i=1 %to %sysfunc(countw(&vars));
-  %let var=%scan(&vars,&i);
-  %let fmt=%mf_getvarformat(&libds,&var,force=1);
-  "&var:&fmt|"
-%end;
-  ;
-  put '|'
-%do i=1 %to %sysfunc(countw(&vars));
-  "---|"
-%end;
-  ;
-run;
-
-/* write out the data */
-data _null_;
-  file &fref mod dlm='|' lrecl=32767;
-  set &libds ;
-  length line $32767;
-  line=cats('|',%mf_getvarlist(&libds,dlm=%str(,'|',)),'|');
-  put line;
-run;
-
-%if %upcase(&showlog)=YES %then %do;
-  options ps=max;
-  data _null_;
-    infile &fref;
-    input;
-    putlog _infile_;
-  run;
-%end;
-
-%mend mp_mdtablewrite;/**
   @file
   @brief Logs the time the macro was executed in a control dataset.
   @details If the dataset does not exist, it is created.  Usage:
@@ -7975,13 +8278,15 @@ select distinct memname into: table_list separated by ' '
   sort it before performing operations such as merges / joins etc.
   That said, there are a few edge cases where it can be desirable:
 
-    @li To improve performance for particular scenarios
     @li To allow adjacent records to be viewed directly in the dataset
-    @li To reduce dataset size (eg when there are deleted records)
+    @li To apply compression, or to remove deleted records
+    @li To improve performance for specific queries
 
   This macro will only work for BASE (V9) engine libraries.  It works by
   creating a copy of the dataset (without data, WITH constraints) in the same
-  library, appending a sorted view into it, and finally - renaming it.
+  library, appending a sorted view into it, and finally - renaming it.  By
+  default, COMPRESS=CHAR and REUSE=YES will be applied, this behaviour can
+  be adjusted using the `dsoptions=` parameter.
 
   Example usage:
 
@@ -7999,6 +8304,7 @@ select distinct memname into: table_list separated by ' '
   @li mf_getengine.sas
   @li mf_getquotedstr.sas
   @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
   @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_getpk.sas
@@ -8044,9 +8350,13 @@ select distinct memname into: table_list separated by ' '
   %return;
 %end;
 
+/* fallback sortkey is all fields */
+%let sortkey=%mf_getvarlist(&libds);
+
+/* overlay actual sort key if it exists */
 data _null_;
   set work.&tempds1;
-  call symputx('sortkey',pk_fields);
+  call symputx('sortkey',coalescec(pk_fields,symget('sortkey')));
 run;
 
 
@@ -8152,10 +8462,10 @@ run;
             'Did value change? (1/0/-1).  Always -1 for appends and deletes.',
           tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
           tgtvar_nm char(32) label='Target variable name (32 chars)',
-          oldval_num num label='Old (numeric) value',
+          oldval_num num format=best32. label='Old (numeric) value',
-          newval_num num label='New (numeric) value',
+          newval_num num format=best32. label='New (numeric) value',
-          oldval_char char(32767) label='Old (character) value',
+          oldval_char char(32765) label='Old (character) value',
-          newval_char char(32767) label='New (character) value',
+          newval_char char(32765) label='New (character) value',
           constraint pk_mpe_audit
             primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
         );
@@ -8299,18 +8609,18 @@ create table &outds as
     ,b.tgtvar_type length=1
     ,case when b.move_type='D' then b.newval_num
       else a.newval_num
-      end as oldval_num
+      end as oldval_num format=best32.
     ,case when b.move_type='D' then .
       else b.newval_num
-      end as newval_num
+      end as newval_num format=best32.
     ,case when b.move_type='D' then b.newval_char
       else a.newval_char
-      end as oldval_char length=32767
+      end as oldval_char length=32765
     ,case when b.move_type='D' then ''
       else b.newval_char
-      end as newval_char length=32767
+      end as newval_char length=32765
   from &ds4(where=(move_type='O')) as a
-  full join &ds4(where=(move_type ne 'O')) as b
+  right join &ds4(where=(move_type ne 'O')) as b
   on a.tgtvar_nm=b.tgtvar_nm
   and a.key_hash=b.key_hash
   order by move_type, key_hash,is_pk desc, tgtvar_nm;
@@ -9834,7 +10144,7 @@ run;
       prop='Connection.DBMS.Property.SERVER.Name.xmlKey.txt';
       rc=metadata_getprop(uri,prop,server,"");
     end;
-    if server^='' then server='server='!!server;
+    if server^='' then server='server='!!quote(cats(server));
     call symputx('server',server,'l');
 
     /* get SCHEMA value */
@@ -9980,11 +10290,11 @@ run;
   run;
 
   %put NOTE: Executing the following:/; %put NOTE-;
-  %put NOTE- libname &libref TERADATA server=&path schema=&schema ;
+  %put NOTE- libname &libref TERADATA server="&path" schema=&schema ;
   %put NOTe-   authdomain=&authdomain;
   %put NOTE-;
 
-  libname &libref TERADATA server=&path schema=&schema authdomain=&authdomain;
+  libname &libref TERADATA server="&path" schema=&schema authdomain=&authdomain;
 %end;
 %else %if &engine= %then %do;
   %put NOTE: Libref &libref is not registered in metadata;
@@ -12232,7 +12542,7 @@ run;
 data &outattrs;
   keep type name value;
   length type $4 name $256 value $32767;
-  rc1=1;n1=1;type='Prop';
+  rc1=1;n1=1;type='Prop';name='';value='';
   do while(rc1>0);
     rc1=metadata_getnprp("&uri",n1,name,value);
     if rc1>0 then output;
@@ -13705,14 +14015,13 @@ run;
 %if %length(&tree)>0 %then %do;
   /* get tree info */
   %mm_gettree(tree=&tree,inds=&outds, outds=&outds, mDebug=&mDebug)
-    %if %mf_nobs(&outds)=0 %then %do;
+  %if %mf_nobs(&outds)=0 %then %do;
     %put NOTE:  Tree &tree did not exist!!;
     %return;
   %end;
 %end;
 
 
-
 data &outds ;
   set &outds(rename=(treeuri=treeuri_compare));
   length treeuri query stpuri $256;
@@ -13857,7 +14166,7 @@ run;
 data &outds;
   length uri serveruri conn_uri domainuri libname ServerContext AuthDomain
     path_schema usingpkguri type tableuri $256 id $17
-    libdesc $200 libref engine $8 IsDBMSLibname $1
+    libdesc $200 libref engine $8 IsDBMSLibname IsPreassigned $1
     tablename $50 /* metadata table names can be longer than $32 */
     ;
   keep libname libdesc libref engine ServerContext path_schema AuthDomain

base/mf_dedup.sas

@@ -0,0 +1,54 @@
+/**
+  @file
+  @brief de-duplicates a macro string
+  @details Removes all duplicates from a string of words.  A delimeter can be
+  chosen.  Is inspired heavily by this excellent [macro](
+  https://github.com/scottbass/SAS/blob/master/Macro/dedup_mstring.sas) from
+  [Scott Base](https://www.linkedin.com/in/scottbass).  Case sensitive.
+
+  Usage:
+
+      %let str=One two one two and through and through;
+      %put %mf_dedup(&str);
+      %put %mf_dedup(&str,outdlm=%str(,));
+
+  Which returns:
+
+      > One two one and through
+      > One,two,one,and,through
+
+  @param [in] str String to be deduplicated
+  @param [in] indlm= ( ) Delimeter of the input string
+  @param [out] outdlm= ( ) Delimiter of the output string
+
+  <h4> Related Macros </h4>
+  @li mf_trimstr.sas
+  @li mf_wordsinstr1butnotstr2.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_dedup(str
+  ,indlm=%str( )
+  ,outdlm=%str( )
+)/*/STORE SOURCE*/;
+
+%local num word i pos out;
+
+%* loop over each token, searching the target for that token ;
+%let num=%sysfunc(countc(%superq(str),%str(&indlm)));
+%do i=1 %to %eval(&num+1);
+  %let word=%scan(%superq(str),&i,%str(&indlm));
+  %let pos=%sysfunc(indexw(&out,&word,%str(&outdlm)));
+  %if (&pos eq 0) %then %do;
+    %if (&i gt 1) %then %let out=&out%str(&outdlm);
+    %let out=&out&word;
+  %end;
+%end;
+
+%unquote(&out)
+
+%mend mf_dedup;
+
+

base/mf_getfmtlist.sas

@@ -0,0 +1,61 @@
+/**
+  @file
+  @brief Returns a distinct list of formats from a table
+  @details Reads the dataset header and returns a distinct list of formats
+  applied.
+
+        %put NOTE- %mf_getfmtlist(sashelp.prdsale);
+        %put NOTE- %mf_getfmtlist(sashelp.shoes);
+        %put NOTE- %mf_getfmtlist(sashelp.demographics);
+
+  returns:
+
+  > DOLLAR $CHAR W MONNAME
+  > $CHAR BEST DOLLAR
+  > BEST Z $CHAR COMMA PERCENTN
+
+
+  @param [in] libds Two part library.dataset reference.
+
+  <h4> SAS Macros </h4>
+  @li mf_getfmtname.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_getfmtlist(libds
+)/*/STORE SOURCE*/;
+/* declare local vars */
+%local out dsid nvars x rc fmt;
+
+/* open dataset in macro */
+%let dsid=%sysfunc(open(&libds));
+
+/* continue if dataset exists */
+%if &dsid %then %do;
+  /* loop each variable in the dataset */
+  %let nvars=%sysfunc(attrn(&dsid,NVARS));
+  %do x=1 %to &nvars;
+    /* grab format and check it exists */
+    %let fmt=%sysfunc(varfmt(&dsid,&x));
+    %if %quote(&fmt) ne %quote() %then %let fmt=%mf_getfmtname(&fmt);
+    %else %do;
+      /* assign default format depending on variable type */
+      %if %sysfunc(vartype(&dsid, &x))=C %then %let fmt=$CHAR;
+      %else %let fmt=BEST;
+    %end;
+    /* concatenate unique list of formats */
+    %if %sysfunc(indexw(&out,&fmt,%str( )))=0 %then %let out=&out &fmt;
+  %end;
+  %let rc=%sysfunc(close(&dsid));
+%end;
+%else %do;
+  %put &sysmacroname: Unable to open &libds (rc=&dsid);
+  %put &sysmacroname: SYSMSG= %sysfunc(sysmsg());
+  %let rc=%sysfunc(close(&dsid));
+%end;
+/* send them out without spaces or quote markers */
+%do;%unquote(&out)%end;
+%mend mf_getfmtlist;

base/mf_getfmtname.sas

@@ -0,0 +1,44 @@
+/**
+  @file
+  @brief Extracts a format name from a fully defined format
+  @details Converts formats in like $thi3. and th13.2 $THI and TH.
+  Usage:
+
+      %put %mf_getfmtname(8.);
+      %put %mf_getfmtname($4.);
+      %put %mf_getfmtname(comma14.10);
+
+  Returns:
+
+  > W
+  > $CHAR
+  > COMMA
+
+  Note that system defaults are inferred from the values provided.
+
+  @param [in] fmt The fully defined format. If left blank, nothing is returned.
+
+  @returns The name (without width or decimal) of the format.
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_getfmtname(fmt
+)/*/STORE SOURCE*/ /minoperator mindelimiter=' ';
+
+%local out dsid nvars x rc fmt;
+
+/* extract actual format name from the format definition */
+%let fmt=%scan(&fmt,1,.);
+%do %while(%substr(&fmt,%length(&fmt),1) in 1 2 3 4 5 6 7 8 9 0);
+  %if %length(&fmt)=1 %then %let fmt=W;
+  %else %let fmt=%substr(&fmt,1,%length(&fmt)-1);
+%end;
+
+%if &fmt=$ %then %let fmt=$CHAR;
+
+/* send them out without spaces or quote markers */
+%do;%unquote(%upcase(&fmt))%end;
+%mend mf_getfmtname;

base/mf_getquotedstr.sas

@@ -16,8 +16,8 @@
 > "these","words","are","double","quoted"
 
   @param [in] in_str The unquoted, spaced delimited string to transform
-  @param [in] dlm= The delimeter to be applied to the output (default comma)
+  @param [in] dlm= (,) The delimeter to be applied to the output (default comma)
-  @param [in] indlm= (,) The delimeter used for the input (default is space)
+  @param [in] indlm= ( ) The delimeter used for the input (default is space)
   @param [in] quote= (S) The quote mark to apply (S=Single, D=Double, N=None).
     If any other value than uppercase S or D is supplied, then that value will
     be used as the quoting character.
@@ -28,7 +28,10 @@
 **/
 
 
-%macro mf_getquotedstr(IN_STR,DLM=%str(,),QUOTE=S,indlm=%str( )
+%macro mf_getquotedstr(IN_STR
+  ,DLM=%str(,)
+  ,QUOTE=S
+  ,indlm=%str( )
 )/*/STORE SOURCE*/;
   /* credit Rowland Hale  - byte34 is double quote, 39 is single quote */
   %if &quote=S %then %let quote=%qsysfunc(byte(39));

base/mp_ds2md.sas

@@ -29,44 +29,44 @@
 
   Usage:
 
-      %mp_mdtablewrite(libds=sashelp.class,showlog=YES)
+      %mp_ds2md(sashelp.class)
 
+  @param [in] libds the library / dataset to create or read from.
+  @param [out] outref= (mdtable) Fileref to contain the markdown
+  @param [out] showlog= (YES) Set to NO to avoid printing markdown to the log
 
   <h4> SAS Macros </h4>
   @li mf_getvarlist.sas
   @li mf_getvarformat.sas
 
-  @param [in] libds= the library / dataset to create or read from.
-  @param [out] fref= Fileref to contain the markdown. Default=mdtable.
-  @param [out] showlog= set to YES to show the markdown in the log. Default=NO.
-
   @version 9.3
   @author Allan Bowe
 **/
 
-%macro mp_mdtablewrite(
+%macro mp_ds2md(
-  libds=,
+  libds,
-  fref=mdtable,
+  outref=mdtable,
-  showlog=NO
+  showlog=YES
 )/*/STORE SOURCE*/;
 
 /* check fileref is assigned */
-%if %sysfunc(fileref(&fref)) > 0 %then %do;
+%if %sysfunc(fileref(&outref)) > 0 %then %do;
-  filename &fref temp;
+  filename &outref temp;
 %end;
 
 %local vars;
-%let vars=%mf_getvarlist(&libds);
+%let vars=%upcase(%mf_getvarlist(&libds));
 
 /* create the header row */
 data _null_;
-  file &fref;
+  file &outref;
   length line $32767;
+  call missing(line);
   put '|'
 %local i var fmt;
 %do i=1 %to %sysfunc(countw(&vars));
   %let var=%scan(&vars,&i);
-  %let fmt=%mf_getvarformat(&libds,&var,force=1);
+  %let fmt=%lowcase(%mf_getvarformat(&libds,&var,force=1));
   "&var:&fmt|"
 %end;
   ;
@@ -79,20 +79,20 @@ run;
 
 /* write out the data */
 data _null_;
-  file &fref mod dlm='|' lrecl=32767;
+  file &outref mod dlm='|' lrecl=32767;
   set &libds ;
   length line $32767;
-  line=cats('|',%mf_getvarlist(&libds,dlm=%str(,'|',)),'|');
+  line='|`'!!cats(%mf_getvarlist(&libds,dlm=%str(%)!!' `|`'!!cats%()))!!' `|';
   put line;
 run;
 
 %if %upcase(&showlog)=YES %then %do;
   options ps=max;
   data _null_;
-    infile &fref;
+    infile &outref;
     input;
     putlog _infile_;
   run;
 %end;
 
-%mend mp_mdtablewrite;
+%mend mp_ds2md;

base/mp_getformats.sas

@@ -0,0 +1,141 @@
+/**
+  @file
+  @brief Export format definitions
+  @details Formats are exported from the first (if any) catalog entry in the
+  FMTSEARCH path.
+
+  Formats are taken from the library / dataset reference and / or a static
+  format list.
+
+  Example usage:
+
+      %mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable)
+
+  @param [in] lib= (0) The libref for which to return formats.
+  @todo Enable exporting of formats for an entire library
+  @param [in] ds= (0) The dataset from which to obtain format definitions
+  @param [in] fmtlist= (0) A list of additional format names
+  @param [out] outsummary= (work.mp_getformats_summary) Output dataset
+    containing summary definitions - structure taken from dictionary.formats as
+    follows:
+
+  |libname:$8.|memname:$32.|path:$1024.|objname:$32.|fmtname:$32.|fmttype:$1.|source:$1.|minw:best.|mind:best.|maxw:best.|maxd:best.|defw:best.|defd:best.|
+  |---|---|---|---|---|---|---|---|---|---|---|---|---|
+  | | | | |$|F|B|1|0|32767|0|1|0|
+  | | | | |$|I|B|1|0|32767|0|1|0|
+  |` `|` `|/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWIANYDT|$ANYDTIF|I|U|1|0|60|0|19|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWFASCII|$ASCII|F|U|1|0|32767|0|1|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWIASCII|$ASCII|I|U|1|0|32767|0|1|0|
+  | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWFBASE6|$BASE64X|F|U|1|0|32767|0|1|0|
+
+
+  @param [out] outdetail= (0) Provide an output dataset in which to export all
+    the custom format definitions (from proc format CNTLOUT).  Definitions:
+https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#a002473477.htm
+    Sample data:
+
+  |FMTNAME:$32.|START:$16.|END:$16.|LABEL:$256.|MIN:best.|MAX:best.|DEFAULT:best.|LENGTH:best.|FUZZ:best.|PREFIX:$2.|MULT:best.|FILL:$1.|NOEDIT:best.|TYPE:$1.|SEXCL:$1.|EEXCL:$1.|HLO:$13.|DECSEP:$1.|DIG3SEP:$1.|DATATYPE:$8.|LANGUAGE:$8.|
+  |---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
+  |`WHICHPATH `|`0 `|`0 `|`path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|` `|` `|` `|` `|` `|
+  |`WHICHPATH `|`**OTHER** `|`**OTHER** `|`big fat problem if not path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|`O `|` `|` `|` `|` `|
+
+  <h4> SAS Macros </h4>
+  @li mf_dedup.sas
+  @li mf_getfmtlist.sas
+  @li mf_getfmtname.sas
+  @li mf_getquotedstr.sas
+  @li mf_getuniquename.sas
+
+
+  <h4> Related Macros </h4>
+  @li mp_getformats.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_getformats(lib=0
+  ,ds=0
+  ,fmtlist=0
+  ,outsummary=work.mp_getformats_summary
+  ,outdetail=0
+);
+
+%local i fmt allfmts tempds fmtcnt;
+
+%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( )));
+  /* ensure format list contains format _name_ only */
+  %let fmt=%scan(&fmtlist,&i,%str( ));
+  %let fmt=%mf_getfmtname(&fmt);
+  %let allfmts=&allfmts &fmt;
+%end;
+
+%if &ds=0 and &lib ne 0 %then %do;
+  /* grab formats from library */
+  /* to do */
+%end;
+%else %if &ds ne 0 and &lib ne 0 %then %do;
+  /* grab formats from dataset */
+  %let allfmts=%mf_getfmtlist(&lib..&ds) &allfmts;
+%end;
+
+/* ensure list is unique */
+%let allfmts=%mf_dedup(%upcase(&allfmts));
+
+/* create summary table */
+%if %index(&outsummary,.)=0 %then %let outsummary=WORK.&outsummary;
+proc sql;
+create table &outsummary as
+  select * from dictionary.formats
+  where fmtname in (%mf_getquotedstr(&allfmts,quote=D))
+    and fmttype='F';
+
+%if "&outdetail" ne "0" %then %do;
+  /* ensure base table always exists */
+  proc sql;
+  create table &outdetail(
+      FMTNAME char(32)     label='Format name'
+      ,START char(16)     label='Starting value for format'
+      ,END char(16)     label='Ending value for format'
+      ,LABEL char(256)     label='Format value label'
+      ,MIN num length=3     label='Minimum length'
+      ,MAX num length=3     label='Maximum length'
+      ,DEFAULT num length=3     label='Default length'
+      ,LENGTH num length=3     label='Format length'
+      ,FUZZ num     label='Fuzz value'
+      ,PREFIX char(2)     label='Prefix characters'
+      ,MULT num     label='Multiplier'
+      ,FILL char(1)     label='Fill character'
+      ,NOEDIT num length=3     label='Is picture string noedit?'
+      ,TYPE char(1)     label='Type of format'
+      ,SEXCL char(1)     label='Start exclusion'
+      ,EEXCL char(1)     label='End exclusion'
+      ,HLO char(13)     label='Additional information'
+      ,DECSEP char(1)     label='Decimal separator'
+      ,DIG3SEP char(1)     label='Three-digit separator'
+      ,DATATYPE char(8)     label='Date/time/datetime?'
+      ,LANGUAGE char(8)     label='Language for date strings'
+  );
+  /* grab the location of each format */
+  %let fmtcnt=0;
+  data _null_;
+    set &outsummary;
+    if not missing(libname);
+    x+1;
+    call symputx(cats('fmtloc',x),cats(libname,'.',memname),'l');
+    call symputx(cats('fmtname',x),fmtname,'l');
+    call symputx('fmtcnt',x,'l');
+  run;
+  /* export each format and append to the output table */
+  %let tempds=%mf_getuniquename(prefix=mp_getformats);
+  %do i=1 %to &fmtcnt;
+    proc format library=&&fmtloc&i CNTLOUT=&tempds;
+      select &&fmtname&i;
+    run;
+    proc append base=&outdetail data=&tempds;
+    run;
+  %end;
+%end;
+
+%mend mp_getformats;

base/mp_init.sas

@@ -30,19 +30,21 @@
 
 **/
 
-%macro mp_init(prefix=
+%macro mp_init(prefix=SASJS
 )/*/STORE SOURCE*/;
 
   %global
-    &prefix._INIT_NUM   /* initialisation time as numeric             */
+    &prefix._INIT_NUM   /* initialisation time as numeric                   */
-    &prefix._INIT_DTTM  /* initialisation time in E8601DT26.6 format */
+    &prefix._INIT_DTTM  /* initialisation time in E8601DT26.6 format        */
+    &prefix.WORK        /* avoid typing %sysfunc(pathname(work)) every time */
   ;
   %if %eval(&&&prefix._INIT_NUM>0) %then %return;  /* only run once */
 
   data _null_;
     dttm=datetime();
-    call symputx("&prefix._init_num",dttm);
+    call symputx("&prefix._init_num",dttm,'g');
-    call symputx("&prefix._init_dttm",put(dttm,E8601DT26.6));
+    call symputx("&prefix._init_dttm",put(dttm,E8601DT26.6),'g');
+    call symputx("&prefix.work",pathname('WORK'),'g');
   run;
 
   options
@@ -56,6 +58,7 @@
     noquotelenmax           /* avoid warnings for long strings                */
     noreplace               /* avoid overwriting permanent datasets           */
     ps=max                  /* reduce log size slightly                       */
+    ls=max                  /* reduce log even more and avoid word truncation */
     validmemname=COMPATIBLE /* avoid special characters etc in table names    */
     validvarname=V7         /* avoid special characters etc in variable names */
     varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */

base/mp_jsonout.sas

@@ -62,13 +62,13 @@
 %put output location=&jref;
 %if &action=OPEN %then %do;
   options nobomfile;
-  data _null_;file &jref encoding='utf-8';
+  data _null_;file &jref encoding='utf-8' termstr=lf;
     put '{"PROCESSED_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '"';
   run;
 %end;
 %else %if (&action=ARR or &action=OBJ) %then %do;
   options validvarname=upcase;
-  data _null_;file &jref mod encoding='utf-8';
+  data _null_;file &jref mod encoding='utf-8' termstr=lf;
     put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":";
 
   %if &engine=PROCJSON %then %do;
@@ -147,7 +147,7 @@
       run;
       %let ds=&fmtds;
     %end; /* &fmt=Y */
-    data _null_;file &jref mod encoding='utf-8';
+    data _null_;file &jref mod encoding='utf-8' termstr=lf;
       put "["; call symputx('cols',0,'l');
     proc sort
       data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
@@ -192,7 +192,7 @@
     /* write to temp loc to avoid _webout truncation
       - https://support.sas.com/kb/49/325.html */
     filename _sjs temp lrecl=131068 encoding='utf-8';
-    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod;
+    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod termstr=lf;
       set &tempds;
       if _n_>1 then put "," @; put
       %if &action=ARR %then "[" ; %else "{" ;
@@ -219,14 +219,14 @@
       rc = fclose(fileid);
     run;
     filename _sjs clear;
-    data _null_; file &jref mod encoding='utf-8';
+    data _null_; file &jref mod encoding='utf-8' termstr=lf;
       put "]";
     run;
   %end;
 %end;
 
 %else %if &action=CLOSE %then %do;
-  data _null_;file &jref encoding='utf-8' mod;
+  data _null_;file &jref encoding='utf-8' mod termstr=lf;
     put "}";
   run;
 %end;

base/mp_makedata.sas

@@ -64,13 +64,13 @@ data &ds1;
     &n1=ranuni(1)*5000000;
     drop &c1 &n1;
     %let charvars=%mf_getvarlist(&libds,typefilter=C);
-    %do i=1 %to %sysfunc(countw(&charvars));
+    %if &charvars ^= %then %do i=1 %to %sysfunc(countw(&charvars));
       %let col=%scan(&charvars,&i);
       &col=subpad(&c1,1,%mf_getvarlen(&libds,&col));
     %end;
 
     %let numvars=%mf_getvarlist(&libds,typefilter=N);
-    %do i=1 %to %sysfunc(countw(&numvars));
+    %if &numvars ^= %then %do i=1 %to %sysfunc(countw(&numvars));
       %let col=%scan(&numvars,&i);
       &col=&n1;
     %end;

base/mp_sortinplace.sas

@@ -5,13 +5,15 @@
   sort it before performing operations such as merges / joins etc.
   That said, there are a few edge cases where it can be desirable:
 
-    @li To improve performance for particular scenarios
     @li To allow adjacent records to be viewed directly in the dataset
-    @li To reduce dataset size (eg when there are deleted records)
+    @li To apply compression, or to remove deleted records
+    @li To improve performance for specific queries
 
   This macro will only work for BASE (V9) engine libraries.  It works by
   creating a copy of the dataset (without data, WITH constraints) in the same
-  library, appending a sorted view into it, and finally - renaming it.
+  library, appending a sorted view into it, and finally - renaming it.  By
+  default, COMPRESS=CHAR and REUSE=YES will be applied, this behaviour can
+  be adjusted using the `dsoptions=` parameter.
 
   Example usage:
 
@@ -29,6 +31,7 @@
   @li mf_getengine.sas
   @li mf_getquotedstr.sas
   @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
   @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_getpk.sas
@@ -74,9 +77,13 @@
   %return;
 %end;
 
+/* fallback sortkey is all fields */
+%let sortkey=%mf_getvarlist(&libds);
+
+/* overlay actual sort key if it exists */
 data _null_;
   set work.&tempds1;
-  call symputx('sortkey',pk_fields);
+  call symputx('sortkey',coalescec(pk_fields,symget('sortkey')));
 run;
 
 

base/mp_storediffs.sas

@@ -65,10 +65,10 @@
             'Did value change? (1/0/-1).  Always -1 for appends and deletes.',
           tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
           tgtvar_nm char(32) label='Target variable name (32 chars)',
-          oldval_num num label='Old (numeric) value',
+          oldval_num num format=best32. label='Old (numeric) value',
-          newval_num num label='New (numeric) value',
+          newval_num num format=best32. label='New (numeric) value',
-          oldval_char char(32767) label='Old (character) value',
+          oldval_char char(32765) label='Old (character) value',
-          newval_char char(32767) label='New (character) value',
+          newval_char char(32765) label='New (character) value',
           constraint pk_mpe_audit
             primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
         );
@@ -212,18 +212,18 @@ create table &outds as
     ,b.tgtvar_type length=1
     ,case when b.move_type='D' then b.newval_num
       else a.newval_num
-      end as oldval_num
+      end as oldval_num format=best32.
     ,case when b.move_type='D' then .
       else b.newval_num
-      end as newval_num
+      end as newval_num format=best32.
     ,case when b.move_type='D' then b.newval_char
       else a.newval_char
-      end as oldval_char length=32767
+      end as oldval_char length=32765
     ,case when b.move_type='D' then ''
       else b.newval_char
-      end as newval_char length=32767
+      end as newval_char length=32765
   from &ds4(where=(move_type='O')) as a
-  full join &ds4(where=(move_type ne 'O')) as b
+  right join &ds4(where=(move_type ne 'O')) as b
   on a.tgtvar_nm=b.tgtvar_nm
   and a.key_hash=b.key_hash
   order by move_type, key_hash,is_pk desc, tgtvar_nm;

meta/mm_assigndirectlib.sas

@@ -295,7 +295,7 @@ run;
       prop='Connection.DBMS.Property.SERVER.Name.xmlKey.txt';
       rc=metadata_getprop(uri,prop,server,"");
     end;
-    if server^='' then server='server='!!server;
+    if server^='' then server='server='!!quote(cats(server));
     call symputx('server',server,'l');
 
     /* get SCHEMA value */
@@ -441,11 +441,11 @@ run;
   run;
 
   %put NOTE: Executing the following:/; %put NOTE-;
-  %put NOTE- libname &libref TERADATA server=&path schema=&schema ;
+  %put NOTE- libname &libref TERADATA server="&path" schema=&schema ;
   %put NOTe-   authdomain=&authdomain;
   %put NOTE-;
 
-  libname &libref TERADATA server=&path schema=&schema authdomain=&authdomain;
+  libname &libref TERADATA server="&path" schema=&schema authdomain=&authdomain;
 %end;
 %else %if &engine= %then %do;
   %put NOTE: Libref &libref is not registered in metadata;

meta/mm_getdetails.sas

@@ -45,7 +45,7 @@ run;
 data &outattrs;
   keep type name value;
   length type $4 name $256 value $32767;
-  rc1=1;n1=1;type='Prop';
+  rc1=1;n1=1;type='Prop';name='';value='';
   do while(rc1>0);
     rc1=metadata_getnprp("&uri",n1,name,value);
     if rc1>0 then output;

meta/mm_getstps.sas

@@ -63,14 +63,13 @@ run;
 %if %length(&tree)>0 %then %do;
   /* get tree info */
   %mm_gettree(tree=&tree,inds=&outds, outds=&outds, mDebug=&mDebug)
-    %if %mf_nobs(&outds)=0 %then %do;
+  %if %mf_nobs(&outds)=0 %then %do;
     %put NOTE:  Tree &tree did not exist!!;
     %return;
   %end;
 %end;
 
 
-
 data &outds ;
   set &outds(rename=(treeuri=treeuri_compare));
   length treeuri query stpuri $256;

meta/mm_gettables.sas

@@ -39,7 +39,7 @@
 data &outds;
   length uri serveruri conn_uri domainuri libname ServerContext AuthDomain
     path_schema usingpkguri type tableuri $256 id $17
-    libdesc $200 libref engine $8 IsDBMSLibname $1
+    libdesc $200 libref engine $8 IsDBMSLibname IsPreassigned $1
     tablename $50 /* metadata table names can be longer than $32 */
     ;
   keep libname libdesc libref engine ServerContext path_schema AuthDomain

sasjs/sasjsconfig.json

@@ -32,7 +32,9 @@
       "name": "viya",
       "serverUrl": "https://sas.analytium.co.uk",
       "serverType": "SASVIYA",
-      "allowInsecureRequests": false,
+      "httpsAgentOptions": {
+        "allowInsecureRequests": false
+      },
       "appLoc": "/Public/temp/macrocore",
       "macroFolders": [
         "tests/viyaonly"

server/ms_webout.sas

@@ -79,7 +79,7 @@
   OPTIONS NOBOMFILE;
 
   /* setup json */
-  data _null_;file &fref encoding='utf-8';
+  data _null_;file &fref encoding='utf-8' termstr=lf;
   %if %str(&_debug) ge 131 %then %do;
     put '>>weboutBEGIN<<';
   %end;
@@ -108,14 +108,14 @@
       i+1;
       call symputx('wt'!!left(i),name,'l');
       call symputx('wtcnt',i,'l');
-    data _null_; file &fref mod encoding='utf-8';
+    data _null_; file &fref mod encoding='utf-8' termstr=lf;
       put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
       proc contents noprint data=&wt
         out=_data_ (keep=name type length format:);
       run;%let tempds=%scan(&syslast,2,.);
-      data _null_; file &fref mod encoding='utf-8';
+      data _null_; file &fref mod encoding='utf-8' termstr=lf;
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
         nvars=attrn(dsid,'NVARS');
@@ -126,15 +126,15 @@
         put ',"nvars":' nvars;
       %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
       %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
-      data _null_; file &fref mod encoding='utf-8';
+      data _null_; file &fref mod encoding='utf-8' termstr=lf;
         put "}";
     %end;
-    data _null_; file &fref mod encoding='utf-8';
+    data _null_; file &fref mod encoding='utf-8' termstr=lf termstr=lf;
       put "}";
     run;
   %end;
   /* close off json */
-  data _null_;file &fref mod encoding='utf-8';
+  data _null_;file &fref mod encoding='utf-8' termstr=lf;
     _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ",""SYSUSERID"" : ""&sysuserid"" ";
     put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
@@ -147,7 +147,9 @@
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
     put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
-    put ",""SYSPROCESSNAME"" : ""&SYSPROCESSNAME"" ";
+    length SYSPROCESSNAME $512;
+    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
+    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
@@ -156,7 +158,8 @@
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
     put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
-    autoexec=quote(trim(getoption('autoexec')));
+    length autoexec $512;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
     put ',"AUTOEXEC" : ' autoexec;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
     memsize=quote(cats(memsize));

tests/crossplatform/mf_dedup.test.sas

@@ -0,0 +1,23 @@
+/**
+  @file
+  @brief Testing mf_dedup macro
+
+  <h4> SAS Macros </h4>
+  @li mf_dedup.sas
+  @li mp_assert.sas
+
+**/
+
+%let str=One two one two and through and through;
+
+%mp_assert(
+  iftrue=("%mf_dedup(&str)"="One two one and through"),
+  desc=Basic test,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=("%mf_dedup(&str,outdlm=%str(,))"="One,two,one,and,through"),
+  desc=Outdlm test,
+  outds=work.test_results
+)

tests/crossplatform/mf_getfmtlist.test.sas

@@ -0,0 +1,33 @@
+/**
+  @file
+  @brief Testing mf_getfmtlist macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getfmtlist.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(
+    "%mf_getfmtlist(sashelp.prdsale)"="DOLLAR $CHAR W MONNAME"
+  ),
+  desc=Checking basic numeric,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getfmtlist(sashelp.shoes)"="$CHAR BEST DOLLAR"
+  ),
+  desc=Checking basic char,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getfmtlist(sashelp.demographics)"="BEST Z $CHAR COMMA PERCENTN"
+  ),
+  desc=Checking longer numeric,
+  outds=work.test_results
+)

tests/crossplatform/mf_getfmtname.test.sas

@@ -0,0 +1,33 @@
+/**
+  @file
+  @brief Testing mf_getfmtname macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getfmtname.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(
+    "%mf_getfmtname(8.)"="W"
+  ),
+  desc=Checking basic numeric,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getfmtname($4.)"="$CHAR"
+  ),
+  desc=Checking basic char,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getfmtname(comma14.10)"="COMMA"
+  ),
+  desc=Checking longer numeric,
+  outds=work.test_results
+)

tests/crossplatform/mp_ds2md.test.sas

@@ -0,0 +1,34 @@
+/**
+  @file
+  @brief Testing mp_ds2md.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2md.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_ds2md(sashelp.class,outref=md)
+
+data _null_;
+  infile md;
+  input;
+  call symputx(cats('test',_n_),_infile_);
+  if _n_=4 then stop;
+run;
+
+%mp_assert(
+  iftrue=("&test1"="|NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.|"),
+  desc=Checking header row,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test2"="|---|---|---|---|---|"),
+  desc=Checking divider row,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test3"="|`Alfred `|`M `|`14 `|`69 `|`112.5 `|"),
+  desc=Checking data row,
+  outds=work.test_results
+)

tests/crossplatform/mp_getformats.test.sas

@@ -0,0 +1,76 @@
+/**
+  @file
+  @brief Testing mp_getformats.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mp_getformats.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * Test - setup
+  */
+
+%mf_mkdir(&sasjswork/path1)
+%mf_mkdir(&sasjswork/path2)
+
+libname path1 "&sasjswork/path1";
+libname path2 "&sasjswork/path2";
+
+
+PROC FORMAT library=path1;
+  value whichpath 0 = 'path1' other='big fat problem if not path1';
+PROC FORMAT library=path2;
+  value whichpath 0 = 'path2' other='big fat problem if not path2';
+RUN;
+
+
+/** run with path1 path2 FMTSEARCH */
+options insert=(fmtsearch=(path1 path2));
+data _null_;
+  test=0;
+  call symputx('test1',put(test,whichpath.));
+run;
+%mp_assert(
+  iftrue=("&test1"="path1"),
+  desc=Check correct format is applied,
+  outds=work.test_results
+)
+%mp_getformats(fmtlist=WHICHPATH,outsummary=sum,outdetail=detail1)
+%let tst1=0;
+data _null_;
+  set detail1;
+  if fmtname='WHICHPATH' and start='**OTHER**' then call symputx('tst1',label);
+  putlog (_all_)(=);
+run;
+%mp_assert(
+  iftrue=("&tst1"="big fat problem if not path1"),
+  desc=Check correct detail results are applied,
+  outds=work.test_results
+)
+
+/** run with path2 path1 FMTSEARCH */
+options insert=(fmtsearch=(path2 path1));
+data _null_;
+  test=0;
+  call symputx('test2',put(test,whichpath.));
+run;
+%mp_assert(
+  iftrue=("&test2"="path2"),
+  desc=Check correct format is applied,
+  outds=work.test_results
+)
+%mp_getformats(fmtlist=WHICHPATH,outsummary=sum,outdetail=detail2)
+%let tst2=0;
+data _null_;
+  set detail2;
+  if fmtname='WHICHPATH' and start='**OTHER**' then call symputx('tst2',label);
+  putlog (_all_)(=);
+run;
+%mp_assert(
+  iftrue=("&tst2"="big fat problem if not path2"),
+  desc=Check correct detail results are applied,
+  outds=work.test_results
+)

tests/crossplatform/mp_init.test.sas

@@ -15,7 +15,7 @@
 
 %let initial_value=&sasjs_init_num;
 
-%mp_init();
+%mp_init()
 
 %mp_assert(
   iftrue=("&initial_value"="&sasjs_init_num"),

tests/crossplatform/mp_sortinplace.test.sas

@@ -39,4 +39,31 @@ run;
   ),
   desc=Check if sort was appplied,
   outds=work.test_results
+)
+
+/** Test 2 - table without PK  */
+proc sql;
+create table work.example2 as
+  select * from sashelp.classfit;
+%mp_sortinplace(work.example2)
+%mp_assert(
+  iftrue=(
+    %str(&syscc)=%str(0)
+  ),
+  desc=Ensure no errors when no key exists,
+  outds=work.test_results
+)
+
+%let test2=0;
+data _null_;
+  set work.example2;
+  call symputx('test2',name);
+  stop;
+run;
+%mp_assert(
+  iftrue=(
+    %str(&test2)=%str(Alfred)
+  ),
+  desc=Check if sort was appplied when no index exists,
+  outds=work.test_results
 )

tests/crossplatform/mp_storediffs.test.sas

@@ -61,4 +61,42 @@ run;
   checkvals=work.check.val,
   desc=All values have a match,
   test=ALLVALS
+)
+
+/* Test for when there are no actual changes */
+data work.orig work.deleted work.changed work.appended;
+  set sashelp.class;
+  output work.orig;
+run;
+%mp_storediffs(sashelp.class,work.orig,NAME
+  ,delds=work.deleted
+  ,modds=work.changed
+  ,appds=work.appended
+  ,outds=work.final2
+  ,mdebug=1
+)
+%mp_assertdsobs(work.final2,
+  desc=No changes produces 0 records,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+/* Test for deletes only */
+data work.orig work.deleted work.changed work.appended;
+  set sashelp.class;
+  output work.orig;
+  if _n_>5 then output work.deleted;
+run;
+
+%mp_storediffs(sashelp.class,work.orig,NAME
+  ,delds=work.deleted
+  ,modds=work.changed
+  ,appds=work.appended
+  ,outds=work.final3
+  ,mdebug=1
+)
+%mp_assertdsobs(work.final3,
+  desc=Delete has 70 records,
+  test=EQUALS 70,
+  outds=work.test_results
 )