feat: finishing mp_formats and adding a test, including prefix in mp_init, allowing mp_sortinplace to work when there is no primary key, sand other small fixes

base/mp_getformats.sas

@@ -9,12 +9,12 @@
 
   Example usage:
 
-      %mp_getformats(lib=sashelp,ds=prdsale,outds=work.prdfmts)
+      %mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable)
 
   @param [in] lib= (0) The libref for which to return formats.
   @todo Enable exporting of formats for an entire library
   @param [in] ds= (0) The dataset from which to obtain format definitions
-  @param [in] fmtstring= (0) A list of additional format names
+  @param [in] fmtlist= (0) A list of additional format names
   @param [out] outsummary= (work.mp_getformats_summary) Output dataset
     containing summary definitions - structure taken from dictionary.formats as
     follows:
@@ -32,8 +32,10 @@ efw:best.|defd:best.|
 |||||$BINARY|F|B|1|0|32767|0|8|0|
 |||||$BINARY|I|B|1|0|32767|0|8|0|
 
-  @param [out] outdetail= (work.mp_getformats_detail) The output dataset to
-    contain the format definitions (from proc format CNTLOUT).  Sample data:
+  @param [out] outdetail= (0) Provide an output dataset in which to export all
+    the custom format definitions (from proc format CNTLOUT).  Definitions:
+https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#a002473477.htm
+  Sample data:
 
   |NAME $|LENGTH 8|VARNUM 8|LABEL $|FORMAT $49|TYPE $1 |DDTYPE $|
   |---|---|---|---|---|---|---|
@@ -46,6 +48,7 @@ efw:best.|defd:best.|
   @li mf_getfmtlist.sas
   @li mf_getfmtname.sas
   @li mf_getquotedstr.sas
+  @li mf_getuniquename.sas
 
 
   <h4> Related Macros </h4>
@@ -58,15 +61,16 @@ efw:best.|defd:best.|
 
 %macro mp_getformats(lib=0
   ,ds=0
-  ,fmtstring=0
+  ,fmtlist=0
   ,outsummary=work.mp_getformats_summary
-  ,outdetail=work.mp_getformats_detail
+  ,outdetail=0
 );
 
-%local i fmt allfmts;
-%if "&fmtstring" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtstring,,%str( )));
+%local i fmt allfmts tempds fmtcnt;
+
+%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,,%str( )));
   /* ensure format list contains format _name_ only */
-  %let fmt=%scan(&fmtstring,&i,%str( ));
+  %let fmt=%scan(&fmtlist,&i,%str( ));
   %let fmt=%mf_getfmtname(&fmt);
   %let allfmts=&allfmts &fmt;
 %end;
@@ -83,12 +87,59 @@ efw:best.|defd:best.|
 /* ensure list is unique */
 %let allfmts=%mf_dedup(%upcase(&allfmts));
 
-/* now we have the formats, lets figure out the search path */
+/* create summary table */
+%if %index(&outsummary,.)=0 %then %let outsummary=WORK.&outsummary;
 proc sql;
 create table &outsummary as
   select * from dictionary.formats
   where fmtname in (%mf_getquotedstr(&allfmts,quote=D))
+    and fmttype='F';
 
-
+%if "&outdetail" ne "0" %then %do;
+  /* ensure base table always exists */
+  proc sql;
+  create table &outdetail(
+      FMTNAME char(32)     label='Format name'
+      ,START char(16)     label='Starting value for format'
+      ,END char(16)     label='Ending value for format'
+      ,LABEL char(256)     label='Format value label'
+      ,MIN num length=3     label='Minimum length'
+      ,MAX num length=3     label='Maximum length'
+      ,DEFAULT num length=3     label='Default length'
+      ,LENGTH num length=3     label='Format length'
+      ,FUZZ num     label='Fuzz value'
+      ,PREFIX char(2)     label='Prefix characters'
+      ,MULT num     label='Multiplier'
+      ,FILL char(1)     label='Fill character'
+      ,NOEDIT num length=3     label='Is picture string noedit?'
+      ,TYPE char(1)     label='Type of format'
+      ,SEXCL char(1)     label='Start exclusion'
+      ,EEXCL char(1)     label='End exclusion'
+      ,HLO char(13)     label='Additional information'
+      ,DECSEP char(1)     label='Decimal separator'
+      ,DIG3SEP char(1)     label='Three-digit separator'
+      ,DATATYPE char(8)     label='Date/time/datetime?'
+      ,LANGUAGE char(8)     label='Language for date strings'
+  );
+  /* grab the location of each format */
+  %let fmtcnt=0;
+  data _null_;
+    set &outsummary;
+    if not missing(libname);
+    x+1;
+    call symputx(cats('fmtloc',x),cats(libname,'.',memname),'l');
+    call symputx(cats('fmtname',x),fmtname,'l');
+    call symputx('fmtcnt',x,'l');
+  run;
+  /* export each format and append to the output table */
+  %let tempds=%mf_getuniquename(prefix=mp_getformats);
+  %do i=1 %to &fmtcnt;
+    proc format library=&&fmtloc&i CNTLOUT=&tempds;
+      select &&fmtname&i;
+    run;
+    proc append base=&outdetail data=&tempds;
+    run;
+  %end;
+%end;
 
 %mend mp_getformats;

base/mp_init.sas

@@ -30,7 +30,7 @@
 
 **/
 
-%macro mp_init(prefix=
+%macro mp_init(prefix=SASJS
 )/*/STORE SOURCE*/;
 
   %global

base/mp_sortinplace.sas

@@ -5,13 +5,15 @@
   sort it before performing operations such as merges / joins etc.
   That said, there are a few edge cases where it can be desirable:
 
-    @li To improve performance for particular scenarios
     @li To allow adjacent records to be viewed directly in the dataset
-    @li To reduce dataset size (eg when there are deleted records)
+    @li To apply compression, or to remove deleted records
+    @li To improve performance for specific queries
 
   This macro will only work for BASE (V9) engine libraries.  It works by
   creating a copy of the dataset (without data, WITH constraints) in the same
-  library, appending a sorted view into it, and finally - renaming it.
+  library, appending a sorted view into it, and finally - renaming it.  By
+  default, COMPRESS=CHAR and REUSE=YES will be applied, this behaviour can
+  be adjusted using the `dsoptions=` parameter.
 
   Example usage:
 
@@ -29,6 +31,7 @@
   @li mf_getengine.sas
   @li mf_getquotedstr.sas
   @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
   @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_getpk.sas
@@ -74,6 +77,10 @@
   %return;
 %end;
 
+/* fallback sortkey is all fields */
+%let sortkey=%mf_getvarlist(&libds);
+
+/* overlay actual sort key if it exists */
 data _null_;
   set work.&tempds1;
   call symputx('sortkey',pk_fields);

base/mp_storediffs.sas

@@ -65,10 +65,10 @@
             'Did value change? (1/0/-1).  Always -1 for appends and deletes.',
           tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
           tgtvar_nm char(32) label='Target variable name (32 chars)',
-          oldval_num num label='Old (numeric) value',
-          newval_num num label='New (numeric) value',
-          oldval_char char(32767) label='Old (character) value',
-          newval_char char(32767) label='New (character) value',
+          oldval_num num format=best32. label='Old (numeric) value',
+          newval_num num format=best32. label='New (numeric) value',
+          oldval_char char(32765) label='Old (character) value',
+          newval_char char(32765) label='New (character) value',
           constraint pk_mpe_audit
             primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
         );
@@ -212,18 +212,18 @@ create table &outds as
     ,b.tgtvar_type length=1
     ,case when b.move_type='D' then b.newval_num
       else a.newval_num
-      end as oldval_num
+      end as oldval_num format=best32.
     ,case when b.move_type='D' then .
       else b.newval_num
-      end as newval_num
+      end as newval_num format=best32.
     ,case when b.move_type='D' then b.newval_char
       else a.newval_char
-      end as oldval_char length=32767
+      end as oldval_char length=32765
     ,case when b.move_type='D' then ''
       else b.newval_char
-      end as newval_char length=32767
+      end as newval_char length=32765
   from &ds4(where=(move_type='O')) as a
-  full join &ds4(where=(move_type ne 'O')) as b
+  right join &ds4(where=(move_type ne 'O')) as b
   on a.tgtvar_nm=b.tgtvar_nm
   and a.key_hash=b.key_hash
   order by move_type, key_hash,is_pk desc, tgtvar_nm;