feat: enabling informats to be ingested with the mp_loadformat macro

base/mp_cntlout.sas

@@ -28,6 +28,7 @@
 
   <h4> Related Macros </h4>
   @li mf_getvarformat.sas
+  @li mp_aligndecimal.sas
   @li mp_getformats.sas
   @li mp_loadformat.sas
   @li mp_ds2fmtds.sas
@@ -69,13 +70,13 @@ run;
 data &cntlout;
   if 0 then set &ddlds;
   set &cntlds;
-  if type="N" then do;
+  if type in ("I","N") then do; /* numeric (in)format */
-    start=cats(start);
+    %mp_aligndecimal(start,width=16)
-    end=cats(end);
+    %mp_aligndecimal(end,width=16)
   end;
 run;
 proc sort;
-  by fmtname start;
+  by type fmtname start;
 run;
 
 proc sql;

base/mp_loadformat.sas

@@ -45,6 +45,7 @@
   <h4> Related Macros </h4>
   @li mddl_dc_difftable.sas
   @li mddl_dc_locktable.sas
+  @li mp_aligndecimal.sas
   @li mp_loadformat.test.sas
   @li mp_lockanytable.sas
   @li mp_stackdiffs.sas
@@ -134,7 +135,16 @@ run;
   * First, extract only relevant formats from the catalog
   */
 proc sql noprint;
-select distinct upcase(fmtname) into: fmtlist separated by ' ' from &libds;
+select distinct
+  case
+    when type='N' then upcase(fmtname)
+    when type='C' then cats('$',upcase(fmtname))
+    when type='I' then cats('@',upcase(fmtname))
+    when type='J' then cats('@$',upcase(fmtname))
+    else "&sysmacroname:UNHANDLED"
+  end
+  into: fmtlist separated by ' '
+  from &libds;
 
 %mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
 
@@ -146,16 +156,24 @@ select distinct upcase(fmtname) into: fmtlist separated by ' ' from &libds;
 data &inlibds;
   length &delete_col $3;
   if 0 then set &template;
+  length start end $10000;
   set &libds;
   if &delete_col='' then &delete_col='No';
   fmtname=upcase(fmtname);
+  type=upcase(type);
   if missing(type) then do;
-    if substr(fmtname,1,1)='$' then type='C';
+    if substr(fmtname,1,1)='@' then do;
-    else type='N';
+      if substr(fmtname,2,1)='$' then type='J';
+      else type='I';
+    end;
+    else do;
+      if substr(fmtname,1,1)='$' then type='C';
+      else type='N';
+    end;
   end;
-  if type='N' then do;
+  if type in ('N','I') then do;
-    start=cats(start);
+    %mp_aligndecimal(start,width=16)
-    end=cats(end);
+    %mp_aligndecimal(end,width=16)
   end;
 run;
 
@@ -169,9 +187,10 @@ create table &outds_add(drop=&delete_col) as
   left join &base_fmts b
   on a.fmtname=b.fmtname
     and a.start=b.start
+    and a.type=b.type
   where b.fmtname is null
     and upcase(a.&delete_col) ne "YES"
-  order by fmtname, start;;
+  order by type, fmtname, start;
 
 /**
   * Identify deleted records
@@ -182,8 +201,9 @@ create table &outds_del(drop=&delete_col) as
   inner join &base_fmts b
   on a.fmtname=b.fmtname
     and a.start=b.start
+    and a.type=b.type
   where upcase(a.&delete_col)="YES"
-  order by fmtname, start;
+  order by type, fmtname, start;
 
 /**
   * Identify modified records
@@ -194,8 +214,9 @@ create table &outds_mod (drop=&delete_col) as
   inner join &base_fmts b
   on a.fmtname=b.fmtname
     and a.start=b.start
+    and a.type=b.type
   where upcase(a.&delete_col) ne "YES"
-  order by fmtname, start;
+  order by type, fmtname, start;
 
 options ibufsize=&ibufsize;
 
@@ -212,13 +233,13 @@ options ibufsize=&ibufsize;
       &outds_add(in=add)
       &outds_del(in=del);
     if not del and not mod;
-    by fmtname start;
+    by type fmtname start;
   run;
   data &stagedata;
     set &ds1 &outds_mod;
   run;
   proc sort;
-    by fmtname start;
+    by type fmtname start;
   run;
 %end;
 /* mp abort needs to run outside of conditional blocks */
@@ -266,7 +287,7 @@ options ibufsize=&ibufsize;
 
     %mp_storediffs(&libcat-FC
       ,&base_fmts
-      ,FMTNAME START
+      ,TYPE FMTNAME START
       ,delds=&outds_del
       ,modds=&outds_mod
       ,appds=&outds_add

ddl/mddl_sas_cntlout.sas

@@ -12,14 +12,20 @@
 
 proc sql;
 create table &libds(
-    FMTNAME char(32)      label='Format name'
+    TYPE char(1)         label='Type of format'
+    ,FMTNAME char(32)      label='Format name'
     /*
       to accommodate larger START values, mp_loadformat.sas will need the
       SQL dependency removed (proc sql needs to accommodate 3 index values in
       a 32767 ibufsize limit)
     */
     ,START char(10000)    label='Starting value for format'
-    ,END char(32767)      label='Ending value for format'
+    /*
+      Keep lengths of START and END the same to avoid this err:
+      "Start is greater than end:  -<."
+      Similar usage note: https://support.sas.com/kb/69/330.html
+    */
+    ,END char(10000)      label='Ending value for format'
     ,LABEL char(32767)    label='Format value label'
     ,MIN num length=3     label='Minimum length'
     ,MAX num length=3     label='Maximum length'
@@ -30,7 +36,6 @@ create table &libds(
     ,MULT num             label='Multiplier'
     ,FILL char(1)         label='Fill character'
     ,NOEDIT num length=3  label='Is picture string noedit?'
-    ,TYPE char(1)         label='Type of format'
     ,SEXCL char(1)        label='Start exclusion'
     ,EEXCL char(1)        label='End exclusion'
     ,HLO char(13)         label='Additional information'

tests/base/mp_loadformat.test.sas

@@ -1,9 +1,11 @@
 /**
   @file
   @brief Testing mp_loadformat.sas macro
+  @details first test regular formats, then informats
 
   <h4> SAS Macros </h4>
   @li mddl_dc_difftable.sas
+  @li mp_aligndecimal.sas
   @li mp_loadformat.sas
   @li mp_assert.sas
   @li mp_assertscope.sas
@@ -15,8 +17,10 @@ libname perm (work);
 
 %mddl_dc_difftable(libds=perm.audit)
 
+/* set up regular formats */
 data work.loadfmts;
-  length fmtname $32;
+  /* matching start / end lengths (to baseds) are important */
+  length fmtname $32 start end $10000;
   eexcl='Y';
   type='N';
   do i=1 to 100;
@@ -24,7 +28,9 @@ data work.loadfmts;
     do j=1 to 100;
       start=cats(j);
       end=cats(j+1);
-      label= cats('Dummy ',start);
+      %mp_aligndecimal(start,width=16)
+      %mp_aligndecimal(end,width=16)
+      label= cats('Numeric Format ',start);
       output;
     end;
   end;
@@ -42,6 +48,8 @@ data work.stagedata;
   else if _n_<350 then do;
     start=cats(_n_);
     end=cats(_n_+1);
+    %mp_aligndecimal(start,width=16)
+    %mp_aligndecimal(end,width=16)
     label='newval'!!cats(_N_);
   end;
   else stop;
@@ -91,3 +99,118 @@ run;
   desc=Test 1 - diffs were found,
   outds=work.test_results
 )
+
+/* set up a mix of formats */
+data work.loadfmts2;
+  length fmtname $32 start end $10000;
+  eexcl='Y';
+  type='J';
+  do i=1 to 3;
+    fmtname=cats('SASJS_CI_',i,'X');
+    do j=1 to 4;
+      start=cats(j);
+      end=start;
+      label= cats('Char INFORMAT ',start);
+      output;
+    end;
+  end;
+  type='I';
+  do i=1 to 3;
+    fmtname=cats('SASJS_NI_',i,'X');
+    do j=1 to 4;
+      start=cats(j);
+      end=cats(j+1);
+      %mp_aligndecimal(start,width=16)
+      %mp_aligndecimal(end,width=16)
+      label= cats(ranuni(0));
+      output;
+    end;
+  end;
+  type='N';
+  do i=1 to 3;
+    fmtname=cats('SASJS_NF_',i,'X');
+    do j=1 to 4;
+      start=cats(j);
+      end=cats(j+1);
+      %mp_aligndecimal(start,width=16)
+      %mp_aligndecimal(end,width=16)
+      label= cats('Numeric Format ',start);
+      output;
+    end;
+  end;
+  type='C';
+  do i=1 to 3;
+    fmtname=cats('SASJS_CF_',i,'X');
+    do j=1 to 4;
+      start=cats(j);
+      end=start;
+      label= cats('Char Format ',start);
+      output;
+    end;
+  end;
+  drop i j;
+run;
+proc format cntlin=work.loadfmts2 library=perm.testcat2;
+run;
+
+/* make some test data */
+data work.stagedata2;
+  set work.loadfmts2;
+  where type in ('I','J');
+  eexcl='Y';
+  if type='I' then do;
+    i+1;
+    if i<3 then deleteme='Yes';
+    else if i<7 then label= cats(ranuni(0)*100);
+    else if i<12 then do;
+      /* new values */
+      z=ranuni(0)*1000000;
+      start=cats(z);
+      end=cats(z+1);
+      %mp_aligndecimal(start,width=16)
+      %mp_aligndecimal(end,width=16)
+      label= cats(ranuni(0)*100);
+    end;
+    if i<12 then output;
+  end;
+  else do;
+    j+1;
+    if j<3 then deleteme='Yes';
+    else if j<7 then label= cats(ranuni(0)*100);
+    else if j<12 then do;
+      start= cats("NEWVAL",start);
+      end=start;
+      label= "NEWVAL "||cats(ranuni(0)*100);
+    end;
+    if j<12 then output;
+  end;
+
+run;
+
+%mp_loadformat(perm.testcat2
+  ,work.stagedata2
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_test2
+  ,outds_del=del_test2
+  ,outds_mod=mod_test2
+  ,mdebug=1
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(del_test2)=4),
+  desc=Test 2 - delete obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(mod_test2)=8),
+  desc=Test 2 - mod obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(add_test2)=10),
+  desc=Test 2 - add obs,
+  outds=work.test_results
+)