From 4f481ec8b4cd46472bef6380c0fe413759be1274 Mon Sep 17 00:00:00 2001
From: Allan <allan@4gl.io>
Date: Wed, 21 Jun 2023 16:41:46 +0100
Subject: [PATCH] fix: adding support for multilabel and notsorted formats
 included additional test job covering multiple scenarios.  Closes #337

---
 all.sas                                       | 122 ++++++++++-------
 base/mp_cntlout.sas                           |  14 +-
 base/mp_ds2md.sas                             |   1 +
 base/mp_loadformat.sas                        |  67 ++++-----
 base/mp_storediffs.sas                        |   6 +-
 ddl/mddl_sas_cntlout.sas                      |  34 +++--
 ...rmat.test.sas => mp_loadformat.test.1.sas} |  97 +++++++------
 tests/base/mp_loadformat.test.2.sas           | 128 ++++++++++++++++++
 8 files changed, 328 insertions(+), 141 deletions(-)
 rename tests/base/{mp_loadformat.test.sas => mp_loadformat.test.1.sas} (67%)
 create mode 100644 tests/base/mp_loadformat.test.2.sas

diff --git a/all.sas b/all.sas
index 329b1b5..fabe602 100644
--- a/all.sas
+++ b/all.sas
@@ -4161,16 +4161,24 @@ proc format lib=&libcat cntlout=&cntlds;
 %end;
 run;
 
-data &cntlout;
+data &cntlout/nonote2err;
   if 0 then set &ddlds;
   set &cntlds;
-  if type in ("I","N") then do; /* numeric (in)format */
+  by type fmtname notsorted;
+
+  /* align the numeric values to avoid overlapping ranges */
+  if type in ("I","N") then do;
     %mp_aligndecimal(start,width=16)
     %mp_aligndecimal(end,width=16)
   end;
+
+  /* create row marker. Data cannot be sorted without it! */
+  if first.fmtname then fmtrow=0;
+  fmtrow+1;
+
 run;
 proc sort;
-  by type fmtname start;
+  by type fmtname fmtrow;
 run;
 
 proc sql;
@@ -6014,6 +6022,7 @@ run;
   options ps=max lrecl=max;
   data _null_;
     infile &outref;
+    if _n_=1 then putlog "# &libds" /;
     input;
     putlog _infile_;
   run;
@@ -10119,19 +10128,20 @@ select distinct lowcase(memname)
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
-  @li mddl_sas_cntlout.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
+  @li mp_md5.sas
   @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
   @li mddl_dc_difftable.sas
   @li mddl_dc_locktable.sas
-  @li mp_loadformat.test.sas
+  @li mp_loadformat.test.1.sas
+  @li mp_loadformat.test.2.sas
   @li mp_lockanytable.sas
   @li mp_stackdiffs.sas
 
@@ -10164,13 +10174,6 @@ select distinct lowcase(memname)
   %let &var=%upcase(&prefix._&var);
 %end;
 
-/*
-format values can be up to 32767 wide.  SQL joins on such a wide column can
-cause buffer issues.  Update ibufsize and reset at the end.
-*/
-%let ibufsize=%sysfunc(getoption(ibufsize));
-options ibufsize=32767 ;
-
 /* in DC, format catalogs maybe specified in the libds with a -FC extension */
 %let libcat=%scan(&libcat,1,-);
 
@@ -10233,16 +10236,24 @@ select distinct
 
 %mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
 
+/* get a hash of the row */
+%local cvars nvars;
+%let cvars=TYPE FMTNAME START END LABEL PREFIX FILL SEXCL EEXCL HLO DECSEP
+  DIG3SEP DATATYPE LANGUAGE;
+%let nvars=FMTROW MIN MAX DEFAULT LENGTH FUZZ MULT NOEDIT;
+data &base_fmts/note2err;
+  set &base_fmts;
+  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
+run;
 
 /**
   * Ensure input table and base_formats have consistent lengths and types
   */
-%mddl_sas_cntlout(libds=&template)
-data &inlibds;
-  length &delete_col $3;
-  if 0 then set &template;
-  length start end $10000;
+data &inlibds/nonote2err;
+  length &delete_col $3 FMTROW 8 start end label $32767;
+  if 0 then set &base_fmts;
   set &libds;
+  by type fmtname notsorted;
   if &delete_col='' then &delete_col='No';
   fmtname=upcase(fmtname);
   type=upcase(type);
@@ -10260,6 +10271,14 @@ data &inlibds;
     %mp_aligndecimal(start,width=16)
     %mp_aligndecimal(end,width=16)
   end;
+
+  /* update row marker - retain new var as fmtrow may already be in libds */
+  if first.fmtname then row=1;
+  else row+1;
+  drop row;
+  fmtrow=row;
+
+  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
 run;
 
 /**
@@ -10270,12 +10289,10 @@ create table &outds_add(drop=&delete_col) as
   select a.*
   from &inlibds a
   left join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
-    and a.type=b.type
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where b.fmtname is null
     and upcase(a.&delete_col) ne "YES"
-  order by type, fmtname, start;
+  order by type, fmtname, fmtrow;
 
 /**
   * Identify deleted records
@@ -10284,11 +10301,9 @@ create table &outds_del(drop=&delete_col) as
   select a.*
   from &inlibds a
   inner join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
-    and a.type=b.type
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where upcase(a.&delete_col)="YES"
-  order by type, fmtname, start;
+  order by type, fmtname, fmtrow;
 
 /**
   * Identify modified records
@@ -10297,13 +10312,10 @@ create table &outds_mod (drop=&delete_col) as
   select a.*
   from &inlibds a
   inner join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
-    and a.type=b.type
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where upcase(a.&delete_col) ne "YES"
-  order by type, fmtname, start;
-
-options ibufsize=&ibufsize;
+    and a.fmthash ne b.fmthash
+  order by type, fmtname, fmtrow;
 
 %mp_abort(
   iftrue=(&syscc ne 0)
@@ -10312,19 +10324,21 @@ options ibufsize=&ibufsize;
 )
 
 %if &loadtarget=YES %then %do;
+  /* new records plus base records that are not deleted or modified */
   data &ds1;
     merge &base_fmts(in=base)
       &outds_mod(in=mod)
       &outds_add(in=add)
       &outds_del(in=del);
     if not del and not mod;
-    by type fmtname start;
+    by type fmtname fmtrow;
   run;
+  /* add back the modified records */
   data &stagedata;
     set &ds1 &outds_mod;
   run;
   proc sort;
-    by type fmtname start;
+    by type fmtname fmtrow;
   run;
 %end;
 /* mp abort needs to run outside of conditional blocks */
@@ -10372,7 +10386,7 @@ options ibufsize=&ibufsize;
 
     %mp_storediffs(&libcat-FC
       ,&base_fmts
-      ,TYPE FMTNAME START
+      ,TYPE FMTNAME FMTROW
       ,delds=&outds_del
       ,modds=&outds_mod
       ,appds=&outds_add
@@ -12796,9 +12810,9 @@ run;
 
 %if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do;
   /* this is a format catalog - cannot query cols directly */
-  %let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH"
-    ,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO"
-    ,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
+  %let vlist="TYPE","FMTNAME","FMTROW","START","END","LABEL","MIN","MAX"
+    ,"DEFAULT","LENGTH","FUZZ","PREFIX","MULT","FILL","NOEDIT","SEXCL"
+    ,"EEXCL","HLO","DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
 %end;
 %else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
 
@@ -14076,22 +14090,18 @@ ods package close;
 
 %macro mddl_sas_cntlout(libds=WORK.CNTLOUT);
 
-proc sql;
-create table &libds(
-    TYPE char(1)         label='Type of format'
-    ,FMTNAME char(32)      label='Format name'
-    /*
-      to accommodate larger START values, mp_loadformat.sas will need the
-      SQL dependency removed (proc sql needs to accommodate 3 index values in
-      a 32767 ibufsize limit)
-    */
-    ,START char(10000)    label='Starting value for format'
+  proc sql;
+  create table &libds(
+    TYPE char(1) label='Type of format - either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
+    ,FMTNAME char(32)     label='Format name'
+    ,FMTROW num label='CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
+    ,START char(32767)    label='Starting value for format'
     /*
       Keep lengths of START and END the same to avoid this err:
       "Start is greater than end:  -<."
       Similar usage note: https://support.sas.com/kb/69/330.html
     */
-    ,END char(10000)      label='Ending value for format'
+    ,END char(32767)      label='Ending value for format'
     ,LABEL char(32767)    label='Format value label'
     ,MIN num length=3     label='Minimum length'
     ,MAX num length=3     label='Maximum length'
@@ -14104,12 +14114,24 @@ create table &libds(
     ,NOEDIT num length=3  label='Is picture string noedit?'
     ,SEXCL char(1)        label='Start exclusion'
     ,EEXCL char(1)        label='End exclusion'
-    ,HLO char(13)         label='Additional information'
+    ,HLO char(13)         label='Additional information. M=MultiLabel'
     ,DECSEP char(1)       label='Decimal separator'
     ,DIG3SEP char(1)      label='Three-digit separator'
     ,DATATYPE char(8)     label='Date/time/datetime?'
     ,LANGUAGE char(8)     label='Language for date strings'
-);
+  );
+
+  %local lib;
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create
+      pk_cntlout=(type fmtname fmtrow)
+      /nomiss unique;
+  quit;
 
 %mend mddl_sas_cntlout;
 /**
diff --git a/base/mp_cntlout.sas b/base/mp_cntlout.sas
index 8e56cc8..0329360 100644
--- a/base/mp_cntlout.sas
+++ b/base/mp_cntlout.sas
@@ -67,16 +67,24 @@ proc format lib=&libcat cntlout=&cntlds;
 %end;
 run;
 
-data &cntlout;
+data &cntlout/nonote2err;
   if 0 then set &ddlds;
   set &cntlds;
-  if type in ("I","N") then do; /* numeric (in)format */
+  by type fmtname notsorted;
+
+  /* align the numeric values to avoid overlapping ranges */
+  if type in ("I","N") then do;
     %mp_aligndecimal(start,width=16)
     %mp_aligndecimal(end,width=16)
   end;
+
+  /* create row marker. Data cannot be sorted without it! */
+  if first.fmtname then fmtrow=0;
+  fmtrow+1;
+
 run;
 proc sort;
-  by type fmtname start;
+  by type fmtname fmtrow;
 run;
 
 proc sql;
diff --git a/base/mp_ds2md.sas b/base/mp_ds2md.sas
index 7febabe..15b5401 100644
--- a/base/mp_ds2md.sas
+++ b/base/mp_ds2md.sas
@@ -95,6 +95,7 @@ run;
   options ps=max lrecl=max;
   data _null_;
     infile &outref;
+    if _n_=1 then putlog "# &libds" /;
     input;
     putlog _infile_;
   run;
diff --git a/base/mp_loadformat.sas b/base/mp_loadformat.sas
index 1945544..517d059 100644
--- a/base/mp_loadformat.sas
+++ b/base/mp_loadformat.sas
@@ -34,19 +34,20 @@
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
-  @li mddl_sas_cntlout.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
+  @li mp_md5.sas
   @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
   @li mddl_dc_difftable.sas
   @li mddl_dc_locktable.sas
-  @li mp_loadformat.test.sas
+  @li mp_loadformat.test.1.sas
+  @li mp_loadformat.test.2.sas
   @li mp_lockanytable.sas
   @li mp_stackdiffs.sas
 
@@ -79,13 +80,6 @@
   %let &var=%upcase(&prefix._&var);
 %end;
 
-/*
-format values can be up to 32767 wide.  SQL joins on such a wide column can
-cause buffer issues.  Update ibufsize and reset at the end.
-*/
-%let ibufsize=%sysfunc(getoption(ibufsize));
-options ibufsize=32767 ;
-
 /* in DC, format catalogs maybe specified in the libds with a -FC extension */
 %let libcat=%scan(&libcat,1,-);
 
@@ -148,16 +142,24 @@ select distinct
 
 %mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
 
+/* get a hash of the row */
+%local cvars nvars;
+%let cvars=TYPE FMTNAME START END LABEL PREFIX FILL SEXCL EEXCL HLO DECSEP
+  DIG3SEP DATATYPE LANGUAGE;
+%let nvars=FMTROW MIN MAX DEFAULT LENGTH FUZZ MULT NOEDIT;
+data &base_fmts/note2err;
+  set &base_fmts;
+  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
+run;
 
 /**
   * Ensure input table and base_formats have consistent lengths and types
   */
-%mddl_sas_cntlout(libds=&template)
-data &inlibds;
-  length &delete_col $3;
-  if 0 then set &template;
-  length start end $10000;
+data &inlibds/nonote2err;
+  length &delete_col $3 FMTROW 8 start end label $32767;
+  if 0 then set &base_fmts;
   set &libds;
+  by type fmtname notsorted;
   if &delete_col='' then &delete_col='No';
   fmtname=upcase(fmtname);
   type=upcase(type);
@@ -175,6 +177,14 @@ data &inlibds;
     %mp_aligndecimal(start,width=16)
     %mp_aligndecimal(end,width=16)
   end;
+
+  /* update row marker - retain new var as fmtrow may already be in libds */
+  if first.fmtname then row=1;
+  else row+1;
+  drop row;
+  fmtrow=row;
+
+  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
 run;
 
 /**
@@ -185,12 +195,10 @@ create table &outds_add(drop=&delete_col) as
   select a.*
   from &inlibds a
   left join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
-    and a.type=b.type
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where b.fmtname is null
     and upcase(a.&delete_col) ne "YES"
-  order by type, fmtname, start;
+  order by type, fmtname, fmtrow;
 
 /**
   * Identify deleted records
@@ -199,11 +207,9 @@ create table &outds_del(drop=&delete_col) as
   select a.*
   from &inlibds a
   inner join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
-    and a.type=b.type
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where upcase(a.&delete_col)="YES"
-  order by type, fmtname, start;
+  order by type, fmtname, fmtrow;
 
 /**
   * Identify modified records
@@ -212,13 +218,10 @@ create table &outds_mod (drop=&delete_col) as
   select a.*
   from &inlibds a
   inner join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
-    and a.type=b.type
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where upcase(a.&delete_col) ne "YES"
-  order by type, fmtname, start;
-
-options ibufsize=&ibufsize;
+    and a.fmthash ne b.fmthash
+  order by type, fmtname, fmtrow;
 
 %mp_abort(
   iftrue=(&syscc ne 0)
@@ -227,19 +230,21 @@ options ibufsize=&ibufsize;
 )
 
 %if &loadtarget=YES %then %do;
+  /* new records plus base records that are not deleted or modified */
   data &ds1;
     merge &base_fmts(in=base)
       &outds_mod(in=mod)
       &outds_add(in=add)
       &outds_del(in=del);
     if not del and not mod;
-    by type fmtname start;
+    by type fmtname fmtrow;
   run;
+  /* add back the modified records */
   data &stagedata;
     set &ds1 &outds_mod;
   run;
   proc sort;
-    by type fmtname start;
+    by type fmtname fmtrow;
   run;
 %end;
 /* mp abort needs to run outside of conditional blocks */
@@ -287,7 +292,7 @@ options ibufsize=&ibufsize;
 
     %mp_storediffs(&libcat-FC
       ,&base_fmts
-      ,TYPE FMTNAME START
+      ,TYPE FMTNAME FMTROW
       ,delds=&outds_del
       ,modds=&outds_mod
       ,appds=&outds_add
diff --git a/base/mp_storediffs.sas b/base/mp_storediffs.sas
index e3714aa..631453c 100644
--- a/base/mp_storediffs.sas
+++ b/base/mp_storediffs.sas
@@ -147,9 +147,9 @@ run;
 
 %if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do;
   /* this is a format catalog - cannot query cols directly */
-  %let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH"
-    ,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO"
-    ,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
+  %let vlist="TYPE","FMTNAME","FMTROW","START","END","LABEL","MIN","MAX"
+    ,"DEFAULT","LENGTH","FUZZ","PREFIX","MULT","FILL","NOEDIT","SEXCL"
+    ,"EEXCL","HLO","DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
 %end;
 %else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
 
diff --git a/ddl/mddl_sas_cntlout.sas b/ddl/mddl_sas_cntlout.sas
index a204c9f..f7b3ece 100644
--- a/ddl/mddl_sas_cntlout.sas
+++ b/ddl/mddl_sas_cntlout.sas
@@ -11,22 +11,18 @@
 
 %macro mddl_sas_cntlout(libds=WORK.CNTLOUT);
 
-proc sql;
-create table &libds(
-    TYPE char(1)         label='Type of format'
-    ,FMTNAME char(32)      label='Format name'
-    /*
-      to accommodate larger START values, mp_loadformat.sas will need the
-      SQL dependency removed (proc sql needs to accommodate 3 index values in
-      a 32767 ibufsize limit)
-    */
-    ,START char(10000)    label='Starting value for format'
+  proc sql;
+  create table &libds(
+    TYPE char(1) label='Type of format - either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
+    ,FMTNAME char(32)     label='Format name'
+    ,FMTROW num label='CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
+    ,START char(32767)    label='Starting value for format'
     /*
       Keep lengths of START and END the same to avoid this err:
       "Start is greater than end:  -<."
       Similar usage note: https://support.sas.com/kb/69/330.html
     */
-    ,END char(10000)      label='Ending value for format'
+    ,END char(32767)      label='Ending value for format'
     ,LABEL char(32767)    label='Format value label'
     ,MIN num length=3     label='Minimum length'
     ,MAX num length=3     label='Maximum length'
@@ -39,11 +35,23 @@ create table &libds(
     ,NOEDIT num length=3  label='Is picture string noedit?'
     ,SEXCL char(1)        label='Start exclusion'
     ,EEXCL char(1)        label='End exclusion'
-    ,HLO char(13)         label='Additional information'
+    ,HLO char(13)         label='Additional information. M=MultiLabel'
     ,DECSEP char(1)       label='Decimal separator'
     ,DIG3SEP char(1)      label='Three-digit separator'
     ,DATATYPE char(8)     label='Date/time/datetime?'
     ,LANGUAGE char(8)     label='Language for date strings'
-);
+  );
+
+  %local lib;
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create
+      pk_cntlout=(type fmtname fmtrow)
+      /nomiss unique;
+  quit;
 
 %mend mddl_sas_cntlout;
diff --git a/tests/base/mp_loadformat.test.sas b/tests/base/mp_loadformat.test.1.sas
similarity index 67%
rename from tests/base/mp_loadformat.test.sas
rename to tests/base/mp_loadformat.test.1.sas
index 9925784..5d14075 100644
--- a/tests/base/mp_loadformat.test.sas
+++ b/tests/base/mp_loadformat.test.1.sas
@@ -6,6 +6,7 @@
   <h4> SAS Macros </h4>
   @li mddl_dc_difftable.sas
   @li mp_aligndecimal.sas
+  @li mp_cntlout.sas
   @li mp_loadformat.sas
   @li mp_assert.sas
   @li mp_assertscope.sas
@@ -23,9 +24,9 @@ data work.loadfmts;
   length fmtname $32 start end $10000;
   eexcl='Y';
   type='N';
-  do i=1 to 100;
-    fmtname=cats('SASJS_',i,'X');
-    do j=1 to 100;
+  do i=1 to 10;
+    fmtname=cats('SASJS_',put(i,z4.),'X');
+    do j=1 to 20;
       start=cats(j);
       end=cats(j+1);
       %mp_aligndecimal(start,width=16)
@@ -38,21 +39,32 @@ run;
 proc format cntlin=work.loadfmts library=perm.testcat;
 run;
 
+/*
+  use actual format data as test baseline, as proc format adds attributes eg
+  min/max etc
+*/
+%mp_cntlout(libcat=perm.testcat,cntlout=work.loadfmts2)
+
 /* make some test data */
 data work.stagedata;
-  set work.loadfmts;
-  type='N';
-  eexcl='Y';
-  if _n_<150 then deleteme='Yes';
-  else if _n_<250 then label='mod'!!cats(_n_);
-  else if _n_<350 then do;
+  set work.loadfmts2 end=lastobs;
+  by type fmtname;
+
+  if lastobs then do;
+    output;
+    fmtname='NEWFMT'!!cats(_n_,'x'); /* 1 new record */
     start=cats(_n_);
     end=cats(_n_+1);
     %mp_aligndecimal(start,width=16)
     %mp_aligndecimal(end,width=16)
-    label='newval'!!cats(_N_);
+    label='newval'!!cats(_N_,'X');
+    output;
+    stop;
   end;
-  else stop;
+  else if last.fmtname then deleteme='Yes'; /* 9 deletions */
+  else if first.fmtname then label='modified '!!cats(_n_); /* 10 changes */
+
+  output;
 run;
 
 /* load the above */
@@ -71,22 +83,22 @@ run;
 %mp_assertscope(COMPARE)
 
 %mp_assert(
-  iftrue=(%mf_nobs(del_test1)=149),
+  iftrue=(%mf_nobs(del_test1)=9),
   desc=Test 1 - delete obs,
   outds=work.test_results
 )
 %mp_assert(
-  iftrue=(%mf_nobs(add_test1)=100),
+  iftrue=(%mf_nobs(add_test1)=1),
   desc=Test 1 - add obs,
   outds=work.test_results
 )
 %mp_assert(
-  iftrue=(%mf_nobs(mod_test1)=100),
+  iftrue=(%mf_nobs(mod_test1)=10),
   desc=Test 1 - mod obs,
   outds=work.test_results
 )
 %mp_assert(
-  iftrue=(%mf_nobs(perm.audit)=7329),
+  iftrue=(%mf_nobs(perm.audit)=440),
   desc=Test 1 - audit table updated,
   outds=work.test_results
 )
@@ -101,7 +113,7 @@ run;
 )
 
 /* set up a mix of formats */
-data work.loadfmts2;
+data work.loadfmts3;
   length fmtname $32 start end $10000;
   eexcl='Y';
   type='J';
@@ -150,45 +162,47 @@ data work.loadfmts2;
   end;
   drop i j;
 run;
-proc format cntlin=work.loadfmts2 library=perm.testcat2;
+proc format cntlin=work.loadfmts3 library=perm.testcat3;
 run;
+%mp_cntlout(libcat=perm.testcat3,cntlout=work.loadfmts4)
 
 /* make some test data */
-data work.stagedata2;
-  set work.loadfmts2;
+data work.stagedata3;
+  set work.loadfmts4;
   where type in ('I','J');
-  eexcl='Y';
+  by type fmtname notsorted;
   if type='I' then do;
-    i+1;
-    if i<3 then deleteme='Yes';
-    else if i<7 then label= cats(ranuni(0)*100);
-    else if i<12 then do;
-      /* new values */
+    if last.fmtname then do;
+      deleteme='Yes'; /* 3 deletions */
+      output;
+    end;
+    else if fmtrow le 3 then do; /* 9 changed values */
       z=ranuni(0)*1000000;
       start=cats(z);
       end=cats(z+1);
       %mp_aligndecimal(start,width=16)
       %mp_aligndecimal(end,width=16)
-      label= cats(ranuni(0)*100);
+      output;
     end;
-    if i<12 then output;
   end;
   else do;
-    j+1;
-    if j<3 then deleteme='Yes';
-    else if j<7 then label= cats(ranuni(0)*100);
-    else if j<12 then do;
-      start= cats("NEWVAL",start);
-      end=start;
-      label= "NEWVAL "||cats(ranuni(0)*100);
+    if last.fmtname then do;
+      output; /* 6 new records */
+      x=_n_;
+      x+1;start=cats("mod",x);end=start;label='newlabel1';output;
+      x+1;start=cats("mod",x);end=start;label='newlabel2';output;
+    end;
+    else if fmtrow le 3 then do; /* 9 more changed values */
+      start= cats("mod",_n_);
+      end=start;
+      label= "mod "||cats(ranuni(0)*100);
+      output;
     end;
-    if j<12 then output;
   end;
-
 run;
 
-%mp_loadformat(perm.testcat2
-  ,work.stagedata2
+%mp_loadformat(perm.testcat3
+  ,work.stagedata3
   ,loadtarget=YES
   ,auditlibds=perm.audit
   ,locklibds=0
@@ -200,17 +214,18 @@ run;
 )
 
 %mp_assert(
-  iftrue=(%mf_nobs(del_test2)=4),
+  iftrue=(%mf_nobs(del_test2)=3),
   desc=Test 2 - delete obs,
   outds=work.test_results
 )
 %mp_assert(
-  iftrue=(%mf_nobs(mod_test2)=8),
+  iftrue=(%mf_nobs(mod_test2)=18),
   desc=Test 2 - mod obs,
   outds=work.test_results
 )
 %mp_assert(
-  iftrue=(%mf_nobs(add_test2)=10),
+  iftrue=(%mf_nobs(add_test2)=6),
   desc=Test 2 - add obs,
   outds=work.test_results
 )
+
diff --git a/tests/base/mp_loadformat.test.2.sas b/tests/base/mp_loadformat.test.2.sas
new file mode 100644
index 0000000..e3ebb8b
--- /dev/null
+++ b/tests/base/mp_loadformat.test.2.sas
@@ -0,0 +1,128 @@
+/**
+  @file
+  @brief Testing mp_loadformat.sas macro for multilabel formats
+  @details Multilabel records can be complete duplicates!!  Also, the order is
+  important.
+
+  The provided formats create a table as follows:
+
+|TYPE:$1.|FMTNAME:$32.|START:$10000.|END:$10000.|LABEL:$32767.|MIN:best.|
+MAX:best.|DEFAULT:best.|LENGTH:best.|FUZZ:best.|PREFIX:$2.|MULT:best.|FILL:$1.|
+NOEDIT:best.|SEXCL:$1.|EEXCL:$1.|HLO:$13.|DECSEP:$1.|DIG3SEP:$1.|DATATYPE:$8.|
+LANGUAGE:$8.|
+|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
+---|---|
+|`C `|`GENDERML `|` `|` `|`Total people `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`1 `|`1 `|`Male `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`1 `|`1 `|`Total people `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`2 `|`2 `|`Female `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`2 `|`2 `|`Female `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`2 `|`2 `|`Thormale `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`C `|`GENDERML `|`2 `|`2 `|`Total people `|`1 `|`40 `|`12 `|`12 `|`0 `|` `|`0 `|` `|`0 `|`N `|`N `|`M `|` `|` `|` `|` `|
+|`N `|`AGEMLA `|`1 `|`4 `|`Preschool `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLA `|`1 `|`18 `|`Children `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLA `|`19 `|`120 `|`Adults `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLB `|`1 `|`4 `|`Preschool `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLB `|`1 `|`18 `|`Children `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLB `|`19 `|`120 `|`Adults `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLC `|`1 `|`18 `|`Children `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLC `|`1 `|`4 `|`Preschool `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+|`N `|`AGEMLC `|`19 `|`120 `|`Adults `|`1 `|`40 `|`9 `|`9 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`SM `|` `|` `|` `|` `|
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_cntlout.sas
+  @li mp_loadformat.sas
+  @li mp_assert.sas
+  @li mp_assertdsobs.sas
+  @li mp_ds2md.sas
+
+**/
+
+/* prep format catalog */
+libname perm (work);
+
+/* create some multilable formats */
+%let cat1=perm.test1;
+proc format library=&cat1;
+  value $genderml (multilabel notsorted)
+    '1'='Male'
+    '2'='Female'
+    '2'='Female'
+    '2'='Farmale'
+    '1','2',' '='Total people';
+  value agemla (multilabel)
+    1-4='Preschool'
+    1-18='Children'
+    19-120='Adults';
+  value agemlb (multilabel)
+    19-120='Adults'
+    1-18='Children'
+    1-4='Preschool';
+  value agemlc (multilabel notsorted)
+    19-120='Adults'
+    1-18='Children'
+    1-4='Preschool';
+run;
+
+%mp_cntlout(libcat=&cat1,cntlout=work.cntlout1)
+%mp_assertdsobs(work.cntlout1,
+  desc=Has 16 records,
+  test=EQUALS 16
+)
+
+data work.stagedata3;
+  set work.cntlout1;
+  if fmtname='AGEMLA' and label ne 'Preschool' then deleteme='Yes';
+  if fmtname='AGEMLB' and label = 'Preschool' then label='Kids';
+  if fmtname='GENDERML' and label='Farmale' then output;
+  output;
+run;
+
+
+%mp_loadformat(&cat1
+  ,work.stagedata3
+  ,loadtarget=YES
+  ,auditlibds=perm.audit
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_test1
+  ,outds_del=del_test1
+  ,outds_mod=mod_test1
+  ,mdebug=1
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(del_test1)=2),
+  desc=Test 1 - deleted obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(mod_test1)=4),
+  desc=Test 1 - mod obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(add_test1)=1),
+  desc=Test 1 - add obs,
+  outds=work.test_results
+)
+
+/* now check the order of the notsorted format */
+%mp_cntlout(libcat=&cat1,cntlout=work.cntlout2)
+
+%let check1=0;
+%let check2=0;
+data test;
+  set work.cntlout2;
+  where fmtname='GENDERML';
+  if _n_=4 and label='Farmale' then call symputx('check1',1);
+  if _n_=5 and label='Farmale' then call symputx('check2',1);
+run;
+%mp_assert(
+  iftrue=(&check1=1 and &check2=1),
+  desc=Ensuring Farmale values retain their order,
+  outds=work.test_results
+)
+
+%mp_ds2md(work.cntlout2)
\ No newline at end of file