fix: mp_loadformat updates by FMTROW

Previously, FMTROW was not being honoured when adding / deleting individual format records.  Updated tests and added additional validations to ensure FMTROW is provided correctly at the input stage.

.all-contributorsrc

@@ -153,6 +153,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "andyjessen",
+      "name": "andyjessen",
+      "avatar_url": "https://avatars.githubusercontent.com/u/62343929?v=4",
+      "profile": "https://github.com/andyjessen",
+      "contributions": [
+        "doc"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,4 +15,4 @@ What code changes have been made to achieve the intent.
 - [ ] Code is formatted correctly (`sasjs lint`).
 - [ ] Any new functionality has been unit tested.
 - [ ] All unit tests are passing (`sasjs test`).
-- [ ] `all.sas` has been regenerated (`python3 build.py`)
+- [ ] The PR desc or underlying commits follow the [Conventional Commit](https://www.conventionalcommits.org) standard

README.md

@@ -248,7 +248,7 @@ The following repositories are also worth checking out:
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-14-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-15-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -275,6 +275,9 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/henrik-forsell"><img src="https://avatars.githubusercontent.com/u/109935936?v=4?s=100" width="100px;" alt="Henrik Forsell"/><br /><sub><b>Henrik Forsell</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=henrik-forsell" title="Documentation">📖</a></td>
       <td align="center" valign="top" width="14.28%"><a href="http://rudvfaden.github.io/"><img src="https://avatars.githubusercontent.com/u/2445577?v=4?s=100" width="100px;" alt="Rud Faden"/><br /><sub><b>Rud Faden</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rudvfaden" title="Code">💻</a></td>
     </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/andyjessen"><img src="https://avatars.githubusercontent.com/u/62343929?v=4?s=100" width="100px;" alt="andyjessen"/><br /><sub><b>andyjessen</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=andyjessen" title="Documentation">📖</a></td>
+    </tr>
   </tbody>
 </table>
 

all.sas

@@ -1684,7 +1684,7 @@ Usage:
 
   Usage:
 
-      %put mf_isblank(&var);
+      %put %mf_isblank(&var);
 
   inspiration:
   https://support.sas.com/resources/papers/proceedings09/022-2009.pdf
@@ -4189,8 +4189,8 @@ data &cntlout/nonote2err;
   end;
 
   /* create row marker. Data cannot be sorted without it! */
-  if first.fmtname then fmtrow=0;
-  fmtrow+1;
+  if first.fmtname then fmtrow=1;
+  else fmtrow+1;
 
 run;
 proc sort;
@@ -10153,6 +10153,9 @@ select distinct lowcase(memname)
   format, to prevent loss of data - UNLESS the input dataset contains a marker
   column, specifying that a particular row needs to be deleted (`delete_col=`).
 
+  Positions of formats are made using the FMTROW variable - this must be present
+  and unique (on TYPE / FMTNAME / FMTROW).
+
   This macro can also be used to identify which records would be (or were)
   considered new, modified or deleted (`loadtarget=`) by creating the following
   tables:
@@ -10161,7 +10164,7 @@ select distinct lowcase(memname)
   @li work.outds_del
   @li work.outds_mod
 
-  For example usage, see mp_loadformat.test.sas
+  For example usage, see test (under Related Macros)
 
   @param [in] libcat The format catalog to be loaded
   @param [in] libds The staging table to load
@@ -10178,12 +10181,15 @@ select distinct lowcase(memname)
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_existvar.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
+  @li mp_md5.sas
   @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
@@ -10227,6 +10233,16 @@ select distinct lowcase(memname)
 %let libcat=%scan(&libcat,1,-);
 
 /* perform input validations */
+%mp_abort(
+  iftrue=(%mf_existds(&libds)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(&libds could not be found)
+)
+%mp_abort(
+  iftrue=(%mf_existvar(&libds,FMTROW)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(FMTROW not found in &libds)
+)
 %let err=0;
 %let msg=0;
 data _null_;
@@ -10247,13 +10263,6 @@ data _null_;
       stop;
     end;
   end;
-  else if name='LIBDS' then do;
-    if exist(value) le 0 then do;
-      call symputx('msg',"Unable to open staging table: "!!value);
-      call symputx('err',1);
-      stop;
-    end;
-  end;
   else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
   and missing(value) then do;
     call symputx('msg',"missing value in var: "!!name);
@@ -10261,6 +10270,14 @@ data _null_;
     stop;
   end;
 run;
+data _null_;
+  set &libds;
+  if missing(fmtrow) then do;
+    call symputx('msg',"missing fmtrow in format: "!!FMTNAME);
+    call symputx('err',1);
+    stop;
+  end;
+run;
 
 %mp_abort(
   iftrue=(&err ne 0)
@@ -10268,6 +10285,15 @@ run;
   ,msg=%str(&msg)
 )
 
+%local cnt;
+proc sql noprint;
+select count(distinct catx('|',type,fmtname,fmtrow)) into: cnt from &libds;
+%mp_abort(
+  iftrue=(&cnt ne %mf_nobs(&libds))
+  ,mac=&sysmacroname
+  ,msg=%str(Non-unique primary key on &libds)
+)
+
 /**
   * First, extract only relevant formats from the catalog
   */
@@ -10321,12 +10347,6 @@ data &inlibds/nonote2err;
     %mp_aligndecimal(end,width=16)
   end;
 
-  /* update row marker - retain new var as fmtrow may already be in libds */
-  if first.fmtname then row=1;
-  else row+1;
-  drop row;
-  fmtrow=row;
-
   fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
 run;
 
@@ -14176,6 +14196,22 @@ ods package close;
   (given various practical restrictions) are described here to enable
   consistency when dealing with format data.
 
+  The HLO variable may have a number of values, documented here due to the
+  256 char label description length limit:
+
+    F=Standard format/informat.
+    H=Range ending value is HIGH.
+    I=Numeric informat.
+    J=Justification for an informat.
+    L=Range starting value is LOW.
+    M=MultiLabel.
+    N=Format or informat has no ranges, including no OTHER= range.
+    O=Range is OTHER.
+    R=ROUND option is in effect.
+    S=Specifies that NOTSORTED is in effect.
+    U=Specifies that the UPCASE option for an informat be used.
+
+
 **/
 
 
@@ -14183,9 +14219,11 @@ ods package close;
 
   proc sql;
   create table &libds(
-    TYPE char(1) label='Type of format - either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
+    TYPE char(1) label=
+'Format Type: either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
     ,FMTNAME char(32)     label='Format name'
-    ,FMTROW num label='CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
+    ,FMTROW num label=
+'CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
     ,START char(32767)    label='Starting value for format'
     /*
       Keep lengths of START and END the same to avoid this err:
@@ -14205,18 +14243,8 @@ ods package close;
     ,NOEDIT num length=3  label='Is picture string noedit?'
     ,SEXCL char(1)        label='Start exclusion'
     ,EEXCL char(1)        label='End exclusion'
-    ,HLO char(13)         label='Additional information.
-F=Standard format/informat.
-H=Range ending value is HIGH.
-I=Numeric informat.
-J=Justification for an informat.
-L=Range starting value is LOW.
-M=MultiLabel.
-N=Format or informat has no ranges, including no OTHER= range.
-O=Range is OTHER.
-R=ROUND option is in effect.
-S=Specifies that NOTSORTED is in effect.
-U=Specifies that the UPCASE option for an informat be used.'
+    ,HLO char(13) label=
+'More info: https://core.sasjs.io/mddl__sas__cntlout_8sas_source.html'
     ,DECSEP char(1)       label='Decimal separator'
     ,DIG3SEP char(1)      label='Three-digit separator'
     ,DATATYPE char(8)     label='Date/time/datetime?'
@@ -14550,7 +14578,7 @@ run;
   %end;
 %end;
 %else %if &engine=ODBC %then %do;
-  &mD.%put NOTE: Retrieving ODBC connection details;
+  %&mD.put NOTE: Retrieving ODBC connection details;
   data _null_;
     length connx_uri conprop_uri value datasource up_uri schema domprop_uri authdomain $256.;
     call missing (of _all_);

base/mf_isblank.sas

@@ -7,7 +7,7 @@
 
   Usage:
 
-      %put mf_isblank(&var);
+      %put %mf_isblank(&var);
 
   inspiration:
   https://support.sas.com/resources/papers/proceedings09/022-2009.pdf

base/mp_cntlout.sas

@@ -79,8 +79,8 @@ data &cntlout/nonote2err;
   end;
 
   /* create row marker. Data cannot be sorted without it! */
-  if first.fmtname then fmtrow=0;
-  fmtrow+1;
+  if first.fmtname then fmtrow=1;
+  else fmtrow+1;
 
 run;
 proc sort;

base/mp_loadformat.sas

@@ -9,6 +9,9 @@
   format, to prevent loss of data - UNLESS the input dataset contains a marker
   column, specifying that a particular row needs to be deleted (`delete_col=`).
 
+  Positions of formats are made using the FMTROW variable - this must be present
+  and unique (on TYPE / FMTNAME / FMTROW).
+
   This macro can also be used to identify which records would be (or were)
   considered new, modified or deleted (`loadtarget=`) by creating the following
   tables:
@@ -17,7 +20,7 @@
   @li work.outds_del
   @li work.outds_mod
 
-  For example usage, see mp_loadformat.test.sas
+  For example usage, see test (under Related Macros)
 
   @param [in] libcat The format catalog to be loaded
   @param [in] libds The staging table to load
@@ -34,12 +37,15 @@
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_existvar.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
+  @li mp_md5.sas
   @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
@@ -83,6 +89,16 @@
 %let libcat=%scan(&libcat,1,-);
 
 /* perform input validations */
+%mp_abort(
+  iftrue=(%mf_existds(&libds)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(&libds could not be found)
+)
+%mp_abort(
+  iftrue=(%mf_existvar(&libds,FMTROW)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(FMTROW not found in &libds)
+)
 %let err=0;
 %let msg=0;
 data _null_;
@@ -103,13 +119,6 @@ data _null_;
       stop;
     end;
   end;
-  else if name='LIBDS' then do;
-    if exist(value) le 0 then do;
-      call symputx('msg',"Unable to open staging table: "!!value);
-      call symputx('err',1);
-      stop;
-    end;
-  end;
   else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
   and missing(value) then do;
     call symputx('msg',"missing value in var: "!!name);
@@ -117,6 +126,14 @@ data _null_;
     stop;
   end;
 run;
+data _null_;
+  set &libds;
+  if missing(fmtrow) then do;
+    call symputx('msg',"missing fmtrow in format: "!!FMTNAME);
+    call symputx('err',1);
+    stop;
+  end;
+run;
 
 %mp_abort(
   iftrue=(&err ne 0)
@@ -124,6 +141,15 @@ run;
   ,msg=%str(&msg)
 )
 
+%local cnt;
+proc sql noprint;
+select count(distinct catx('|',type,fmtname,fmtrow)) into: cnt from &libds;
+%mp_abort(
+  iftrue=(&cnt ne %mf_nobs(&libds))
+  ,mac=&sysmacroname
+  ,msg=%str(Non-unique primary key on &libds)
+)
+
 /**
   * First, extract only relevant formats from the catalog
   */
@@ -177,12 +203,6 @@ data &inlibds/nonote2err;
     %mp_aligndecimal(end,width=16)
   end;
 
-  /* update row marker - retain new var as fmtrow may already be in libds */
-  if first.fmtname then row=1;
-  else row+1;
-  drop row;
-  fmtrow=row;
-
   fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
 run;
 

ddl/mddl_sas_cntlout.sas

@@ -6,6 +6,22 @@
   (given various practical restrictions) are described here to enable
   consistency when dealing with format data.
 
+  The HLO variable may have a number of values, documented here due to the
+  256 char label description length limit:
+
+    F=Standard format/informat.
+    H=Range ending value is HIGH.
+    I=Numeric informat.
+    J=Justification for an informat.
+    L=Range starting value is LOW.
+    M=MultiLabel.
+    N=Format or informat has no ranges, including no OTHER= range.
+    O=Range is OTHER.
+    R=ROUND option is in effect.
+    S=Specifies that NOTSORTED is in effect.
+    U=Specifies that the UPCASE option for an informat be used.
+
+
 **/
 
 
@@ -13,9 +29,11 @@
 
   proc sql;
   create table &libds(
-    TYPE char(1) label='Type of format - either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
+    TYPE char(1) label=
+'Format Type: either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
     ,FMTNAME char(32)     label='Format name'
-    ,FMTROW num label='CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
+    ,FMTROW num label=
+'CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
     ,START char(32767)    label='Starting value for format'
     /*
       Keep lengths of START and END the same to avoid this err:
@@ -35,18 +53,8 @@
     ,NOEDIT num length=3  label='Is picture string noedit?'
     ,SEXCL char(1)        label='Start exclusion'
     ,EEXCL char(1)        label='End exclusion'
-    ,HLO char(13)         label='Additional information.
-F=Standard format/informat.
-H=Range ending value is HIGH.
-I=Numeric informat.
-J=Justification for an informat.
-L=Range starting value is LOW.
-M=MultiLabel.
-N=Format or informat has no ranges, including no OTHER= range.
-O=Range is OTHER.
-R=ROUND option is in effect.
-S=Specifies that NOTSORTED is in effect.
-U=Specifies that the UPCASE option for an informat be used.'
+    ,HLO char(13) label=
+'More info: https://core.sasjs.io/mddl__sas__cntlout_8sas_source.html'
     ,DECSEP char(1)       label='Decimal separator'
     ,DIG3SEP char(1)      label='Three-digit separator'
     ,DATATYPE char(8)     label='Date/time/datetime?'

meta/mm_assigndirectlib.sas

@@ -210,7 +210,7 @@ run;
   %end;
 %end;
 %else %if &engine=ODBC %then %do;
-  &mD.%put NOTE: Retrieving ODBC connection details;
+  %&mD.put NOTE: Retrieving ODBC connection details;
   data _null_;
     length connx_uri conprop_uri value datasource up_uri schema domprop_uri authdomain $256.;
     call missing (of _all_);

sasjs/sasjsconfig.json

@@ -67,7 +67,7 @@
     },
     {
       "name": "server",
-      "serverUrl": "https://sas9.4gl.io",
+      "serverUrl": "https://sas.4gl.io",
       "serverType": "SASJS",
       "httpsAgentOptions": {
         "allowInsecureRequests": false

tests/base/mp_loadformat.test.1.sas

@@ -189,8 +189,10 @@ data work.stagedata3;
     if last.fmtname then do;
       output; /* 6 new records */
       x=_n_;
-      x+1;start=cats("mod",x);end=start;label='newlabel1';output;
-      x+1;start=cats("mod",x);end=start;label='newlabel2';output;
+      x+1;start=cats("mod",x);end=start;label='newlabel1';fmtrow=fmtrow+1;
+      output;
+      x+1;start=cats("mod",x);end=start;label='newlabel2';fmtrow=fmtrow+2;
+      output;
     end;
     else if fmtrow le 3 then do; /* 9 more changed values */
       start= cats("mod",_n_);

tests/base/mp_loadformat.test.2.sas

@@ -58,6 +58,9 @@ proc format library=&cat1;
   value agemlb (multilabel)
     19-120='Adults'
     1-18='Children'
+    0-1='Preschool'
+    1-2='Preschool'
+    2-3='Preschool'
     1-4='Preschool';
   value agemlc (multilabel notsorted)
     19-120='Adults'
@@ -67,16 +70,19 @@ run;
 
 %mp_cntlout(libcat=&cat1,cntlout=work.cntlout1)
 %mp_assertdsobs(work.cntlout1,
-  desc=Has 16 records,
-  test=EQUALS 16
+  desc=Has 19 records,
+  test=EQUALS 19
 )
 
 data work.stagedata3;
   set work.cntlout1;
   if fmtname='AGEMLA' and label ne 'Preschool' then deleteme='Yes';
   if fmtname='AGEMLB' and label = 'Preschool' then label='Kids';
-  if fmtname='GENDERML' and label='Farmale' then output;
-  output;
+  if fmtname='GENDERML' and label='Farmale' then do;
+    output;
+    fmtrow=101; output;
+  end;
+  else output;
 run;
 
 
@@ -113,14 +119,17 @@ run;
 
 %let check1=0;
 %let check2=0;
+%let check3=0;
 data test;
   set work.cntlout2;
   where fmtname='GENDERML';
+  putlog fmtrow= label=;
   if _n_=4 and label='Farmale' then call symputx('check1',1);
-  if _n_=5 and label='Farmale' then call symputx('check2',1);
+  if _n_=5 and label ne 'Farmale' then call symputx('check2',1);
+  if _n_=8 and label = 'Farmale' then call symputx('check3',1);
 run;
 %mp_assert(
-  iftrue=(&check1=1 and &check2=1),
+  iftrue=(&check1=1 and &check2=1 and &check3=1),
   desc=Ensuring Farmale values retain their order,
   outds=work.test_results
 )