chore: updating all.sas

.all-contributorsrc

@@ -153,15 +153,6 @@
       "contributions": [
         "code"
       ]
-    },
-    {
-      "login": "andyjessen",
-      "name": "andyjessen",
-      "avatar_url": "https://avatars.githubusercontent.com/u/62343929?v=4",
-      "profile": "https://github.com/andyjessen",
-      "contributions": [
-        "doc"
-      ]
     }
   ],
   "contributorsPerLine": 7,

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,4 +15,4 @@ What code changes have been made to achieve the intent.
 - [ ] Code is formatted correctly (`sasjs lint`).
 - [ ] Any new functionality has been unit tested.
 - [ ] All unit tests are passing (`sasjs test`).
-- [ ] The PR desc or underlying commits follow the [Conventional Commit](https://www.conventionalcommits.org) standard
+- [ ] `all.sas` has been regenerated (`python3 build.py`)

README.md

@@ -248,7 +248,7 @@ The following repositories are also worth checking out:
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-15-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-14-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -275,9 +275,6 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/henrik-forsell"><img src="https://avatars.githubusercontent.com/u/109935936?v=4?s=100" width="100px;" alt="Henrik Forsell"/><br /><sub><b>Henrik Forsell</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=henrik-forsell" title="Documentation">📖</a></td>
       <td align="center" valign="top" width="14.28%"><a href="http://rudvfaden.github.io/"><img src="https://avatars.githubusercontent.com/u/2445577?v=4?s=100" width="100px;" alt="Rud Faden"/><br /><sub><b>Rud Faden</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rudvfaden" title="Code">💻</a></td>
     </tr>
-    <tr>
-      <td align="center" valign="top" width="14.28%"><a href="https://github.com/andyjessen"><img src="https://avatars.githubusercontent.com/u/62343929?v=4?s=100" width="100px;" alt="andyjessen"/><br /><sub><b>andyjessen</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=andyjessen" title="Documentation">📖</a></td>
-    </tr>
   </tbody>
 </table>
 

all.sas

@@ -1684,7 +1684,7 @@ Usage:
 
   Usage:
 
-      %put %mf_isblank(&var);
+      %put mf_isblank(&var);
 
   inspiration:
   https://support.sas.com/resources/papers/proceedings09/022-2009.pdf
@@ -4189,8 +4189,8 @@ data &cntlout/nonote2err;
   end;
 
   /* create row marker. Data cannot be sorted without it! */
-  if first.fmtname then fmtrow=1;
-  else fmtrow+1;
+  if first.fmtname then fmtrow=0;
+  fmtrow+1;
 
 run;
 proc sort;
@@ -4861,7 +4861,7 @@ data;run;
 data &out_ds(compress=no
     keep=file_or_folder filepath filename ext msg directory level
   );
-  length directory filepath $2000 fref fref2 $8 file_or_folder $6 filename $255
+  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
     ext $20 msg $200 foption $16;
   if _n_=1 then call missing(of _all_);
   retain level &level;
@@ -5541,21 +5541,13 @@ data _null_;
     header = cats(coalescec(varlabel(dsid,i),varnm),dlm);
   %end;
   %else %if &headerformat=SASJS %then %do;
-    vlen=varlen(dsid,i);
-    if vartype(dsid,i)='C' then header=cats(varnm,':$char',vlen,'.');
+    if vartype(dsid,i)='C' then header=cats(varnm,':$char',varlen(dsid,i),'.');
     else do;
       vfmt=coalescec(varfmt(dsid,i),'0');
       fmttype=mcf_getfmttype(vfmt);
       if fmttype='DATE' then header=cats(varnm,':date9.');
       else if fmttype='DATETIME' then header=cats(varnm,':E8601DT26.6');
       else if fmttype='TIME' then header=cats(varnm,':TIME12.');
-      /**
-        * there is not much point importing a short length numeric like this,
-        * eg with best4., as the resulting variable will still be stored as
-        * length 8.  We need a length or format statement to ensure variable
-        * is creatd with the smaller length...
-        **/
-      else if vlen<8 then header=cats(varnm,':best',vlen,'.');
       else header=cats(varnm,':best.');
     end;
   %end;
@@ -5582,7 +5574,6 @@ data _null_;
   set &ds end=last;
 %do i=1 %to &vcnt;
   %let var=%scan(&varlist,&i);
-  %local vlen&i;
   %if %mf_getvartype(&ds,&var)=C %then %do;
     %let dsv1=%mf_getuniquename(prefix=csvcol1_);
     %let dsv2=%mf_getuniquename(prefix=csvcol2_);
@@ -6386,14 +6377,15 @@ drop table &ds1, &ds2;
 /**
   * Sanitise the values based on valid value lists, then strip out
   * quotes, commas, periods and spaces.
+  * Only numeric values should remain
   */
 %local reason_cd nobs;
 %let nobs=0;
 data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
-  set &inds end=last;
-  length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000;
+  set &inds;
+  length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000;
   drop tmp;
 
   /* quick check to ensure column exists */
@@ -6409,8 +6401,7 @@ data &outds;
   end;
 
   /* need to open the dataset to get the column type */
-  retain dsid;
-  if _n_=1 then dsid=open("&targetds","i");
+  dsid=open("&targetds","i");
   if dsid>0 then do;
     vnum=varnum(dsid,VARIABLE_NM);
     if vnum<1 then do;
@@ -6420,19 +6411,11 @@ data &outds;
       call symputx('reason_cd',reason_cd,'l');
       call symputx('nobs',_n_,'l');
       output;
-      goto endstep;
+      return;
     end;
     /* now we can get the type */
     else vtype=vartype(dsid,vnum);
   end;
-  else do;
-      REASON_CD=cats("Could not open &targetds");
-      putlog REASON_CD= dsid=;
-      call symputx('reason_cd',reason_cd,'l');
-      call symputx('nobs',_n_,'l');
-      output;
-      stop;
-  end;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
@@ -6467,40 +6450,15 @@ data &outds;
   end;
 
   /* special missing logic */
-  if vtype='N' & OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE') then do;
-    if cats(upcase(raw_value)) in (
+  if vtype='N'
+    and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE')
+    and cats(upcase(raw_value)) in (
       '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N'
       '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._'
     )
-    then do;
-      /* valid numeric - exit data step loop */
-      return;
-    end;
-    else if subpad(upcase(raw_value),1,1) in (
-      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
-      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
-    )
-    then do;
-      /* check if the raw_value contains a valid variable NAME */
-      vnum=varnum(dsid,subpad(raw_value,1,32));
-      if vnum>0 then do;
-        /* now we can get the type */
-        vtype2=vartype(dsid,vnum);
-        /* check type matches */
-        if vtype2=vtype then do;
-          /* valid target var - exit loop */
-          return;
-        end;
-        else do;
-          REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
-          putlog REASON_CD= dsid=;
-          call symputx('reason_cd',reason_cd,'l');
-          call symputx('nobs',_n_,'l');
-          output;
-          goto endstep;
-        end;
-      end;
-    end;
+  then do;
+    /* valid numeric - exit data step loop */
+    return;
   end;
 
   /* special logic */
@@ -6522,32 +6480,6 @@ data &outds;
     if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
       tmp=scan(raw_value1,i,',');
       if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
-        if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in (
-          'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
-          'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
-        )
-        then do;
-          /* check if the raw_value contains a valid variable NAME */
-          /* is not valid syntax for IN or NOT IN */
-          vnum=varnum(dsid,subpad(tmp,1,32));
-          if vnum>0 then do;
-            /* now we can get the type */
-            vtype2=vartype(dsid,vnum);
-            /* check type matches */
-            if vtype2=vtype then do;
-              /* valid target var - exit loop */
-              return;
-            end;
-            else do;
-              REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
-              putlog REASON_CD= dsid=;
-              call symputx('reason_cd',reason_cd,'l');
-              call symputx('nobs',_n_,'l');
-              output;
-              goto endstep;
-            end;
-          end;
-        end;
         REASON_CD='Non Numeric value provided';
         putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
         call symputx('reason_cd',reason_cd,'l');
@@ -6572,42 +6504,14 @@ data &outds;
 
   /* output records that contain values other than digits and spaces */
   if notdigit(compress(raw_value3,' '))>0 then do;
-    if vtype='C' and subpad(upcase(raw_value),1,1) in (
-      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
-      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
-    )
-    then do;
-      /* check if the raw_value contains a valid variable NAME */
-      vnum=varnum(dsid,subpad(raw_value,1,32));
-      if vnum>0 then do;
-        /* now we can get the type */
-        vtype2=vartype(dsid,vnum);
-        /* check type matches */
-        if vtype2=vtype then do;
-          /* valid target var - exit loop */
-          return;
-        end;
-        else do;
-          REASON_CD=cats("Compared Char Type (",vtype2,") is not (",vtype,")");
-          putlog REASON_CD= dsid=;
-          call symputx('reason_cd',reason_cd,'l');
-          call symputx('nobs',_n_,'l');
-          output;
-          goto endstep;
-        end;
-      end;
-    end;
-
     putlog raw_value3= $hex32.;
     REASON_CD=cats('Invalid RAW_VALUE:',raw_value);
-    putlog (_all_)(=);
+    putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');
     output;
   end;
 
-  endstep:
-  if last then rc=close(dsid);
 run;
 
 
@@ -10249,9 +10153,6 @@ select distinct lowcase(memname)
   format, to prevent loss of data - UNLESS the input dataset contains a marker
   column, specifying that a particular row needs to be deleted (`delete_col=`).
 
-  Positions of formats are made using the FMTROW variable - this must be present
-  and unique (on TYPE / FMTNAME / FMTROW).
-
   This macro can also be used to identify which records would be (or were)
   considered new, modified or deleted (`loadtarget=`) by creating the following
   tables:
@@ -10260,7 +10161,7 @@ select distinct lowcase(memname)
   @li work.outds_del
   @li work.outds_mod
 
-  For example usage, see test (under Related Macros)
+  For example usage, see mp_loadformat.test.sas
 
   @param [in] libcat The format catalog to be loaded
   @param [in] libds The staging table to load
@@ -10277,15 +10178,12 @@ select distinct lowcase(memname)
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
-  @li mf_existds.sas
-  @li mf_existvar.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
-  @li mp_md5.sas
   @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
@@ -10329,16 +10227,6 @@ select distinct lowcase(memname)
 %let libcat=%scan(&libcat,1,-);
 
 /* perform input validations */
-%mp_abort(
-  iftrue=(%mf_existds(&libds)=0)
-  ,mac=&sysmacroname
-  ,msg=%str(&libds could not be found)
-)
-%mp_abort(
-  iftrue=(%mf_existvar(&libds,FMTROW)=0)
-  ,mac=&sysmacroname
-  ,msg=%str(FMTROW not found in &libds)
-)
 %let err=0;
 %let msg=0;
 data _null_;
@@ -10359,6 +10247,13 @@ data _null_;
       stop;
     end;
   end;
+  else if name='LIBDS' then do;
+    if exist(value) le 0 then do;
+      call symputx('msg',"Unable to open staging table: "!!value);
+      call symputx('err',1);
+      stop;
+    end;
+  end;
   else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
   and missing(value) then do;
     call symputx('msg',"missing value in var: "!!name);
@@ -10366,14 +10261,6 @@ data _null_;
     stop;
   end;
 run;
-data _null_;
-  set &libds;
-  if missing(fmtrow) then do;
-    call symputx('msg',"missing fmtrow in format: "!!FMTNAME);
-    call symputx('err',1);
-    stop;
-  end;
-run;
 
 %mp_abort(
   iftrue=(&err ne 0)
@@ -10381,15 +10268,6 @@ run;
   ,msg=%str(&msg)
 )
 
-%local cnt;
-proc sql noprint;
-select count(distinct catx('|',type,fmtname,fmtrow)) into: cnt from &libds;
-%mp_abort(
-  iftrue=(&cnt ne %mf_nobs(&libds))
-  ,mac=&sysmacroname
-  ,msg=%str(Non-unique primary key on &libds)
-)
-
 /**
   * First, extract only relevant formats from the catalog
   */
@@ -10443,6 +10321,12 @@ data &inlibds/nonote2err;
     %mp_aligndecimal(end,width=16)
   end;
 
+  /* update row marker - retain new var as fmtrow may already be in libds */
+  if first.fmtname then row=1;
+  else row+1;
+  drop row;
+  fmtrow=row;
+
   fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
 run;
 
@@ -12475,7 +12359,6 @@ run;
   @li mp_coretable.sas
   @li mp_stackdiffs.test.sas
   @li mp_storediffs.sas
-  @li mp_stripdiffs.sas
 
   @todo The current approach assumes that a variable called KEY_HASH is not on
     the base table.  This part will need to be refactored (eg using
@@ -12934,7 +12817,6 @@ select distinct tgtvar_nm into: missvars separated by ' '
   <h4> Related Macros </h4>
   @li mp_stackdiffs.sas
   @li mp_storediffs.test.sas
-  @li mp_stripdiffs.sas
 
   @version 9.2
   @author Allan Bowe
@@ -13054,7 +12936,7 @@ data &ds4;
 run;
 
 %if "&loadref"="0" %then %let loadref=%sysfunc(uuidgen());
-%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime(),8.6);
+%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime());
 %let libds=%upcase(&libds);
 
 /* join orig vals for modified & deleted */
@@ -13384,260 +13266,6 @@ run;
 
 %mend mp_streamfile;
 /**
-  @file
-  @brief Generates a stage dataset to revert diffs tracked in an audit table
-  @details A big benefit of tracking data changes in an audit table is that
-    those changes can be subsequently reverted if necessary!
-
-    This macro prepares a staging dataset containing those differences - eg for:
-
-    @li deleted rows - these are re-inserted
-    @li changed rows - differences are reverted
-    @li added rows - marked with `_____DELETE__THIS__RECORD_____="YES"`
-
-    These changes are NOT applied to the base table - a staging dataset is
-    simply prepared for an ETL process to action.  In Data Controller, this
-    dataset is used directly as an input to the APPROVE process (so that the
-    reversion diffs can be reviewed prior to being applied).
-
-
-  @param [in] libds Base library.dataset (will not be modified).  The library
-    must be assigned.
-  @param [in] loadref Unique identifier for the version to be reverted.  This
-    change, plus ALL SUBSEQUENT CHANGES, will be reverted in the output table.
-  @param [in] difftable The dataset containing the diffs.  Definition available
-    in mddl_dc_difftable.sas
-  @param [in] filtervar= (0) If provided, the contents of this macro variable
-    will be applied as an additional filter against &libds
-  @param [out] outds= (work.mp_stripdiffs) Output table containing the diffs.
-    Has the same format as the base datset, plus a
-    `_____DELETE__THIS__RECORD_____` variable.
-  @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
-
-  <h4> SAS Macros </h4>
-  @li mf_getuniquefileref.sas
-  @li mf_getuniquename.sas
-  @li mf_getvarlist.sas
-  @li mf_islibds.sas
-  @li mf_wordsinstr1butnotstr2.sas
-  @li mp_abort.sas
-
-  <h4> Related Macros </h4>
-  @li mddl_dc_difftable.sas
-  @li mp_stackdiffs.sas
-  @li mp_storediffs.sas
-  @li mp_stripdiffs.test.sas
-
-  @version 9.2
-  @author Allan Bowe
-**/
-/** @cond */
-
-%macro mp_stripdiffs(libds
-  ,loadref
-  ,difftable
-  ,filtervar=0
-  ,outds=work.mp_stripdiffs
-  ,mdebug=0
-)/*/STORE SOURCE*/;
-%local dbg;
-%if &mdebug=1 %then %do;
-  %put &sysmacroname entry vars:;
-  %put _local_;
-%end;
-%else %let dbg=*;
-
-%let libds=%upcase(&libds);
-
-/* safety checks */
-%mp_abort(iftrue= (&syscc ne 0)
-  ,mac=&sysmacroname
-  ,msg=%str(SYSCC=&syscc on entry.  Clean session required!)
-)
-%let libds=%upcase(&libds);
-%mp_abort(iftrue= (%mf_islibds(&libds)=0)
-  ,mac=&sysmacroname
-  ,msg=%str(Invalid library.dataset reference - %superq(libds))
-)
-
-/* set up unique and temporary vars */
-%local ds1 ds2 ds3 ds4 ds5 fref1 filterstr;
-%let fref1=%mf_getuniquefileref();
-%if &filtervar ne 0 %then %let filterstr=%superq(&filtervar);
-%else %let filterstr=%str(1=1);
-
-/* get timestamp of the diff to be reverted */
-%local ts;
-proc sql noprint;
-select put(processed_dttm,datetime19.6) into: ts
-  from &difftable where load_ref="&loadref";
-%mp_abort(iftrue= (&sqlobs=0)
-  ,mac=&sysmacroname
-  ,msg=%str(Load ref %superq(loadref) not found!)
-)
-
-/* extract diffs for this base table from this timestamp onwards */
-%let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_diffs));
-create table &ds1 (drop=libref dsn) as
-  select * from &difftable
-  where upcase(cats(libref))="%scan(&libds,1,.)"
-  and upcase(cats(dsn))="%scan(&libds,2,.)"
-  and processed_dttm ge "&ts"dt
-  order by processed_dttm desc, key_hash, is_pk;
-
-/* extract key values only */
-%let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_pks));
-%local keyhash processed;
-%let keyhash=%upcase(%mf_getuniquename(prefix=mpsdvar_keyhash));
-%let processed=%upcase(%mf_getuniquename(prefix=mpsdvar_processed));
-create table &ds2 as
-  select key_hash as &keyhash,
-    tgtvar_nm,
-    tgtvar_type,
-    coalescec(oldval_char,newval_char) as charval,
-    coalesce(oldval_num, newval_num) as numval,
-    processed_dttm as &processed
-  from &ds1
-  where is_pk=1
-  order by &keyhash, &processed;
-
-/* grab pk values */
-%local pk;
-select distinct upcase(tgtvar_nm) into: pk separated by ' ' from &ds2;
-
-%let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_keychar));
-proc transpose data=&ds2(where=(tgtvar_type='C'))
-    out=&ds3(drop=_name_);
-  by &keyhash &processed;
-  id TGTVAR_NM;
-  var charval;
-run;
-
-%let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_keynum));
-proc transpose data=&ds2(where=(tgtvar_type='N'))
-    out=&ds4(drop=_name_);
-  by &keyhash &processed;
-  id TGTVAR_NM;
-  var numval;
-run;
-/* shorten the lengths */
-%mp_ds2squeeze(&ds3,outds=&ds3)
-%mp_ds2squeeze(&ds4,outds=&ds4)
-
-/* now merge to get all key values and de-dup */
-%let ds5=%upcase(work.%mf_getuniquename(prefix=mpsd_merged));
-data &ds5;
-  length &keyhash $32 &processed 8;
-  merge &ds3 &ds4;
-  by &keyhash &processed;
-  if not missing(&keyhash);
-run;
-proc sort data=&ds5 nodupkey;
-  by &pk;
-run;
-
-/* join to base table for preliminary stage DS */
-proc sql;
-create table &outds as select "No " as _____DELETE__THIS__RECORD_____
-  %do x=1 %to %sysfunc(countw(&pk,%str( )));
-    ,a.%scan(&pk,&x,%str( ))
-  %end;
-  %local notpkcols;
-  %let notpkcols=%upcase(%mf_getvarlist(&libds));
-  %let notpkcols=%mf_wordsinstr1butnotstr2(str1=&notpkcols,str2=&pk);
-  %do x=1 %to %sysfunc(countw(&notpkcols,%str( )));
-    ,b.%scan(&notpkcols,&x,%str( ))
-  %end;
-  from &ds5 a
-  left join &libds (where=(&filterstr)) b
-  on 1=1
-%do x=1 %to %sysfunc(countw(&pk,%str( )));
-  and a.%scan(&pk,&x,%str( ))=b.%scan(&pk,&x,%str( ))
-%end;
-;
-
-/* create SAS code to apply to stage_ds */
-data _null_;
-  set &ds1;
-  file &fref1 lrecl=33000;
-  length charval $32767;
-  if _n_=1 then put 'proc sql noprint;';
-  by descending processed_dttm key_hash is_pk;
-  if move_type='M' then do;
-    if first.key_hash then do;
-      put "update &outds set " @@;
-    end;
-    if IS_PK=0 then do;
-      put "  " tgtvar_nm '=' @@;
-      cnt=count(oldval_char,'"');
-      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
-      if tgtvar_type='C' then put charval @@;
-      else put oldval_num @@;
-      if not last.is_pk then put ',';
-    end;
-    else do;
-      if first.is_pk then put "  where 1=1 " @@;
-      put "  and " tgtvar_nm '=' @@;
-      cnt=count(oldval_char,'"');
-      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
-      if tgtvar_type='C' then put charval @@;
-      else put oldval_num @@;
-    end;
-  end;
-  else if move_type='A' then do;
-    if first.key_hash then do;
-      put "update &outds set _____DELETE__THIS__RECORD_____='Yes' where 1=1 "@@;
-    end;
-    /* gating if - as only need PK now */
-    if is_pk=1;
-    put '  AND ' tgtvar_nm '=' @@;
-    cnt=count(newval_char,'"');
-    charval=quote(trim(substr(newval_char,1,32765-cnt)));
-    if tgtvar_type='C' then put charval @@;
-    else put newval_num @@;
-  end;
-  else if move_type='D' then do;
-    if first.key_hash then do;
-      put "update &outds set _____DELETE__THIS__RECORD_____='No' " @@;
-    end;
-    if IS_PK=0 then do;
-      put "  ," tgtvar_nm '=' @@;
-      cnt=count(oldval_char,'"');
-      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
-      if tgtvar_type='C' then put charval @@;
-      else put oldval_num @@;
-    end;
-    else do;
-      if first.is_pk then put "  where 1=1 " @@;
-      put "  and " tgtvar_nm '=' @@;
-      cnt=count(oldval_char,'"');
-      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
-      if tgtvar_type='C' then put charval @@;
-      else put oldval_num @@;
-    end;
-  end;
-  if last.key_hash then put ';';
-run;
-
-/* apply the modification statements */
-%inc &fref1/source2 lrecl=33000;
-
-%if &mdebug=0 %then %do;
-  proc sql;
-  drop table &ds1, &ds2, &ds3, &ds4, &ds5;
-  file &fref1 clear;
-%end;
-%else %do;
-  data _null_;
-    infile &fref1;
-    input;
-    if _n_=1 then putlog "Contents of SQL adjustments";
-    putlog _infile_;
-  run;
-%end;
-
-%mend mp_stripdiffs;
-/** @endcond *//**
   @file
   @brief Runs arbitrary code for a specified amount of time
   @details Executes a series of procs and data steps to enable performance
@@ -14548,22 +14176,6 @@ ods package close;
   (given various practical restrictions) are described here to enable
   consistency when dealing with format data.
 
-  The HLO variable may have a number of values, documented here due to the
-  256 char label description length limit:
-
-    F=Standard format/informat.
-    H=Range ending value is HIGH.
-    I=Numeric informat.
-    J=Justification for an informat.
-    L=Range starting value is LOW.
-    M=MultiLabel.
-    N=Format or informat has no ranges, including no OTHER= range.
-    O=Range is OTHER.
-    R=ROUND option is in effect.
-    S=Specifies that NOTSORTED is in effect.
-    U=Specifies that the UPCASE option for an informat be used.
-
-
 **/
 
 
@@ -14571,11 +14183,9 @@ ods package close;
 
   proc sql;
   create table &libds(
-    TYPE char(1) label=
-'Format Type: either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
+    TYPE char(1) label='Type of format - either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
     ,FMTNAME char(32)     label='Format name'
-    ,FMTROW num label=
-'CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
+    ,FMTROW num label='CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
     ,START char(32767)    label='Starting value for format'
     /*
       Keep lengths of START and END the same to avoid this err:
@@ -14595,8 +14205,18 @@ ods package close;
     ,NOEDIT num length=3  label='Is picture string noedit?'
     ,SEXCL char(1)        label='Start exclusion'
     ,EEXCL char(1)        label='End exclusion'
-    ,HLO char(13) label=
-'More info: https://core.sasjs.io/mddl__sas__cntlout_8sas_source.html'
+    ,HLO char(13)         label='Additional information.
+F=Standard format/informat.
+H=Range ending value is HIGH.
+I=Numeric informat.
+J=Justification for an informat.
+L=Range starting value is LOW.
+M=MultiLabel.
+N=Format or informat has no ranges, including no OTHER= range.
+O=Range is OTHER.
+R=ROUND option is in effect.
+S=Specifies that NOTSORTED is in effect.
+U=Specifies that the UPCASE option for an informat be used.'
     ,DECSEP char(1)       label='Decimal separator'
     ,DIG3SEP char(1)      label='Three-digit separator'
     ,DATATYPE char(8)     label='Date/time/datetime?'
@@ -14930,7 +14550,7 @@ run;
   %end;
 %end;
 %else %if &engine=ODBC %then %do;
-  %&mD.put NOTE: Retrieving ODBC connection details;
+  &mD.%put NOTE: Retrieving ODBC connection details;
   data _null_;
     length connx_uri conprop_uri value datasource up_uri schema domprop_uri authdomain $256.;
     call missing (of _all_);
@@ -18229,11 +17849,10 @@ run;
 
   @param [in] user= the metadata user to return groups for.  Leave blank for all
     groups.
-  @param [in] repo= (foundation) the metadata repository that contains the
-    user/group information
-  @param [in] mDebug= (0) set to 1 to show debug messages in the log
-  @param [out] outds= (work.mm_getgroups) The dataset to create that contains
-    the list of groups
+  @param [in] repo= the metadata repository that contains the user/group
+    information
+  @param [in] mDebug= set to 1 to show debug messages in the log
+  @param [out] outds= the dataset to create that contains the list of groups
 
   @returns outds  dataset containing all groups in a column named "metagroup"
     - groupuri
@@ -29789,103 +29408,6 @@ Usage:
 
 %mend mx_getcode;
 /**
-  @file
-  @brief Fetches all groups or the groups for a particular member
-  @details  When building applications that run on multiple flavours of SAS, it
-  is convenient to use a single macro (like this one) to fetch the groups
-  regardless of the flavour of SAS being used
-
-  The alternative would be to compile a generic macro in target-specific
-  folders (SASVIYA, SAS9 and SASJS).  This avoids compiling unnecessary macros
-  at the expense of a more complex sasjsconfig.json setup.
-
-
-  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
-  @param [in] user= (0) Provide the username on which to filter
-  @param [in] uid= (0) Provide the userid on which to filter
-  @param [in] repo= (foundation) SAS9 only, choose the metadata repo to query
-  @param [in] access_token_var= (ACCESS_TOKEN) VIYA only.
-    The global macro variable to contain the access token
-  @param [in] grant_type= (sas_services) VIYA only.
-    Valid values are "password" or "authorization_code" (unquoted).
-  @param [out] outds= (work.mx_getgroups) This output dataset will contain the
-    list of groups. Format:
-|GROUPNAME:$32.|GROUPDESC:$256.|GROUPURI:best.|
-|---|---|---|
-|`SomeGroup `|`A group `|`1`|
-|`Another Group`|`this is a different group`|`2`|
-|`admin`|`Administrators `|`3`|
-
-  <h4> SAS Macros </h4>
-  @li mf_getplatform.sas
-  @li mm_getgroups.sas
-  @li ms_getgroups.sas
-  @li mv_getgroups.sas
-  @li mv_getusergroups.sas
-
-**/
-
-%macro mx_getgroups(
-  mdebug=0,
-  user=0,
-  uid=0,
-  repo=foundation,
-  access_token_var=ACCESS_TOKEN,
-  grant_type=sas_services,
-  outds=work.mx_getgroups
-)/*/STORE SOURCE*/;
-%local platform name shortloc;
-%let platform=%mf_getplatform();
-
-%if &platform=SASJS %then %do;
-  %ms_getgroups(
-    user=&user,
-    uid=&uid,
-    outds=&outds,
-    mdebug=&mdebug
-  )
-  data &outds;
-    length groupuri groupname $32 groupdesc $128 ;
-    set &outds;
-    keep groupuri groupname groupdesc;
-    groupuri=cats(groupid);
-    groupname=name;
-    groupdesc=description;
-  run;
-  proc sort; by groupname; run;
-%end;
-%else %if &platform=SAS9 or &platform=SASMETA %then %do;
-  %if &user=0 %then %let user=;
-  %mm_getGroups(
-    user=&user
-    ,outds=&outds
-    ,repo=&repo
-    ,mDebug=&mdebug
-  )
-  proc sort data=&outds; by groupname; run;
-%end;
-%else %if &platform=SASVIYA %then %do;
-  %if &user=0 %then %do;
-    %mv_getgroups(access_token_var=&access_token_var
-      ,grant_type=&grant_type
-      ,outds=&outds
-    )
-  %end;
-  %else %do;
-    %mv_getusergroups(&user
-      ,outds=&outds
-      ,access_token_var=&access_token_var
-      ,grant_type=&grant_type
-    )
-  %end;
-  proc sort
-      data=&outds(rename=(id=groupuri name=groupname description=groupdesc))
-      out=&outds (keep=groupuri groupname groupdesc);
-    by groupname;
-  run;
-%end;
-
-%mend mx_getgroups;/**
   @file
   @brief Will execute a SASjs web service on SAS 9, Viya or SASjs Server
   @details Prepares the input files and retrieves the resulting datasets from

base/mf_isblank.sas

@@ -7,7 +7,7 @@
 
   Usage:
 
-      %put %mf_isblank(&var);
+      %put mf_isblank(&var);
 
   inspiration:
   https://support.sas.com/resources/papers/proceedings09/022-2009.pdf

base/mp_cntlout.sas

@@ -79,8 +79,8 @@ data &cntlout/nonote2err;
   end;
 
   /* create row marker. Data cannot be sorted without it! */
-  if first.fmtname then fmtrow=1;
-  else fmtrow+1;
+  if first.fmtname then fmtrow=0;
+  fmtrow+1;
 
 run;
 proc sort;

base/mp_dirlist.sas

@@ -85,7 +85,7 @@ data;run;
 data &out_ds(compress=no
     keep=file_or_folder filepath filename ext msg directory level
   );
-  length directory filepath $2000 fref fref2 $8 file_or_folder $6 filename $255
+  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
     ext $20 msg $200 foption $16;
   if _n_=1 then call missing(of _all_);
   retain level &level;

base/mp_ds2csv.sas

@@ -118,21 +118,13 @@ data _null_;
     header = cats(coalescec(varlabel(dsid,i),varnm),dlm);
   %end;
   %else %if &headerformat=SASJS %then %do;
-    vlen=varlen(dsid,i);
-    if vartype(dsid,i)='C' then header=cats(varnm,':$char',vlen,'.');
+    if vartype(dsid,i)='C' then header=cats(varnm,':$char',varlen(dsid,i),'.');
     else do;
       vfmt=coalescec(varfmt(dsid,i),'0');
       fmttype=mcf_getfmttype(vfmt);
       if fmttype='DATE' then header=cats(varnm,':date9.');
       else if fmttype='DATETIME' then header=cats(varnm,':E8601DT26.6');
       else if fmttype='TIME' then header=cats(varnm,':TIME12.');
-      /**
-        * there is not much point importing a short length numeric like this,
-        * eg with best4., as the resulting variable will still be stored as
-        * length 8.  We need a length or format statement to ensure variable
-        * is creatd with the smaller length...
-        **/
-      else if vlen<8 then header=cats(varnm,':best',vlen,'.');
       else header=cats(varnm,':best.');
     end;
   %end;
@@ -159,7 +151,6 @@ data _null_;
   set &ds end=last;
 %do i=1 %to &vcnt;
   %let var=%scan(&varlist,&i);
-  %local vlen&i;
   %if %mf_getvartype(&ds,&var)=C %then %do;
     %let dsv1=%mf_getuniquename(prefix=csvcol1_);
     %let dsv2=%mf_getuniquename(prefix=csvcol2_);

base/mp_filtercheck.sas

@@ -86,14 +86,15 @@
 /**
   * Sanitise the values based on valid value lists, then strip out
   * quotes, commas, periods and spaces.
+  * Only numeric values should remain
   */
 %local reason_cd nobs;
 %let nobs=0;
 data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
-  set &inds end=last;
-  length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000;
+  set &inds;
+  length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000;
   drop tmp;
 
   /* quick check to ensure column exists */
@@ -109,8 +110,7 @@ data &outds;
   end;
 
   /* need to open the dataset to get the column type */
-  retain dsid;
-  if _n_=1 then dsid=open("&targetds","i");
+  dsid=open("&targetds","i");
   if dsid>0 then do;
     vnum=varnum(dsid,VARIABLE_NM);
     if vnum<1 then do;
@@ -120,19 +120,11 @@ data &outds;
       call symputx('reason_cd',reason_cd,'l');
       call symputx('nobs',_n_,'l');
       output;
-      goto endstep;
+      return;
     end;
     /* now we can get the type */
     else vtype=vartype(dsid,vnum);
   end;
-  else do;
-      REASON_CD=cats("Could not open &targetds");
-      putlog REASON_CD= dsid=;
-      call symputx('reason_cd',reason_cd,'l');
-      call symputx('nobs',_n_,'l');
-      output;
-      stop;
-  end;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
@@ -167,40 +159,15 @@ data &outds;
   end;
 
   /* special missing logic */
-  if vtype='N' & OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE') then do;
-    if cats(upcase(raw_value)) in (
+  if vtype='N'
+    and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE')
+    and cats(upcase(raw_value)) in (
       '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N'
       '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._'
     )
-    then do;
-      /* valid numeric - exit data step loop */
-      return;
-    end;
-    else if subpad(upcase(raw_value),1,1) in (
-      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
-      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
-    )
-    then do;
-      /* check if the raw_value contains a valid variable NAME */
-      vnum=varnum(dsid,subpad(raw_value,1,32));
-      if vnum>0 then do;
-        /* now we can get the type */
-        vtype2=vartype(dsid,vnum);
-        /* check type matches */
-        if vtype2=vtype then do;
-          /* valid target var - exit loop */
-          return;
-        end;
-        else do;
-          REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
-          putlog REASON_CD= dsid=;
-          call symputx('reason_cd',reason_cd,'l');
-          call symputx('nobs',_n_,'l');
-          output;
-          goto endstep;
-        end;
-      end;
-    end;
+  then do;
+    /* valid numeric - exit data step loop */
+    return;
   end;
 
   /* special logic */
@@ -222,32 +189,6 @@ data &outds;
     if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
       tmp=scan(raw_value1,i,',');
       if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
-        if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in (
-          'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
-          'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
-        )
-        then do;
-          /* check if the raw_value contains a valid variable NAME */
-          /* is not valid syntax for IN or NOT IN */
-          vnum=varnum(dsid,subpad(tmp,1,32));
-          if vnum>0 then do;
-            /* now we can get the type */
-            vtype2=vartype(dsid,vnum);
-            /* check type matches */
-            if vtype2=vtype then do;
-              /* valid target var - exit loop */
-              return;
-            end;
-            else do;
-              REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
-              putlog REASON_CD= dsid=;
-              call symputx('reason_cd',reason_cd,'l');
-              call symputx('nobs',_n_,'l');
-              output;
-              goto endstep;
-            end;
-          end;
-        end;
         REASON_CD='Non Numeric value provided';
         putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
         call symputx('reason_cd',reason_cd,'l');
@@ -272,42 +213,14 @@ data &outds;
 
   /* output records that contain values other than digits and spaces */
   if notdigit(compress(raw_value3,' '))>0 then do;
-    if vtype='C' and subpad(upcase(raw_value),1,1) in (
-      'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
-      'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
-    )
-    then do;
-      /* check if the raw_value contains a valid variable NAME */
-      vnum=varnum(dsid,subpad(raw_value,1,32));
-      if vnum>0 then do;
-        /* now we can get the type */
-        vtype2=vartype(dsid,vnum);
-        /* check type matches */
-        if vtype2=vtype then do;
-          /* valid target var - exit loop */
-          return;
-        end;
-        else do;
-          REASON_CD=cats("Compared Char Type (",vtype2,") is not (",vtype,")");
-          putlog REASON_CD= dsid=;
-          call symputx('reason_cd',reason_cd,'l');
-          call symputx('nobs',_n_,'l');
-          output;
-          goto endstep;
-        end;
-      end;
-    end;
-
     putlog raw_value3= $hex32.;
     REASON_CD=cats('Invalid RAW_VALUE:',raw_value);
-    putlog (_all_)(=);
+    putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
     call symputx('reason_cd',reason_cd,'l');
     call symputx('nobs',_n_,'l');
     output;
   end;
 
-  endstep:
-  if last then rc=close(dsid);
 run;
 
 

base/mp_loadformat.sas

@@ -9,9 +9,6 @@
   format, to prevent loss of data - UNLESS the input dataset contains a marker
   column, specifying that a particular row needs to be deleted (`delete_col=`).
 
-  Positions of formats are made using the FMTROW variable - this must be present
-  and unique (on TYPE / FMTNAME / FMTROW).
-
   This macro can also be used to identify which records would be (or were)
   considered new, modified or deleted (`loadtarget=`) by creating the following
   tables:
@@ -20,7 +17,7 @@
   @li work.outds_del
   @li work.outds_mod
 
-  For example usage, see test (under Related Macros)
+  For example usage, see mp_loadformat.test.sas
 
   @param [in] libcat The format catalog to be loaded
   @param [in] libds The staging table to load
@@ -37,15 +34,12 @@
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
-  @li mf_existds.sas
-  @li mf_existvar.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
-  @li mp_md5.sas
   @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
@@ -89,16 +83,6 @@
 %let libcat=%scan(&libcat,1,-);
 
 /* perform input validations */
-%mp_abort(
-  iftrue=(%mf_existds(&libds)=0)
-  ,mac=&sysmacroname
-  ,msg=%str(&libds could not be found)
-)
-%mp_abort(
-  iftrue=(%mf_existvar(&libds,FMTROW)=0)
-  ,mac=&sysmacroname
-  ,msg=%str(FMTROW not found in &libds)
-)
 %let err=0;
 %let msg=0;
 data _null_;
@@ -119,6 +103,13 @@ data _null_;
       stop;
     end;
   end;
+  else if name='LIBDS' then do;
+    if exist(value) le 0 then do;
+      call symputx('msg',"Unable to open staging table: "!!value);
+      call symputx('err',1);
+      stop;
+    end;
+  end;
   else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
   and missing(value) then do;
     call symputx('msg',"missing value in var: "!!name);
@@ -126,14 +117,6 @@ data _null_;
     stop;
   end;
 run;
-data _null_;
-  set &libds;
-  if missing(fmtrow) then do;
-    call symputx('msg',"missing fmtrow in format: "!!FMTNAME);
-    call symputx('err',1);
-    stop;
-  end;
-run;
 
 %mp_abort(
   iftrue=(&err ne 0)
@@ -141,15 +124,6 @@ run;
   ,msg=%str(&msg)
 )
 
-%local cnt;
-proc sql noprint;
-select count(distinct catx('|',type,fmtname,fmtrow)) into: cnt from &libds;
-%mp_abort(
-  iftrue=(&cnt ne %mf_nobs(&libds))
-  ,mac=&sysmacroname
-  ,msg=%str(Non-unique primary key on &libds)
-)
-
 /**
   * First, extract only relevant formats from the catalog
   */
@@ -203,6 +177,12 @@ data &inlibds/nonote2err;
     %mp_aligndecimal(end,width=16)
   end;
 
+  /* update row marker - retain new var as fmtrow may already be in libds */
+  if first.fmtname then row=1;
+  else row+1;
+  drop row;
+  fmtrow=row;
+
   fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
 run;
 

base/mp_stackdiffs.sas

@@ -197,7 +197,6 @@
   @li mp_coretable.sas
   @li mp_stackdiffs.test.sas
   @li mp_storediffs.sas
-  @li mp_stripdiffs.sas
 
   @todo The current approach assumes that a variable called KEY_HASH is not on
     the base table.  This part will need to be refactored (eg using

base/mp_storediffs.sas

@@ -64,7 +64,6 @@
   <h4> Related Macros </h4>
   @li mp_stackdiffs.sas
   @li mp_storediffs.test.sas
-  @li mp_stripdiffs.sas
 
   @version 9.2
   @author Allan Bowe
@@ -184,7 +183,7 @@ data &ds4;
 run;
 
 %if "&loadref"="0" %then %let loadref=%sysfunc(uuidgen());
-%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime(),8.6);
+%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime());
 %let libds=%upcase(&libds);
 
 /* join orig vals for modified & deleted */

base/mp_stripdiffs.sas

@@ -1,255 +0,0 @@
-/**
-  @file
-  @brief Generates a stage dataset to revert diffs tracked in an audit table
-  @details A big benefit of tracking data changes in an audit table is that
-    those changes can be subsequently reverted if necessary!
-
-    This macro prepares a staging dataset containing those differences - eg for:
-
-    @li deleted rows - these are re-inserted
-    @li changed rows - differences are reverted
-    @li added rows - marked with `_____DELETE__THIS__RECORD_____="YES"`
-
-    These changes are NOT applied to the base table - a staging dataset is
-    simply prepared for an ETL process to action.  In Data Controller, this
-    dataset is used directly as an input to the APPROVE process (so that the
-    reversion diffs can be reviewed prior to being applied).
-
-
-  @param [in] libds Base library.dataset (will not be modified).  The library
-    must be assigned.
-  @param [in] loadref Unique identifier for the version to be reverted.  This
-    change, plus ALL SUBSEQUENT CHANGES, will be reverted in the output table.
-  @param [in] difftable The dataset containing the diffs.  Definition available
-    in mddl_dc_difftable.sas
-  @param [in] filtervar= (0) If provided, the contents of this macro variable
-    will be applied as an additional filter against &libds
-  @param [out] outds= (work.mp_stripdiffs) Output table containing the diffs.
-    Has the same format as the base datset, plus a
-    `_____DELETE__THIS__RECORD_____` variable.
-  @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
-
-  <h4> SAS Macros </h4>
-  @li mf_getuniquefileref.sas
-  @li mf_getuniquename.sas
-  @li mf_getvarlist.sas
-  @li mf_islibds.sas
-  @li mf_wordsinstr1butnotstr2.sas
-  @li mp_abort.sas
-
-  <h4> Related Macros </h4>
-  @li mddl_dc_difftable.sas
-  @li mp_stackdiffs.sas
-  @li mp_storediffs.sas
-  @li mp_stripdiffs.test.sas
-
-  @version 9.2
-  @author Allan Bowe
-**/
-/** @cond */
-
-%macro mp_stripdiffs(libds
-  ,loadref
-  ,difftable
-  ,filtervar=0
-  ,outds=work.mp_stripdiffs
-  ,mdebug=0
-)/*/STORE SOURCE*/;
-%local dbg;
-%if &mdebug=1 %then %do;
-  %put &sysmacroname entry vars:;
-  %put _local_;
-%end;
-%else %let dbg=*;
-
-%let libds=%upcase(&libds);
-
-/* safety checks */
-%mp_abort(iftrue= (&syscc ne 0)
-  ,mac=&sysmacroname
-  ,msg=%str(SYSCC=&syscc on entry.  Clean session required!)
-)
-%let libds=%upcase(&libds);
-%mp_abort(iftrue= (%mf_islibds(&libds)=0)
-  ,mac=&sysmacroname
-  ,msg=%str(Invalid library.dataset reference - %superq(libds))
-)
-
-/* set up unique and temporary vars */
-%local ds1 ds2 ds3 ds4 ds5 fref1 filterstr;
-%let fref1=%mf_getuniquefileref();
-%if &filtervar ne 0 %then %let filterstr=%superq(&filtervar);
-%else %let filterstr=%str(1=1);
-
-/* get timestamp of the diff to be reverted */
-%local ts;
-proc sql noprint;
-select put(processed_dttm,datetime19.6) into: ts
-  from &difftable where load_ref="&loadref";
-%mp_abort(iftrue= (&sqlobs=0)
-  ,mac=&sysmacroname
-  ,msg=%str(Load ref %superq(loadref) not found!)
-)
-
-/* extract diffs for this base table from this timestamp onwards */
-%let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_diffs));
-create table &ds1 (drop=libref dsn) as
-  select * from &difftable
-  where upcase(cats(libref))="%scan(&libds,1,.)"
-  and upcase(cats(dsn))="%scan(&libds,2,.)"
-  and processed_dttm ge "&ts"dt
-  order by processed_dttm desc, key_hash, is_pk;
-
-/* extract key values only */
-%let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_pks));
-%local keyhash processed;
-%let keyhash=%upcase(%mf_getuniquename(prefix=mpsdvar_keyhash));
-%let processed=%upcase(%mf_getuniquename(prefix=mpsdvar_processed));
-create table &ds2 as
-  select key_hash as &keyhash,
-    tgtvar_nm,
-    tgtvar_type,
-    coalescec(oldval_char,newval_char) as charval,
-    coalesce(oldval_num, newval_num) as numval,
-    processed_dttm as &processed
-  from &ds1
-  where is_pk=1
-  order by &keyhash, &processed;
-
-/* grab pk values */
-%local pk;
-select distinct upcase(tgtvar_nm) into: pk separated by ' ' from &ds2;
-
-%let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_keychar));
-proc transpose data=&ds2(where=(tgtvar_type='C'))
-    out=&ds3(drop=_name_);
-  by &keyhash &processed;
-  id TGTVAR_NM;
-  var charval;
-run;
-
-%let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_keynum));
-proc transpose data=&ds2(where=(tgtvar_type='N'))
-    out=&ds4(drop=_name_);
-  by &keyhash &processed;
-  id TGTVAR_NM;
-  var numval;
-run;
-/* shorten the lengths */
-%mp_ds2squeeze(&ds3,outds=&ds3)
-%mp_ds2squeeze(&ds4,outds=&ds4)
-
-/* now merge to get all key values and de-dup */
-%let ds5=%upcase(work.%mf_getuniquename(prefix=mpsd_merged));
-data &ds5;
-  length &keyhash $32 &processed 8;
-  merge &ds3 &ds4;
-  by &keyhash &processed;
-  if not missing(&keyhash);
-run;
-proc sort data=&ds5 nodupkey;
-  by &pk;
-run;
-
-/* join to base table for preliminary stage DS */
-proc sql;
-create table &outds as select "No " as _____DELETE__THIS__RECORD_____
-  %do x=1 %to %sysfunc(countw(&pk,%str( )));
-    ,a.%scan(&pk,&x,%str( ))
-  %end;
-  %local notpkcols;
-  %let notpkcols=%upcase(%mf_getvarlist(&libds));
-  %let notpkcols=%mf_wordsinstr1butnotstr2(str1=&notpkcols,str2=&pk);
-  %do x=1 %to %sysfunc(countw(&notpkcols,%str( )));
-    ,b.%scan(&notpkcols,&x,%str( ))
-  %end;
-  from &ds5 a
-  left join &libds (where=(&filterstr)) b
-  on 1=1
-%do x=1 %to %sysfunc(countw(&pk,%str( )));
-  and a.%scan(&pk,&x,%str( ))=b.%scan(&pk,&x,%str( ))
-%end;
-;
-
-/* create SAS code to apply to stage_ds */
-data _null_;
-  set &ds1;
-  file &fref1 lrecl=33000;
-  length charval $32767;
-  if _n_=1 then put 'proc sql noprint;';
-  by descending processed_dttm key_hash is_pk;
-  if move_type='M' then do;
-    if first.key_hash then do;
-      put "update &outds set " @@;
-    end;
-    if IS_PK=0 then do;
-      put "  " tgtvar_nm '=' @@;
-      cnt=count(oldval_char,'"');
-      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
-      if tgtvar_type='C' then put charval @@;
-      else put oldval_num @@;
-      if not last.is_pk then put ',';
-    end;
-    else do;
-      if first.is_pk then put "  where 1=1 " @@;
-      put "  and " tgtvar_nm '=' @@;
-      cnt=count(oldval_char,'"');
-      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
-      if tgtvar_type='C' then put charval @@;
-      else put oldval_num @@;
-    end;
-  end;
-  else if move_type='A' then do;
-    if first.key_hash then do;
-      put "update &outds set _____DELETE__THIS__RECORD_____='Yes' where 1=1 "@@;
-    end;
-    /* gating if - as only need PK now */
-    if is_pk=1;
-    put '  AND ' tgtvar_nm '=' @@;
-    cnt=count(newval_char,'"');
-    charval=quote(trim(substr(newval_char,1,32765-cnt)));
-    if tgtvar_type='C' then put charval @@;
-    else put newval_num @@;
-  end;
-  else if move_type='D' then do;
-    if first.key_hash then do;
-      put "update &outds set _____DELETE__THIS__RECORD_____='No' " @@;
-    end;
-    if IS_PK=0 then do;
-      put "  ," tgtvar_nm '=' @@;
-      cnt=count(oldval_char,'"');
-      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
-      if tgtvar_type='C' then put charval @@;
-      else put oldval_num @@;
-    end;
-    else do;
-      if first.is_pk then put "  where 1=1 " @@;
-      put "  and " tgtvar_nm '=' @@;
-      cnt=count(oldval_char,'"');
-      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
-      if tgtvar_type='C' then put charval @@;
-      else put oldval_num @@;
-    end;
-  end;
-  if last.key_hash then put ';';
-run;
-
-/* apply the modification statements */
-%inc &fref1/source2 lrecl=33000;
-
-%if &mdebug=0 %then %do;
-  proc sql;
-  drop table &ds1, &ds2, &ds3, &ds4, &ds5;
-  file &fref1 clear;
-%end;
-%else %do;
-  data _null_;
-    infile &fref1;
-    input;
-    if _n_=1 then putlog "Contents of SQL adjustments";
-    putlog _infile_;
-  run;
-%end;
-
-%mend mp_stripdiffs;
-/** @endcond */

ddl/mddl_sas_cntlout.sas

@@ -6,22 +6,6 @@
   (given various practical restrictions) are described here to enable
   consistency when dealing with format data.
 
-  The HLO variable may have a number of values, documented here due to the
-  256 char label description length limit:
-
-    F=Standard format/informat.
-    H=Range ending value is HIGH.
-    I=Numeric informat.
-    J=Justification for an informat.
-    L=Range starting value is LOW.
-    M=MultiLabel.
-    N=Format or informat has no ranges, including no OTHER= range.
-    O=Range is OTHER.
-    R=ROUND option is in effect.
-    S=Specifies that NOTSORTED is in effect.
-    U=Specifies that the UPCASE option for an informat be used.
-
-
 **/
 
 
@@ -29,11 +13,9 @@
 
   proc sql;
   create table &libds(
-    TYPE char(1) label=
-'Format Type: either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
+    TYPE char(1) label='Type of format - either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)'
     ,FMTNAME char(32)     label='Format name'
-    ,FMTROW num label=
-'CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
+    ,FMTROW num label='CALCULATED Position of record by FMTNAME (reqd for multilabel formats)'
     ,START char(32767)    label='Starting value for format'
     /*
       Keep lengths of START and END the same to avoid this err:
@@ -53,8 +35,18 @@
     ,NOEDIT num length=3  label='Is picture string noedit?'
     ,SEXCL char(1)        label='Start exclusion'
     ,EEXCL char(1)        label='End exclusion'
-    ,HLO char(13) label=
-'More info: https://core.sasjs.io/mddl__sas__cntlout_8sas_source.html'
+    ,HLO char(13)         label='Additional information.
+F=Standard format/informat.
+H=Range ending value is HIGH.
+I=Numeric informat.
+J=Justification for an informat.
+L=Range starting value is LOW.
+M=MultiLabel.
+N=Format or informat has no ranges, including no OTHER= range.
+O=Range is OTHER.
+R=ROUND option is in effect.
+S=Specifies that NOTSORTED is in effect.
+U=Specifies that the UPCASE option for an informat be used.'
     ,DECSEP char(1)       label='Decimal separator'
     ,DIG3SEP char(1)      label='Three-digit separator'
     ,DATATYPE char(8)     label='Date/time/datetime?'

meta/mm_assigndirectlib.sas

@@ -210,7 +210,7 @@ run;
   %end;
 %end;
 %else %if &engine=ODBC %then %do;
-  %&mD.put NOTE: Retrieving ODBC connection details;
+  &mD.%put NOTE: Retrieving ODBC connection details;
   data _null_;
     length connx_uri conprop_uri value datasource up_uri schema domprop_uri authdomain $256.;
     call missing (of _all_);

meta/mm_getgroups.sas

@@ -11,11 +11,10 @@
 
   @param [in] user= the metadata user to return groups for.  Leave blank for all
     groups.
-  @param [in] repo= (foundation) the metadata repository that contains the
-    user/group information
-  @param [in] mDebug= (0) set to 1 to show debug messages in the log
-  @param [out] outds= (work.mm_getgroups) The dataset to create that contains
-    the list of groups
+  @param [in] repo= the metadata repository that contains the user/group
+    information
+  @param [in] mDebug= set to 1 to show debug messages in the log
+  @param [out] outds= the dataset to create that contains the list of groups
 
   @returns outds  dataset containing all groups in a column named "metagroup"
     - groupuri

sasjs/sasjsconfig.json

@@ -67,7 +67,7 @@
     },
     {
       "name": "server",
-      "serverUrl": "https://sas.4gl.io",
+      "serverUrl": "https://sas9.4gl.io",
       "serverType": "SASJS",
       "httpsAgentOptions": {
         "allowInsecureRequests": false

tests/base/mp_ds2csv.test.2.sas

@@ -1,43 +0,0 @@
-/**
-  @file
-  @brief Testing mp_ds2csv.sas macro
-
-  <h4> SAS Macros </h4>
-  @li mp_ds2csv.sas
-  @li mp_assert.sas
-  @li mp_assertscope.sas
-
-**/
-
-data work.shortnum;
-  length a 3 b 4 c 8;
-  a=1;b=2;c=3;
-  output;
-  stop;
-run;
-
-/**
-  * Test 1 - default CSV
-  */
-
-%mp_ds2csv(work.shortnum,outfile="&sasjswork/test1.csv",headerformat=SASJS)
-
-%let test1b=FAIL;
-data _null_;
-  infile "&sasjswork/test1.csv";
-  input;
-  list;
-  if _n_=1 then call symputx('test1a',_infile_);
-  else if _infile_=:'1,2,3' then call symputx('test1b','PASS');
-run;
-
-%mp_assert(
-  iftrue=("&test1a"="A:best3. B:best4. C:best."),
-  desc=Checking header row Test 1,
-  outds=work.test_results
-)
-%mp_assert(
-  iftrue=("&test1b"="PASS"),
-  desc=Checking data row Test 1,
-  outds=work.test_results
-)

tests/base/mp_filtercheck.test.sas

@@ -53,10 +53,7 @@ AND,AND,1,age,=,.A
 AND,AND,1,height,<,.B
 AND,AND,1,age,IN,"(.a,.b,.)"
 AND,AND,1,age,IN,"(.A)"
-AND,AND,1,AGE,=,AGE
-AND,AND,1,AGE,<,Weight
-AND,AND,1,AGE,BETWEEN,"HEIGHT AND WEIGHT"
-AND,OR,2,Name,=,name
+
 ;;;;
 run;
 
@@ -207,26 +204,3 @@ run;
   outds=work.test_results
 )
 %let syscc=0;
-
-
-/* invalid IN value (cannot use var names) */
-data work.inds;
-  infile datalines4 dsd;
-  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:$4000.;
-datalines4;
-AND,AND,1,AGE,NOT IN,"(height, age)"
-;;;;
-run;
-
-%mp_filtercheck(work.inds,
-  targetds=work.class,
-  outds=work.badrecords,
-  abort=NO
-)
-%let syscc=0;
-%mp_assertdsobs(work.badrecords,
-  desc=Invalid IN syntax,
-  test=HASOBS,
-  outds=work.test_results
-)

tests/base/mp_loadformat.test.1.sas

@@ -189,10 +189,8 @@ data work.stagedata3;
     if last.fmtname then do;
       output; /* 6 new records */
       x=_n_;
-      x+1;start=cats("mod",x);end=start;label='newlabel1';fmtrow=fmtrow+1;
-      output;
-      x+1;start=cats("mod",x);end=start;label='newlabel2';fmtrow=fmtrow+2;
-      output;
+      x+1;start=cats("mod",x);end=start;label='newlabel1';output;
+      x+1;start=cats("mod",x);end=start;label='newlabel2';output;
     end;
     else if fmtrow le 3 then do; /* 9 more changed values */
       start= cats("mod",_n_);

tests/base/mp_loadformat.test.2.sas

@@ -58,9 +58,6 @@ proc format library=&cat1;
   value agemlb (multilabel)
     19-120='Adults'
     1-18='Children'
-    0-1='Preschool'
-    1-2='Preschool'
-    2-3='Preschool'
     1-4='Preschool';
   value agemlc (multilabel notsorted)
     19-120='Adults'
@@ -70,19 +67,16 @@ run;
 
 %mp_cntlout(libcat=&cat1,cntlout=work.cntlout1)
 %mp_assertdsobs(work.cntlout1,
-  desc=Has 19 records,
-  test=EQUALS 19
+  desc=Has 16 records,
+  test=EQUALS 16
 )
 
 data work.stagedata3;
   set work.cntlout1;
   if fmtname='AGEMLA' and label ne 'Preschool' then deleteme='Yes';
   if fmtname='AGEMLB' and label = 'Preschool' then label='Kids';
-  if fmtname='GENDERML' and label='Farmale' then do;
-    output;
-    fmtrow=101; output;
-  end;
-  else output;
+  if fmtname='GENDERML' and label='Farmale' then output;
+  output;
 run;
 
 
@@ -119,17 +113,14 @@ run;
 
 %let check1=0;
 %let check2=0;
-%let check3=0;
 data test;
   set work.cntlout2;
   where fmtname='GENDERML';
-  putlog fmtrow= label=;
   if _n_=4 and label='Farmale' then call symputx('check1',1);
-  if _n_=5 and label ne 'Farmale' then call symputx('check2',1);
-  if _n_=8 and label = 'Farmale' then call symputx('check3',1);
+  if _n_=5 and label='Farmale' then call symputx('check2',1);
 run;
 %mp_assert(
-  iftrue=(&check1=1 and &check2=1 and &check3=1),
+  iftrue=(&check1=1 and &check2=1),
   desc=Ensuring Farmale values retain their order,
   outds=work.test_results
 )

tests/base/mp_stripdiffs.test.sas

@@ -1,106 +0,0 @@
-/**
-  @file
-  @brief Testing mp_stripdiffs.sas macro
-  @details
-
-  <h4> SAS Macros </h4>
-  @li mp_assert.sas
-  @li mp_assertscope.sas
-  @li mp_ds2md.sas
-  @li mp_stripdiffs.sas
-
-**/
-
-/* make an adjustable base dataset */
-/* use a composite key also (name weight) */
-libname libby (work);
-data libby.class;
-  set sashelp.class;
-run;
-
-/* first, store some diffs */
-data work.orig work.deleted work.changed work.appended;
-  set libby.class;
-  if _n_=1 then do;
-    call symputx('delname',name);
-    output work.orig work.deleted;
-  end;
-  else if _n_=2 then do;
-    output work.orig;
-    call symputx('modname',name);
-    call symputx('modval',age);
-    age=99;
-    output work.changed;
-  end;
-  else do;
-    name='Newbie';
-    output work.appended;
-    stop;
-  end;
-run;
-%mp_storediffs(libby.class,work.orig,NAME WEIGHT
-  ,delds=work.deleted
-  ,modds=work.changed
-  ,appds=work.appended
-  ,outds=work.audit
-  ,loadref=UPLOAD1
-  ,mdebug=0
-)
-%mp_ds2md(work.audit)
-%mp_assert(
-  iftrue=(&syscc=0),
-  desc=Checking preparation case,
-  outds=work.test_results
-)
-
-/* apply the changes */
-proc sql;
-delete from libby.class where name in ("&delname","&modname");
-proc append base=libby.class data=work.appended;
-proc append base=libby.class data=work.changed;
-run;
-
-/* now, prepare the revert dataset */
-%mp_assertscope(SNAPSHOT)
-%mp_stripdiffs(libby.class
-  ,UPLOAD1
-  ,work.audit
-  ,outds=work.mp_stripdiffs
-  ,mdebug=1
-)
-%mp_ds2md(work.mp_stripdiffs)
-%mp_assertscope(COMPARE)
-
-%mp_assert(
-  iftrue=(&syscc=0),
-  desc=Checking error condition,
-  outds=work.test_results
-)
-
-%let delpass=0;
-%let modpass=0;
-%let addpass=0;
-data _null_;
-  set work.mp_stripdiffs;
-  if upcase(_____DELETE__THIS__RECORD_____)='NO' and name="&delname"
-  then call symputx('delpass',1);
-  if name="&modname" and age=&modval then call symputx('modpass',1);
-  if upcase(_____DELETE__THIS__RECORD_____)='YES' and name="Newbie"
-  then call symputx('addpass',1);
-run;
-
-%mp_assert(
-  iftrue=(&delpass=1),
-  desc=Ensuring deleted record is back in the dataset,
-  outds=work.test_results
-)
-%mp_assert(
-  iftrue=(&modpass=1),
-  desc=Ensuring modified record now has old value,
-  outds=work.test_results
-)
-%mp_assert(
-  iftrue=(&addpass=1),
-  desc=Ensuring added record is now marked for deletion,
-  outds=work.test_results
-)

tests/x-platform/mx_getgroups.test.sas

@@ -1,41 +0,0 @@
-/**
-  @file
-  @brief Testing mx_getgroups.test.sas macro
-
-  Be sure to run <code>%let mcTestAppLoc=/Public/temp/macrocore;</code> when
-  running in Studio
-
-  <h4> SAS Macros </h4>
-  @li mf_nobs.sas
-  @li mf_getuser.sas
-  @li mp_assert.sas
-  @li mx_getgroups.sas
-
-**/
-
-
-%mx_getgroups(outds=work.test1)
-
-%mp_assert(
-  iftrue=(%mf_nobs(work.test1)>0),
-  desc=groups were found,
-  outds=work.test_results
-)
-%mp_assertcols(work.test1,
-  cols=groupuri groupname groupdesc,
-  test=ALL,
-  desc=check all columns exist
-)
-
-%mx_getgroups(outds=work.test2,user=%mf_getuser())
-
-%mp_assert(
-  iftrue=(%mf_nobs(work.test2)>0),
-  desc=groups for current user were found,
-  outds=work.test_results
-)
-%mp_assertcols(work.test2,
-  cols=groupuri groupname groupdesc,
-  test=ALL,
-  desc=check all columns exist
-)

xplatform/mx_getgroups.sas

@@ -1,98 +0,0 @@
-/**
-  @file
-  @brief Fetches all groups or the groups for a particular member
-  @details  When building applications that run on multiple flavours of SAS, it
-  is convenient to use a single macro (like this one) to fetch the groups
-  regardless of the flavour of SAS being used
-
-  The alternative would be to compile a generic macro in target-specific
-  folders (SASVIYA, SAS9 and SASJS).  This avoids compiling unnecessary macros
-  at the expense of a more complex sasjsconfig.json setup.
-
-
-  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
-  @param [in] user= (0) Provide the username on which to filter
-  @param [in] uid= (0) Provide the userid on which to filter
-  @param [in] repo= (foundation) SAS9 only, choose the metadata repo to query
-  @param [in] access_token_var= (ACCESS_TOKEN) VIYA only.
-    The global macro variable to contain the access token
-  @param [in] grant_type= (sas_services) VIYA only.
-    Valid values are "password" or "authorization_code" (unquoted).
-  @param [out] outds= (work.mx_getgroups) This output dataset will contain the
-    list of groups. Format:
-|GROUPNAME:$32.|GROUPDESC:$256.|GROUPURI:best.|
-|---|---|---|
-|`SomeGroup `|`A group `|`1`|
-|`Another Group`|`this is a different group`|`2`|
-|`admin`|`Administrators `|`3`|
-
-  <h4> SAS Macros </h4>
-  @li mf_getplatform.sas
-  @li mm_getgroups.sas
-  @li ms_getgroups.sas
-  @li mv_getgroups.sas
-  @li mv_getusergroups.sas
-
-**/
-
-%macro mx_getgroups(
-  mdebug=0,
-  user=0,
-  uid=0,
-  repo=foundation,
-  access_token_var=ACCESS_TOKEN,
-  grant_type=sas_services,
-  outds=work.mx_getgroups
-)/*/STORE SOURCE*/;
-%local platform name shortloc;
-%let platform=%mf_getplatform();
-
-%if &platform=SASJS %then %do;
-  %ms_getgroups(
-    user=&user,
-    uid=&uid,
-    outds=&outds,
-    mdebug=&mdebug
-  )
-  data &outds;
-    length groupuri groupname $32 groupdesc $128 ;
-    set &outds;
-    keep groupuri groupname groupdesc;
-    groupuri=cats(groupid);
-    groupname=name;
-    groupdesc=description;
-  run;
-  proc sort; by groupname; run;
-%end;
-%else %if &platform=SAS9 or &platform=SASMETA %then %do;
-  %if &user=0 %then %let user=;
-  %mm_getGroups(
-    user=&user
-    ,outds=&outds
-    ,repo=&repo
-    ,mDebug=&mdebug
-  )
-  proc sort data=&outds; by groupname; run;
-%end;
-%else %if &platform=SASVIYA %then %do;
-  %if &user=0 %then %do;
-    %mv_getgroups(access_token_var=&access_token_var
-      ,grant_type=&grant_type
-      ,outds=&outds
-    )
-  %end;
-  %else %do;
-    %mv_getusergroups(&user
-      ,outds=&outds
-      ,access_token_var=&access_token_var
-      ,grant_type=&grant_type
-    )
-  %end;
-  proc sort
-      data=&outds(rename=(id=groupuri name=groupname description=groupdesc))
-      out=&outds (keep=groupuri groupname groupdesc);
-    by groupname;
-  run;
-%end;
-
-%mend mx_getgroups;