From f7fac501086a8fee98e2b8e55eafe8bca2891069 Mon Sep 17 00:00:00 2001
From: munja <allnbwe@gmail.com>
Date: Sat, 22 Jan 2022 21:16:15 +0100
Subject: [PATCH 1/5] fix: removing deprecated functionality ahead of planned
 breaking change

---
 README.md                       |  3 ++-
 all.sas                         | 18 ++++++++----------
 base/mf_abort.sas               |  2 +-
 base/mp_abort.sas               |  5 +++--
 base/mp_jsonout.sas             |  1 -
 meta/mm_createwebservice.sas    |  3 +--
 meta/mm_updatestpsourcecode.sas | 11 -----------
 meta/mm_webout.sas              |  2 +-
 server/ms_webout.sas            |  2 +-
 viya/mv_createwebservice.sas    |  3 +--
 viya/mv_getaccesstoken.sas      | 32 --------------------------------
 viya/mv_getapptoken.sas         | 23 -----------------------
 viya/mv_getrefreshtoken.sas     | 33 ---------------------------------
 viya/mv_webout.sas              |  2 +-
 14 files changed, 19 insertions(+), 121 deletions(-)
 delete mode 100644 viya/mv_getaccesstoken.sas
 delete mode 100644 viya/mv_getapptoken.sas
 delete mode 100644 viya/mv_getrefreshtoken.sas

diff --git a/README.md b/README.md
index ca08f63..5fb7235 100644
--- a/README.md
+++ b/README.md
@@ -125,6 +125,7 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 - macro names must be lowercase
 - one macro per file
 - prefixes:
+  - _mcf_ for macro compiled functions (proc fcmp)
   - _mf_ for macro functions (can be used in open code).
   - _ml_ for macros that are used to compile LUA modules
   - _mm_ for metadata macros (interface with the metadata server).
@@ -183,6 +184,7 @@ When contributing to this library, it is therefore important to ensure that all
 - All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect.
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
 - The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
+- Use [sasjs lint](https://github.com/sasjs/lint)!
 
 ## General Notes
 
@@ -193,7 +195,6 @@ When contributing to this library, it is therefore important to ensure that all
 We are currently on major release v3.  The following changes are planned when the next major (breaking) release becomes necessary:
 
 * Remove `dbg` parameter from mp_jsonout.sas (implement mdebug instead)
-* Remove `END_DTTM` and `START_DTTM` from mx_webout JSON
 
 ## Star Gazing
 
diff --git a/all.sas b/all.sas
index ca6c030..532b9c2 100644
--- a/all.sas
+++ b/all.sas
@@ -2198,7 +2198,8 @@ Usage:
       if symexist('_debug') then debug=quote(trim(symget('_debug')));
       else debug='""';
       put '>>weboutBEGIN<<';
-      put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
+      put '{"SYSDATE" : "' "&SYSDATE" '"';
+      put ',"SYSTIME" : "' "&SYSTIME" '"';
       put ',"sasjsAbort" : [{';
       put ' "MSG":' msg ;
       put ' ,"MAC": "' "&mac" '"}]';
@@ -2227,7 +2228,7 @@ Usage:
       put ',"SYSVLONG" : ' sysvlong;
       syswarningtext=quote(trim(symget('syswarningtext')));
       put ",""SYSWARNINGTEXT"" : " syswarningtext;
-      put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+      put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
       put "}" @;
       put '>>weboutEND<<';
     run;
@@ -7712,7 +7713,6 @@ filename &tempref clear;
 
 %macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y
   ,engine=DATASTEP
-  ,dbg=0 /* DEPRECATED */
   ,missing=NULL
   ,showmeta=NO
 )/*/STORE SOURCE*/;
@@ -12844,7 +12844,6 @@ data _null_;
   put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
-  put '  ,dbg=0 /* DEPRECATED */ ';
   put '  ,missing=NULL ';
   put '  ,showmeta=NO ';
   put ')/*/STORE SOURCE*/; ';
@@ -13175,7 +13174,7 @@ data _null_;
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
   put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
-  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''" ''; ';
+  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';
   put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
   put '    memsize=quote(cats(memsize)); ';
@@ -16681,7 +16680,7 @@ run;
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
-    put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
     memsize=quote(cats(memsize));
@@ -16988,7 +16987,7 @@ run;
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
-    put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length autoexec $512;
     autoexec=quote(urlencode(trim(getoption('autoexec'))));
     put ',"AUTOEXEC" : ' autoexec;
@@ -17980,7 +17979,6 @@ data _null_;
   put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
-  put '  ,dbg=0 /* DEPRECATED */ ';
   put '  ,missing=NULL ';
   put '  ,showmeta=NO ';
   put ')/*/STORE SOURCE*/; ';
@@ -18368,7 +18366,7 @@ data _null_;
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
   put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
-  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''" ''; ';
+  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';
   put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
   put '    memsize=quote(cats(memsize)); ';
@@ -22227,7 +22225,7 @@ filename &fref1 clear;
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
-    put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
     memsize=quote(cats(memsize));
diff --git a/base/mf_abort.sas b/base/mf_abort.sas
index 91bffcd..caa9281 100644
--- a/base/mf_abort.sas
+++ b/base/mf_abort.sas
@@ -11,7 +11,7 @@
   @cond
 **/
 
-%macro mf_abort(mac=mf_abort.sas, type=deprecated, msg=, iftrue=%str(1=1)
+%macro mf_abort(mac=mf_abort.sas, msg=, iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
   %if not(%eval(%unquote(&iftrue))) %then %return;
diff --git a/base/mp_abort.sas b/base/mp_abort.sas
index 48ed281..40a9f09 100644
--- a/base/mp_abort.sas
+++ b/base/mp_abort.sas
@@ -173,7 +173,8 @@
       if symexist('_debug') then debug=quote(trim(symget('_debug')));
       else debug='""';
       put '>>weboutBEGIN<<';
-      put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
+      put '{"SYSDATE" : "' "&SYSDATE" '"';
+      put ',"SYSTIME" : "' "&SYSTIME" '"';
       put ',"sasjsAbort" : [{';
       put ' "MSG":' msg ;
       put ' ,"MAC": "' "&mac" '"}]';
@@ -202,7 +203,7 @@
       put ',"SYSVLONG" : ' sysvlong;
       syswarningtext=quote(trim(symget('syswarningtext')));
       put ",""SYSWARNINGTEXT"" : " syswarningtext;
-      put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+      put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
       put "}" @;
       put '>>weboutEND<<';
     run;
diff --git a/base/mp_jsonout.sas b/base/mp_jsonout.sas
index 20fefd4..5a3443d 100644
--- a/base/mp_jsonout.sas
+++ b/base/mp_jsonout.sas
@@ -62,7 +62,6 @@
 
 %macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y
   ,engine=DATASTEP
-  ,dbg=0 /* DEPRECATED */
   ,missing=NULL
   ,showmeta=NO
 )/*/STORE SOURCE*/;
diff --git a/meta/mm_createwebservice.sas b/meta/mm_createwebservice.sas
index f9c9cc8..e90f2fd 100644
--- a/meta/mm_createwebservice.sas
+++ b/meta/mm_createwebservice.sas
@@ -95,7 +95,6 @@ data _null_;
   put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
-  put '  ,dbg=0 /* DEPRECATED */ ';
   put '  ,missing=NULL ';
   put '  ,showmeta=NO ';
   put ')/*/STORE SOURCE*/; ';
@@ -426,7 +425,7 @@ data _null_;
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
   put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
-  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''" ''; ';
+  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';
   put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
   put '    memsize=quote(cats(memsize)); ';
diff --git a/meta/mm_updatestpsourcecode.sas b/meta/mm_updatestpsourcecode.sas
index 785fee7..114bcb8 100644
--- a/meta/mm_updatestpsourcecode.sas
+++ b/meta/mm_updatestpsourcecode.sas
@@ -13,9 +13,6 @@
   @param [in] stpcode= the source file (or fileref) containing the SAS code to load
     into the stp.  For multiple files, they should simply be concatenated first.
   @param [in] minify= set to YES in order to strip comments, blank lines, and CRLFs.
-
-  @param frefin= deprecated - a unique fileref is now always used
-  @param frefout= deprecated - a unique fileref is now always used
   @param mDebug= set to 1 to show debug messages in the log
 
   @version 9.3
@@ -30,16 +27,8 @@
   ,stpcode=
   ,minify=NO
   ,mdebug=0
-  /* deprecated */
-  ,frefin=inmeta
-  ,frefout=outmeta
 );
 
-%if &frefin ne inmeta or &frefout ne outmeta %then %do;
-  %put %str(WARN)ING: the frefin and frefout parameters will be deprecated in
-    an upcoming release.;
-%end;
-
 /* first, check if STP exists */
 %local tsuri;
 %let tsuri=stopifempty ;
diff --git a/meta/mm_webout.sas b/meta/mm_webout.sas
index 64370e6..e50cbde 100644
--- a/meta/mm_webout.sas
+++ b/meta/mm_webout.sas
@@ -165,7 +165,7 @@
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
-    put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
     memsize=quote(cats(memsize));
diff --git a/server/ms_webout.sas b/server/ms_webout.sas
index a14c257..9cc8026 100644
--- a/server/ms_webout.sas
+++ b/server/ms_webout.sas
@@ -161,7 +161,7 @@
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
-    put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length autoexec $512;
     autoexec=quote(urlencode(trim(getoption('autoexec'))));
     put ',"AUTOEXEC" : ' autoexec;
diff --git a/viya/mv_createwebservice.sas b/viya/mv_createwebservice.sas
index 7d03359..045690e 100644
--- a/viya/mv_createwebservice.sas
+++ b/viya/mv_createwebservice.sas
@@ -239,7 +239,6 @@ data _null_;
   put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
-  put '  ,dbg=0 /* DEPRECATED */ ';
   put '  ,missing=NULL ';
   put '  ,showmeta=NO ';
   put ')/*/STORE SOURCE*/; ';
@@ -627,7 +626,7 @@ data _null_;
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
   put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
-  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),datetime20.3)" ''" ''; ';
+  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';
   put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
   put '    memsize=quote(cats(memsize)); ';
diff --git a/viya/mv_getaccesstoken.sas b/viya/mv_getaccesstoken.sas
deleted file mode 100644
index 4de3924..0000000
--- a/viya/mv_getaccesstoken.sas
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
-  @file mv_getaccesstoken.sas
-  @brief deprecated - replaced by mv_tokenrefresh.sas
-
-  @version VIYA V.03.04
-  @author Allan Bowe, source: https://github.com/sasjs/core
-
-  <h4> SAS Macros </h4>
-  @li mv_tokenrefresh.sas
-
-**/
-
-%macro mv_getaccesstoken(client_id=someclient
-    ,client_secret=somesecret
-    ,grant_type=authorization_code
-    ,code=
-    ,user=
-    ,pass=
-    ,access_token_var=ACCESS_TOKEN
-    ,refresh_token_var=REFRESH_TOKEN
-  );
-
-%mv_tokenrefresh(client_id=&client_id
-  ,client_secret=&client_secret
-  ,grant_type=&grant_type
-  ,user=&user
-  ,pass=&pass
-  ,access_token_var=&access_token_var
-  ,refresh_token_var=&refresh_token_var
-)
-
-%mend mv_getaccesstoken;
\ No newline at end of file
diff --git a/viya/mv_getapptoken.sas b/viya/mv_getapptoken.sas
deleted file mode 100644
index f7c9b75..0000000
--- a/viya/mv_getapptoken.sas
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
-  @file
-  @brief deprecated - replaced by mv_registerclient.sas
-
-  @version VIYA V.03.04
-  @author Allan Bowe, source: https://github.com/sasjs/core
-
-  <h4> SAS Macros </h4>
-  @li mv_registerclient.sas
-
-**/
-
-%macro mv_getapptoken(client_id=someclient
-    ,client_secret=somesecret
-    ,grant_type=authorization_code
-  );
-
-%mv_registerclient(client_id=&client_id
-  ,client_secret=&client_secret
-  ,grant_type=&grant_type
-)
-
-%mend mv_getapptoken;
\ No newline at end of file
diff --git a/viya/mv_getrefreshtoken.sas b/viya/mv_getrefreshtoken.sas
deleted file mode 100644
index 5ad73d5..0000000
--- a/viya/mv_getrefreshtoken.sas
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
-  @file mv_getrefreshtoken.sas
-  @brief deprecated - replaced by mv_tokenauth.sas
-
-  @version VIYA V.03.04
-  @author Allan Bowe, source: https://github.com/sasjs/core
-
-  <h4> SAS Macros </h4>
-  @li mv_tokenauth.sas
-
-**/
-
-%macro mv_getrefreshtoken(client_id=someclient
-    ,client_secret=somesecret
-    ,grant_type=authorization_code
-    ,code=
-    ,user=
-    ,pass=
-    ,access_token_var=ACCESS_TOKEN
-    ,refresh_token_var=REFRESH_TOKEN
-  );
-
-%mv_tokenauth(client_id=&client_id
-  ,client_secret=&client_secret
-  ,grant_type=&grant_type
-  ,code=&code
-  ,user=&user
-  ,pass=&pass
-  ,access_token_var=&access_token_var
-  ,refresh_token_var=&refresh_token_var
-)
-
-%mend mv_getrefreshtoken;
\ No newline at end of file
diff --git a/viya/mv_webout.sas b/viya/mv_webout.sas
index 4297f7a..7480b4d 100644
--- a/viya/mv_webout.sas
+++ b/viya/mv_webout.sas
@@ -225,7 +225,7 @@
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
-    put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
     memsize=quote(cats(memsize));

From 142b46570d516b8095cbd6fd7ed65935e54d7c3a Mon Sep 17 00:00:00 2001
From: munja <allnbwe@gmail.com>
Date: Sun, 23 Jan 2022 23:26:10 +0100
Subject: [PATCH 2/5] feat: adding mcf_length to mp_getmaxvarlengths BREAKING
 CHANGE: mp_getmaxvarlengths now returns 0 for non-special missings, and will
 use numeric length (as opposed to cast-to-character length) by default

---
 README.md                                     |  7 +-
 base/mp_getmaxvarlengths.sas                  | 69 +++++++++++-----
 base/mp_init.sas                              | 62 +++++++-------
 fcmp/mcf_length.sas                           |  9 ++-
 fcmp/mcf_string2file.sas                      |  5 ++
 tests/crossplatform/mcf_length.test.sas       | 17 ++++
 .../mp_getmaxvarlengths.test.sas              | 80 +++++++++++++++++++
 7 files changed, 196 insertions(+), 53 deletions(-)
 create mode 100644 tests/crossplatform/mp_getmaxvarlengths.test.sas

diff --git a/README.md b/README.md
index 5fb7235..1c142e9 100644
--- a/README.md
+++ b/README.md
@@ -167,7 +167,7 @@ SAS code can contain one of two types of dependency - SAS Macros, and SAS Includ
   @li someprogram.sas FREFTWO
 ```
 
-The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Includes) when creating SAS Jobs and Services.
+The CLI can then extract all the dependencies and insert as precode (SAS Macros) or in a temp engine fileref (SAS Includes) when creating SAS Jobs and Services (and Tests).
 
 When contributing to this library, it is therefore important to ensure that all dependencies are listed in the header in this format.
 
@@ -183,6 +183,7 @@ When contributing to this library, it is therefore important to ensure that all
 - Mandatory parameters should be positional, all optional parameters should be keyword (var=) style.
 - All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect.
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
+- Where global macro variables are absolutely necessary, they should make use of `&sasjs_prefix` - see mp_init.sas
 - The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
 - Use [sasjs lint](https://github.com/sasjs/lint)!
 
@@ -192,9 +193,9 @@ When contributing to this library, it is therefore important to ensure that all
 
 ## Breaking Changes
 
-We are currently on major release v3.  The following changes are planned when the next major (breaking) release becomes necessary:
+We are currently on major release v4.  The following changes are planned when the next major (breaking) release becomes necessary:
 
-* Remove `dbg` parameter from mp_jsonout.sas (implement mdebug instead)
+* (None as yet)
 
 ## Star Gazing
 
diff --git a/base/mp_getmaxvarlengths.sas b/base/mp_getmaxvarlengths.sas
index 644ac10..133f249 100755
--- a/base/mp_getmaxvarlengths.sas
+++ b/base/mp_getmaxvarlengths.sas
@@ -1,28 +1,46 @@
 /**
-  @file mp_getmaxvarlengths.sas
+  @file
   @brief Scans a dataset to find the max length of the variable values
   @details
   This macro will scan a base dataset and produce an output dataset with two
   columns:
 
   - NAME    Name of the base dataset column
-  - MAXLEN Maximum length of the data contained therein.
+  - MAXLEN  Maximum length of the data contained therein.
 
-  Character fields may be allocated very large widths (eg 32000) of which the
-  maximum  value is likely to be much narrower.  This macro was designed to
-  enable a HTML table to be appropriately sized however this could be used as
-  part of a data audit to ensure we aren't over-sizing our tables in relation to
-  the data therein.
+  Character fields are often allocated very large widths (eg 32000) of which the
+  maximum  value is likely to be much narrower.  Identifying such cases can be
+  helpful in the following scenarios:
+
+  @li Enabling a HTML table to be appropriately sized (`num2char=YES`)
+  @li Reducing the size of a dataset to save on storage (mp_ds2squeeze.sas)
+  @li Identifying columns containing nothing but missing values (`MAXLEN=0` in
+    the output table)
+
+  If the entire column is made up of (non-special) missing values then a value
+  of 0 is returned.
 
-  Numeric fields are converted using the relevant format to determine the width.
   Usage:
 
       %mp_getmaxvarlengths(sashelp.class,outds=work.myds)
 
-  @param libds Two part dataset (or view) reference.
-  @param outds= The output dataset to create
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] num2char= (NO) When set to NO, numeric fields are sized according
+    to the number of bytes used (or set to zero in the case of non-special
+    missings). When YES, the numeric field is converted to character (using the
+    format, if available), and that is sized instead, using `lengthn()`.
+  @param [out] outds= The output dataset to create, eg:
+  |NAME:$8.|MAXLEN:best.|
+  |---|---|
+  |`Name `|`7 `|
+  |`Sex `|`1 `|
+  |`Age `|`3 `|
+  |`Height `|`8 `|
+  |`Weight `|`3 `|
 
   <h4> SAS Macros </h4>
+  @li mcf_length.sas
+  @li mf_getuniquename.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
   @li mf_getvarformat.sas
@@ -30,20 +48,32 @@
   @version 9.2
   @author Allan Bowe
 
+  <h4> Related Macros </h4>
+  @li mp_ds2squeeze.sas
+  @li mp_getmaxvarlengths.test.sas
+
 **/
 
 %macro mp_getmaxvarlengths(
-    libds      /* libref.dataset to analyse */
-    ,outds=work.mp_getmaxvarlengths /* name of output dataset to create */
+  libds
+  ,num2char=NO
+  ,outds=work.mp_getmaxvarlengths
 )/*/STORE SOURCE*/;
 
-%local vars x var fmt;
+%local vars prefix x var fmt;
 %let vars=%mf_getvarlist(libds=&libds);
+%let prefix=%substr(%mf_getuniquename(),1,25);
+%let num2char=%upcase(&num2char);
+
+%if &num2char=NO %then %do;
+  /* compile length function for numeric fields */
+  %mcf_length(wrap=YES, insert_cmplib=YES)
+%end;
 
 proc sql;
 create table &outds (rename=(
     %do x=1 %to %sysfunc(countw(&vars,%str( )));
-      ________&x=%scan(&vars,&x)
+      &prefix.&x=%scan(&vars,&x)
     %end;
     ))
   as select
@@ -51,18 +81,21 @@ create table &outds (rename=(
       %let var=%scan(&vars,&x);
       %if &x>1 %then ,;
       %if %mf_getvartype(&libds,&var)=C %then %do;
-        max(length(&var)) as ________&x
+        max(lengthn(&var)) as &prefix.&x
       %end;
-      %else %do;
+      %else %if &num2char=YES %then %do;
         %let fmt=%mf_getvarformat(&libds,&var);
         %put fmt=&fmt;
         %if %str(&fmt)=%str() %then %do;
-          max(length(cats(&var))) as ________&x
+          max(lengthn(cats(&var))) as &prefix.&x
         %end;
         %else %do;
-          max(length(put(&var,&fmt))) as ________&x
+          max(lengthn(put(&var,&fmt))) as &prefix.&x
         %end;
       %end;
+      %else %do;
+        max(mcf_length(&var)) as &prefix.&x
+      %end;
     %end;
   from &libds;
 
diff --git a/base/mp_init.sas b/base/mp_init.sas
index 47fd2eb..28d67d6 100644
--- a/base/mp_init.sas
+++ b/base/mp_init.sas
@@ -33,37 +33,39 @@
 %macro mp_init(prefix=SASJS
 )/*/STORE SOURCE*/;
 
-  %global
-    &prefix._INIT_NUM   /* initialisation time as numeric                   */
-    &prefix._INIT_DTTM  /* initialisation time in E8601DT26.6 format        */
-    &prefix.WORK        /* avoid typing %sysfunc(pathname(work)) every time */
-  ;
-  %if %eval(&&&prefix._INIT_NUM>0) %then %return;  /* only run once */
+%global
+  SASJS_PREFIX       /* the ONLY hard-coded global macro variable in SASjs    */
+  &prefix._INIT_NUM  /* initialisation time as numeric                        */
+  &prefix._INIT_DTTM /* initialisation time in E8601DT26.6 format             */
+  &prefix.WORK       /* avoid typing %sysfunc(pathname(work)) every time      */
+;
+%if %length(&sasjs_prefix>0) %then %return;  /* only run once */
+%let sasjs_prefix=&prefix;
 
-  data _null_;
-    dttm=datetime();
-    call symputx("&prefix._init_num",dttm,'g');
-    call symputx("&prefix._init_dttm",put(dttm,E8601DT26.6),'g');
-    call symputx("&prefix.work",pathname('WORK'),'g');
-  run;
+data _null_;
+  dttm=datetime();
+  call symputx("&sasjs_prefix._init_num",dttm,'g');
+  call symputx("&sasjs_prefix._init_dttm",put(dttm,E8601DT26.6),'g');
+  call symputx("&sasjs_prefix.work",pathname('WORK'),'g');
+run;
 
-  options
-    noautocorrect           /* disallow misspelled procedure names            */
-    compress=CHAR           /* default is none so ensure we have something!   */
-    datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
-    dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
-    %str(err)orcheck=STRICT /* catch errs in libname/filename statements      */
-    fmterr                  /* ensure err when a format cannot be found       */
-    mergenoby=%str(ERR)OR   /* throw err when a merge has no BY variables     */
-    missing=.               /* changing this can cause hard to detect errs    */
-    noquotelenmax           /* avoid warnings for long strings                */
-    noreplace               /* avoid overwriting permanent datasets           */
-    ps=max                  /* reduce log size slightly                       */
-    ls=max                  /* reduce log even more and avoid word truncation */
-    validmemname=COMPATIBLE /* avoid special characters etc in table names    */
-    validvarname=V7         /* avoid special characters etc in variable names */
-    varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
-    varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
-  ;
+options
+  noautocorrect           /* disallow misspelled procedure names            */
+  compress=CHAR           /* default is none so ensure we have something!   */
+  datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
+  dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
+  %str(err)orcheck=STRICT /* catch errs in libname/filename statements      */
+  fmterr                  /* ensure err when a format cannot be found       */
+  mergenoby=%str(ERR)OR   /* throw err when a merge has no BY variables     */
+  missing=.               /* changing this can cause hard to detect errs    */
+  noquotelenmax           /* avoid warnings for long strings                */
+  noreplace               /* avoid overwriting permanent datasets           */
+  ps=max                  /* reduce log size slightly                       */
+  ls=max                  /* reduce log even more and avoid word truncation */
+  validmemname=COMPATIBLE /* avoid special characters etc in table names    */
+  validvarname=V7         /* avoid special characters etc in variable names */
+  varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
+  varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
+;
 
 %mend mp_init;
\ No newline at end of file
diff --git a/fcmp/mcf_length.sas b/fcmp/mcf_length.sas
index b6688bd..43b2bdb 100644
--- a/fcmp/mcf_length.sas
+++ b/fcmp/mcf_length.sas
@@ -39,6 +39,9 @@
   @param [out] pkg= (utils) The output package in which to create the function.
     Uses a 3 part format:  libref.catalog.package
 
+  <h4> SAS Macros </h4>
+  @li mf_existfunction.sas
+
   <h4> Related Macros </h4>
   @li mcf_length.test.sas
 
@@ -51,13 +54,15 @@
   ,pkg=UTILS
 )/*/STORE SOURCE*/;
 
+%if %mf_existfunction(mcf_length)=1 %then %return;
+
 %if &wrap=YES  %then %do;
   proc fcmp outlib=&lib..&cat..&pkg;
 %end;
 
 function mcf_length(var);
-  if missing(var) then len=0;
-  else if trunc(var,3)=var then len=3;
+  if var=. then len=0;
+  else if missing(var) or trunc(var,3)=var then len=3;
   else if trunc(var,4)=var then len=4;
   else if trunc(var,5)=var then len=5;
   else if trunc(var,6)=var then len=6;
diff --git a/fcmp/mcf_string2file.sas b/fcmp/mcf_string2file.sas
index dbfcbb9..f41b30c 100644
--- a/fcmp/mcf_string2file.sas
+++ b/fcmp/mcf_string2file.sas
@@ -39,6 +39,9 @@
   @param [out] pkg= (utils) The output package in which to create the function.
     Uses a 3 part format:  libref.catalog.package
 
+  <h4> SAS Macros </h4>
+  @li mf_existfunction.sas
+
 **/
 
 %macro mcf_string2file(wrap=NO
@@ -48,6 +51,8 @@
   ,pkg=UTILS
 )/*/STORE SOURCE*/;
 
+%if %mf_existfunction(mcf_string2file)=1 %then %return;
+
 %if &wrap=YES  %then %do;
   proc fcmp outlib=&lib..&cat..&pkg;
 %end;
diff --git a/tests/crossplatform/mcf_length.test.sas b/tests/crossplatform/mcf_length.test.sas
index be1f3e6..90a7587 100644
--- a/tests/crossplatform/mcf_length.test.sas
+++ b/tests/crossplatform/mcf_length.test.sas
@@ -12,6 +12,7 @@
 
 data test;
   call symputx('null',mcf_length(.));
+  call symputx('special',mcf_length(._))
   call symputx('three',mcf_length(1));
   call symputx('four',mcf_length(10000000));
   call symputx('five',mcf_length(12345678));
@@ -24,6 +25,10 @@ run;
   iftrue=(%str(&null)=%str(0)),
   desc=Check if NULL returns 0
 )
+%mp_assert(
+  iftrue=(%str(&special)=%str(3)),
+  desc=Check if special missing ._ returns 3
+)
 %mp_assert(
   iftrue=(%str(&three)=%str(3)),
   desc=Check for length 3
@@ -47,4 +52,16 @@ run;
 %mp_assert(
   iftrue=(%str(&eight)=%str(8)),
   desc=Check for length 8
+)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check syscc=0 before re-initialisation
+)
+
+/* test 2 - compile again test for warnings */
+%mcf_length(wrap=YES, insert_cmplib=YES)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check syscc=0 after re-initialisation
 )
\ No newline at end of file
diff --git a/tests/crossplatform/mp_getmaxvarlengths.test.sas b/tests/crossplatform/mp_getmaxvarlengths.test.sas
new file mode 100644
index 0000000..2ec974f
--- /dev/null
+++ b/tests/crossplatform/mp_getmaxvarlengths.test.sas
@@ -0,0 +1,80 @@
+/**
+  @file
+  @brief Testing mp_getmaxvarlengths macro
+
+  <h4> SAS Macros </h4>
+  @li mp_getmaxvarlengths.sas
+  @li mp_assert.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
+
+**/
+
+
+/* regular usage */
+%mp_assertscope(SNAPSHOT)
+%mp_getmaxvarlengths(sashelp.class,outds=work.myds)
+%mp_assertscope(COMPARE,desc=checking scope leakage on mp_getmaxvarlengths)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errs
+)
+%mp_assertdsobs(work.myds,
+  desc=Has 5 records,
+  test=EQUALS 5
+)
+data work.errs;
+  set work.myds;
+  if name='Name' and maxlen ne 7 then output;
+  if name='Sex' and maxlen ne 1 then output;
+  if name='Age' and maxlen ne 3 then output;
+  if name='Height' and maxlen ne 8 then output;
+  if name='Weight' and maxlen ne 3 then output;
+run;
+data _null_;
+  set work.errs;
+  putlog (_all_)(=);
+run;
+
+%mp_assertdsobs(work.errs,
+  desc=Err table has 0 records,
+  test=EQUALS 0
+)
+
+/* test2 */
+data work.test2;
+  length a 3 b 5;
+  a=1/3;
+  b=1/3;
+  c=1/3;
+  d=._;
+  e=.;
+  output;
+  output;
+run;
+%mp_getmaxvarlengths(work.test2,outds=work.myds2)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errs in second test (with nulls)
+)
+%mp_assertdsobs(work.myds2,
+  desc=Has 5 records,
+  test=EQUALS 5
+)
+data work.errs2;
+  set work.myds2;
+  if name='a' and maxlen ne 3 then output;
+  if name='b' and maxlen ne 5 then output;
+  if name='c' and maxlen ne 8 then output;
+  if name='d' and maxlen ne 3 then output;
+  if name='e' and maxlen ne 0 then output;
+run;
+data _null_;
+  set work.errs2;
+  putlog (_all_)(=);
+run;
+
+%mp_assertdsobs(work.errs2,
+  desc=Err table has 0 records,
+  test=EQUALS 0
+)
\ No newline at end of file

From c3b89c7f7d9c568bf8afc71ac841070cb773fe1f Mon Sep 17 00:00:00 2001
From: munja <allnbwe@gmail.com>
Date: Mon, 24 Jan 2022 11:17:21 +0100
Subject: [PATCH 3/5] feat: mp_ds2squeeze macro

---
 base/mp_assertscope.sas                    |   4 +-
 base/mp_ds2squeeze.sas                     | 118 +++++++++++++++++++++
 base/mp_init.sas                           |   4 +-
 tests/crossplatform/mp_ds2squeeze.test.sas |  43 ++++++++
 4 files changed, 166 insertions(+), 3 deletions(-)
 create mode 100644 base/mp_ds2squeeze.sas
 create mode 100644 tests/crossplatform/mp_ds2squeeze.test.sas

diff --git a/base/mp_assertscope.sas b/base/mp_assertscope.sas
index 38d6a54..b1a9bd3 100644
--- a/base/mp_assertscope.sas
+++ b/base/mp_assertscope.sas
@@ -29,7 +29,7 @@
     @li COMPARE - compare the current macro variables against previous values
   @param [in] scope= (GLOBAL) The scope of the variables to be checked.  This
     corresponds to the values in the SCOPE column in `sashelp.vmacro`.
-  @param [in] desc= (Testing variable scope) The user provided test description
+  @param [in] desc= (Testing scope leakage) The user provided test description
   @param [in,out] scopeds= (work.mp_assertscope) The dataset to contain the
     scope snapshot
   @param [out] outds= (work.test_results) The output dataset to contain the
@@ -51,7 +51,7 @@
 **/
 
 %macro mp_assertscope(action,
-  desc=0,
+  desc=Testing Scope Leakage,
   scope=GLOBAL,
   scopeds=work.mp_assertscope,
   outds=work.test_results
diff --git a/base/mp_ds2squeeze.sas b/base/mp_ds2squeeze.sas
new file mode 100644
index 0000000..a729fdb
--- /dev/null
+++ b/base/mp_ds2squeeze.sas
@@ -0,0 +1,118 @@
+/**
+  @file
+  @brief Create a smaller version of a dataset, without data loss
+  @details This macro will scan the input dataset and create a new one, that
+  has the minimum variable lengths needed to store the data without data loss.
+
+  Inspiration was taken from [How to Reduce the Disk Space Required by a
+  SAS® Data Set](https://www.lexjansen.com/nesug/nesug06/io/io18.pdf) by
+  Selvaratnam Sridharma.  The end of the referenced paper presents a macro named
+  "squeeze", hence the nomenclature.
+
+  Usage:
+
+      data big;
+        length my big $32000;
+        do i=1 to 1e4;
+          my=repeat('oh my',100);
+          big='dawg';
+          special=._;
+          output;
+        end;
+      run;
+
+      %mp_ds2squeeze(work.big,outds=work.smaller)
+
+  The following will also be printed to the log (exact values may differ
+  depending on your OS and COMPRESS settings):
+
+  > MP_DS2SQUEEZE: work.big was  625MB
+  > MP_DS2SQUEEZE: work.smaller is    5MB
+
+  @param [in] libds The library.dataset to be squeezed
+  @param [out] outds= (work.mp_ds2squeeze) The squeezed dataset to create
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> SAS Macros </h4>
+  @li mf_getfilesize.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_getmaxvarlengths.sas
+
+  <h4> Related Programs </h4>
+  @li mp_ds2squeeze.test.sas
+
+  @version 9.3
+  @author Allan Bowe
+**/
+
+%macro mp_ds2squeeze(
+  libds,
+  outds=work.work.mp_ds2squeeze,
+  mdebug=0
+)/*/STORE SOURCE*/;
+%local dbg source;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %do;
+  %let dbg=*;
+  %let source=/source2;
+%end;
+
+%local optval ds fref;
+%let ds=%mf_getuniquename();
+%let fref=%mf_getuniquefileref();
+
+%mp_getmaxvarlengths(&libds,outds=&ds)
+
+data _null_;
+  set &ds end=last;
+  file &fref;
+  /* grab the types */
+  retain dsid;
+  if _n_=1 then dsid=open("&libds",'is');
+  if dsid le 0 then do;
+    msg=sysmsg();
+    put msg=;
+    stop;
+  end;
+  type=vartype(dsid,varnum(dsid, name));
+  if last then rc=close(dsid);
+  /* write out the length statement */
+  if _n_=1 then put 'length ';
+  length len $6;
+  if type='C' then do;
+    if maxlen=0 then len='$1';
+    else len=cats('$',maxlen);
+  end;
+  else do;
+    if maxlen=0 then len='3';
+    else len=maxlen;
+  end;
+  put '  ' name ' ' len;
+  if last then put ';';
+run;
+
+/* configure varlenchk - as we are explicitly shortening the variables */
+%let optval=%sysfunc(getoption(varlenchk));
+options varlenchk=NOWARN;
+
+data &outds;
+  %inc &fref &source;
+  set &libds;
+run;
+
+options varlenchk=&optval;
+
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &ds;
+  filename &fref clear;
+%end;
+
+%put &sysmacroname: &libds was %mf_getfilesize(libds=&libds,format=yes);
+%put &sysmacroname: &outds is %mf_getfilesize(libds=&outds,format=yes);
+
+%mend mp_ds2squeeze;
\ No newline at end of file
diff --git a/base/mp_init.sas b/base/mp_init.sas
index 28d67d6..734f555 100644
--- a/base/mp_init.sas
+++ b/base/mp_init.sas
@@ -33,13 +33,15 @@
 %macro mp_init(prefix=SASJS
 )/*/STORE SOURCE*/;
 
+%if %symexist(SASJS_PREFIX) %then %return;  /* only run once */
+
 %global
   SASJS_PREFIX       /* the ONLY hard-coded global macro variable in SASjs    */
   &prefix._INIT_NUM  /* initialisation time as numeric                        */
   &prefix._INIT_DTTM /* initialisation time in E8601DT26.6 format             */
   &prefix.WORK       /* avoid typing %sysfunc(pathname(work)) every time      */
 ;
-%if %length(&sasjs_prefix>0) %then %return;  /* only run once */
+
 %let sasjs_prefix=&prefix;
 
 data _null_;
diff --git a/tests/crossplatform/mp_ds2squeeze.test.sas b/tests/crossplatform/mp_ds2squeeze.test.sas
new file mode 100644
index 0000000..db8c246
--- /dev/null
+++ b/tests/crossplatform/mp_ds2squeeze.test.sas
@@ -0,0 +1,43 @@
+/**
+  @file
+  @brief Testing mp_ds2squeeze.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mp_ds2squeeze.sas
+
+**/
+
+data big;
+  length my big $32000;
+  do i=1 to 1e4;
+    my=repeat('oh my',100);
+    big='dawg';
+    special=._;
+    missn=.;
+    missc='';
+    output;
+  end;
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mp_ds2squeeze(work.big,outds=work.smaller)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking syscc
+)
+%mp_assert(
+  iftrue=(%mf_getvarlen(work.smaller,missn)=3),
+  desc=Check missing numeric is 3
+)
+%mp_assert(
+  iftrue=(%mf_getvarlen(work.smaller,special)=3),
+  desc=Check missing special numeric is 3
+)
+%mp_assert(
+  iftrue=(%mf_getvarlen(work.smaller,missc)=1),
+  desc=Check missing char is 1
+)

From 0e03b06a4be7745dcd9abe1e9ee6d5c51af864a6 Mon Sep 17 00:00:00 2001
From: munja <allnbwe@gmail.com>
Date: Mon, 24 Jan 2022 12:53:36 +0100
Subject: [PATCH 4/5] fix: adjustments to ensure the tests work, also building
 all.sas

---
 all.sas                                 | 368 ++++++++++++++----------
 base/mp_ds2squeeze.sas                  |   2 +-
 tests/crossplatform/mcf_length.test.sas |   2 +-
 viya/mv_deletefoldermember.sas          |   1 +
 viya/mv_deletejes.sas                   |   1 +
 5 files changed, 223 insertions(+), 151 deletions(-)

diff --git a/all.sas b/all.sas
index 532b9c2..baaf4a3 100644
--- a/all.sas
+++ b/all.sas
@@ -29,7 +29,7 @@ options noquotelenmax;
   @cond
 **/
 
-%macro mf_abort(mac=mf_abort.sas, type=deprecated, msg=, iftrue=%str(1=1)
+%macro mf_abort(mac=mf_abort.sas, msg=, iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
   %if not(%eval(%unquote(&iftrue))) %then %return;
@@ -3036,7 +3036,7 @@ run;
     @li COMPARE - compare the current macro variables against previous values
   @param [in] scope= (GLOBAL) The scope of the variables to be checked.  This
     corresponds to the values in the SCOPE column in `sashelp.vmacro`.
-  @param [in] desc= (Testing variable scope) The user provided test description
+  @param [in] desc= (Testing scope leakage) The user provided test description
   @param [in,out] scopeds= (work.mp_assertscope) The dataset to contain the
     scope snapshot
   @param [out] outds= (work.test_results) The output dataset to contain the
@@ -3058,7 +3058,7 @@ run;
 **/
 
 %macro mp_assertscope(action,
-  desc=0,
+  desc=Testing Scope Leakage,
   scope=GLOBAL,
   scopeds=work.mp_assertscope,
   outds=work.test_results
@@ -5028,6 +5028,123 @@ run;
 %end;
 
 %mend mp_ds2md;/**
+  @file
+  @brief Create a smaller version of a dataset, without data loss
+  @details This macro will scan the input dataset and create a new one, that
+  has the minimum variable lengths needed to store the data without data loss.
+
+  Inspiration was taken from [How to Reduce the Disk Space Required by a
+  SAS® Data Set](https://www.lexjansen.com/nesug/nesug06/io/io18.pdf) by
+  Selvaratnam Sridharma.  The end of the referenced paper presents a macro named
+  "squeeze", hence the nomenclature.
+
+  Usage:
+
+      data big;
+        length my big $32000;
+        do i=1 to 1e4;
+          my=repeat('oh my',100);
+          big='dawg';
+          special=._;
+          output;
+        end;
+      run;
+
+      %mp_ds2squeeze(work.big,outds=work.smaller)
+
+  The following will also be printed to the log (exact values may differ
+  depending on your OS and COMPRESS settings):
+
+  > MP_DS2SQUEEZE: work.big was  625MB
+  > MP_DS2SQUEEZE: work.smaller is    5MB
+
+  @param [in] libds The library.dataset to be squeezed
+  @param [out] outds= (work.mp_ds2squeeze) The squeezed dataset to create
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> SAS Macros </h4>
+  @li mf_getfilesize.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_getmaxvarlengths.sas
+
+  <h4> Related Programs </h4>
+  @li mp_ds2squeeze.test.sas
+
+  @version 9.3
+  @author Allan Bowe
+**/
+
+%macro mp_ds2squeeze(
+  libds,
+  outds=work.work.mp_ds2squeeze,
+  mdebug=0
+)/*/STORE SOURCE*/;
+%local dbg source;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %do;
+  %let dbg=*;
+  %let source=/source2;
+%end;
+
+%local optval ds fref;
+%let ds=%mf_getuniquename();
+%let fref=%mf_getuniquefileref();
+
+%mp_getmaxvarlengths(&libds,outds=&ds)
+
+data _null_;
+  set &ds end=last;
+  file &fref;
+  /* grab the types */
+  retain dsid;
+  if _n_=1 then dsid=open("&libds",'is');
+  if dsid le 0 then do;
+    msg=sysmsg();
+    put msg=;
+    stop;
+  end;
+  type=vartype(dsid,varnum(dsid, name));
+  if last then rc=close(dsid);
+  /* write out the length statement */
+  if _n_=1 then put 'length ';
+  length len $6;
+  if type='C' then do;
+    if maxlen=0 then len='$1';
+    else len=cats('$',maxlen);
+  end;
+  else do;
+    if maxlen=0 then len='3';
+    else len=cats(maxlen);
+  end;
+  put '  ' name ' ' len;
+  if last then put ';';
+run;
+
+/* configure varlenchk - as we are explicitly shortening the variables */
+%let optval=%sysfunc(getoption(varlenchk));
+options varlenchk=NOWARN;
+
+data &outds;
+  %inc &fref &source;
+  set &libds;
+run;
+
+options varlenchk=&optval;
+
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &ds;
+  filename &fref clear;
+%end;
+
+%put &sysmacroname: &libds was %mf_getfilesize(libds=&libds,format=yes);
+%put &sysmacroname: &outds is %mf_getfilesize(libds=&outds,format=yes);
+
+%mend mp_ds2squeeze;/**
   @file
   @brief Checks an input filter table for validity
   @details Performs checks on the input table to ensure it arrives in the
@@ -6703,30 +6820,48 @@ create table &outsummary as
 %end;
 
 %mend mp_getformats;/**
-  @file mp_getmaxvarlengths.sas
+  @file
   @brief Scans a dataset to find the max length of the variable values
   @details
   This macro will scan a base dataset and produce an output dataset with two
   columns:
 
   - NAME    Name of the base dataset column
-  - MAXLEN Maximum length of the data contained therein.
+  - MAXLEN  Maximum length of the data contained therein.
 
-  Character fields may be allocated very large widths (eg 32000) of which the
-  maximum  value is likely to be much narrower.  This macro was designed to
-  enable a HTML table to be appropriately sized however this could be used as
-  part of a data audit to ensure we aren't over-sizing our tables in relation to
-  the data therein.
+  Character fields are often allocated very large widths (eg 32000) of which the
+  maximum  value is likely to be much narrower.  Identifying such cases can be
+  helpful in the following scenarios:
+
+  @li Enabling a HTML table to be appropriately sized (`num2char=YES`)
+  @li Reducing the size of a dataset to save on storage (mp_ds2squeeze.sas)
+  @li Identifying columns containing nothing but missing values (`MAXLEN=0` in
+    the output table)
+
+  If the entire column is made up of (non-special) missing values then a value
+  of 0 is returned.
 
-  Numeric fields are converted using the relevant format to determine the width.
   Usage:
 
       %mp_getmaxvarlengths(sashelp.class,outds=work.myds)
 
-  @param libds Two part dataset (or view) reference.
-  @param outds= The output dataset to create
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] num2char= (NO) When set to NO, numeric fields are sized according
+    to the number of bytes used (or set to zero in the case of non-special
+    missings). When YES, the numeric field is converted to character (using the
+    format, if available), and that is sized instead, using `lengthn()`.
+  @param [out] outds= The output dataset to create, eg:
+  |NAME:$8.|MAXLEN:best.|
+  |---|---|
+  |`Name `|`7 `|
+  |`Sex `|`1 `|
+  |`Age `|`3 `|
+  |`Height `|`8 `|
+  |`Weight `|`3 `|
 
   <h4> SAS Macros </h4>
+  @li mcf_length.sas
+  @li mf_getuniquename.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
   @li mf_getvarformat.sas
@@ -6734,20 +6869,32 @@ create table &outsummary as
   @version 9.2
   @author Allan Bowe
 
+  <h4> Related Macros </h4>
+  @li mp_ds2squeeze.sas
+  @li mp_getmaxvarlengths.test.sas
+
 **/
 
 %macro mp_getmaxvarlengths(
-    libds      /* libref.dataset to analyse */
-    ,outds=work.mp_getmaxvarlengths /* name of output dataset to create */
+  libds
+  ,num2char=NO
+  ,outds=work.mp_getmaxvarlengths
 )/*/STORE SOURCE*/;
 
-%local vars x var fmt;
+%local vars prefix x var fmt;
 %let vars=%mf_getvarlist(libds=&libds);
+%let prefix=%substr(%mf_getuniquename(),1,25);
+%let num2char=%upcase(&num2char);
+
+%if &num2char=NO %then %do;
+  /* compile length function for numeric fields */
+  %mcf_length(wrap=YES, insert_cmplib=YES)
+%end;
 
 proc sql;
 create table &outds (rename=(
     %do x=1 %to %sysfunc(countw(&vars,%str( )));
-      ________&x=%scan(&vars,&x)
+      &prefix.&x=%scan(&vars,&x)
     %end;
     ))
   as select
@@ -6755,18 +6902,21 @@ create table &outds (rename=(
       %let var=%scan(&vars,&x);
       %if &x>1 %then ,;
       %if %mf_getvartype(&libds,&var)=C %then %do;
-        max(length(&var)) as ________&x
+        max(lengthn(&var)) as &prefix.&x
       %end;
-      %else %do;
+      %else %if &num2char=YES %then %do;
         %let fmt=%mf_getvarformat(&libds,&var);
         %put fmt=&fmt;
         %if %str(&fmt)=%str() %then %do;
-          max(length(cats(&var))) as ________&x
+          max(lengthn(cats(&var))) as &prefix.&x
         %end;
         %else %do;
-          max(length(put(&var,&fmt))) as ________&x
+          max(lengthn(put(&var,&fmt))) as &prefix.&x
         %end;
       %end;
+      %else %do;
+        max(mcf_length(&var)) as &prefix.&x
+      %end;
     %end;
   from &libds;
 
@@ -7616,38 +7766,42 @@ filename &tempref clear;
 %macro mp_init(prefix=SASJS
 )/*/STORE SOURCE*/;
 
-  %global
-    &prefix._INIT_NUM   /* initialisation time as numeric                   */
-    &prefix._INIT_DTTM  /* initialisation time in E8601DT26.6 format        */
-    &prefix.WORK        /* avoid typing %sysfunc(pathname(work)) every time */
-  ;
-  %if %eval(&&&prefix._INIT_NUM>0) %then %return;  /* only run once */
+%if %symexist(SASJS_PREFIX) %then %return;  /* only run once */
 
-  data _null_;
-    dttm=datetime();
-    call symputx("&prefix._init_num",dttm,'g');
-    call symputx("&prefix._init_dttm",put(dttm,E8601DT26.6),'g');
-    call symputx("&prefix.work",pathname('WORK'),'g');
-  run;
+%global
+  SASJS_PREFIX       /* the ONLY hard-coded global macro variable in SASjs    */
+  &prefix._INIT_NUM  /* initialisation time as numeric                        */
+  &prefix._INIT_DTTM /* initialisation time in E8601DT26.6 format             */
+  &prefix.WORK       /* avoid typing %sysfunc(pathname(work)) every time      */
+;
 
-  options
-    noautocorrect           /* disallow misspelled procedure names            */
-    compress=CHAR           /* default is none so ensure we have something!   */
-    datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
-    dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
-    %str(err)orcheck=STRICT /* catch errs in libname/filename statements      */
-    fmterr                  /* ensure err when a format cannot be found       */
-    mergenoby=%str(ERR)OR   /* throw err when a merge has no BY variables     */
-    missing=.               /* changing this can cause hard to detect errs    */
-    noquotelenmax           /* avoid warnings for long strings                */
-    noreplace               /* avoid overwriting permanent datasets           */
-    ps=max                  /* reduce log size slightly                       */
-    ls=max                  /* reduce log even more and avoid word truncation */
-    validmemname=COMPATIBLE /* avoid special characters etc in table names    */
-    validvarname=V7         /* avoid special characters etc in variable names */
-    varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
-    varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
-  ;
+%let sasjs_prefix=&prefix;
+
+data _null_;
+  dttm=datetime();
+  call symputx("&sasjs_prefix._init_num",dttm,'g');
+  call symputx("&sasjs_prefix._init_dttm",put(dttm,E8601DT26.6),'g');
+  call symputx("&sasjs_prefix.work",pathname('WORK'),'g');
+run;
+
+options
+  noautocorrect           /* disallow misspelled procedure names            */
+  compress=CHAR           /* default is none so ensure we have something!   */
+  datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
+  dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
+  %str(err)orcheck=STRICT /* catch errs in libname/filename statements      */
+  fmterr                  /* ensure err when a format cannot be found       */
+  mergenoby=%str(ERR)OR   /* throw err when a merge has no BY variables     */
+  missing=.               /* changing this can cause hard to detect errs    */
+  noquotelenmax           /* avoid warnings for long strings                */
+  noreplace               /* avoid overwriting permanent datasets           */
+  ps=max                  /* reduce log size slightly                       */
+  ls=max                  /* reduce log even more and avoid word truncation */
+  validmemname=COMPATIBLE /* avoid special characters etc in table names    */
+  validvarname=V7         /* avoid special characters etc in variable names */
+  varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
+  varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
+;
 
 %mend mp_init;/**
   @file mp_jsonout.sas
@@ -16389,9 +16543,6 @@ run;
   @param [in] stpcode= the source file (or fileref) containing the SAS code to load
     into the stp.  For multiple files, they should simply be concatenated first.
   @param [in] minify= set to YES in order to strip comments, blank lines, and CRLFs.
-
-  @param frefin= deprecated - a unique fileref is now always used
-  @param frefout= deprecated - a unique fileref is now always used
   @param mDebug= set to 1 to show debug messages in the log
 
   @version 9.3
@@ -16406,16 +16557,8 @@ run;
   ,stpcode=
   ,minify=NO
   ,mdebug=0
-  /* deprecated */
-  ,frefin=inmeta
-  ,frefout=outmeta
 );
 
-%if &frefin ne inmeta or &frefout ne outmeta %then %do;
-  %put %str(WARN)ING: the frefin and frefout parameters will be deprecated in
-    an upcoming release.;
-%end;
-
 /* first, check if STP exists */
 %local tsuri;
 %let tsuri=stopifempty ;
@@ -18671,6 +18814,7 @@ libname &libref1a JSON fileref=&fname1a;
 %let found=0;
 %put Getting object uri from &libref1a..items;
 data _null_;
+  length contenttype name $1000;
   set &libref1a..items;
   if contenttype="&contenttype" and upcase(name)="%upcase(&name)" then do;
     call symputx('uri',uri,'l');
@@ -18818,6 +18962,7 @@ libname &libref1a JSON fileref=&fname1a;
 %let found=0;
 %put Getting object uri from &libref1a..items;
 data _null_;
+  length contenttype name $1000;
   set &libref1a..items;
   if contenttype='jobDefinition' and upcase(name)="%upcase(&name)" then do;
     call symputx('uri',cats("&base_uri",uri),'l');
@@ -18990,59 +19135,6 @@ filename &fname2 clear;
 libname &libref1 clear;
 
 %mend mv_deleteviyafolder;/**
-  @file mv_getaccesstoken.sas
-  @brief deprecated - replaced by mv_tokenrefresh.sas
-
-  @version VIYA V.03.04
-  @author Allan Bowe, source: https://github.com/sasjs/core
-
-  <h4> SAS Macros </h4>
-  @li mv_tokenrefresh.sas
-
-**/
-
-%macro mv_getaccesstoken(client_id=someclient
-    ,client_secret=somesecret
-    ,grant_type=authorization_code
-    ,code=
-    ,user=
-    ,pass=
-    ,access_token_var=ACCESS_TOKEN
-    ,refresh_token_var=REFRESH_TOKEN
-  );
-
-%mv_tokenrefresh(client_id=&client_id
-  ,client_secret=&client_secret
-  ,grant_type=&grant_type
-  ,user=&user
-  ,pass=&pass
-  ,access_token_var=&access_token_var
-  ,refresh_token_var=&refresh_token_var
-)
-
-%mend mv_getaccesstoken;/**
-  @file
-  @brief deprecated - replaced by mv_registerclient.sas
-
-  @version VIYA V.03.04
-  @author Allan Bowe, source: https://github.com/sasjs/core
-
-  <h4> SAS Macros </h4>
-  @li mv_registerclient.sas
-
-**/
-
-%macro mv_getapptoken(client_id=someclient
-    ,client_secret=somesecret
-    ,grant_type=authorization_code
-  );
-
-%mv_registerclient(client_id=&client_id
-  ,client_secret=&client_secret
-  ,grant_type=&grant_type
-)
-
-%mend mv_getapptoken;/**
   @file mv_getclients.sas
   @brief Get a list of Viya Clients
   @details First, be sure you have an access token (which requires an app token).
@@ -20369,38 +20461,6 @@ filename &fname0 clear;
 
 %mend mv_getjobstate;
 /**
-  @file mv_getrefreshtoken.sas
-  @brief deprecated - replaced by mv_tokenauth.sas
-
-  @version VIYA V.03.04
-  @author Allan Bowe, source: https://github.com/sasjs/core
-
-  <h4> SAS Macros </h4>
-  @li mv_tokenauth.sas
-
-**/
-
-%macro mv_getrefreshtoken(client_id=someclient
-    ,client_secret=somesecret
-    ,grant_type=authorization_code
-    ,code=
-    ,user=
-    ,pass=
-    ,access_token_var=ACCESS_TOKEN
-    ,refresh_token_var=REFRESH_TOKEN
-  );
-
-%mv_tokenauth(client_id=&client_id
-  ,client_secret=&client_secret
-  ,grant_type=&grant_type
-  ,code=&code
-  ,user=&user
-  ,pass=&pass
-  ,access_token_var=&access_token_var
-  ,refresh_token_var=&refresh_token_var
-)
-
-%mend mv_getrefreshtoken;/**
   @file mv_getusergroups.sas
   @brief Creates a dataset with a list of groups for a particular user
   @details If using outside of Viya SPRE, then an access token is needed.
@@ -22718,6 +22778,9 @@ run;
   @param [out] pkg= (utils) The output package in which to create the function.
     Uses a 3 part format:  libref.catalog.package
 
+  <h4> SAS Macros </h4>
+  @li mf_existfunction.sas
+
   <h4> Related Macros </h4>
   @li mcf_length.test.sas
 
@@ -22730,13 +22793,15 @@ run;
   ,pkg=UTILS
 )/*/STORE SOURCE*/;
 
+%if %mf_existfunction(mcf_length)=1 %then %return;
+
 %if &wrap=YES  %then %do;
   proc fcmp outlib=&lib..&cat..&pkg;
 %end;
 
 function mcf_length(var);
-  if missing(var) then len=0;
-  else if trunc(var,3)=var then len=3;
+  if var=. then len=0;
+  else if missing(var) or trunc(var,3)=var then len=3;
   else if trunc(var,4)=var then len=4;
   else if trunc(var,5)=var then len=5;
   else if trunc(var,6)=var then len=6;
@@ -22892,6 +22957,9 @@ endsub;
   @param [out] pkg= (utils) The output package in which to create the function.
     Uses a 3 part format:  libref.catalog.package
 
+  <h4> SAS Macros </h4>
+  @li mf_existfunction.sas
+
 **/
 
 %macro mcf_string2file(wrap=NO
@@ -22901,6 +22969,8 @@ endsub;
   ,pkg=UTILS
 )/*/STORE SOURCE*/;
 
+%if %mf_existfunction(mcf_string2file)=1 %then %return;
+
 %if &wrap=YES  %then %do;
   proc fcmp outlib=&lib..&cat..&pkg;
 %end;
diff --git a/base/mp_ds2squeeze.sas b/base/mp_ds2squeeze.sas
index a729fdb..01bf766 100644
--- a/base/mp_ds2squeeze.sas
+++ b/base/mp_ds2squeeze.sas
@@ -89,7 +89,7 @@ data _null_;
   end;
   else do;
     if maxlen=0 then len='3';
-    else len=maxlen;
+    else len=cats(maxlen);
   end;
   put '  ' name ' ' len;
   if last then put ';';
diff --git a/tests/crossplatform/mcf_length.test.sas b/tests/crossplatform/mcf_length.test.sas
index 90a7587..532a6c2 100644
--- a/tests/crossplatform/mcf_length.test.sas
+++ b/tests/crossplatform/mcf_length.test.sas
@@ -12,7 +12,7 @@
 
 data test;
   call symputx('null',mcf_length(.));
-  call symputx('special',mcf_length(._))
+  call symputx('special',mcf_length(._));
   call symputx('three',mcf_length(1));
   call symputx('four',mcf_length(10000000));
   call symputx('five',mcf_length(12345678));
diff --git a/viya/mv_deletefoldermember.sas b/viya/mv_deletefoldermember.sas
index d857ae1..332c815 100644
--- a/viya/mv_deletefoldermember.sas
+++ b/viya/mv_deletefoldermember.sas
@@ -117,6 +117,7 @@ libname &libref1a JSON fileref=&fname1a;
 %let found=0;
 %put Getting object uri from &libref1a..items;
 data _null_;
+  length contenttype name $1000;
   set &libref1a..items;
   if contenttype="&contenttype" and upcase(name)="%upcase(&name)" then do;
     call symputx('uri',uri,'l');
diff --git a/viya/mv_deletejes.sas b/viya/mv_deletejes.sas
index a0c8a54..4a88d97 100644
--- a/viya/mv_deletejes.sas
+++ b/viya/mv_deletejes.sas
@@ -114,6 +114,7 @@ libname &libref1a JSON fileref=&fname1a;
 %let found=0;
 %put Getting object uri from &libref1a..items;
 data _null_;
+  length contenttype name $1000;
   set &libref1a..items;
   if contenttype='jobDefinition' and upcase(name)="%upcase(&name)" then do;
     call symputx('uri',cats("&base_uri",uri),'l');

From 16489a9494aaadbf3db0759391565daa833132dd Mon Sep 17 00:00:00 2001
From: munja <allnbwe@gmail.com>
Date: Mon, 24 Jan 2022 13:12:31 +0100
Subject: [PATCH 5/5] fix: missing macro dependency in mp_ds2squeeze.test.sas

---
 README.md                                  | 2 +-
 all.sas                                    | 1 +
 base/mp_ds2squeeze.sas                     | 1 +
 tests/crossplatform/mp_ds2squeeze.test.sas | 1 +
 4 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1c142e9..ded8a4c 100644
--- a/README.md
+++ b/README.md
@@ -193,7 +193,7 @@ When contributing to this library, it is therefore important to ensure that all
 
 ## Breaking Changes
 
-We are currently on major release v4.  The following changes are planned when the next major (breaking) release becomes necessary:
+We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major (breaking) release becomes necessary:
 
 * (None as yet)
 
diff --git a/all.sas b/all.sas
index baaf4a3..dfc509b 100644
--- a/all.sas
+++ b/all.sas
@@ -5056,6 +5056,7 @@ run;
   depending on your OS and COMPRESS settings):
 
   > MP_DS2SQUEEZE: work.big was  625MB
+
   > MP_DS2SQUEEZE: work.smaller is    5MB
 
   @param [in] libds The library.dataset to be squeezed
diff --git a/base/mp_ds2squeeze.sas b/base/mp_ds2squeeze.sas
index 01bf766..a9c1e16 100644
--- a/base/mp_ds2squeeze.sas
+++ b/base/mp_ds2squeeze.sas
@@ -27,6 +27,7 @@
   depending on your OS and COMPRESS settings):
 
   > MP_DS2SQUEEZE: work.big was  625MB
+
   > MP_DS2SQUEEZE: work.smaller is    5MB
 
   @param [in] libds The library.dataset to be squeezed
diff --git a/tests/crossplatform/mp_ds2squeeze.test.sas b/tests/crossplatform/mp_ds2squeeze.test.sas
index db8c246..6383813 100644
--- a/tests/crossplatform/mp_ds2squeeze.test.sas
+++ b/tests/crossplatform/mp_ds2squeeze.test.sas
@@ -3,6 +3,7 @@
   @brief Testing mp_ds2squeeze.sas macro
 
   <h4> SAS Macros </h4>
+  @li mf_getvarlen.sas
   @li mp_assert.sas
   @li mp_assertscope.sas
   @li mp_ds2squeeze.sas