Merge pull request #316 from sasjs/gitinfo

feat: new gitreleaseinfo macro and associated test

.github/vpn/config.ovpn

@@ -1,30 +0,0 @@
-cipher AES-256-CBC
-setenv FORWARD_COMPATIBLE 1
-client
-server-poll-timeout 4
-nobind
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 443 tcp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-remote vpn.analytium.co.uk 1194 udp
-dev tun
-dev-type tun
-ns-cert-type server
-setenv opt tls-version-min 1.0 or-highest
-reneg-sec 604800
-sndbuf 0
-rcvbuf 0
-# NOTE: LZO commands are pushed by the Access Server at connect time.
-# NOTE: The below line doesn't disable LZO.
-comp-lzo no
-verb 3
-setenv PUSH_PEER_INFO
-
-ca ca.crt
-cert user.crt
-key user.key
-tls-auth tls.key 1

.github/workflows/run-tests.yml

@@ -21,31 +21,6 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
 
-      - name: Write VPN Files
-        run: |
-          echo "$CA_CRT" > .github/vpn/ca.crt
-          echo "$USER_CRT" > .github/vpn/user.crt
-          echo "$USER_KEY" > .github/vpn/user.key
-          echo "$TLS_KEY" > .github/vpn/tls.key
-        shell: bash
-        env:
-          CA_CRT: ${{ secrets.CA_CRT}}
-          USER_CRT: ${{ secrets.USER_CRT }}
-          USER_KEY: ${{ secrets.USER_KEY }}
-          TLS_KEY: ${{ secrets.TLS_KEY }}
-
-      - name: Install Open VPN
-        run: |
-          sudo apt install apt-transport-https
-          sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
-          sudo apt-key add openvpn-repo-pkg-key.pub
-          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-focal.list
-          sudo apt update
-          sudo apt install openvpn3
-
-      - name: Start Open VPN 3
-        run: openvpn3 session-start --config .github/vpn/config.ovpn
-
       - name: Install Doxygen
         run: sudo apt-get install doxygen
 
@@ -78,7 +53,5 @@ jobs:
           SECRET: ${{secrets.SECRET}}
           SAS_USERNAME: ${{secrets.SAS_USERNAME}}
           SAS_PASSWORD: ${{secrets.SAS_PASSWORD}}
-          SERVER_URL: ${{secrets.SERVER_URL}}
-          SERVER_TYPE: ${{secrets.SERVER_TYPE}}
           ACCESS_TOKEN: ${{secrets.ACCESS_TOKEN}}
           REFRESH_TOKEN: ${{secrets.REFRESH_TOKEN}}

README.md

@@ -237,6 +237,7 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 The following repositories are also worth checking out:
 
+* [xieliaing/SAS](https://github.com/xieliaing/SAS)
 * [SASJedi/sas-macros](https://github.com/SASJedi/sas-macros)
 * [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
 * [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)

all.sas

@@ -4112,11 +4112,14 @@ proc sql;
       %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
       %mp_createconstraints(inds=work.constraints,outds=created,execute=YES)
 
-  @param inds= The input table containing the constraint info
+  @param inds= (work.mp_getconstraints) The input table containing the
-  @param outds= a table containing the create statements (create_statement column)
+    constraint info
-  @param execute= `YES|NO` - default is NO. To actually create, use YES.
+  @param outds= (work.mp_createconstraints) A table containing the create
+    statements (create_statement column)
+  @param execute= (NO) To actually create, use YES.
 
-  <h4> SAS Macros </h4>
+  <h4> Related Files </h4>
+  @li mp_getconstraints.sas
 
   @version 9.2
   @author Allan Bowe
@@ -4124,7 +4127,7 @@ proc sql;
 **/
 
 %macro mp_createconstraints(inds=mp_getconstraints
-  ,outds=mp_createconstraints
+  ,outds=work.mp_createconstraints
   ,execute=NO
 )/*/STORE SOURCE*/;
 
@@ -4158,7 +4161,8 @@ data &outds;
   output;
 run;
 
-%mend mp_createconstraints;/**
+%mend mp_createconstraints;
+/**
   @file mp_createwebservice.sas
   @brief Create a web service in SAS 9, Viya or SASjs Server
   @details This is actually a wrapper for mx_createwebservice.sas, remaining
@@ -4450,6 +4454,58 @@ run;
   %end;
   %else %put &sysmacroname: &folder: is not a valid / accessible folder. ;
 %mend mp_deletefolder;/**
+  @file mp_dictionary.sas
+  @brief Creates a portal (libref) into the SQL Dictionary Views
+  @details Provide a libref and the macro will create a series of views against
+  each view in the special PROC SQL dictionary libref.
+
+  This is useful if you would like to visualise (navigate) the views in a SAS
+  client such as Base SAS, Enterprise Guide, or Studio (or [Data Controller](
+  https://datacontroller.io)).
+
+  It works by extracting the dictionary.dictionaries view into
+  YOURLIB.dictionaries, then uses that to create a YOURLIB.{viewName} for every
+  other dictionary.view, eg:
+
+      proc sql;
+      create view YOURLIB.columns as select * from dictionary.columns;
+
+  Usage:
+
+      libname demo "/lib/directory";
+      %mp_dictionary(lib=demo)
+
+  Or, to just create them in WORK:
+
+      %mp_dictionary()
+
+  If you'd just like to browse the dictionary data model, you can also check
+  out [this article](https://rawsas.com/dictionary-of-dictionaries/).
+
+  ![](https://user-images.githubusercontent.com/4420615/188278365-2987db97-0594-4a39-ac81-dbacdef5cdc8.png)
+
+  @param lib= (WORK) The libref in which to create the views
+
+  <h4> Related Files </h4>
+  @li mp_dictionary.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_dictionary(lib=WORK)/*/STORE SOURCE*/;
+  %local list i mem;
+  proc sql noprint;
+  create view &lib..dictionaries as select * from dictionary.dictionaries;
+  select distinct memname into: list separated by ' '  from &lib..dictionaries;
+  %do i=1 %to %sysfunc(countw(&list,%str( )));
+    %let mem=%scan(&list,&i,%str( ));
+    create view &lib..&mem as select * from dictionary.&mem;
+  %end;
+  quit;
+%mend mp_dictionary;
+/**
   @file
   @brief Returns all files and subdirectories within a specified parent
   @details When used with getattrs=NO, is not OS specific (uses dopen / dread).
@@ -4478,6 +4534,9 @@ run;
   @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
     subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
     recursion, set to MAX.
+  @param [in] showparent= (NO) By default, the initial parent directory is not
+    part of the results.  Set to YES to include it.  For this record only,
+    directory=filepath.
   @param [out] outds= (work.mp_dirlist) The output dataset to create
   @param [out] getattrs= (NO)  If getattrs=YES then the doptname / foptname
     functions are used to scan all properties - any characters that are not
@@ -4514,6 +4573,7 @@ run;
     , fref=0
     , outds=work.mp_dirlist
     , getattrs=NO
+    , showparent=NO
     , maxdepth=0
     , level=0 /* The level of recursion to perform.  For internal use only. */
 )/*/STORE SOURCE*/;
@@ -4596,6 +4656,15 @@ data &out_ds(compress=no
     output;
   end;
   rc = dclose(did);
+  %if &showparent=YES and &level=0 %then %do;
+    filepath=directory;
+    file_or_folder='folder';
+    ext='';
+    filename=scan(directory,-1,'/\');
+    msg='';
+    level=&level;
+    output;
+  %end;
   stop;
 run;
 
@@ -4683,6 +4752,9 @@ run;
   data _null_;
     set &out_ds;
     where file_or_folder='folder';
+  %if &showparent=YES and &level=0 %then %do;
+    if filepath ne directory;
+  %end;
     length code $10000;
     code=cats('%nrstr(%mp_dirlist(path=',filepath,",outds=&outds"
       ,",getattrs=&getattrs,level=%eval(&level+1),maxdepth=&maxdepth))");
@@ -5698,7 +5770,7 @@ data _null_;
 run;
 
 %if %upcase(&showlog)=YES %then %do;
-  options ps=max;
+  options ps=max lrecl=max;
   data _null_;
     infile &outref;
     input;
@@ -5706,7 +5778,8 @@ run;
   run;
 %end;
 
-%mend mp_ds2md;/**
+%mend mp_ds2md;
+/**
   @file
   @brief Create a smaller version of a dataset, without data loss
   @details This macro will scan the input dataset and create a new one, that
@@ -5826,6 +5899,116 @@ options varlenchk=&optval;
 %put &sysmacroname: &outds is %mf_getfilesize(libds=&outds,format=yes);
 
 %mend mp_ds2squeeze;/**
+  @file
+  @brief Export dataset metadata to a single output table
+  @details Exports the dataset attributes and enginehost information, then
+  converts the datasets into a single output table in the following format:
+
+|ODS_TABLE:$10.|NAME:$100.|VALUE:$1000.|
+|---|---|---|
+|`ATTRIBUTES `|`Data Set Name `|`SASHELP.CLASS `|
+|`ATTRIBUTES `|`Observations `|`19 `|
+|`ATTRIBUTES `|`Member Type `|`DATA `|
+|`ATTRIBUTES `|`Variables `|`5 `|
+|`ATTRIBUTES `|`Engine `|`V9 `|
+|`ATTRIBUTES `|`Indexes `|`0 `|
+|`ATTRIBUTES `|`Created `|`06/08/2020 00:59:14 `|
+|`ATTRIBUTES `|`Observation Length `|`40 `|
+|`ATTRIBUTES `|`Last Modified `|`06/08/2020 00:59:14 `|
+|`ATTRIBUTES `|`Deleted Observations `|`0 `|
+|`ATTRIBUTES `|`Protection `|`. `|
+|`ATTRIBUTES `|`Compressed `|`NO `|
+|`ATTRIBUTES `|`Data Set Type `|`. `|
+|`ATTRIBUTES `|`Sorted `|`NO `|
+|`ATTRIBUTES `|`Label `|`Student Data `|
+|`ATTRIBUTES `|`Data Representation `|`SOLARIS_X86_64, LINUX_X86_64, ALPHA_TRU64, LINUX_IA64 `|
+|`ATTRIBUTES `|`Encoding `|`us-ascii  ASCII (ANSI) `|
+|`ENGINEHOST `|`Data Set Page Size `|`65536 `|
+|`ENGINEHOST `|`Number of Data Set Pages `|`1 `|
+|`ENGINEHOST `|`First Data Page `|`1 `|
+|`ENGINEHOST `|`Max Obs per Page `|`1632 `|
+|`ENGINEHOST `|`Obs in First Data Page `|`19 `|
+|`ENGINEHOST `|`Number of Data Set Repairs `|`0 `|
+|`ENGINEHOST `|`Filename `|`/opt/sas/sas9/SASHome/SASFoundation/9.4/sashelp/class.sas7bdat `|
+|`ENGINEHOST `|`Release Created `|`9.0401M7 `|
+|`ENGINEHOST `|`Host Created `|`Linux `|
+|`ENGINEHOST `|`Inode Number `|`28314616 `|
+|`ENGINEHOST `|`Access Permission `|`rw-r--r-- `|
+|`ENGINEHOST `|`Owner Name `|`sas `|
+|`ENGINEHOST `|`File Size `|`128KB `|
+|`ENGINEHOST `|`File Size (bytes) `|`131072 `|
+
+  Example usage:
+
+      %mp_dsmeta(sashelp.class,outds=work.mymeta)
+      proc print data=work.mymeta;
+      run;
+
+  For more details on creating datasets from PROC CONTENTS check out this
+  excellent [paper](
+  https://support.sas.com/resources/papers/proceedings14/1549-2014.pdf) by
+  [Louise Hadden](https://www.linkedin.com/in/louisehadden/).
+
+  @param libds The library.dataset to export the metadata for
+  @param outds= (work.dsmeta) The output table to contain the metadata
+
+  <h4> Related Files </h4>
+  @li mp_dsmeta.test.sas
+  @li mp_getcols.sas
+  @li mp_getdbml.sas
+  @li mp_getddl.sas
+  @li mp_getformats.sas
+  @li mp_getpk.sas
+  @li mp_guesspk.sas
+
+**/
+
+%macro mp_dsmeta(libds,outds=work.dsmeta);
+
+%local ds1 ds2;
+data;run; %let ds1=&syslast;
+data;run; %let ds2=&syslast;
+
+/* setup the ODS capture */
+ods output attributes=&ds1 enginehost=&ds2;
+
+/* export the metadata */
+proc contents data=&libds;
+run;
+
+/* load it into a single table */
+data &outds (keep=ods_table name value);
+  length ods_table $10 name label2 label1 label $100
+    value cvalue cvalue1 cvalue2 $1000
+    nvalue nvalue1 nvalue2 8;
+  if _n_=1 then call missing (of _all_);
+  * putlog (_all_)(=);
+  set &ds1 (in=atrs) &ds2 (in=eng);
+  if atrs then do;
+    ods_table='ATTRIBUTES';
+    name=coalescec(label1,label);
+    value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.));
+    output;
+    if label2 ne '' then do;
+    name=label2;
+    value=coalescec(cvalue2,put(nvalue2,best.));
+    output;
+    end;
+  end;
+  else if eng then do;
+    ods_table='ENGINEHOST';
+    name=coalescec(label1,label);
+    value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.));
+    output;
+  end;
+run;
+
+proc sql;
+drop table &ds1, &ds2;
+
+%mend mp_dsmeta;
+
+/**
   @file
   @brief Checks an input filter table for validity
   @details Performs checks on the input table to ensure it arrives in the
@@ -7985,6 +8168,80 @@ create table &outds as
 )
 
 %mend mp_getpk;
+/**
+  @file
+  @brief Pulls latest release info from a GIT repository
+  @details Useful for grabbing the latest version number or other attributes
+  from a GIT server.  Supported providers are GitLab and GitHub. Pull requests
+  are welcome if you'd like to see additional providers!
+
+  Note that each provider provides slightly different JSON output.  Therefore
+  the macro simply extracts the JSON and assigns the libname (using the JSON
+  engine).
+
+  Example usage (eg, to grab latest release version from github):
+
+      %mp_gitreleaseinfo(GITHUB,sasjs/core,outlib=mylibref)
+
+      data _null_;
+        set mylibref.root;
+        putlog TAG_NAME=;
+      run;
+
+  @param [in] provider The GIT provider for the release info.  Accepted values:
+    @li GITLAB
+    @li GITHUB - Tables include root, assets, author, alldata
+  @param [in] project The link to the repository.  This has different formats
+    depending on the vendor:
+    @li GITHUB - org/repo, eg sasjs/core
+    @li GITLAB - project, eg 1343223
+  @param [in] server= (0) If your repo is self-hosted, then provide the domain
+    here.  Otherwise it will default to the provider domain (eg gitlab.com).
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outlib= (GITREL) The JSON-engine libref to be created, which will
+    point at the returned JSON
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+
+  <h4> Related Files </h4>
+  @li mp_gitreleaseinfo.test.sas
+
+**/
+
+%macro mp_gitreleaseinfo(provider,project,server=0,outlib=GITREL,mdebug=0);
+%local url fref;
+
+%let provider=%upcase(&provider);
+
+%if &provider=GITHUB %then %do;
+  %if "&server"="0" %then %let server=https://api.github.com;
+  %let url=&server/repos/&project/releases/latest;
+%end;
+%else %if &provider=GITLAB %then %do;
+  %if "&server"="0" %then %let server=https://gitlab.com;
+  %let url=&server/api/v4/projects/&project/releases;
+%end;
+
+%let fref=%mf_getuniquefileref();
+
+proc http method='GET' out=&fref url="&url";
+%if &mdebug=1 %then %do;
+  debug level = 3;
+%end;
+run;
+
+libname &outlib JSON fileref=&fref;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fref;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_gitreleaseinfo;
 /**
   @file
   @brief Performs a text substitution on a file
@@ -8387,7 +8644,7 @@ run;
         put hashkey=;
       run;
 
-  ![sas md5 hash dataset log results](https://i.imgur.com/MqF98vk.png)
+  ![sas md5 hash dataset log results](https://i.4gl.io/1/KorUKoyE05.png/raw)
 
   <h4> SAS Macros </h4>
   @li mf_getattrn.sas
@@ -8397,11 +8654,12 @@ run;
 
   <h4> Related Files </h4>
   @li mp_hashdataset.test.sas
+  @li mp_hashdirectory.sas
 
   @param [in] libds dataset to hash
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
-  @param [in] iftrue= A condition under which the macro should be executed.
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
-  @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
+  @param [out] outds= (work._data_) The output dataset to create. This
   will contain one column (hashkey) with one observation (a $hex32.
   representation of the input hash)
   |hashkey:$32.|
@@ -8464,6 +8722,168 @@ run;
   run;
 %end;
 %mend mp_hashdataset;
+/**
+  @file
+  @brief Returns a unique hash for each file in a directory
+  @details Hashes each file in each directory, and then hashes the hashes to
+  create a hash for each directory also.
+
+  This makes use of the new `hashing_file()` and `hashing` functions, available
+  since 9.4m6. Interestingly, these can even be used in pure macro, eg:
+
+      %put %sysfunc(hashing_file(md5,/path/to/file.blob,0));
+
+  Actual usage:
+
+      %let fpath=/some/directory;
+
+      %mp_hashdirectory(&fpath,outds=myhash,maxdepth=2)
+
+      data _null_;
+        set work.myhash;
+        put (_all_)(=);
+      run;
+
+  Whilst files are hashed in their entirety, the logic for creating a folder
+  hash is as follows:
+
+  @li Sort the files by filename (case sensitive, uppercase then lower)
+  @li Take the first 100 hashes, concatenate and hash
+  @li Concatenate this hash with another 100 hashes and hash again
+  @li Continue until the end of the folder.  This is the folder hash
+  @li If a folder contains other folders, start from the bottom of the tree -
+    the folder hashes cascade upwards so you know immediately if there is a
+    change in a sub/sub directory
+  @li If the folder has no content (empty) then it is ignored. No hash created.
+  @li If the file is empty, it is also ignored / no hash created.
+
+  <h4> SAS Macros </h4>
+  @li mp_dirlist.sas
+
+  <h4> Related Files </h4>
+  @li mp_hashdataset.sas
+  @li mp_hashdirectory.test.sas
+  @li mp_md5.sas
+
+  @param [in] inloc Full filepath of the file to be hashed (unquoted)
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
+    subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
+  @param [in] method= (MD5) the hashing method to use.  Available options:
+    @li MD5
+    @li SH1
+    @li SHA256
+    @li SHA384
+    @li SHA512
+    @li CRC32
+  @param [out] outds= (work.mp_hashdirectory) The output dataset.  Contains:
+    @li directory - the parent folder
+    @li file_hash - the hash output
+    @li hash_duration - how long the hash took (first hash always takes longer)
+    @li file_path - /full/path/to/each/file.ext
+    @li file_or_folder - contains either "file" or "folder"
+    @li level - the depth of the directory (top level is 0)
+
+  @version 9.4m6
+  @author Allan Bowe
+**/
+
+%macro mp_hashdirectory(inloc,
+  outds=work.mp_hashdirectory,
+  method=MD5,
+  maxdepth=0,
+  iftrue=%str(1=1)
+)/*/STORE SOURCE*/;
+
+%local curlevel tempds ;
+
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+/* get the directory listing */
+%mp_dirlist(path=&inloc, outds=&outds, maxdepth=&maxdepth, showparent=YES)
+
+/* create the hashes */
+data &outds;
+  set &outds (rename=(filepath=file_path));
+  length FILE_HASH $32 HASH_DURATION 8;
+  keep directory file_hash hash_duration file_path file_or_folder level;
+
+  ts=datetime();
+  if file_or_folder='file' then do;
+    /* if file is empty, hashing_file will break - so ignore / delete */
+    length fname val $8;
+    drop fname val fid is_empty;
+    rc=filename(fname,file_path);
+    fid=fopen(fname);
+    if fid > 0 then do;
+      rc=fread(fid);
+      is_empty=fget(fid,val);
+    end;
+    rc=fclose(fid);
+    rc=filename(fname);
+    if is_empty ne 0 then delete;
+    else file_hash=hashing_file("&method",cats(file_path),0);
+  end;
+  hash_duration=datetime()-ts;
+run;
+
+proc sort data=&outds ;
+  by descending level directory file_path;
+run;
+
+data _null_;
+  set &outds;
+  call symputx('maxlevel',level,'l');
+  stop;
+run;
+
+/* now hash the hashes to populate folder hashes, starting from the bottom */
+%do curlevel=&maxlevel %to 0 %by -1;
+  data work._data_ (keep=directory file_hash);
+    set &outds;
+    where level=&curlevel;
+    by descending level directory file_path;
+    length str $32767 tmp_hash $32;
+    retain str tmp_hash ;
+    /* reset vars when starting a new directory */
+    if first.directory then do;
+      str='';
+      tmp_hash='';
+      i=0;
+    end;
+    /* hash each chunk of 100 file paths */
+    i+1;
+    str=cats(str,file_hash);
+    if mod(i,100)=0 or last.directory then do;
+      tmp_hash=hashing("&method",cats(tmp_hash,str));
+      str='';
+    end;
+    /* output the hash at directory level */
+    if last.directory then do;
+      file_hash=tmp_hash;
+      output;
+    end;
+    if last.level then stop;
+  run;
+  %let tempds=&syslast;
+  /* join the hash back into the main table */
+  proc sql undo_policy=none;
+  create table &outds as
+    select a.directory
+      ,coalesce(b.file_hash,a.file_hash) as file_hash
+      ,a.hash_duration
+      ,a.file_path
+      ,a.file_or_folder
+      ,a.level
+    from &outds a
+    left join &tempds b
+    on a.file_path=b.directory
+    order by level desc, directory, file_path;
+  drop table &tempds;
+%end;
+
+%mend mp_hashdirectory;
 /**
   @file
   @brief Performs a wrapped \%include
@@ -8793,7 +9213,7 @@ options
     call symputx(cats('label',_n_),coalescec(label,name),'l');
     /* overwritten when fmt=Y and a custom format exists in catalog */
     if typelong='num' then call symputx(cats('fmtlen',_n_),200,'l');
-    else call symputx(cats('fmtlen',_n_),min(32767,ceil((length+3)*1.5)),'l');
+    else call symputx(cats('fmtlen',_n_),min(32767,ceil((length+10)*1.5)),'l');
   run;
 
   %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
@@ -8849,8 +9269,8 @@ options
       %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
       proc sql noprint;
       create table &tmpds1 as
-          select cats(libname,'.',memname) as fmtcat,
+          select cats(libname,'.',memname) as FMTCAT,
-          fmtname
+          FMTNAME
         from dictionary.formats
         where fmttype='F' and libname is not null
           and fmtname in (select format from &colinfo where format is not null)
@@ -8875,7 +9295,7 @@ options
 
       proc sql;
       create table &tmpds4 as
-        select a.*, b.length as maxw
+        select a.*, b.length as MAXW
         from &colinfo a
         left join &tmpds2 b
         on cats(a.format)=cats(upcase(b.fmtname))
@@ -8886,7 +9306,7 @@ options
         call symputx(
           cats('fmtlen',_n_),
           /* vars need extra padding due to JSON escaping of special chars */
-          min(32767,ceil((max(length,maxw)+3)*1.5))
+          min(32767,ceil((max(length,maxw)+10)*1.5))
           ,'l'
         );
       run;
@@ -8961,7 +9381,7 @@ options
       format _numeric_ bart.;
     %do i=1 %to &numcols;
       %if &&typelong&i=char or &fmt=Y %then %do;
-        if findc(&&name&i,'"\'!!'0A0D09000E0F01021011'x) then do;
+        if findc(&&name&i,'"\'!!'0A0D09000E0F010210111A'x) then do;
           &&name&i='"'!!trim(
             prxchange('s/"/\\"/',-1,        /* double quote */
             prxchange('s/\x0A/\n/',-1,      /* new line */
@@ -8974,8 +9394,9 @@ options
             prxchange('s/\x02/\\u0002/',-1, /* STX */
             prxchange('s/\x10/\\u0010/',-1, /* DLE */
             prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+            prxchange('s/\x1A/\\u001A/',-1, /* SUB */
             prxchange('s/\\/\\\\/',-1,&&name&i)
-          ))))))))))))!!'"';
+          )))))))))))))!!'"';
         end;
         else &&name&i=quote(cats(&&name&i));
       %end;
@@ -15288,7 +15709,7 @@ data _null_;
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
   put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
   put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
-  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); ';
+  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
@@ -15344,8 +15765,8 @@ data _null_;
   put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
   put '      proc sql noprint; ';
   put '      create table &tmpds1 as ';
-  put '          select cats(libname,''.'',memname) as fmtcat, ';
+  put '          select cats(libname,''.'',memname) as FMTCAT, ';
-  put '          fmtname ';
+  put '          FMTNAME ';
   put '        from dictionary.formats ';
   put '        where fmttype=''F'' and libname is not null ';
   put '          and fmtname in (select format from &colinfo where format is not null) ';
@@ -15370,7 +15791,7 @@ data _null_;
   put ' ';
   put '      proc sql; ';
   put '      create table &tmpds4 as ';
-  put '        select a.*, b.length as maxw ';
+  put '        select a.*, b.length as MAXW ';
   put '        from &colinfo a ';
   put '        left join &tmpds2 b ';
   put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
@@ -15381,7 +15802,7 @@ data _null_;
   put '        call symputx( ';
   put '          cats(''fmtlen'',_n_), ';
   put '          /* vars need extra padding due to JSON escaping of special chars */ ';
-  put '          min(32767,ceil((max(length,maxw)+3)*1.5)) ';
+  put '          min(32767,ceil((max(length,maxw)+10)*1.5)) ';
   put '          ,''l'' ';
   put '        ); ';
   put '      run; ';
@@ -15456,7 +15877,7 @@ data _null_;
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; ';
+  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
   put '          &&name&i=''"''!!trim( ';
   put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
   put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
@@ -15469,8 +15890,9 @@ data _null_;
   put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
   put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
   put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
   put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '          ))))))))))))!!''"''; ';
+  put '          )))))))))))))!!''"''; ';
   put '        end; ';
   put '        else &&name&i=quote(cats(&&name&i)); ';
   put '      %end; ';
@@ -17719,7 +18141,7 @@ data &outds;
     rc5=metadata_getattr(tsuri,"Name",servercontext);
   end;
   else do;
-    put "%str(ERR)OR: could not find " pgm;
+    put "%str(ERR)OR: could not find " path;
     put (_all_)(=);
   end;
   &md.put (_all_)(=);
@@ -20309,7 +20731,7 @@ data _null_;
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
   put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
   put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
-  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); ';
+  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
@@ -20365,8 +20787,8 @@ data _null_;
   put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
   put '      proc sql noprint; ';
   put '      create table &tmpds1 as ';
-  put '          select cats(libname,''.'',memname) as fmtcat, ';
+  put '          select cats(libname,''.'',memname) as FMTCAT, ';
-  put '          fmtname ';
+  put '          FMTNAME ';
   put '        from dictionary.formats ';
   put '        where fmttype=''F'' and libname is not null ';
   put '          and fmtname in (select format from &colinfo where format is not null) ';
@@ -20391,7 +20813,7 @@ data _null_;
   put ' ';
   put '      proc sql; ';
   put '      create table &tmpds4 as ';
-  put '        select a.*, b.length as maxw ';
+  put '        select a.*, b.length as MAXW ';
   put '        from &colinfo a ';
   put '        left join &tmpds2 b ';
   put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
@@ -20402,7 +20824,7 @@ data _null_;
   put '        call symputx( ';
   put '          cats(''fmtlen'',_n_), ';
   put '          /* vars need extra padding due to JSON escaping of special chars */ ';
-  put '          min(32767,ceil((max(length,maxw)+3)*1.5)) ';
+  put '          min(32767,ceil((max(length,maxw)+10)*1.5)) ';
   put '          ,''l'' ';
   put '        ); ';
   put '      run; ';
@@ -20477,7 +20899,7 @@ data _null_;
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; ';
+  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
   put '          &&name&i=''"''!!trim( ';
   put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
   put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
@@ -20490,8 +20912,9 @@ data _null_;
   put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
   put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
   put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
   put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '          ))))))))))))!!''"''; ';
+  put '          )))))))))))))!!''"''; ';
   put '        end; ';
   put '        else &&name&i=quote(cats(&&name&i)); ';
   put '      %end; ';
@@ -21821,6 +22244,66 @@ run;
   %end;
 
 %mend mfv_existfolder;/**
+  @file mfv_existsashdat.sas
+  @brief Checks whether a CAS sashdat dataset exists in persistent storage.
+  @details Can be used in open code, eg as follows:
+
+      %if %mfv_existsashdat(libds=casuser.sometable) %then %put  yes it does!;
+
+  The function uses `dosubl()` to run the `table.fileinfo` action, for the
+  specified library, filtering for `*.sashdat` tables.  The results are stored
+  in a WORK table (&outprefix._&lib). If that table already exists, it is
+  queried instead, to avoid the dosubl() performance hit.
+
+  To force a rescan, just use a new `&outprefix` value, or delete the table(s)
+  before running the function.
+
+  @param libds library.dataset
+  @param outprefix= (work.mfv_existsashdat) Used to store the current HDATA
+    tables to improve subsequent query performance.  This reference is a prefix
+    and is converted to `&prefix._{libref}`
+
+  @return output returns 1 or 0
+
+  @version 0.2
+  @author Mathieu Blauw
+**/
+
+%macro mfv_existsashdat(libds,outprefix=work.mfv_existsashdat
+);
+%local rc dsid name lib ds;
+%let lib=%upcase(%scan(&libds,1,'.'));
+%let ds=%upcase(%scan(&libds,-1,'.'));
+
+/* if table does not exist, create it */
+%if %sysfunc(exist(&outprefix._&lib)) ne 1 %then %do;
+  %let rc=%sysfunc(dosubl(%nrstr(
+    /* Read in table list (once per &lib per session) */
+    proc cas;
+      table.fileinfo result=source_list /caslib="&lib";
+      val=findtable(source_list);
+      saveresult val dataout=&outprefix._&lib;
+    quit;
+    /* Only keep name, without file extension */
+    data &outprefix._&lib;
+      set &outprefix._&lib(where=(Name like '%.sashdat') keep=Name);
+      Name=upcase(scan(Name,1,'.'));
+    run;
+  )));
+%end;
+
+/* Scan table for hdat existence */
+%let dsid=%sysfunc(open(&outprefix._&lib(where=(name="&ds"))));
+%syscall set(dsid);
+%let rc = %sysfunc(fetch(&dsid));
+%let rc = %sysfunc(close(&dsid));
+
+/* Return result */
+%if "%trim(&name)"="%trim(&ds)" %then 1;
+%else 0;
+
+%mend mfv_existsashdat;
+/**
   @file
   @brief Creates a file in SAS Drive
   @details Creates a file in SAS Drive and adds the appropriate content type.
@@ -21846,7 +22329,8 @@ run;
   @param [in] contentdisp= (inline) Content Disposition. Example values:
     @li inline
     @li attachment
-
+  @param [in] ctype= (0) Set a default HTTP Content-Type header to be returned
+    with the file when the content is retrieved from the Files service.
   @param [in] access_token_var= The global macro variable to contain the access
     token, if using authorization_code grant type.
   @param [in] grant_type= (sas_services) Valid values are:
@@ -21874,6 +22358,7 @@ run;
     ,inref=
     ,intype=BINARY
     ,contentdisp=inline
+    ,ctype=0
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
     ,mdebug=0
@@ -21925,8 +22410,10 @@ filename &fref filesrvc
   folderPath="&path"
   filename="&name"
   cdisp="&contentdisp"
+%if "&ctype" ne "0" %then %do;
+  ctype="&ctype"
+%end;
   lrecl=1048544;
-
 %if &intype=BINARY %then %do;
   %mp_binarycopy(inref=&inref, outref=&fref)
 %end;
@@ -22163,14 +22650,25 @@ options noquotelenmax;
 
   @param path= The full path (on SAS Drive) where the job will be created
   @param name= The name of the job
-  @param desc= The description of the job
+  @param desc= (Created by the mv_createjob.sas macro) The job description
   @param precode= Space separated list of filerefs, pointing to the code that
     needs to be attached to the beginning of the job
-  @param code= Fileref(s) of the actual code to be added
+  @param code= (ft15f001) Fileref(s) of the actual code to be added
-  @param access_token_var= The global macro variable to contain the access token
+  @param access_token_var= (ACCESS_TOKEN) Global macro variable containing the
-  @param grant_type= valid values are "password" or "authorization_code"
+    access token
-    (unquoted). The default is authorization_code.
+  @param grant_type= (sas_services) Valid values:
-  @param replace= select NO to avoid replacing any existing job in that location
+    @li sas_services
+    @li detect
+    @li authorization_code
+    @li password
+  @param replace= (YES) select NO to avoid replacing any existing job
+  @param addjesbeginendmacros= (false) Relates to the `_addjesbeginendmacros`
+    setting.  Normally this would always be false however due to a Viya bug
+    (https://github.com/sasjs/cli/issues/1229) this is now configurable.  Valid
+    values:
+    @li true
+    @li false
+    @li 0 - this will prevent the flag from being set (job will default to true)
   @param contextname= Choose a specific context on which to run the Job.  Leave
     blank to use the default context.  From Viya 3.5 it is possible to configure
     a shared context - see
@@ -22191,6 +22689,7 @@ https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5p
     ,replace=YES
     ,debug=0
     ,contextname=
+    ,addjesbeginendmacros=false
   );
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
@@ -22314,19 +22813,29 @@ run;
 %end;
 
 /* set up the body of the request to create the service */
-%local fname3;
+%local fname3 comma;
 %let fname3=%mf_getuniquefileref();
 data _null_;
   file &fname3 TERMSTR=' ';
   length string $32767;
   string=cats('{"version": 0,"name":"'
     ,"&name"
-    ,'","type":"Compute","parameters":[{"name":"_addjesbeginendmacros"'
+    ,'","type":"Compute","parameters":['
-    ,',"type":"CHARACTER","defaultValue":"false"}');
+%if &addjesbeginendmacros ne 0 %then %do;
+    ,'{"name":"_addjesbeginendmacros"'
+    ,',"type":"CHARACTER","defaultValue":"'
+    ,"&addjesbeginendmacros"
+    ,'"}'
+  %let comma=%str(,);
+%end;
+  );
   context=quote(cats(symget('contextname')));
   if context ne '""' then do;
-    string=cats(string,',{"version": 1,"name": "_contextName","defaultValue":'
+    string=cats(string
-      ,context,',"type":"CHARACTER","label":"Context Name","required": false}');
+      ,"&comma"
+      ,'{"version": 1,"name": "_contextName","defaultValue":'
+      ,context,',"type":"CHARACTER","label":"Context Name","required": false}'
+    );
   end;
   string=cats(string,'],"code":"');
   put string;
@@ -22772,7 +23281,7 @@ data _null_;
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
   put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
   put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
-  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); ';
+  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
@@ -22828,8 +23337,8 @@ data _null_;
   put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
   put '      proc sql noprint; ';
   put '      create table &tmpds1 as ';
-  put '          select cats(libname,''.'',memname) as fmtcat, ';
+  put '          select cats(libname,''.'',memname) as FMTCAT, ';
-  put '          fmtname ';
+  put '          FMTNAME ';
   put '        from dictionary.formats ';
   put '        where fmttype=''F'' and libname is not null ';
   put '          and fmtname in (select format from &colinfo where format is not null) ';
@@ -22854,7 +23363,7 @@ data _null_;
   put ' ';
   put '      proc sql; ';
   put '      create table &tmpds4 as ';
-  put '        select a.*, b.length as maxw ';
+  put '        select a.*, b.length as MAXW ';
   put '        from &colinfo a ';
   put '        left join &tmpds2 b ';
   put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
@@ -22865,7 +23374,7 @@ data _null_;
   put '        call symputx( ';
   put '          cats(''fmtlen'',_n_), ';
   put '          /* vars need extra padding due to JSON escaping of special chars */ ';
-  put '          min(32767,ceil((max(length,maxw)+3)*1.5)) ';
+  put '          min(32767,ceil((max(length,maxw)+10)*1.5)) ';
   put '          ,''l'' ';
   put '        ); ';
   put '      run; ';
@@ -22940,7 +23449,7 @@ data _null_;
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; ';
+  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
   put '          &&name&i=''"''!!trim( ';
   put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
   put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
@@ -22953,8 +23462,9 @@ data _null_;
   put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
   put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
   put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
   put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '          ))))))))))))!!''"''; ';
+  put '          )))))))))))))!!''"''; ';
   put '        end; ';
   put '        else &&name&i=quote(cats(&&name&i)); ';
   put '      %end; ';

base/mp_createconstraints.sas

@@ -18,11 +18,14 @@
       %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
       %mp_createconstraints(inds=work.constraints,outds=created,execute=YES)
 
-  @param inds= The input table containing the constraint info
+  @param inds= (work.mp_getconstraints) The input table containing the
-  @param outds= a table containing the create statements (create_statement column)
+    constraint info
-  @param execute= `YES|NO` - default is NO. To actually create, use YES.
+  @param outds= (work.mp_createconstraints) A table containing the create
+    statements (create_statement column)
+  @param execute= (NO) To actually create, use YES.
 
-  <h4> SAS Macros </h4>
+  <h4> Related Files </h4>
+  @li mp_getconstraints.sas
 
   @version 9.2
   @author Allan Bowe
@@ -30,7 +33,7 @@
 **/
 
 %macro mp_createconstraints(inds=mp_getconstraints
-  ,outds=mp_createconstraints
+  ,outds=work.mp_createconstraints
   ,execute=NO
 )/*/STORE SOURCE*/;
 
@@ -64,4 +67,4 @@ data &outds;
   output;
 run;
 
-%mend mp_createconstraints;
+%mend mp_createconstraints;

base/mp_dictionary.sas

@@ -0,0 +1,52 @@
+/**
+  @file mp_dictionary.sas
+  @brief Creates a portal (libref) into the SQL Dictionary Views
+  @details Provide a libref and the macro will create a series of views against
+  each view in the special PROC SQL dictionary libref.
+
+  This is useful if you would like to visualise (navigate) the views in a SAS
+  client such as Base SAS, Enterprise Guide, or Studio (or [Data Controller](
+  https://datacontroller.io)).
+
+  It works by extracting the dictionary.dictionaries view into
+  YOURLIB.dictionaries, then uses that to create a YOURLIB.{viewName} for every
+  other dictionary.view, eg:
+
+      proc sql;
+      create view YOURLIB.columns as select * from dictionary.columns;
+
+  Usage:
+
+      libname demo "/lib/directory";
+      %mp_dictionary(lib=demo)
+
+  Or, to just create them in WORK:
+
+      %mp_dictionary()
+
+  If you'd just like to browse the dictionary data model, you can also check
+  out [this article](https://rawsas.com/dictionary-of-dictionaries/).
+
+  ![](https://user-images.githubusercontent.com/4420615/188278365-2987db97-0594-4a39-ac81-dbacdef5cdc8.png)
+
+  @param lib= (WORK) The libref in which to create the views
+
+  <h4> Related Files </h4>
+  @li mp_dictionary.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_dictionary(lib=WORK)/*/STORE SOURCE*/;
+  %local list i mem;
+  proc sql noprint;
+  create view &lib..dictionaries as select * from dictionary.dictionaries;
+  select distinct memname into: list separated by ' '  from &lib..dictionaries;
+  %do i=1 %to %sysfunc(countw(&list,%str( )));
+    %let mem=%scan(&list,&i,%str( ));
+    create view &lib..&mem as select * from dictionary.&mem;
+  %end;
+  quit;
+%mend mp_dictionary;

base/mp_dirlist.sas

@@ -27,6 +27,9 @@
   @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
     subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
     recursion, set to MAX.
+  @param [in] showparent= (NO) By default, the initial parent directory is not
+    part of the results.  Set to YES to include it.  For this record only,
+    directory=filepath.
   @param [out] outds= (work.mp_dirlist) The output dataset to create
   @param [out] getattrs= (NO)  If getattrs=YES then the doptname / foptname
     functions are used to scan all properties - any characters that are not
@@ -63,6 +66,7 @@
     , fref=0
     , outds=work.mp_dirlist
     , getattrs=NO
+    , showparent=NO
     , maxdepth=0
     , level=0 /* The level of recursion to perform.  For internal use only. */
 )/*/STORE SOURCE*/;
@@ -145,6 +149,15 @@ data &out_ds(compress=no
     output;
   end;
   rc = dclose(did);
+  %if &showparent=YES and &level=0 %then %do;
+    filepath=directory;
+    file_or_folder='folder';
+    ext='';
+    filename=scan(directory,-1,'/\');
+    msg='';
+    level=&level;
+    output;
+  %end;
   stop;
 run;
 
@@ -232,6 +245,9 @@ run;
   data _null_;
     set &out_ds;
     where file_or_folder='folder';
+  %if &showparent=YES and &level=0 %then %do;
+    if filepath ne directory;
+  %end;
     length code $10000;
     code=cats('%nrstr(%mp_dirlist(path=',filepath,",outds=&outds"
       ,",getattrs=&getattrs,level=%eval(&level+1),maxdepth=&maxdepth))");

base/mp_ds2md.sas

@@ -92,7 +92,7 @@ data _null_;
 run;
 
 %if %upcase(&showlog)=YES %then %do;
-  options ps=max;
+  options ps=max lrecl=max;
   data _null_;
     infile &outref;
     input;
@@ -100,4 +100,4 @@ run;
   run;
 %end;
 
-%mend mp_ds2md;
+%mend mp_ds2md;

base/mp_dsmeta.sas

@@ -0,0 +1,110 @@
+/**
+  @file
+  @brief Export dataset metadata to a single output table
+  @details Exports the dataset attributes and enginehost information, then
+  converts the datasets into a single output table in the following format:
+
+|ODS_TABLE:$10.|NAME:$100.|VALUE:$1000.|
+|---|---|---|
+|`ATTRIBUTES `|`Data Set Name `|`SASHELP.CLASS `|
+|`ATTRIBUTES `|`Observations `|`19 `|
+|`ATTRIBUTES `|`Member Type `|`DATA `|
+|`ATTRIBUTES `|`Variables `|`5 `|
+|`ATTRIBUTES `|`Engine `|`V9 `|
+|`ATTRIBUTES `|`Indexes `|`0 `|
+|`ATTRIBUTES `|`Created `|`06/08/2020 00:59:14 `|
+|`ATTRIBUTES `|`Observation Length `|`40 `|
+|`ATTRIBUTES `|`Last Modified `|`06/08/2020 00:59:14 `|
+|`ATTRIBUTES `|`Deleted Observations `|`0 `|
+|`ATTRIBUTES `|`Protection `|`. `|
+|`ATTRIBUTES `|`Compressed `|`NO `|
+|`ATTRIBUTES `|`Data Set Type `|`. `|
+|`ATTRIBUTES `|`Sorted `|`NO `|
+|`ATTRIBUTES `|`Label `|`Student Data `|
+|`ATTRIBUTES `|`Data Representation `|`SOLARIS_X86_64, LINUX_X86_64, ALPHA_TRU64, LINUX_IA64 `|
+|`ATTRIBUTES `|`Encoding `|`us-ascii  ASCII (ANSI) `|
+|`ENGINEHOST `|`Data Set Page Size `|`65536 `|
+|`ENGINEHOST `|`Number of Data Set Pages `|`1 `|
+|`ENGINEHOST `|`First Data Page `|`1 `|
+|`ENGINEHOST `|`Max Obs per Page `|`1632 `|
+|`ENGINEHOST `|`Obs in First Data Page `|`19 `|
+|`ENGINEHOST `|`Number of Data Set Repairs `|`0 `|
+|`ENGINEHOST `|`Filename `|`/opt/sas/sas9/SASHome/SASFoundation/9.4/sashelp/class.sas7bdat `|
+|`ENGINEHOST `|`Release Created `|`9.0401M7 `|
+|`ENGINEHOST `|`Host Created `|`Linux `|
+|`ENGINEHOST `|`Inode Number `|`28314616 `|
+|`ENGINEHOST `|`Access Permission `|`rw-r--r-- `|
+|`ENGINEHOST `|`Owner Name `|`sas `|
+|`ENGINEHOST `|`File Size `|`128KB `|
+|`ENGINEHOST `|`File Size (bytes) `|`131072 `|
+
+  Example usage:
+
+      %mp_dsmeta(sashelp.class,outds=work.mymeta)
+      proc print data=work.mymeta;
+      run;
+
+  For more details on creating datasets from PROC CONTENTS check out this
+  excellent [paper](
+  https://support.sas.com/resources/papers/proceedings14/1549-2014.pdf) by
+  [Louise Hadden](https://www.linkedin.com/in/louisehadden/).
+
+  @param libds The library.dataset to export the metadata for
+  @param outds= (work.dsmeta) The output table to contain the metadata
+
+  <h4> Related Files </h4>
+  @li mp_dsmeta.test.sas
+  @li mp_getcols.sas
+  @li mp_getdbml.sas
+  @li mp_getddl.sas
+  @li mp_getformats.sas
+  @li mp_getpk.sas
+  @li mp_guesspk.sas
+
+**/
+
+%macro mp_dsmeta(libds,outds=work.dsmeta);
+
+%local ds1 ds2;
+data;run; %let ds1=&syslast;
+data;run; %let ds2=&syslast;
+
+/* setup the ODS capture */
+ods output attributes=&ds1 enginehost=&ds2;
+
+/* export the metadata */
+proc contents data=&libds;
+run;
+
+/* load it into a single table */
+data &outds (keep=ods_table name value);
+  length ods_table $10 name label2 label1 label $100
+    value cvalue cvalue1 cvalue2 $1000
+    nvalue nvalue1 nvalue2 8;
+  if _n_=1 then call missing (of _all_);
+  * putlog (_all_)(=);
+  set &ds1 (in=atrs) &ds2 (in=eng);
+  if atrs then do;
+    ods_table='ATTRIBUTES';
+    name=coalescec(label1,label);
+    value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.));
+    output;
+    if label2 ne '' then do;
+    name=label2;
+    value=coalescec(cvalue2,put(nvalue2,best.));
+    output;
+    end;
+  end;
+  else if eng then do;
+    ods_table='ENGINEHOST';
+    name=coalescec(label1,label);
+    value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.));
+    output;
+  end;
+run;
+
+proc sql;
+drop table &ds1, &ds2;
+
+%mend mp_dsmeta;
+

base/mp_gitreleaseinfo.sas

@@ -0,0 +1,74 @@
+/**
+  @file
+  @brief Pulls latest release info from a GIT repository
+  @details Useful for grabbing the latest version number or other attributes
+  from a GIT server.  Supported providers are GitLab and GitHub. Pull requests
+  are welcome if you'd like to see additional providers!
+
+  Note that each provider provides slightly different JSON output.  Therefore
+  the macro simply extracts the JSON and assigns the libname (using the JSON
+  engine).
+
+  Example usage (eg, to grab latest release version from github):
+
+      %mp_gitreleaseinfo(GITHUB,sasjs/core,outlib=mylibref)
+
+      data _null_;
+        set mylibref.root;
+        putlog TAG_NAME=;
+      run;
+
+  @param [in] provider The GIT provider for the release info.  Accepted values:
+    @li GITLAB
+    @li GITHUB - Tables include root, assets, author, alldata
+  @param [in] project The link to the repository.  This has different formats
+    depending on the vendor:
+    @li GITHUB - org/repo, eg sasjs/core
+    @li GITLAB - project, eg 1343223
+  @param [in] server= (0) If your repo is self-hosted, then provide the domain
+    here.  Otherwise it will default to the provider domain (eg gitlab.com).
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outlib= (GITREL) The JSON-engine libref to be created, which will
+    point at the returned JSON
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+
+  <h4> Related Files </h4>
+  @li mp_gitreleaseinfo.test.sas
+
+**/
+
+%macro mp_gitreleaseinfo(provider,project,server=0,outlib=GITREL,mdebug=0);
+%local url fref;
+
+%let provider=%upcase(&provider);
+
+%if &provider=GITHUB %then %do;
+  %if "&server"="0" %then %let server=https://api.github.com;
+  %let url=&server/repos/&project/releases/latest;
+%end;
+%else %if &provider=GITLAB %then %do;
+  %if "&server"="0" %then %let server=https://gitlab.com;
+  %let url=&server/api/v4/projects/&project/releases;
+%end;
+
+%let fref=%mf_getuniquefileref();
+
+proc http method='GET' out=&fref url="&url";
+%if &mdebug=1 %then %do;
+  debug level = 3;
+%end;
+run;
+
+libname &outlib JSON fileref=&fref;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fref;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_gitreleaseinfo;

base/mp_hashdataset.sas

@@ -11,7 +11,7 @@
         put hashkey=;
       run;
 
-  ![sas md5 hash dataset log results](https://i.imgur.com/MqF98vk.png)
+  ![sas md5 hash dataset log results](https://i.4gl.io/1/KorUKoyE05.png/raw)
 
   <h4> SAS Macros </h4>
   @li mf_getattrn.sas
@@ -21,11 +21,12 @@
 
   <h4> Related Files </h4>
   @li mp_hashdataset.test.sas
+  @li mp_hashdirectory.sas
 
   @param [in] libds dataset to hash
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
-  @param [in] iftrue= A condition under which the macro should be executed.
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
-  @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
+  @param [out] outds= (work._data_) The output dataset to create. This
   will contain one column (hashkey) with one observation (a $hex32.
   representation of the input hash)
   |hashkey:$32.|

base/mp_hashdirectory.sas

@@ -0,0 +1,162 @@
+/**
+  @file
+  @brief Returns a unique hash for each file in a directory
+  @details Hashes each file in each directory, and then hashes the hashes to
+  create a hash for each directory also.
+
+  This makes use of the new `hashing_file()` and `hashing` functions, available
+  since 9.4m6. Interestingly, these can even be used in pure macro, eg:
+
+      %put %sysfunc(hashing_file(md5,/path/to/file.blob,0));
+
+  Actual usage:
+
+      %let fpath=/some/directory;
+
+      %mp_hashdirectory(&fpath,outds=myhash,maxdepth=2)
+
+      data _null_;
+        set work.myhash;
+        put (_all_)(=);
+      run;
+
+  Whilst files are hashed in their entirety, the logic for creating a folder
+  hash is as follows:
+
+  @li Sort the files by filename (case sensitive, uppercase then lower)
+  @li Take the first 100 hashes, concatenate and hash
+  @li Concatenate this hash with another 100 hashes and hash again
+  @li Continue until the end of the folder.  This is the folder hash
+  @li If a folder contains other folders, start from the bottom of the tree -
+    the folder hashes cascade upwards so you know immediately if there is a
+    change in a sub/sub directory
+  @li If the folder has no content (empty) then it is ignored. No hash created.
+  @li If the file is empty, it is also ignored / no hash created.
+
+  <h4> SAS Macros </h4>
+  @li mp_dirlist.sas
+
+  <h4> Related Files </h4>
+  @li mp_hashdataset.sas
+  @li mp_hashdirectory.test.sas
+  @li mp_md5.sas
+
+  @param [in] inloc Full filepath of the file to be hashed (unquoted)
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
+    subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
+  @param [in] method= (MD5) the hashing method to use.  Available options:
+    @li MD5
+    @li SH1
+    @li SHA256
+    @li SHA384
+    @li SHA512
+    @li CRC32
+  @param [out] outds= (work.mp_hashdirectory) The output dataset.  Contains:
+    @li directory - the parent folder
+    @li file_hash - the hash output
+    @li hash_duration - how long the hash took (first hash always takes longer)
+    @li file_path - /full/path/to/each/file.ext
+    @li file_or_folder - contains either "file" or "folder"
+    @li level - the depth of the directory (top level is 0)
+
+  @version 9.4m6
+  @author Allan Bowe
+**/
+
+%macro mp_hashdirectory(inloc,
+  outds=work.mp_hashdirectory,
+  method=MD5,
+  maxdepth=0,
+  iftrue=%str(1=1)
+)/*/STORE SOURCE*/;
+
+%local curlevel tempds ;
+
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+/* get the directory listing */
+%mp_dirlist(path=&inloc, outds=&outds, maxdepth=&maxdepth, showparent=YES)
+
+/* create the hashes */
+data &outds;
+  set &outds (rename=(filepath=file_path));
+  length FILE_HASH $32 HASH_DURATION 8;
+  keep directory file_hash hash_duration file_path file_or_folder level;
+
+  ts=datetime();
+  if file_or_folder='file' then do;
+    /* if file is empty, hashing_file will break - so ignore / delete */
+    length fname val $8;
+    drop fname val fid is_empty;
+    rc=filename(fname,file_path);
+    fid=fopen(fname);
+    if fid > 0 then do;
+      rc=fread(fid);
+      is_empty=fget(fid,val);
+    end;
+    rc=fclose(fid);
+    rc=filename(fname);
+    if is_empty ne 0 then delete;
+    else file_hash=hashing_file("&method",cats(file_path),0);
+  end;
+  hash_duration=datetime()-ts;
+run;
+
+proc sort data=&outds ;
+  by descending level directory file_path;
+run;
+
+data _null_;
+  set &outds;
+  call symputx('maxlevel',level,'l');
+  stop;
+run;
+
+/* now hash the hashes to populate folder hashes, starting from the bottom */
+%do curlevel=&maxlevel %to 0 %by -1;
+  data work._data_ (keep=directory file_hash);
+    set &outds;
+    where level=&curlevel;
+    by descending level directory file_path;
+    length str $32767 tmp_hash $32;
+    retain str tmp_hash ;
+    /* reset vars when starting a new directory */
+    if first.directory then do;
+      str='';
+      tmp_hash='';
+      i=0;
+    end;
+    /* hash each chunk of 100 file paths */
+    i+1;
+    str=cats(str,file_hash);
+    if mod(i,100)=0 or last.directory then do;
+      tmp_hash=hashing("&method",cats(tmp_hash,str));
+      str='';
+    end;
+    /* output the hash at directory level */
+    if last.directory then do;
+      file_hash=tmp_hash;
+      output;
+    end;
+    if last.level then stop;
+  run;
+  %let tempds=&syslast;
+  /* join the hash back into the main table */
+  proc sql undo_policy=none;
+  create table &outds as
+    select a.directory
+      ,coalesce(b.file_hash,a.file_hash) as file_hash
+      ,a.hash_duration
+      ,a.file_path
+      ,a.file_or_folder
+      ,a.level
+    from &outds a
+    left join &tempds b
+    on a.file_path=b.directory
+    order by level desc, directory, file_path;
+  drop table &tempds;
+%end;
+
+%mend mp_hashdirectory;

base/mp_jsonout.sas

@@ -146,7 +146,7 @@
     call symputx(cats('label',_n_),coalescec(label,name),'l');
     /* overwritten when fmt=Y and a custom format exists in catalog */
     if typelong='num' then call symputx(cats('fmtlen',_n_),200,'l');
-    else call symputx(cats('fmtlen',_n_),min(32767,ceil((length+3)*1.5)),'l');
+    else call symputx(cats('fmtlen',_n_),min(32767,ceil((length+10)*1.5)),'l');
   run;
 
   %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
@@ -202,8 +202,8 @@
       %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
       proc sql noprint;
       create table &tmpds1 as
-          select cats(libname,'.',memname) as fmtcat,
+          select cats(libname,'.',memname) as FMTCAT,
-          fmtname
+          FMTNAME
         from dictionary.formats
         where fmttype='F' and libname is not null
           and fmtname in (select format from &colinfo where format is not null)
@@ -228,7 +228,7 @@
 
       proc sql;
       create table &tmpds4 as
-        select a.*, b.length as maxw
+        select a.*, b.length as MAXW
         from &colinfo a
         left join &tmpds2 b
         on cats(a.format)=cats(upcase(b.fmtname))
@@ -239,7 +239,7 @@
         call symputx(
           cats('fmtlen',_n_),
           /* vars need extra padding due to JSON escaping of special chars */
-          min(32767,ceil((max(length,maxw)+3)*1.5))
+          min(32767,ceil((max(length,maxw)+10)*1.5))
           ,'l'
         );
       run;
@@ -314,7 +314,7 @@
       format _numeric_ bart.;
     %do i=1 %to &numcols;
       %if &&typelong&i=char or &fmt=Y %then %do;
-        if findc(&&name&i,'"\'!!'0A0D09000E0F01021011'x) then do;
+        if findc(&&name&i,'"\'!!'0A0D09000E0F010210111A'x) then do;
           &&name&i='"'!!trim(
             prxchange('s/"/\\"/',-1,        /* double quote */
             prxchange('s/\x0A/\n/',-1,      /* new line */
@@ -327,8 +327,9 @@
             prxchange('s/\x02/\\u0002/',-1, /* STX */
             prxchange('s/\x10/\\u0010/',-1, /* DLE */
             prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+            prxchange('s/\x1A/\\u001A/',-1, /* SUB */
             prxchange('s/\\/\\\\/',-1,&&name&i)
-          ))))))))))))!!'"';
+          )))))))))))))!!'"';
         end;
         else &&name&i=quote(cats(&&name&i));
       %end;

meta/mm_createwebservice.sas

@@ -169,7 +169,7 @@ data _null_;
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
   put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
   put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
-  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); ';
+  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
@@ -225,8 +225,8 @@ data _null_;
   put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
   put '      proc sql noprint; ';
   put '      create table &tmpds1 as ';
-  put '          select cats(libname,''.'',memname) as fmtcat, ';
+  put '          select cats(libname,''.'',memname) as FMTCAT, ';
-  put '          fmtname ';
+  put '          FMTNAME ';
   put '        from dictionary.formats ';
   put '        where fmttype=''F'' and libname is not null ';
   put '          and fmtname in (select format from &colinfo where format is not null) ';
@@ -251,7 +251,7 @@ data _null_;
   put ' ';
   put '      proc sql; ';
   put '      create table &tmpds4 as ';
-  put '        select a.*, b.length as maxw ';
+  put '        select a.*, b.length as MAXW ';
   put '        from &colinfo a ';
   put '        left join &tmpds2 b ';
   put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
@@ -262,7 +262,7 @@ data _null_;
   put '        call symputx( ';
   put '          cats(''fmtlen'',_n_), ';
   put '          /* vars need extra padding due to JSON escaping of special chars */ ';
-  put '          min(32767,ceil((max(length,maxw)+3)*1.5)) ';
+  put '          min(32767,ceil((max(length,maxw)+10)*1.5)) ';
   put '          ,''l'' ';
   put '        ); ';
   put '      run; ';
@@ -337,7 +337,7 @@ data _null_;
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; ';
+  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
   put '          &&name&i=''"''!!trim( ';
   put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
   put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
@@ -350,8 +350,9 @@ data _null_;
   put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
   put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
   put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
   put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '          ))))))))))))!!''"''; ';
+  put '          )))))))))))))!!''"''; ';
   put '        end; ';
   put '        else &&name&i=quote(cats(&&name&i)); ';
   put '      %end; ';

meta/mm_getstpinfo.sas

@@ -58,7 +58,7 @@ data &outds;
     rc5=metadata_getattr(tsuri,"Name",servercontext);
   end;
   else do;
-    put "%str(ERR)OR: could not find " pgm;
+    put "%str(ERR)OR: could not find " path;
     put (_all_)(=);
   end;
   &md.put (_all_)(=);

server/ms_createwebservice.sas

@@ -170,7 +170,7 @@ data _null_;
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
   put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
   put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
-  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); ';
+  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
@@ -226,8 +226,8 @@ data _null_;
   put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
   put '      proc sql noprint; ';
   put '      create table &tmpds1 as ';
-  put '          select cats(libname,''.'',memname) as fmtcat, ';
+  put '          select cats(libname,''.'',memname) as FMTCAT, ';
-  put '          fmtname ';
+  put '          FMTNAME ';
   put '        from dictionary.formats ';
   put '        where fmttype=''F'' and libname is not null ';
   put '          and fmtname in (select format from &colinfo where format is not null) ';
@@ -252,7 +252,7 @@ data _null_;
   put ' ';
   put '      proc sql; ';
   put '      create table &tmpds4 as ';
-  put '        select a.*, b.length as maxw ';
+  put '        select a.*, b.length as MAXW ';
   put '        from &colinfo a ';
   put '        left join &tmpds2 b ';
   put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
@@ -263,7 +263,7 @@ data _null_;
   put '        call symputx( ';
   put '          cats(''fmtlen'',_n_), ';
   put '          /* vars need extra padding due to JSON escaping of special chars */ ';
-  put '          min(32767,ceil((max(length,maxw)+3)*1.5)) ';
+  put '          min(32767,ceil((max(length,maxw)+10)*1.5)) ';
   put '          ,''l'' ';
   put '        ); ';
   put '      run; ';
@@ -338,7 +338,7 @@ data _null_;
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; ';
+  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
   put '          &&name&i=''"''!!trim( ';
   put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
   put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
@@ -351,8 +351,9 @@ data _null_;
   put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
   put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
   put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
   put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '          ))))))))))))!!''"''; ';
+  put '          )))))))))))))!!''"''; ';
   put '        end; ';
   put '        else &&name&i=quote(cats(&&name&i)); ';
   put '      %end; ';

tests/base/mp_dictionary.test.sas

@@ -0,0 +1,26 @@
+/**
+  @file
+  @brief Testing mp_dictionary.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_dictionary.sas
+  @li mp_assert.sas
+
+**/
+
+libname test (work);
+%mp_dictionary(lib=test)
+
+proc sql;
+create table work.compare1 as select * from test.styles;
+create table work.compare2 as select * from dictionary.styles;
+
+proc compare base=compare1 compare=compare2;
+run;
+%put _all_;
+
+%mp_assert(
+  iftrue=(%mf_existds(&sysinfo)=0),
+  desc=Compare was exact,
+  outds=work.test_results
+)

tests/base/mp_dsmeta.test.sas

@@ -0,0 +1,32 @@
+/**
+  @file
+  @brief Testing mp_dsmeta.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mp_dsmeta.sas
+
+**/
+
+data work.Example;
+  set sashelp.vmacro;
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mp_dsmeta(work.example,outds=work.test)
+%mp_assertscope(COMPARE)
+
+proc sql noprint;
+select count(*) into: nobs from work.test;
+select count(distinct ods_table) into: tnobs from work.test;
+
+%mp_assert(
+  iftrue=(&tnobs=2),
+  desc=Check that both ATTRIBUTES and ENGINEHOST are provided
+)
+%mp_assert(
+  iftrue=(&nobs>10),
+  desc=Check that sufficient details are provided
+)
+

tests/base/mp_gitreleaseinfo.test.sas

@@ -0,0 +1,30 @@
+/**
+  @file
+  @brief Testing mp_gitreleaseinfo.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_gitreleaseinfo.sas
+  @li mp_assert.sas
+
+**/
+
+
+%mp_gitreleaseinfo(github,sasjs/core,outlib=mylibref,mdebug=1)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=mp_gitreleaseinfo runs without errors,
+  outds=work.test_results
+)
+
+data _null_;
+  set mylibref.author;
+  putlog (_all_)(=);
+  call symputx('author',login);
+run;
+
+%mp_assert(
+  iftrue=(&author=sasjsbot),
+  desc=release info extracted successfully,
+  outds=work.test_results
+)

tests/base/mp_hashdirectory.test.sas

@@ -0,0 +1,133 @@
+/**
+  @file
+  @brief Testing mp_hashdirectory.sas macro
+
+
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mf_nobs.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mp_hashdirectory.sas
+
+**/
+
+/* set up a directory to hash */
+%let fpath=%sysfunc(pathname(work))/testdir;
+
+%mf_mkdir(&fpath)
+%mf_mkdir(&fpath/sub1)
+%mf_mkdir(&fpath/sub2)
+%mf_mkdir(&fpath/sub1/subsub)
+
+/* note - the path in the file means the hash is different in each run */
+%macro makefile(path,name);
+  data _null_;
+    file "&path/&name" termstr=lf;
+    put "This file is located at:";
+    put "&path";
+    put "and it is called:";
+    put "&name";
+  run;
+%mend makefile;
+
+%macro spawner(path);
+  %do x=1 %to 5;
+    %makefile(&path,file&x..txt)
+  %end;
+%mend spawner;
+
+%spawner(&fpath)
+%spawner(&fpath/sub1)
+%spawner(&fpath/sub1/subsub)
+
+
+%mp_assertscope(SNAPSHOT)
+%mp_hashdirectory(&fpath,outds=work.hashes,maxdepth=MAX)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=No errors,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.hashes)=19),
+  desc=record created for each entry,
+  outds=work.test_results
+)
+
+proc sql;
+select count(*) into: misscheck
+  from work.hashes
+  where file_hash is missing;
+
+%mp_assert(
+  iftrue=(&misscheck=1),
+  desc=Only one missing hash - the empty directory,
+  outds=work.test_results
+)
+
+data _null_;
+  set work.hashes;
+  if directory=file_path then call symputx('tophash',file_hash);
+run;
+
+%mp_assert(
+  iftrue=(%length(&tophash)=32),
+  desc=ensure valid top level hash created,
+  outds=work.test_results
+)
+
+/* now change a file and re-hash */
+data _null_;
+  file "&fpath/sub1/subsub/file1.txt" termstr=lf;
+  put "This file has changed!";
+run;
+
+%mp_hashdirectory(&fpath,outds=work.hashes2,maxdepth=MAX)
+
+data _null_;
+  set work.hashes2;
+  if directory=file_path then call symputx('tophash2',file_hash);
+run;
+
+%mp_assert(
+  iftrue=(&tophash ne &tophash2),
+  desc=ensure the changing of the hash results in a new value,
+  outds=work.test_results
+)
+
+/* now change it back and see if it matches */
+data _null_;
+  file "&fpath/sub1/subsub/file1.txt" termstr=lf;
+    put "This file is located at:";
+    put "&fpath/sub1/subsub";
+    put "and it is called:";
+    put "file1.txt";
+  run;
+run;
+
+%mp_hashdirectory(&fpath,outds=work.hashes3,maxdepth=MAX)
+
+data _null_;
+  set work.hashes3;
+  if directory=file_path then call symputx('tophash3',file_hash);
+run;
+
+%mp_assert(
+  iftrue=(&tophash=&tophash3),
+  desc=ensure the same files result in the same hash,
+  outds=work.test_results
+)
+
+/* dump contents for debugging */
+data _null_;
+  set work.hashes;
+  put file_hash file_path;
+run;
+data _null_;
+  set work.hashes2;
+  put file_hash file_path;
+run;

viya/mfv_existsashdat.sas

@@ -0,0 +1,60 @@
+/**
+  @file mfv_existsashdat.sas
+  @brief Checks whether a CAS sashdat dataset exists in persistent storage.
+  @details Can be used in open code, eg as follows:
+
+      %if %mfv_existsashdat(libds=casuser.sometable) %then %put  yes it does!;
+
+  The function uses `dosubl()` to run the `table.fileinfo` action, for the
+  specified library, filtering for `*.sashdat` tables.  The results are stored
+  in a WORK table (&outprefix._&lib). If that table already exists, it is
+  queried instead, to avoid the dosubl() performance hit.
+
+  To force a rescan, just use a new `&outprefix` value, or delete the table(s)
+  before running the function.
+
+  @param libds library.dataset
+  @param outprefix= (work.mfv_existsashdat) Used to store the current HDATA
+    tables to improve subsequent query performance.  This reference is a prefix
+    and is converted to `&prefix._{libref}`
+
+  @return output returns 1 or 0
+
+  @version 0.2
+  @author Mathieu Blauw
+**/
+
+%macro mfv_existsashdat(libds,outprefix=work.mfv_existsashdat
+);
+%local rc dsid name lib ds;
+%let lib=%upcase(%scan(&libds,1,'.'));
+%let ds=%upcase(%scan(&libds,-1,'.'));
+
+/* if table does not exist, create it */
+%if %sysfunc(exist(&outprefix._&lib)) ne 1 %then %do;
+  %let rc=%sysfunc(dosubl(%nrstr(
+    /* Read in table list (once per &lib per session) */
+    proc cas;
+      table.fileinfo result=source_list /caslib="&lib";
+      val=findtable(source_list);
+      saveresult val dataout=&outprefix._&lib;
+    quit;
+    /* Only keep name, without file extension */
+    data &outprefix._&lib;
+      set &outprefix._&lib(where=(Name like '%.sashdat') keep=Name);
+      Name=upcase(scan(Name,1,'.'));
+    run;
+  )));
+%end;
+
+/* Scan table for hdat existence */
+%let dsid=%sysfunc(open(&outprefix._&lib(where=(name="&ds"))));
+%syscall set(dsid);
+%let rc = %sysfunc(fetch(&dsid));
+%let rc = %sysfunc(close(&dsid));
+
+/* Return result */
+%if "%trim(&name)"="%trim(&ds)" %then 1;
+%else 0;
+
+%mend mfv_existsashdat;

viya/mv_createfile.sas

@@ -24,7 +24,8 @@
   @param [in] contentdisp= (inline) Content Disposition. Example values:
     @li inline
     @li attachment
-
+  @param [in] ctype= (0) Set a default HTTP Content-Type header to be returned
+    with the file when the content is retrieved from the Files service.
   @param [in] access_token_var= The global macro variable to contain the access
     token, if using authorization_code grant type.
   @param [in] grant_type= (sas_services) Valid values are:
@@ -52,6 +53,7 @@
     ,inref=
     ,intype=BINARY
     ,contentdisp=inline
+    ,ctype=0
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
     ,mdebug=0
@@ -103,8 +105,10 @@ filename &fref filesrvc
   folderPath="&path"
   filename="&name"
   cdisp="&contentdisp"
+%if "&ctype" ne "0" %then %do;
+  ctype="&ctype"
+%end;
   lrecl=1048544;
-
 %if &intype=BINARY %then %do;
   %mp_binarycopy(inref=&inref, outref=&fref)
 %end;

viya/mv_createjob.sas

@@ -34,14 +34,25 @@
 
   @param path= The full path (on SAS Drive) where the job will be created
   @param name= The name of the job
-  @param desc= The description of the job
+  @param desc= (Created by the mv_createjob.sas macro) The job description
   @param precode= Space separated list of filerefs, pointing to the code that
     needs to be attached to the beginning of the job
-  @param code= Fileref(s) of the actual code to be added
+  @param code= (ft15f001) Fileref(s) of the actual code to be added
-  @param access_token_var= The global macro variable to contain the access token
+  @param access_token_var= (ACCESS_TOKEN) Global macro variable containing the
-  @param grant_type= valid values are "password" or "authorization_code"
+    access token
-    (unquoted). The default is authorization_code.
+  @param grant_type= (sas_services) Valid values:
-  @param replace= select NO to avoid replacing any existing job in that location
+    @li sas_services
+    @li detect
+    @li authorization_code
+    @li password
+  @param replace= (YES) select NO to avoid replacing any existing job
+  @param addjesbeginendmacros= (false) Relates to the `_addjesbeginendmacros`
+    setting.  Normally this would always be false however due to a Viya bug
+    (https://github.com/sasjs/cli/issues/1229) this is now configurable.  Valid
+    values:
+    @li true
+    @li false
+    @li 0 - this will prevent the flag from being set (job will default to true)
   @param contextname= Choose a specific context on which to run the Job.  Leave
     blank to use the default context.  From Viya 3.5 it is possible to configure
     a shared context - see
@@ -62,6 +73,7 @@ https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5p
     ,replace=YES
     ,debug=0
     ,contextname=
+    ,addjesbeginendmacros=false
   );
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
@@ -185,19 +197,29 @@ run;
 %end;
 
 /* set up the body of the request to create the service */
-%local fname3;
+%local fname3 comma;
 %let fname3=%mf_getuniquefileref();
 data _null_;
   file &fname3 TERMSTR=' ';
   length string $32767;
   string=cats('{"version": 0,"name":"'
     ,"&name"
-    ,'","type":"Compute","parameters":[{"name":"_addjesbeginendmacros"'
+    ,'","type":"Compute","parameters":['
-    ,',"type":"CHARACTER","defaultValue":"false"}');
+%if &addjesbeginendmacros ne 0 %then %do;
+    ,'{"name":"_addjesbeginendmacros"'
+    ,',"type":"CHARACTER","defaultValue":"'
+    ,"&addjesbeginendmacros"
+    ,'"}'
+  %let comma=%str(,);
+%end;
+  );
   context=quote(cats(symget('contextname')));
   if context ne '""' then do;
-    string=cats(string,',{"version": 1,"name": "_contextName","defaultValue":'
+    string=cats(string
-      ,context,',"type":"CHARACTER","label":"Context Name","required": false}');
+      ,"&comma"
+      ,'{"version": 1,"name": "_contextName","defaultValue":'
+      ,context,',"type":"CHARACTER","label":"Context Name","required": false}'
+    );
   end;
   string=cats(string,'],"code":"');
   put string;

viya/mv_createwebservice.sas

@@ -312,7 +312,7 @@ data _null_;
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
   put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
   put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
-  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); ';
+  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
@@ -368,8 +368,8 @@ data _null_;
   put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
   put '      proc sql noprint; ';
   put '      create table &tmpds1 as ';
-  put '          select cats(libname,''.'',memname) as fmtcat, ';
+  put '          select cats(libname,''.'',memname) as FMTCAT, ';
-  put '          fmtname ';
+  put '          FMTNAME ';
   put '        from dictionary.formats ';
   put '        where fmttype=''F'' and libname is not null ';
   put '          and fmtname in (select format from &colinfo where format is not null) ';
@@ -394,7 +394,7 @@ data _null_;
   put ' ';
   put '      proc sql; ';
   put '      create table &tmpds4 as ';
-  put '        select a.*, b.length as maxw ';
+  put '        select a.*, b.length as MAXW ';
   put '        from &colinfo a ';
   put '        left join &tmpds2 b ';
   put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
@@ -405,7 +405,7 @@ data _null_;
   put '        call symputx( ';
   put '          cats(''fmtlen'',_n_), ';
   put '          /* vars need extra padding due to JSON escaping of special chars */ ';
-  put '          min(32767,ceil((max(length,maxw)+3)*1.5)) ';
+  put '          min(32767,ceil((max(length,maxw)+10)*1.5)) ';
   put '          ,''l'' ';
   put '        ); ';
   put '      run; ';
@@ -480,7 +480,7 @@ data _null_;
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; ';
+  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; ';
   put '          &&name&i=''"''!!trim( ';
   put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
   put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
@@ -493,8 +493,9 @@ data _null_;
   put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
   put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
   put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ ';
   put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '          ))))))))))))!!''"''; ';
+  put '          )))))))))))))!!''"''; ';
   put '        end; ';
   put '        else &&name&i=quote(cats(&&name&i)); ';
   put '      %end; ';