Merge pull request #409 from sasjs/build/disable-npm-scripts

Disable npm scripts

.github/vpn/config.ovpn

@@ -3,10 +3,12 @@ client
 tls-client
 dev tun
 # this will connect with whatever proto DNS tells us (https://community.openvpn.net/openvpn/ticket/934)
-proto tcp
-remote vpn.4gl.io 7494
+proto udp
+remote vpn.4gl.io 7194
 resolv-retry infinite
-cipher AES-256-CBC
+# this will fallback from udp6 to udp4 as well
+connect-timeout 5
+data-ciphers AES-256-CBC:AES-256-GCM
 auth SHA256
 script-security 2
 keepalive 10 120

.github/workflows/run-tests.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     strategy:
       matrix:
@@ -34,6 +34,10 @@ jobs:
           USER_KEY: ${{ secrets.USER_KEY }}
           TLS_KEY: ${{ secrets.TLS_KEY }}
 
+      - name: Chmod VPN files
+        run: |
+          chmod 600 .github/vpn/ca.crt .github/vpn/user.crt .github/vpn/user.key .github/vpn/tls.key
+
       - name: Install Open VPN
         run: |
           sudo apt install apt-transport-https
@@ -42,8 +46,13 @@ jobs:
           sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
           sudo apt update
           sudo apt install openvpn3=17~betaUb22042+jammy
+
       - name: Start Open VPN 3
         run: openvpn3 session-start --config .github/vpn/config.ovpn
+
+      - name: Fetch SASJS server
+        run: curl ${{ secrets.SASJS_SERVER_URL }}/SASjsApi/info
+
       - name: Install Doxygen
         run: sudo apt-get install doxygen
 

.npmrc

@@ -0,0 +1 @@
+ignore-scripts=true

README.md

@@ -1,19 +1,12 @@
 # Macro Core
-[![npm package][npm-image]][npm-url]
-[![Github Workflow][githubworkflow-image]][githubworkflow-url]
+
+[![npm package](https://img.shields.io/npm/v/@sasjs/core.svg)](http://npmjs.org/package/@sasjs/core)
+[![Github Workflow](https://github.com/sasjs/core/actions/workflows/main.yml/badge.svg)](https://github.com/sasjs/core/blob/main/.github/workflows/main.yml)
 ![npm](https://img.shields.io/npm/dt/@sasjs/core)
 ![GitHub top language](https://img.shields.io/github/languages/top/sasjs/core)
 [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sasjs/core)](https://github.com/sasjs/core/issues?q=is%3Aissue+is%3Aclosed)
 [![GitHub issues](https://img.shields.io/github/issues-raw/sasjs/core)](https://github.com/sasjs/core/issues)
 ![total lines](https://tokei.rs/b1/github/sasjs/core)
-[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-908a85?logo=gitpod)](https://gitpod.io/#https://github.com/sasjs/core)
-
-
-[npm-image]:https://img.shields.io/npm/v/@sasjs/core.svg
-[npm-url]:http://npmjs.org/package/@sasjs/core
-[githubworkflow-image]:https://github.com/sasjs/core/actions/workflows/main.yml/badge.svg
-[githubworkflow-url]:https://github.com/sasjs/core/blob/main/.github/workflows/main.yml
-[dependency-url]:https://github.com/sasjs/core/blob/main/package.json
 
 
 Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/.github/CONTRIBUTING.md) are welcome.
@@ -147,16 +140,17 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 - macro names must be lowercase
 - one macro per file
 - prefixes:
-  - _mcf_ for macro compiled functions (proc fcmp)
-  - _mddl_ for macros containing DDL (Data Definition Language)
-  - _mf_ for macro functions (can be used in open code).
-  - _ml_ for macros that are used to compile LUA modules
-  - _mm_ for metadata macros (interface with the metadata server).
-  - _mmx_ for macros that use metadata and are XCMD enabled (working on both windows and unix)
-  - _mp_ for macro procedures (which generate sas code)
-  - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
-  - _mv_ for macro procedures that will only work in Viya
-  - _mx_ for macros that work on Viya, SAS 9 EBI and SASjs Server
+  - _mcf__: macro compiled functions (proc fcmp)
+  - _mddl__: macros containing DDL (Data Definition Language)
+  - _mf__: macro functions (can be used in open code).
+  - _mfv__: macro functions that work only in Viya
+  - _ml__: macros that are used to compile LUA modules
+  - _mm__: metadata macros (interface with the metadata server).
+  - _mmx__: macros that use metadata and are XCMD enabled (working on both windows and unix)
+  - _mp__: macro procedures (which generate sas code)
+  - _ms__: macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
+  - _mv__: macro procedures that will only work in Viya
+  - _mx__: macros that work on Viya, SAS 9 EBI and SASjs Server
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
@@ -219,12 +213,13 @@ When contributing to this library, it is therefore important to ensure that all
 
 We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major/breaking release (v5) becomes necessary:
 
-* mf_getuniquelibref.sas to have the deprecated maxtried parameter removed (no longer needed)
-* mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
-* `insert_cmplib` option of mcf_xxx macros will be deprecated (the option is now checked automatically with value inserted only if needed)
+* mcf_xxx macros to have `insert_cmplib` option deprecated (the option is now checked automatically with value inserted only if needed)
 * mcf_xxx macros to have `wrap=` option defaulted to YES for convenience.  Set this option explicitly to avoid issues.
-* mp_getddl.sas to be renamed to mp_ds2ddl.sas (consistent with other ds2xxx macros).  A wrapper macro is already in place, and you are able to use this immediately.  The default for SHOWLOG will also be YES instead of NO.
+* mf_getuniquelibref.sas to have the deprecated maxtries parameter removed (no longer needed)
+* mp_abort.sas will have the redundant type= parameter removed.
 * mp_coretable.sas will be replaced by the standalone macros in the `ddl` folder (which are already available)
+* mp_getddl.sas to be renamed to mp_ds2ddl.sas (consistent with other ds2xxx macros).  A wrapper macro is already in place, and you are able to use this immediately.  The default for SHOWLOG will also be YES instead of NO.
+* mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
 
 ## Star Gazing
 
@@ -237,6 +232,7 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 The following repositories are also worth checking out:
 
 * [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
+* [Criptic/sas_snippets](https://github.com/Criptic/sas_snippets)
 * [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)
 * [KatjaGlassConsulting/SMILE-SmartSASMacros](https://github.com/KatjaGlassConsulting/SMILE-SmartSASMacros)
 * [paul-canals/toolbox](https://github.com/paul-canals/toolbox)

base/mf_getuniquelibref.sas

@@ -26,7 +26,7 @@
   @author Allan Bowe
 **/
 
-%macro mf_getuniquelibref(prefix=mclib,maxtries=1000);
+%macro mf_getuniquelibref(prefix=mc,maxtries=1000);
   %local x;
 
   %if ( %length(&prefix) gt 7 ) %then %do;

base/mf_mimetype.sas

@@ -0,0 +1,436 @@
+/**
+  @file
+  @brief Returns a mime type from a file extension
+  @details Provide a file extension and get a mime type.
+    The mappings were derived from this source:
+    https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
+
+
+
+  @param [in] ext The file extension (unquoted)
+  @return output The mime type
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_mimetype(
+    ext
+)/*/STORE SOURCE*/ /minoperator mindelimiter=' ';
+
+%let ext=%lowcase(&ext);
+
+%if &ext in (sas txt text conf def list log)
+%then %do;%str(text/plain)%end;
+%else %if &ext=xlsx %then %do;
+%str(application/vnd.openxmlformats-officedocument.spreadsheetml.sheet)%end;
+%else %if &ext in (xls xlm xla xlc xlt xlw)
+%then %do;%str(application/vnd.ms-excel)%end;
+%else %if &ext=xlsm
+%then %do;%str(application/vnd.ms-excel.sheet.macroenabled.12)%end;
+%else %if &ext=xlsb
+%then %do;%str(application/vnd.ms-excel.sheet.binary.macroenabled.12)%end;
+%else %if &ext in (css csv html n3 sgml vcard)
+%then %do;%str(text/&ext)%end;
+%else %if &ext in (avif bmp cgm gif ief jxl ktx png sgi tiff webp)
+%then %do;%str(image/&ext)%end;
+%else %if &ext in (exi gxf ipfix json mbox mp21 mxf oda oxps pdf rtf sdp wasm
+  xml yang zip)
+%then %do;%str(application/&ext)%end;
+%else %if &ext in (jpeg jpg jpe) %then %do;%str(image/jpeg)%end;
+%else %if &ext in (mp4 mp4v mpg4) %then %do;%str(video/mp4)%end;
+%else %if &ext in (otf ttf woff woff2) %then %do;%str(font/&ext)%end;
+%else %if &ext in (mpeg mpg mpe m1v m2v)
+%then %do;%str(video/mpeg)%end;
+%else %if &ext in (h261 h263 h264 jpm mj2 webm)
+%then %do;%str(video/&ext)%end;
+%else %if &ext in (f4v fli flv m4v mng smv)
+%then %do;%str(video/x-&ext)%end;
+%else %if &ext in (3ds cmx pcx rgb tga)
+%then %do;%str(image/x-&ext)%end;
+%else %if &ext in (asm nfo opml sfv)
+%then %do;%str(text/x-&ext)%end;
+%else %if &ext in (aac caf flac wav)
+%then %do;%str(audio/x-&ext)%end;
+%else %if &ext in (ts m2t m2ts mts)
+%then %do;%str(video/mp2t)%end;
+%else %if &ext in (pfa pfb pfm afm)
+%then %do;%str(application/x-font-type1)%end;
+%else %if &ext in (oga ogg spx opus)
+%then %do;%str(audio/ogg)%end;
+%else %if &ext in (mid midi kar rmi)
+%then %do;%str(audio/midi)%end;
+%else %if &ext in (onetoc onetoc2 onetmp onepkg)
+%then %do;%str(application/onenote)%end;
+%else %if &ext in (mxml xhvml xvml xvm)
+%then %do;%str(application/xv+xml)%end;
+%else %if &ext in (f for f77 f90)
+%then %do;%str(text/x-fortran)%end;
+%else %if &ext in (wmf wmz emf emz)
+%then %do;%str(application/x-msmetafile)%end;
+%else %if &ext in (exe dll com bat msi)
+%then %do;%str(application/x-msdownload)%end;
+%else %if &ext in (bin dms lrf mar so dist distz pkg bpk dump elc deploy)
+%then %do;%str(application/octet-stream)%end;
+%else %if &ext in (atom atomcat atomsvc ccxml davmount emma gml gpx inkml mads
+  mathml metalink mets mods omdoc pls rdf rsd rss sbml shf smil sru ssdl ssml
+  tei wsdl wspolicy xaml xenc xhtml xop xslt xspf yin)
+%then %do;%str(application/&ext+xml)%end;
+%else %if &ext in (dir dcr dxr cst cct cxt w3d fgd swa)
+%then %do;%str(application/x-director)%end;
+%else %if &ext in (z1 z2 z3 z4 z5 z6 z7 z8)
+%then %do;%str(application/x-zmachine)%end;
+%else %if &ext in (c cc cxx cpp h hh dic)
+%then %do;%str(text/x-c)%end;
+%else %if &ext in (mpga mp2 mp2a mp3 m2a m3a)
+%then %do;%str(audio/mpeg)%end;
+%else %if &ext in (t tr roff man me ms)
+%then %do;%str(text/troff)%end;
+%else %if &ext in (cbr cba cbt cbz cb7)
+%then %do;%str(application/x-cbr)%end;
+%else %if &ext in (fh fhc fh4 fh5 fh7)
+%then %do;%str(image/x-freehand)%end;
+%else %if &ext in (aab x32 u32 vox)
+%then %do;%str(application/x-authorware-bin)%end;
+%else %if &ext in (uvi uvvi uvg uvvg)
+%then %do;%str(image/vnd.dece.graphic)%end;
+%else %if &ext in (cdx cif cmdf cml csml xyz)
+%then %do;%str(chemical/x-&ext)%end;
+%else %if &ext in (aif aiff aifc) %then %do;%str(audio/x-aiff)%end;
+%else %if &ext in (ma nb mb) %then %do;%str(application/mathematica)%end;
+%else %if &ext in (mvb m13 m14) %then %do;%str(application/x-msmediaview)%end;
+%else %if &ext in (msh mesh silo) %then %do;%str(model/mesh)%end;
+%else %if &ext in (uri uris urls) %then %do;%str(text/uri-list)%end;
+%else %if &ext in (mkv mk3d mks) %then %do;%str(video/x-matroska)%end;
+%else %if &ext=ez %then %do;%str(application/andrew-inset)%end;
+%else %if &ext=aw %then %do;%str(application/applixware)%end;
+%else %if &ext=cdmia %then %do;%str(application/cdmi-capability)%end;
+%else %if &ext=cdmic %then %do;%str(application/cdmi-container)%end;
+%else %if &ext=cdmid %then %do;%str(application/cdmi-domain)%end;
+%else %if &ext=cdmio %then %do;%str(application/cdmi-object)%end;
+%else %if &ext=cdmiq %then %do;%str(application/cdmi-queue)%end;
+%else %if &ext=cu %then %do;%str(application/cu-seeme)%end;
+%else %if &ext=dssc %then %do;%str(application/dssc+der)%end;
+%else %if &ext=xdssc %then %do;%str(application/dssc+xml)%end;
+%else %if &ext=ecma %then %do;%str(application/ecmascript)%end;
+%else %if &ext=epub %then %do;%str(application/epub+zip)%end;
+%else %if &ext=pfr %then %do;%str(application/font-tdpfr)%end;
+%else %if &ext=stk %then %do;%str(application/hyperstudio)%end;
+%else %if &ext=ink %then %do;%str(application/inkml+xml)%end;
+%else %if &ext=jar %then %do;%str(application/java-archive)%end;
+%else %if &ext=ser %then %do;%str(application/java-serialized-object)%end;
+%else %if &ext=class %then %do;%str(application/java-vm)%end;
+%else %if &ext=jsonml %then %do;%str(application/jsonml+json)%end;
+%else %if &ext=lostxml %then %do;%str(application/lost+xml)%end;
+%else %if &ext=hqx %then %do;%str(application/mac-binhex40)%end;
+%else %if &ext=cpt %then %do;%str(application/mac-compactpro)%end;
+%else %if &ext=mrc %then %do;%str(application/marc)%end;
+%else %if &ext=mrcx %then %do;%str(application/marcxml+xml)%end;
+%else %if &ext=mscml %then %do;%str(application/mediaservercontrol+xml)%end;
+%else %if &ext=meta4 %then %do;%str(application/metalink4+xml)%end;
+%else %if &ext=m21 %then %do;%str(application/mp21)%end;
+%else %if &ext=mp4s %then %do;%str(application/mp4)%end;
+%else %if &ext=doc %then %do;%str(application/msword)%end;
+%else %if &ext=dot %then %do;%str(application/msword)%end;
+%else %if &ext=opf %then %do;%str(application/oebps-package+xml)%end;
+%else %if &ext=ogx %then %do;%str(application/ogg)%end;
+%else %if &ext=xer %then %do;%str(application/patch-ops-error+xml)%end;
+%else %if &ext=pgp %then %do;%str(application/pgp-encrypted)%end;
+%else %if &ext=asc %then %do;%str(application/pgp-signature)%end;
+%else %if &ext=sig %then %do;%str(application/pgp-signature)%end;
+%else %if &ext=prf %then %do;%str(application/pics-rules)%end;
+%else %if &ext=p10 %then %do;%str(application/pkcs10)%end;
+%else %if &ext=p7m %then %do;%str(application/pkcs7-mime)%end;
+%else %if &ext=p7c %then %do;%str(application/pkcs7-mime)%end;
+%else %if &ext=p7s %then %do;%str(application/pkcs7-signature)%end;
+%else %if &ext=p8 %then %do;%str(application/pkcs8)%end;
+%else %if &ext=ac %then %do;%str(application/pkix-attr-cert)%end;
+%else %if &ext=cer %then %do;%str(application/pkix-cert)%end;
+%else %if &ext=crl %then %do;%str(application/pkix-crl)%end;
+%else %if &ext=pkipath %then %do;%str(application/pkix-pkipath)%end;
+%else %if &ext=pki %then %do;%str(application/pkixcmp)%end;
+%else %if &ext=cww %then %do;%str(application/prs.cww)%end;
+%else %if &ext=pskcxml %then %do;%str(application/pskc+xml)%end;
+%else %if &ext=rif %then %do;%str(application/reginfo+xml)%end;
+%else %if &ext=rnc %then %do;%str(application/relax-ng-compact-syntax)%end;
+%else %if &ext=rld %then %do;%str(application/resource-lists-diff+xml)%end;
+%else %if &ext=rl %then %do;%str(application/resource-lists+xml)%end;
+%else %if &ext=gbr %then %do;%str(application/rpki-ghostbusters)%end;
+%else %if &ext=mft %then %do;%str(application/rpki-manifest)%end;
+%else %if &ext=roa %then %do;%str(application/rpki-roa)%end;
+%else %if &ext=scq %then %do;%str(application/scvp-cv-request)%end;
+%else %if &ext=scs %then %do;%str(application/scvp-cv-response)%end;
+%else %if &ext=spq %then %do;%str(application/scvp-vp-request)%end;
+%else %if &ext=spp %then %do;%str(application/scvp-vp-response)%end;
+%else %if &ext=setpay %then %do;%str(application/set-payment-initiation)%end;
+%else %if &ext=setreg %then %do;%str(application/set-registration-initiation)%end;
+%else %if &ext=smi %then %do;%str(application/smil+xml)%end;
+%else %if &ext=rq %then %do;%str(application/sparql-query)%end;
+%else %if &ext=srx %then %do;%str(application/sparql-results+xml)%end;
+%else %if &ext=gram %then %do;%str(application/srgs)%end;
+%else %if &ext=grxml %then %do;%str(application/srgs+xml)%end;
+%else %if &ext=teicorpus %then %do;%str(application/tei+xml)%end;
+%else %if &ext=tfi %then %do;%str(application/thraud+xml)%end;
+%else %if &ext=tsd %then %do;%str(application/timestamped-data)%end;
+%else %if &ext=vxml %then %do;%str(application/voicexml+xml)%end;
+%else %if &ext=wgt %then %do;%str(application/widget)%end;
+%else %if &ext=hlp %then %do;%str(application/winhlp)%end;
+%else %if &ext=7z %then %do;%str(application/x-7z-compressed)%end;
+%else %if &ext=abw %then %do;%str(application/x-abiword)%end;
+%else %if &ext=ace %then %do;%str(application/x-ace-compressed)%end;
+%else %if &ext=dmg %then %do;%str(application/x-apple-diskimage)%end;
+%else %if &ext=aam %then %do;%str(application/x-authorware-map)%end;
+%else %if &ext=aas %then %do;%str(application/x-authorware-seg)%end;
+%else %if &ext=bcpio %then %do;%str(application/x-bcpio)%end;
+%else %if &ext=torrent %then %do;%str(application/x-bittorrent)%end;
+%else %if &ext=blb %then %do;%str(application/x-blorb)%end;
+%else %if &ext=blorb %then %do;%str(application/x-blorb)%end;
+%else %if &ext=bz %then %do;%str(application/x-bzip)%end;
+%else %if &ext=bz2 %then %do;%str(application/x-bzip2)%end;
+%else %if &ext=boz %then %do;%str(application/x-bzip2)%end;
+%else %if &ext=vcd %then %do;%str(application/x-cdlink)%end;
+%else %if &ext=cfs %then %do;%str(application/x-cfs-compressed)%end;
+%else %if &ext=chat %then %do;%str(application/x-chat)%end;
+%else %if &ext=pgn %then %do;%str(application/x-chess-pgn)%end;
+%else %if &ext=nsc %then %do;%str(application/x-conference)%end;
+%else %if &ext=cpio %then %do;%str(application/x-cpio)%end;
+%else %if &ext=csh %then %do;%str(application/x-csh)%end;
+%else %if &ext=deb %then %do;%str(application/x-debian-package)%end;
+%else %if &ext=udeb %then %do;%str(application/x-debian-package)%end;
+%else %if &ext=dgc %then %do;%str(application/x-dgc-compressed)%end;
+%else %if &ext=wad %then %do;%str(application/x-doom)%end;
+%else %if &ext=ncx %then %do;%str(application/x-dtbncx+xml)%end;
+%else %if &ext=dtb %then %do;%str(application/x-dtbook+xml)%end;
+%else %if &ext=res %then %do;%str(application/x-dtbresource+xml)%end;
+%else %if &ext=dvi %then %do;%str(application/x-dvi)%end;
+%else %if &ext=evy %then %do;%str(application/x-envoy)%end;
+%else %if &ext=eva %then %do;%str(application/x-eva)%end;
+%else %if &ext=bdf %then %do;%str(application/x-font-bdf)%end;
+%else %if &ext=gsf %then %do;%str(application/x-font-ghostscript)%end;
+%else %if &ext=psf %then %do;%str(application/x-font-linux-psf)%end;
+%else %if &ext=pcf %then %do;%str(application/x-font-pcf)%end;
+%else %if &ext=snf %then %do;%str(application/x-font-snf)%end;
+%else %if &ext=arc %then %do;%str(application/x-freearc)%end;
+%else %if &ext=spl %then %do;%str(application/x-futuresplash)%end;
+%else %if &ext=gca %then %do;%str(application/x-gca-compressed)%end;
+%else %if &ext=ulx %then %do;%str(application/x-glulx)%end;
+%else %if &ext=gnumeric %then %do;%str(application/x-gnumeric)%end;
+%else %if &ext=gramps %then %do;%str(application/x-gramps-xml)%end;
+%else %if &ext=gtar %then %do;%str(application/x-gtar)%end;
+%else %if &ext=hdf %then %do;%str(application/x-hdf)%end;
+%else %if &ext=install %then %do;%str(application/x-install-instructions)%end;
+%else %if &ext=iso %then %do;%str(application/x-iso9660-image)%end;
+%else %if &ext=jnlp %then %do;%str(application/x-java-jnlp-file)%end;
+%else %if &ext=latex %then %do;%str(application/x-latex)%end;
+%else %if &ext=lzh %then %do;%str(application/x-lzh-compressed)%end;
+%else %if &ext=lha %then %do;%str(application/x-lzh-compressed)%end;
+%else %if &ext=mie %then %do;%str(application/x-mie)%end;
+%else %if &ext=prc %then %do;%str(application/x-mobipocket-ebook)%end;
+%else %if &ext=mobi %then %do;%str(application/x-mobipocket-ebook)%end;
+%else %if &ext=application %then %do;%str(application/x-ms-application)%end;
+%else %if &ext=lnk %then %do;%str(application/x-ms-shortcut)%end;
+%else %if &ext=wmd %then %do;%str(application/x-ms-wmd)%end;
+%else %if &ext=wmz %then %do;%str(application/x-ms-wmz)%end;
+%else %if &ext=xbap %then %do;%str(application/x-ms-xbap)%end;
+%else %if &ext=mdb %then %do;%str(application/x-msaccess)%end;
+%else %if &ext=obd %then %do;%str(application/x-msbinder)%end;
+%else %if &ext=crd %then %do;%str(application/x-mscardfile)%end;
+%else %if &ext=clp %then %do;%str(application/x-msclip)%end;
+%else %if &ext=mny %then %do;%str(application/x-msmoney)%end;
+%else %if &ext=pub %then %do;%str(application/x-mspublisher)%end;
+%else %if &ext=scd %then %do;%str(application/x-msschedule)%end;
+%else %if &ext=trm %then %do;%str(application/x-msterminal)%end;
+%else %if &ext=wri %then %do;%str(application/x-mswrite)%end;
+%else %if &ext=nc %then %do;%str(application/x-netcdf)%end;
+%else %if &ext=cdf %then %do;%str(application/x-netcdf)%end;
+%else %if &ext=nzb %then %do;%str(application/x-nzb)%end;
+%else %if &ext=p12 %then %do;%str(application/x-pkcs12)%end;
+%else %if &ext=pfx %then %do;%str(application/x-pkcs12)%end;
+%else %if &ext=p7b %then %do;%str(application/x-pkcs7-certificates)%end;
+%else %if &ext=spc %then %do;%str(application/x-pkcs7-certificates)%end;
+%else %if &ext=p7r %then %do;%str(application/x-pkcs7-certreqresp)%end;
+%else %if &ext=rar %then %do;%str(application/x-rar-compressed)%end;
+%else %if &ext=ris %then %do;%str(application/x-research-info-systems)%end;
+%else %if &ext=sh %then %do;%str(application/x-sh)%end;
+%else %if &ext=shar %then %do;%str(application/x-shar)%end;
+%else %if &ext=swf %then %do;%str(application/x-shockwave-flash)%end;
+%else %if &ext=xap %then %do;%str(application/x-silverlight-app)%end;
+%else %if &ext=sql %then %do;%str(application/x-sql)%end;
+%else %if &ext=sit %then %do;%str(application/x-stuffit)%end;
+%else %if &ext=sitx %then %do;%str(application/x-stuffitx)%end;
+%else %if &ext=srt %then %do;%str(application/x-subrip)%end;
+%else %if &ext=sv4cpio %then %do;%str(application/x-sv4cpio)%end;
+%else %if &ext=sv4crc %then %do;%str(application/x-sv4crc)%end;
+%else %if &ext=t3 %then %do;%str(application/x-t3vm-image)%end;
+%else %if &ext=gam %then %do;%str(application/x-tads)%end;
+%else %if &ext=tar %then %do;%str(application/x-tar)%end;
+%else %if &ext=tcl %then %do;%str(application/x-tcl)%end;
+%else %if &ext=tex %then %do;%str(application/x-tex)%end;
+%else %if &ext=tfm %then %do;%str(application/x-tex-tfm)%end;
+%else %if &ext=texinfo %then %do;%str(application/x-texinfo)%end;
+%else %if &ext=texi %then %do;%str(application/x-texinfo)%end;
+%else %if &ext=obj %then %do;%str(application/x-tgif)%end;
+%else %if &ext=ustar %then %do;%str(application/x-ustar)%end;
+%else %if &ext=src %then %do;%str(application/x-wais-source)%end;
+%else %if &ext=der %then %do;%str(application/x-x509-ca-cert)%end;
+%else %if &ext=crt %then %do;%str(application/x-x509-ca-cert)%end;
+%else %if &ext=fig %then %do;%str(application/x-xfig)%end;
+%else %if &ext=xlf %then %do;%str(application/x-xliff+xml)%end;
+%else %if &ext=xpi %then %do;%str(application/x-xpinstall)%end;
+%else %if &ext=xz %then %do;%str(application/x-xz)%end;
+%else %if &ext=xdf %then %do;%str(application/xcap-diff+xml)%end;
+%else %if &ext=xht %then %do;%str(application/xhtml+xml)%end;
+%else %if &ext=xsl %then %do;%str(application/xml)%end;
+%else %if &ext=dtd %then %do;%str(application/xml-dtd)%end;
+%else %if &ext=xpl %then %do;%str(application/xproc+xml)%end;
+%else %if &ext=adp %then %do;%str(audio/adpcm)%end;
+%else %if &ext=au %then %do;%str(audio/basic)%end;
+%else %if &ext=snd %then %do;%str(audio/basic)%end;
+%else %if &ext=m4a %then %do;%str(audio/mp4)%end;
+%else %if &ext=mp4a %then %do;%str(audio/mp4)%end;
+%else %if &ext=s3m %then %do;%str(audio/s3m)%end;
+%else %if &ext=sil %then %do;%str(audio/silk)%end;
+%else %if &ext=uva %then %do;%str(audio/vnd.dece.audio)%end;
+%else %if &ext=uvva %then %do;%str(audio/vnd.dece.audio)%end;
+%else %if &ext=eol %then %do;%str(audio/vnd.digital-winds)%end;
+%else %if &ext=dra %then %do;%str(audio/vnd.dra)%end;
+%else %if &ext=dts %then %do;%str(audio/vnd.dts)%end;
+%else %if &ext=dtshd %then %do;%str(audio/vnd.dts.hd)%end;
+%else %if &ext=lvp %then %do;%str(audio/vnd.lucent.voice)%end;
+%else %if &ext=pya %then %do;%str(audio/vnd.ms-playready.media.pya)%end;
+%else %if &ext=ecelp4800 %then %do;%str(audio/vnd.nuera.ecelp4800)%end;
+%else %if &ext=ecelp7470 %then %do;%str(audio/vnd.nuera.ecelp7470)%end;
+%else %if &ext=ecelp9600 %then %do;%str(audio/vnd.nuera.ecelp9600)%end;
+%else %if &ext=rip %then %do;%str(audio/vnd.rip)%end;
+%else %if &ext=weba %then %do;%str(audio/webm)%end;
+%else %if &ext=mka %then %do;%str(audio/x-matroska)%end;
+%else %if &ext=m3u %then %do;%str(audio/x-mpegurl)%end;
+%else %if &ext=wax %then %do;%str(audio/x-ms-wax)%end;
+%else %if &ext=wma %then %do;%str(audio/x-ms-wma)%end;
+%else %if &ext=ra %then %do;%str(audio/x-pn-realaudio)%end;
+%else %if &ext=ram %then %do;%str(audio/x-pn-realaudio)%end;
+%else %if &ext=rmp %then %do;%str(audio/x-pn-realaudio-plugin)%end;
+%else %if &ext=xm %then %do;%str(audio/xm)%end;
+%else %if &ext=ttc %then %do;%str(font/collection)%end;
+%else %if &ext=g3 %then %do;%str(image/g3fax)%end;
+%else %if &ext=btif %then %do;%str(image/prs.btif)%end;
+%else %if &ext=svg %then %do;%str(image/svg+xml)%end;
+%else %if &ext=svgz %then %do;%str(image/svg+xml)%end;
+%else %if &ext=tif %then %do;%str(image/tiff)%end;
+%else %if &ext=psd %then %do;%str(image/vnd.adobe.photoshop)%end;
+%else %if &ext=djv %then %do;%str(image/vnd.djvu)%end;
+%else %if &ext=djvu %then %do;%str(image/vnd.djvu)%end;
+%else %if &ext=sub %then %do;%str(image/vnd.dvb.subtitle)%end;
+%else %if &ext=dwg %then %do;%str(image/vnd.dwg)%end;
+%else %if &ext=dxf %then %do;%str(image/vnd.dxf)%end;
+%else %if &ext=fbs %then %do;%str(image/vnd.fastbidsheet)%end;
+%else %if &ext=fpx %then %do;%str(image/vnd.fpx)%end;
+%else %if &ext=fst %then %do;%str(image/vnd.fst)%end;
+%else %if &ext=mmr %then %do;%str(image/vnd.fujixerox.edmics-mmr)%end;
+%else %if &ext=rlc %then %do;%str(image/vnd.fujixerox.edmics-rlc)%end;
+%else %if &ext=mdi %then %do;%str(image/vnd.ms-modi)%end;
+%else %if &ext=wdp %then %do;%str(image/vnd.ms-photo)%end;
+%else %if &ext=npx %then %do;%str(image/vnd.net-fpx)%end;
+%else %if &ext=wbmp %then %do;%str(image/vnd.wap.wbmp)%end;
+%else %if &ext=xif %then %do;%str(image/vnd.xiff)%end;
+%else %if &ext=ras %then %do;%str(image/x-cmu-raster)%end;
+%else %if &ext=ico %then %do;%str(image/x-icon)%end;
+%else %if &ext=sid %then %do;%str(image/x-mrsid-image)%end;
+%else %if &ext=pct %then %do;%str(image/x-pict)%end;
+%else %if &ext=pic %then %do;%str(image/x-pict)%end;
+%else %if &ext=pnm %then %do;%str(image/x-portable-anymap)%end;
+%else %if &ext=pbm %then %do;%str(image/x-portable-bitmap)%end;
+%else %if &ext=pgm %then %do;%str(image/x-portable-graymap)%end;
+%else %if &ext=ppm %then %do;%str(image/x-portable-pixmap)%end;
+%else %if &ext=xbm %then %do;%str(image/x-xbitmap)%end;
+%else %if &ext=xpm %then %do;%str(image/x-xpixmap)%end;
+%else %if &ext=xwd %then %do;%str(image/x-xwindowdump)%end;
+%else %if &ext=eml %then %do;%str(message/rfc822)%end;
+%else %if &ext=mime %then %do;%str(message/rfc822)%end;
+%else %if &ext=iges %then %do;%str(model/iges)%end;
+%else %if &ext=igs %then %do;%str(model/iges)%end;
+%else %if &ext=dae %then %do;%str(model/vnd.collada+xml)%end;
+%else %if &ext=dwf %then %do;%str(model/vnd.dwf)%end;
+%else %if &ext=gdl %then %do;%str(model/vnd.gdl)%end;
+%else %if &ext=gtw %then %do;%str(model/vnd.gtw)%end;
+%else %if &ext=vtu %then %do;%str(model/vnd.vtu)%end;
+%else %if &ext=vrml %then %do;%str(model/vrml)%end;
+%else %if &ext=wrl %then %do;%str(model/vrml)%end;
+%else %if &ext=x3db %then %do;%str(model/x3d+binary)%end;
+%else %if &ext=x3dbz %then %do;%str(model/x3d+binary)%end;
+%else %if &ext=x3dv %then %do;%str(model/x3d+vrml)%end;
+%else %if &ext=x3dvz %then %do;%str(model/x3d+vrml)%end;
+%else %if &ext=x3d %then %do;%str(model/x3d+xml)%end;
+%else %if &ext=x3dz %then %do;%str(model/x3d+xml)%end;
+%else %if &ext=appcache %then %do;%str(text/cache-manifest)%end;
+%else %if &ext=ics %then %do;%str(text/calendar)%end;
+%else %if &ext=ifb %then %do;%str(text/calendar)%end;
+%else %if &ext=htm %then %do;%str(text/html)%end;
+%else %if &ext=js %then %do;%str(text/javascript)%end;
+%else %if &ext=mjs %then %do;%str(text/javascript)%end;
+%else %if &ext=dsc %then %do;%str(text/prs.lines.tag)%end;
+%else %if &ext=rtx %then %do;%str(text/richtext)%end;
+%else %if &ext=sgm %then %do;%str(text/sgml)%end;
+%else %if &ext=tsv %then %do;%str(text/tab-separated-values)%end;
+%else %if &ext=ttl %then %do;%str(text/turtle)%end;
+%else %if &ext=curl %then %do;%str(text/vnd.curl)%end;
+%else %if &ext=dcurl %then %do;%str(text/vnd.curl.dcurl)%end;
+%else %if &ext=mcurl %then %do;%str(text/vnd.curl.mcurl)%end;
+%else %if &ext=scurl %then %do;%str(text/vnd.curl.scurl)%end;
+%else %if &ext=sub %then %do;%str(text/vnd.dvb.subtitle)%end;
+%else %if &ext=fly %then %do;%str(text/vnd.fly)%end;
+%else %if &ext=flx %then %do;%str(text/vnd.fmi.flexstor)%end;
+%else %if &ext=gv %then %do;%str(text/vnd.graphviz)%end;
+%else %if &ext=3dml %then %do;%str(text/vnd.in3d.3dml)%end;
+%else %if &ext=spot %then %do;%str(text/vnd.in3d.spot)%end;
+%else %if &ext=jad %then %do;%str(text/vnd.sun.j2me.app-descriptor)%end;
+%else %if &ext=wml %then %do;%str(text/vnd.wap.wml)%end;
+%else %if &ext=wmls %then %do;%str(text/vnd.wap.wmlscript)%end;
+%else %if &ext=s %then %do;%str(text/x-asm)%end;
+%else %if &ext=java %then %do;%str(text/x-java-source)%end;
+%else %if &ext=p %then %do;%str(text/x-pascal)%end;
+%else %if &ext=pas %then %do;%str(text/x-pascal)%end;
+%else %if &ext=etx %then %do;%str(text/x-setext)%end;
+%else %if &ext=uu %then %do;%str(text/x-uuencode)%end;
+%else %if &ext=vcs %then %do;%str(text/x-vcalendar)%end;
+%else %if &ext=vcf %then %do;%str(text/x-vcard)%end;
+%else %if &ext=3gp %then %do;%str(video/3gpp)%end;
+%else %if &ext=3g2 %then %do;%str(video/3gpp2)%end;
+%else %if &ext=jpgv %then %do;%str(video/jpeg)%end;
+%else %if &ext=jpgm %then %do;%str(video/jpm)%end;
+%else %if &ext=mjp2 %then %do;%str(video/mj2)%end;
+%else %if &ext=ogv %then %do;%str(video/ogg)%end;
+%else %if &ext=mov %then %do;%str(video/quicktime)%end;
+%else %if &ext=qt %then %do;%str(video/quicktime)%end;
+%else %if &ext=uvh %then %do;%str(video/vnd.dece.hd)%end;
+%else %if &ext=uvvh %then %do;%str(video/vnd.dece.hd)%end;
+%else %if &ext=uvm %then %do;%str(video/vnd.dece.mobile)%end;
+%else %if &ext=uvvm %then %do;%str(video/vnd.dece.mobile)%end;
+%else %if &ext=uvp %then %do;%str(video/vnd.dece.pd)%end;
+%else %if &ext=uvvp %then %do;%str(video/vnd.dece.pd)%end;
+%else %if &ext=uvs %then %do;%str(video/vnd.dece.sd)%end;
+%else %if &ext=uvvs %then %do;%str(video/vnd.dece.sd)%end;
+%else %if &ext=uvv %then %do;%str(video/vnd.dece.video)%end;
+%else %if &ext=uvvv %then %do;%str(video/vnd.dece.video)%end;
+%else %if &ext=dvb %then %do;%str(video/vnd.dvb.file)%end;
+%else %if &ext=fvt %then %do;%str(video/vnd.fvt)%end;
+%else %if &ext=m4u %then %do;%str(video/vnd.mpegurl)%end;
+%else %if &ext=mxu %then %do;%str(video/vnd.mpegurl)%end;
+%else %if &ext=pyv %then %do;%str(video/vnd.ms-playready.media.pyv)%end;
+%else %if &ext=uvu %then %do;%str(video/vnd.uvvu.mp4)%end;
+%else %if &ext=uvvu %then %do;%str(video/vnd.uvvu.mp4)%end;
+%else %if &ext=viv %then %do;%str(video/vnd.vivo)%end;
+%else %if &ext=asf %then %do;%str(video/x-ms-asf)%end;
+%else %if &ext=asx %then %do;%str(video/x-ms-asf)%end;
+%else %if &ext=vob %then %do;%str(video/x-ms-vob)%end;
+%else %if &ext=wm %then %do;%str(video/x-ms-wm)%end;
+%else %if &ext=wmv %then %do;%str(video/x-ms-wmv)%end;
+%else %if &ext=wmx %then %do;%str(video/x-ms-wmx)%end;
+%else %if &ext=wvx %then %do;%str(video/x-ms-wvx)%end;
+%else %if &ext=avi %then %do;%str(video/x-msvideo)%end;
+%else %if &ext=movie %then %do;%str(video/x-sgi-movie)%end;
+%else %if &ext=ice %then %do;%str(x-conference/x-cooltalk)%end;
+%else %if "&ext"="in" %then %do;%str(text/plain)%end;
+%else %do;%str(application/octet-stream)%end;
+
+%mend mf_mimetype;

base/mp_assertscope.sas

@@ -73,10 +73,6 @@
   ignorelist=,
   outds=work.test_results
 )/*/STORE SOURCE*/;
-%local ds test_result test_comments del add mod ilist;
-%let ilist=%upcase(&sasjs_prefix._FUNCTIONS SYS_PROCHTTP_STATUS_CODE
-  SYS_PROCHTTP_STATUS_CODE SYS_PROCHTTP_STATUS_PHRASE &ignorelist);
-
 /**
   * this sets up the global vars, it will also enter STRICT mode.  If this
   * behaviour is not desired, simply initiate the following global macro
@@ -84,6 +80,10 @@
   */
 %mp_init()
 
+%local ds test_result test_comments del add mod ilist;
+%let ilist=%upcase(&sasjs_prefix._FUNCTIONS SYS_PROCHTTP_STATUS_CODE
+  SYS_PROCHTTP_STATUS_CODE SYS_PROCHTTP_STATUS_PHRASE &ignorelist);
+
 /* get current variables */
 %if &action=SNAPSHOT %then %do;
   proc sql;

base/mp_dirlist.sas

@@ -85,7 +85,7 @@ data;run;
 data &out_ds(compress=no
     keep=file_or_folder filepath filename ext msg directory level
   );
-  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
+  length directory filepath $2000 fref fref2 $8 file_or_folder $6 filename $255
     ext $20 msg $200 foption $16;
   if _n_=1 then call missing(of _all_);
   retain level &level;

base/mp_getcols.sas

@@ -53,10 +53,11 @@ data &outds(keep=name type length varnum format label ddtype fmtname);
     else if formatd=0 then format=cats(fmtname,formatl,'.');
     else format=cats(fmtname,formatl,'.',formatd);
     type='N';
-    if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
+    if format=:'DATETIME' or format=:'E8601DT' or format=:'NLDATM'
+    then ddtype='DATETIME';
     else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
       or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
-      or format=:'MONYY'
+      or format=:'MONYY' or format=:'NLDATE'
     then ddtype='DATE';
     else if format=:'TIME' then ddtype='TIME';
     else ddtype='NUMERIC';

base/mp_init.sas

@@ -70,6 +70,7 @@ options
 %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
   noautocorrect           /* disallow misspelled procedure names            */
   dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
+                          /* turn off with dsoptions=nonote2err             */
 %end;
 ;
 

base/mp_replace.sas

@@ -33,7 +33,8 @@
   LUA, you can also use this macro: mp_gsubfile.sas
 
   @param [in] infile The QUOTED path to the file on which to perform the
-    substitution
+    substitution.  Note that you can extract the pathname from a fileref using
+    the pathname function, eg: `"%sysfunc(pathname(fref))"`;
   @param [in] findvar= Macro variable NAME containing the string to search for
   @param [in] replacevar= Macro variable NAME containing the replacement string
   @param [out] outfile= (0) Optional QUOTED path to the adjusted output file (to
@@ -147,6 +148,6 @@ data _null_;
 run;
 
 /* END */
-%put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) seconds to run;
+/* %put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) secs to run; */
 
 %mend mp_replace;

base/mp_stripdiffs.sas

@@ -8,7 +8,7 @@
 
     @li deleted rows - these are re-inserted
     @li changed rows - differences are reverted
-    @li added rows - these are marked with `_____DELETE_THIS_RECORD_____="YES"`
+    @li added rows - marked with `_____DELETE__THIS__RECORD_____="YES"`
 
     These changes are NOT applied to the base table - a staging dataset is
     simply prepared for an ETL process to action.  In Data Controller, this
@@ -22,16 +22,21 @@
     change, plus ALL SUBSEQUENT CHANGES, will be reverted in the output table.
   @param [in] difftable The dataset containing the diffs.  Definition available
     in mddl_dc_difftable.sas
+  @param [in] filtervar= (0) If provided, the contents of this macro variable
+    will be applied as an additional filter against &libds
   @param [out] outds= (work.mp_stripdiffs) Output table containing the diffs.
     Has the same format as the base datset, plus a
-    `_____DELETE_THIS_RECORD_____` variable.
+    `_____DELETE__THIS__RECORD_____` variable.
   @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
   @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
   @li mf_islibds.sas
+  @li mf_wordsinstr1butnotstr2.sas
   @li mp_abort.sas
+  @li mp_ds2squeeze.sas
 
   <h4> Related Macros </h4>
   @li mddl_dc_difftable.sas
@@ -47,6 +52,7 @@
 %macro mp_stripdiffs(libds
   ,loadref
   ,difftable
+  ,filtervar=0
   ,outds=work.mp_stripdiffs
   ,mdebug=0
 )/*/STORE SOURCE*/;
@@ -70,11 +76,11 @@
   ,msg=%str(Invalid library.dataset reference - %superq(libds))
 )
 
-
-
 /* set up unique and temporary vars */
-%local ds1 ds2 ds3 ds4 ds5 fref1;
+%local ds1 ds2 ds3 ds4 ds5 fref1 filterstr;
 %let fref1=%mf_getuniquefileref();
+%if &filtervar ne 0 %then %let filterstr=%superq(&filtervar);
+%else %let filterstr=%str(1=1);
 
 /* get timestamp of the diff to be reverted */
 %local ts;
@@ -97,30 +103,28 @@ create table &ds1 (drop=libref dsn) as
 
 /* extract key values only */
 %let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_pks));
+%local keyhash processed;
+%let keyhash=%upcase(%mf_getuniquename(prefix=mpsdvar_keyhash));
+%let processed=%upcase(%mf_getuniquename(prefix=mpsdvar_processed));
 create table &ds2 as
-  select key_hash,
+  select key_hash as &keyhash,
     tgtvar_nm,
     tgtvar_type,
     coalescec(oldval_char,newval_char) as charval,
     coalesce(oldval_num, newval_num) as numval,
-    processed_dttm
+    processed_dttm as &processed
   from &ds1
   where is_pk=1
-  order by key_hash, processed_dttm;
+  order by &keyhash, &processed;
 
 /* grab pk values */
 %local pk;
-data _null_;
-  set &ds2;
-  by key_hash;
-  call symputx('pk',catx(' ',symget('pk'),tgtvar_nm),'l');
-  if last.key_hash then stop;
-run;
+select distinct upcase(tgtvar_nm) into: pk separated by ' ' from &ds2;
 
 %let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_keychar));
 proc transpose data=&ds2(where=(tgtvar_type='C'))
     out=&ds3(drop=_name_);
-  by KEY_HASH;
+  by &keyhash &processed;
   id TGTVAR_NM;
   var charval;
 run;
@@ -128,7 +132,7 @@ run;
 %let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_keynum));
 proc transpose data=&ds2(where=(tgtvar_type='N'))
     out=&ds4(drop=_name_);
-  by KEY_HASH;
+  by &keyhash &processed;
   id TGTVAR_NM;
   var numval;
 run;
@@ -136,19 +140,32 @@ run;
 %mp_ds2squeeze(&ds3,outds=&ds3)
 %mp_ds2squeeze(&ds4,outds=&ds4)
 
+/* now merge to get all key values and de-dup */
 %let ds5=%upcase(work.%mf_getuniquename(prefix=mpsd_merged));
 data &ds5;
+  length &keyhash $32 &processed 8;
   merge &ds3 &ds4;
-  by key_hash;
-  if not missing(key_hash);
+  by &keyhash &processed;
+  if not missing(&keyhash);
+run;
+proc sort data=&ds5 nodupkey;
+  by &pk;
 run;
 
 /* join to base table for preliminary stage DS */
 proc sql;
-create table &outds as select "No " as _____DELETE_THIS_RECORD_____,
-    b.*
+create table &outds as select "No " as _____DELETE__THIS__RECORD_____
+  %do x=1 %to %sysfunc(countw(&pk,%str( )));
+    ,a.%scan(&pk,&x,%str( ))
+  %end;
+  %local notpkcols;
+  %let notpkcols=%upcase(%mf_getvarlist(&libds));
+  %let notpkcols=%mf_wordsinstr1butnotstr2(str1=&notpkcols,str2=&pk);
+  %do x=1 %to %sysfunc(countw(&notpkcols,%str( )));
+    ,b.%scan(&notpkcols,&x,%str( ))
+  %end;
   from &ds5 a
-  inner join &libds b
+  left join &libds (where=(&filterstr)) b
   on 1=1
 %do x=1 %to %sysfunc(countw(&pk,%str( )));
   and a.%scan(&pk,&x,%str( ))=b.%scan(&pk,&x,%str( ))
@@ -158,7 +175,8 @@ create table &outds as select "No " as _____DELETE_THIS_RECORD_____,
 /* create SAS code to apply to stage_ds */
 data _null_;
   set &ds1;
-  file &fref1;
+  file &fref1 lrecl=33000;
+  length charval $32767;
   if _n_=1 then put 'proc sql noprint;';
   by descending processed_dttm key_hash is_pk;
   if move_type='M' then do;
@@ -167,7 +185,8 @@ data _null_;
     end;
     if IS_PK=0 then do;
       put "  " tgtvar_nm '=' @@;
-      charval=quote(cats(oldval_char));
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
       if tgtvar_type='C' then put charval @@;
       else put oldval_num @@;
       if not last.is_pk then put ',';
@@ -175,36 +194,49 @@ data _null_;
     else do;
       if first.is_pk then put "  where 1=1 " @@;
       put "  and " tgtvar_nm '=' @@;
-      charval=quote(cats(oldval_char));
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
       if tgtvar_type='C' then put charval @@;
       else put oldval_num @@;
     end;
   end;
   else if move_type='A' then do;
     if first.key_hash then do;
-      put "update &outds set _____DELETE_THIS_RECORD_____='Yes'  where 1=1 " @@;
+      put "update &outds set _____DELETE__THIS__RECORD_____='Yes' where 1=1 "@@;
     end;
     /* gating if - as only need PK now */
     if is_pk=1;
     put '  AND ' tgtvar_nm '=' @@;
-    charval=quote(cats(newval_char));
+    cnt=count(newval_char,'"');
+    charval=quote(trim(substr(newval_char,1,32765-cnt)));
     if tgtvar_type='C' then put charval @@;
     else put newval_num @@;
   end;
   else if move_type='D' then do;
     if first.key_hash then do;
-      put "insert into &outds set _____DELETE_THIS_RECORD_____='No' " @@;
+      put "update &outds set _____DELETE__THIS__RECORD_____='No' " @@;
     end;
+    if IS_PK=0 then do;
       put "  ," tgtvar_nm '=' @@;
-    charval=quote(cats(oldval_char));
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
       if tgtvar_type='C' then put charval @@;
       else put oldval_num @@;
     end;
+    else do;
+      if first.is_pk then put "  where 1=1 " @@;
+      put "  and " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+  end;
   if last.key_hash then put ';';
 run;
 
 /* apply the modification statements */
-%inc &fref1/source2;
+%inc &fref1/source2 lrecl=33000;
 
 %if &mdebug=0 %then %do;
   proc sql;

fcmp/mcf_getfmttype.sas

@@ -80,8 +80,8 @@ function mcf_getfmttype(fmtnm $) $8;
     /* apply lookups */
     if cats(fmt) in ('DATETIME','B8601DN','B8601DN','B8601DT','B8601DT'
       ,'B8601DZ','B8601DZ','DATEAMPM','DTDATE','DTMONYY','DTWKDATX','DTYEAR'
-      ,'DTYYQC','E8601DN','E8601DN','E8601DT','E8601DT','E8601DZ','E8601DZ')
-      then return('DATETIME');
+      ,'DTYYQC','E8601DN','E8601DN','E8601DT','E8601DT','E8601DZ','E8601DZ'
+      ,'NLDATM') then return('DATETIME');
     else if fmt in ('DATE','YYMMDD','B8601DA','B8601DA','DAY','DDMMYY'
       ,'DDMMYYB','DDMMYYC','DDMMYYD','DDMMYYN','DDMMYYP','DDMMYYS','DDMMYYx'
       ,'DOWNAME','E8601DA','E8601DA','JULDAY','JULIAN','MMDDYY','MMDDYYB'
@@ -92,7 +92,7 @@ function mcf_getfmttype(fmtnm $) $8;
       ,'YYMMD','YYMMDDB','YYMMDDC','YYMMDDD','YYMMDDN','YYMMDDP','YYMMDDS'
       ,'YYMMDDx','YYMMN','YYMMP','YYMMS','YYMMx','YYMON','YYQ','YYQC','YYQD'
       ,'YYQN','YYQP','YYQR','YYQRC','YYQRD','YYQRN','YYQRP','YYQRS','YYQRx'
-      ,'YYQS','YYQx','YYQZ') then return('DATE');
+      ,'YYQS','YYQx','YYQZ','NLDATE') then return('DATE');
     else if fmt in ('TIME','B8601LZ','B8601LZ','B8601TM','B8601TM','B8601TZ'
       ,'B8601TZ','E8601LZ','E8601LZ','E8601TM','E8601TM','E8601TZ','E8601TZ'
       ,'HHMM','HOUR','MMSS','TIMEAMPM','TOD') then return('TIME');

meta/mm_assigndirectlib.sas

@@ -54,7 +54,8 @@
 
 %local cur_engine;
 %let cur_engine=%mf_getengine(&libref);
-%if &cur_engine ne META and &cur_engine ne %then %do;
+%if &cur_engine ne META and &cur_engine ne and %length(&open_passthrough)=0
+%then %do;
   %put NOTE:  &libref already has a direct (&cur_engine) libname connection;
   %return;
 %end;

meta/mm_getdetails.sas

@@ -1,22 +1,25 @@
 /**
   @file mm_getdetails.sas
   @brief extracts metadata attributes and associations for a particular uri
-
   @param [in] uri the metadata object for which to return
     attributes / associations
+  @param [in] sortoptions= Enables sorting of the output datasets, for example,
+    `SORTSEQ=LINGUISTIC`
   @param [out] outattrs= (work.attributes)
     The dataset to create that contains the list of attributes
   @param [out] outassocs= (work.associations)
     The dataset to contain the list of associations
 
-  @version 9.2
-  @author Allan Bowe
+  <h4> Related Files </h4>
+  @li mm_getobjects.sas
+  @li mm_gettypes.sas
 
 **/
 
 %macro mm_getdetails(uri
   ,outattrs=work.attributes
   ,outassocs=work.associations
+  ,sortoptions=
 )/*/STORE SOURCE*/;
 
 data &outassocs;
@@ -41,7 +44,7 @@ data &outassocs;
     n1+1;
   end;
 run;
-proc sort;
+proc sort &sortoptions;
   by assoc name;
 run;
 
@@ -61,7 +64,7 @@ data &outattrs;
     n1+1;
   end;
 run;
-proc sort;
+proc sort &sortoptions;
   by type name;
 run;
 

package.json

@@ -7,7 +7,7 @@
     "Viya",
     "SASjs"
   ],
-  "author": "Allan Bowe <support@macropeople.com>",
+  "author": "Allan Bowe",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/sasjs/core.git"
@@ -26,13 +26,10 @@
   "homepage": "https://core.sasjs.io",
   "main": "index.js",
   "scripts": {
-    "build": "sasjs cbd -t server",
-    "docs": "sasjs doc -t docsonly && ./sasjs/utils/build.sh",
-    "test": "sasjs test -t server",
-    "lint": "sasjs lint",
+    "build": "npx @sasjs/cli cbd -t server",
+    "docs": "npx @sasjs/cli doc -t docsonly && ./sasjs/utils/build.sh",
+    "test": "npx @sasjs/cli test -t server",
+    "lint": "npx @sasjs/cli lint",
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
-  },
-  "devDependencies": {
-    "@sasjs/cli": "^4.4.1"
   }
 }

sasjs/sasjsconfig.json

@@ -5,7 +5,10 @@
     "ddl",
     "fcmp",
     "lua",
+    "meta",
+    "metax",
     "server",
+    "viya",
     "xplatform",
     "tests/base",
     "tests/ddlonly",
@@ -37,8 +40,11 @@
         "allowInsecureRequests": false
       },
       "appLoc": "/Public/app/macrocore",
+      "deployConfig": {
+        "deployServicePack": true,
+        "deployScripts": []
+      },
       "macroFolders": [
-        "viya",
         "tests/viyaonly"
       ],
       "contextName": "SAS Job Execution compute context"
@@ -52,8 +58,6 @@
       },
       "appLoc": "/Shared Data/temp/macrocore",
       "macroFolders": [
-        "meta",
-        "metax",
         "tests/sas9only"
       ],
       "programFolders": [],
@@ -78,7 +82,6 @@
         "deployScripts": []
       },
       "macroFolders": [
-        "server",
         "tests/serveronly"
       ]
     },

sasjs/utils/build.sh

@@ -1,22 +1,27 @@
 #!/bin/bash
 ####################################################################
-# PROJECT: Macro Core Docs Build                                   #
-# To execute, use the npm command (npm run docs)                   #
+# PROJECT: SASjs Core Docs Build
+# To execute, use the npm command (npm run docs)
+# Target repo will have github action to create sitemap
+# https://github.com/marketplace/actions/generate-sitemap
 ####################################################################
 
 # refresh github pages site
 rm -rf sasjsbuild/docsite
 git clone git@github.com:sasjs/core.github.io.git sasjsbuild/docsite
-rm -rf sasjsbuild/docsite/*
-mv sasjsbuild/docs/* sasjsbuild/docsite/
+rm -rf sasjsbuild/docsite/*.html
+rm -rf sasjsbuild/docsite/*.js
+rm -rf sasjsbuild/docsite/*.png
+rm -rf sasjsbuild/docsite/*.dot
+rm -rf sasjsbuild/docsite/*.css
+rm -rf sasjsbuild/docsite/*.svg
+rm -rf search
+cp -R sasjsbuild/docs/* sasjsbuild/docsite/
 cd sasjsbuild/docsite/
+git config user.name sasjs
 echo 'core.sasjs.io' > CNAME
 git add .
 git commit -m "build.sh build on $(date +%F:%H:%M:%S)"
 git push
-npx sitemap-generator-cli https://core.sasjs.io
-git add .
-git commit -m "adding sitemap"
-git push
 
 echo "check it out:  https://sasjs.github.io/core.github.io/files.html"

sasjs/utils/create_sas_package.sas

@@ -8,6 +8,12 @@
 
     Requires the server to have SSH keys.
 
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mp_gitadd.sas
+  @li mp_gitreleaseinfo.sas
+  @li mp_gitstatus.sas
+
   <h4> SAS Macros </h4>
   @li mp_gitadd.sas
   @li mp_gitreleaseinfo.sas
@@ -124,11 +130,15 @@ data members(compress=char);
   keep name name2 path;
 run;
 
+proc sort data=members;
+  by name name2;
+run;
+
 %let temp_options = %sysfunc(getoption(source)) %sysfunc(getoption(notes));
 options nosource nonotes;
 data _null_;
   set members;
-  by name notsorted;
+  by name;
 
   ord + first.name;
 

server/ms_triggerstp.sas

@@ -0,0 +1,231 @@
+/**
+  @file
+  @brief Triggers a SASjs Server STP using the /SASjsApi/stp/trigger endpoint
+  @details Triggers the STP and returns the sessionId
+
+  Example:
+
+      %ms_triggerstp(/some/stored/program
+        ,debug=131
+        ,outds=work.myresults
+      )
+
+  @param [in] pgm The full path to the Stored Program in SASjs Drive (_program
+    parameter)
+  @param [in] debug= (131) The value to supply to the _debug URL parameter
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] inputparams=(_null_) A dataset containing name/value pairs in the
+    following format:
+    |name:$32|value:$10000|
+    |---|---|
+    |stpmacname|some value|
+    |mustbevalidname|can be anything, oops, %abort!!|
+  @param [in] inputfiles= (_null_) A dataset containing fileref/name/filename in
+    the following format:
+    |fileref:$8|name:$32|filename:$256|
+    |---|---|--|
+    |someref|some_name|some_filename.xls|
+    |fref2|another_file|zyx_v2.csv|
+  @param [in] expiresaftermins= (15) The number of minutes to retain the session
+    folder after the session ends.
+
+  @param [out] outds= (work.ms_triggerstp) Set to the name of a dataset to
+  contain the sessionId. If this dataset already exists, and contains the
+  sessionId, it will be appended to.
+  Format:
+    |sessionId:$36|
+    |---|
+    |20241028074744-54132-1730101664824|
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+  @li mp_dropmembers.sas
+  @li mf_nobs.sas
+
+**/
+
+%macro ms_triggerstp(pgm
+    ,debug=131
+    ,inputparams=_null_
+    ,inputfiles=_null_
+    ,expiresAfterMins=15
+    ,outds=work.ms_triggerstp
+    ,mdebug=0
+  );
+  %local dbg mainref authref boundary libref triggered_sid;
+  %let mainref=%mf_getuniquefileref();
+  %let authref=%mf_getuniquefileref();
+  %let boundary=%mf_getuniquename();
+  %if &inputparams=0 %then %let inputparams=_null_;
+
+  %if &mdebug=1 %then %do;
+    %put &sysmacroname entry vars:;
+    %put _local_;
+  %end;
+  %else %let dbg=*;
+
+
+  %mp_abort(iftrue=("&pgm"="")
+    ,mac=&sysmacroname
+    ,msg=%str(Program not provided)
+  )
+  %mp_abort(iftrue=("&outds"="")
+    ,mac=&sysmacroname
+    ,msg=%str(Output dataset not provided)
+  )
+
+  /* avoid sending bom marker to API */
+  %local optval;
+  %let optval=%sysfunc(getoption(bomfile));
+  options nobomfile;
+
+  /* Add params to the content */
+  data _null_;
+    file &mainref termstr=crlf lrecl=32767 mod;
+    length line $1000 name $32 value $32767;
+    if _n_=1 then call missing(of _all_);
+    set &inputparams;
+    put "--&boundary";
+    line=cats('Content-Disposition: form-data; name="',name,'"');
+    put line;
+    put ;
+    put value;
+  run;
+
+  /* parse input file list */
+  %local webcount;
+  %let webcount=0;
+  data _null_;
+    set &inputfiles end=last;
+    length fileref $8 name $32 filename $256;
+    call symputx(cats('webref',_n_),fileref,'l');
+    call symputx(cats('webname',_n_),name,'l');
+    call symputx(cats('webfilename',_n_),filename,'l');
+    if last then do;
+      call symputx('webcount',_n_);
+      call missing(of _all_);
+    end;
+  run;
+
+  /* write out the input files to the content */
+  %local i;
+  %do i=1 %to &webcount;
+    data _null_;
+      file &mainref termstr=crlf lrecl=32767 mod;
+      infile &&webref&i lrecl=32767;
+      if _n_ = 1 then do;
+        length line $32767;
+        line=cats(
+          'Content-Disposition: form-data; name="'
+          ,"&&webname&i"
+          ,'"; filename="'
+          ,"&&webfilename&i"
+          ,'"'
+        );
+        put "--&boundary";
+        put line;
+        put "Content-Type: text/plain";
+        put ;
+      end;
+      input;
+      put _infile_; /* add the actual file to be sent */
+    run;
+  %end;
+
+  /* Add footer to the content */
+  data _null_;
+    file &mainref termstr=crlf mod;
+    put / "--&boundary--";
+  run;
+
+  data _null_;
+    file &authref lrecl=1000;
+    infile "&_sasjs_tokenfile" lrecl=1000;
+    input;
+    if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary";
+    put _infile_;
+  run;
+
+  %if &mdebug=1 %then %do;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** authref=&authref content *****";
+      infile &authref;
+      input;
+      put _infile_;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** mainref=&mainref content *****";
+      infile &mainref;
+      input;
+      put _infile_;
+    run;
+  %end;
+
+  %local resp_path outref;
+  %let resp_path=%sysfunc(pathname(work))/%mf_getuniquename();
+  %let outref=%mf_getuniquefileref();
+  filename &outref "&resp_path" lrecl=32767;
+
+  /* prepare request*/
+  proc http method='POST' headerin=&authref in=&mainref out=&outref
+    url="&_sasjs_apiserverurl/SASjsApi/stp/trigger?%trim(
+      )_program=&pgm%str(&)_debug=131%str(&)expiresAfterMins=&expiresaftermins";
+  %if &mdebug=1 %then %do;
+    debug level=2;
+  %end;
+  run;
+
+  %if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  or &mdebug=1
+  %then %do;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** outref=&outref content *****";
+      infile &outref;
+      input;
+      putlog _infile_;
+    run;
+  %end;
+  %mp_abort(
+    iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200
+        and &SYS_PROCHTTP_STATUS_CODE ne 201)
+    ,mac=&sysmacroname
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+
+  /* reset options */
+  options &optval;
+
+  %let libref=%mf_getuniquelibref();
+  libname &libref JSON fileref=&outref;
+  %let triggered_sid=%mf_getuniquename(prefix=triggered_sid_);
+
+  data work.&triggered_sid (keep=sessionid);
+    set &libref..root;
+
+    %if &mdebug=1 %then %do;
+      putlog (_all_)(=);
+    %end;
+  run;
+
+  %if %mf_nobs(work.&triggered_sid)>0 %then %do;
+    proc append base=&outds data=work.&triggered_sid;
+    run;
+  %end;
+
+  %if &mdebug=1 %then %do;
+    %put &sysmacroname exit vars:;
+    %put _local_;
+  %end;
+  %else %do;
+    /* clear refs */
+    filename &authref;
+    filename &mainref;
+    filename &outref;
+    libname &libref clear;
+    /* and remove temp dataset */
+    %mp_dropmembers(&triggered_sid,libref=work);
+  %end;
+
+%mend ms_triggerstp;

tests/base/mf_mimetype.test.sas

@@ -0,0 +1,16 @@
+/**
+  @file
+  @brief Testing mf_mimetype macro
+
+  <h4> SAS Macros </h4>
+  @li mf_mimetype.sas
+  @li mp_assert.sas
+
+**/
+
+
+%mp_assert(
+  iftrue=("%mf_mimetype(XLS)"="application/vnd.ms-excel",
+  desc=Checking correct value
+)
+

tests/base/mp_stripdiffs.test.sas

@@ -82,10 +82,10 @@ run;
 %let addpass=0;
 data _null_;
   set work.mp_stripdiffs;
-  if upcase(_____DELETE_THIS_RECORD_____)='NO' and name="&delname"
+  if upcase(_____DELETE__THIS__RECORD_____)='NO' and name="&delname"
   then call symputx('delpass',1);
   if name="&modname" and age=&modval then call symputx('modpass',1);
-  if upcase(_____DELETE_THIS_RECORD_____)='YES' and name="Newbie"
+  if upcase(_____DELETE__THIS__RECORD_____)='YES' and name="Newbie"
   then call symputx('addpass',1);
 run;
 

tests/serveronly/ms_triggerstp.test.sas

@@ -0,0 +1,90 @@
+/**
+  @file
+  @brief Testing ms_triggerstp.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li ms_createfile.sas
+  @li ms_triggerstp.sas
+  @li mf_existds.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertcols.sas
+  @li mf_getvartype.sas
+  @li ms_deletefile.sas
+
+**/
+
+/* first, create multiple STPs to run */
+filename stpcode1 temp;
+data _null_;
+  file stpcode1;
+  put '%put hello world;';
+  put '%put _all_;';
+  put 'data _null_; file _webout1; put "triggerstp test 1";run;';
+run;
+filename stpcode2 temp;
+data _null_;
+  file stpcode2;
+  put '%put Lorem Ipsum;';
+  put '%put _all_;';
+  put 'data _null_; file _webout2; put "triggerstp test 2";run;';
+run;
+options mprint;
+%let fname1=%mf_getuniquename();
+%let fname2=%mf_getuniquename();
+
+%ms_createfile(/sasjs/tests/&fname1..sas
+  ,inref=stpcode1
+  ,mdebug=1
+)
+%ms_createfile(/sasjs/tests/&fname2..sas
+  ,inref=stpcode2
+)
+
+%mp_assertscope(SNAPSHOT)
+  %ms_triggerstp(/sasjs/tests/&fname1
+    ,debug=131
+    ,outds=work.mySessions
+  )
+  %ms_triggerstp(/sasjs/tests/&fname2
+    ,outds=work.mySessions
+  )
+%mp_assertscope(COMPARE
+                ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN)
+
+%mp_assert(iftrue=%str(%mf_existds(work.mySessions)=1)
+          ,desc=Testing output exists
+          ,outds=work.test_results
+)
+
+%mp_assertdsobs(work.mySessions,
+  test=EQUALS 2,
+  desc=Testing observations,
+  outds=work.test_results
+)
+%mp_assertcols(work.mySessions,
+  cols=sessionid,
+  test=ALL,
+  desc=Testing column exists,
+  outds=work.test_results
+)
+
+data _null_;
+  retain contentCheck 1;
+  set work.mySessions end=last;
+  if missing(sessionID) then contentCheck = 0;
+  if last then do;
+    call symputx("contentCheck",contentCheck,"l");
+  end;
+run;
+%let typeCheck = %mf_getvartype(work.mySessions,sessionid);
+
+%mp_assert(iftrue=%str(&typeCheck = C and &contentCheck = 1)
+          ,desc=Testing type and content of output
+          ,outds=work.test_results
+)
+
+%ms_deletefile(/sasjs/tests/&fname1..sas)
+%ms_deletefile(/sasjs/tests/&fname2..sas)

tests/testinit.sas

@@ -10,7 +10,7 @@
 **/
 
 /* location in metadata or SAS Drive for temporary files */
-%let mcTestAppLoc=/tmp/tests/sasjs/core/%mf_uid();
+%let mcTestAppLoc=/Public/testresults/sasjs_core/%mf_uid();
 
 /* set defaults */
 %mp_init()

tests/viyaonly/mfv_getfolderpath.test.sas

@@ -0,0 +1,31 @@
+/**
+  @file
+  @brief Testing mfv_getfolderpath macro function
+
+  <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mfv_getfolderpath.sas
+  @li mfv_getpathuri.sas
+  @li mp_assert.sas
+  @li mv_createfolder.sas
+
+**/
+
+options mprint sgen;
+
+%let folder=%mf_uid();
+/* create a folder */
+%mv_createfolder(path=&mcTestAppLoc/&folder)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=no errs on folder creation
+)
+
+%let uri=%mfv_getpathuri(&mcTestAppLoc/&folder);
+%put %mfv_getfolderpath(&uri);
+
+%mp_assert(
+  iftrue=("%mfv_getfolderpath(&uri)"="&mcTestAppLoc/&folder"),
+  desc=Check if correct folder was returned
+)
+

tests/viyaonly/mfv_getpathuri.test.sas

@@ -0,0 +1,35 @@
+/**
+  @file
+  @brief Testing mfv_getpathuri macro function
+
+  <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mfv_getpathuri.sas
+  @li mp_assert.sas
+  @li mv_createfile.sas
+
+**/
+
+options mprint sgen;
+
+%let file=%mf_uid();
+
+/* create a file */
+filename somefile temp;
+data _null_;
+  file somefile;
+  put 'hello testings';
+run;
+%let path=&mcTestAppLoc/temp;
+%mv_createfile(path=&path, name=&file..txt,inref=somefile)
+
+
+%mp_assert(
+  iftrue=(%mfv_existfile(&path/&file..txt)=1),
+  desc=Check if created file exists
+)
+
+%mp_assert(
+  iftrue=(%length(%mfv_getpathuri(&path/&file..txt))>0),
+  desc=Check that a URI was returned
+)

tests/viyaonly/mv_createfile.test.sas

@@ -6,6 +6,7 @@
   @li mf_uid.sas
   @li mfv_existfile.sas
   @li mp_assert.sas
+  @li mp_assertscope.sas
   @li mv_createfile.sas
 
 
@@ -21,22 +22,90 @@ data _null_;
   file somefile;
   put 'hello testings';
 run;
-%mv_createfile(path=&mcTestAppLoc/temp, name=&file..txt,inref=somefile)
+%mp_assertscope(SNAPSHOT)
+%mv_createfile(path=&mcTestAppLoc, name=&file..txt,inref=somefile,mdebug=1)
+%mp_assertscope(COMPARE
+  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADP2LEN MCLIB0_JADPNUM
+    MCLIB0_JADVLEN MCLIB2_JADP1LEN
+    SASJSPROCESSMODE SASJS_STPSRV_HEADER_LOC
+    MCLIB2_JADP2LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
+)
 
 %mp_assert(
-  iftrue=(%mfv_existfile(&mcTestAppLoc/temp/&file..txt)=1),
+  iftrue=(%mfv_existfile(&mcTestAppLoc/&file..txt)=1),
   desc=Check if created file exists
 )
 
-%put TEST 2 - dataset upload ;
+%put TEST 2 - html file;
+filename f2 temp;
+data _null_;
+  file f2;
+  put '<html><body><p>Hello world</p></body></html>';
+run;
+%mv_createfile(path=&mcTestAppLoc, name=test.html,inref=f2,mdebug=1)
+
+%mp_assert(
+  iftrue=(%mfv_existfile(&mcTestAppLoc/test.html)=1),
+  desc=Check if created file exists
+)
+
+%put TEST 3 - dataset upload ;
 data temp;
 x=1;
 run;
 filename ds "%sysfunc(pathname(work))/temp.sas7bdat";
 
-%mv_createfile(path=&mcTestAppLoc/temp, name=&file..sas7bdat,inref=ds)
+%mv_createfile(path=&mcTestAppLoc, name=&file..sas7bdat,inref=ds,mdebug=1)
 
 %mp_assert(
-  iftrue=(%mfv_existfile(&mcTestAppLoc/temp/&file..sas7bdat)=1),
+  iftrue=(%mfv_existfile(&mcTestAppLoc/&file..sas7bdat)=1),
   desc=Check if created dataset exists
 )
+
+%put TEST 4 - create a .sas file;
+filename f4 temp;
+data _null_;
+  file f4;
+  put '%put hello FromSASStudioBailey; ';
+run;
+%mv_createfile(path=&mcTestAppLoc, name=test4.sas,inref=f4,mdebug=1)
+
+%mp_assert(
+  iftrue=(%mfv_existfile(&mcTestAppLoc/test4.sas)=1),
+  desc=Check if created sas program exists
+)
+
+
+
+%put TEST 5 - reading from files service and writing back;
+filename sendfrom filesrvc folderpath="&mcTestAppLoc" filename='test4.sas';
+
+OPTIONS MERROR SYMBOLGEN MLOGIC MPRINT;
+
+%mv_createfile(path=&mcTestAppLoc,name=test5.sas,inref=sendfrom,mdebug=1) ;
+
+%put TEST 6 - try the find and replace;
+filename f6 temp;
+data _null_;
+  file f6;
+  put '//Hello world!';
+  put 'let var=/some/path/name;';
+run;
+%let in=/some/path/name;
+%let out=/final/destination;
+%mv_createfile(path=&mcTestAppLoc, name=test6.js,inref=f6,mdebug=1,swap=in out)
+
+filename getback filesrvc folderpath="&mcTestAppLoc" filename='test6.js';
+
+%let test6=0;
+data _null_;
+  infile getback;
+  input;
+  if _infile_="let var=&out;" then call symputx('test6',1);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(&test6=1),
+  desc=Check if find & replace worked
+)

tests/viyaonly/mv_createfolder.test.sas

@@ -30,3 +30,19 @@ run;
   iftrue=(&test=1),
   desc=Check if temp folder can be successfully created
 )
+
+/* create a folder without output dataset as part of the original macro */
+%mv_createfolder(path=&mcTestAppLoc/temp/&folder/folder2,outds=folders2)
+
+%let test=0;
+data _null_;
+  set work.folders2;
+  putlog (_all_)(=);
+  if not missing(self_uri) and not missing(parent_uri)
+  then call symputx('test2',1);
+run;
+
+%mp_assert(
+  iftrue=(&test2=1),
+  desc=Check if outds param works
+)

tests/viyaonly/mv_getviyafileextparms.test.sas

@@ -0,0 +1,96 @@
+/**
+  @file
+  @brief Testing mv_getviyafileextparms macro
+
+  <h4> SAS Macros </h4>
+  @li mf_isblank.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mv_getviyafileextparms.sas
+
+**/
+
+options mprint;
+
+%let mvarIgnoreList =
+  MC0_JADP1LEN MC0_JADP2LEN MC0_JADP3LEN MC0_JADPNUM MC0_JADVLEN
+  SASJSPROCESSMODE SASJS_STPSRV_HEADER_LOC;
+
+%put TEST 1 - Test with common extension, requesting only typeDefName parameter;
+%mp_assertscope(SNAPSHOT)
+%mv_getviyafileextparms(ext=txt, typeDefNameVar=viyaTypeDefName)
+%mp_assertscope(COMPARE
+  ,ignorelist=&mvarIgnoreList viyaTypeDefName
+)
+
+%mp_assert(
+  iftrue=(not %mf_isBlank(&viyaTypeDefName)),
+  desc=Check the requested macro variable viyaTypeDefName is not blank.
+)
+
+%put TEST 2 - Test with common extension, requesting only properties parameter;
+%mp_assertscope(SNAPSHOT)
+%mv_getviyafileextparms(ext=html, propertiesVar=viyaProperties)
+%mp_assertscope(COMPARE
+  ,ignorelist=&mvarIgnoreList viyaProperties
+)
+
+%mp_assert(
+  iftrue=(not %mf_isBlank(%superq(viyaProperties))),
+  desc=Check the requested macro variable viyaProperties is not blank.
+)
+
+%put TEST 3 - Test with common extension, requesting only mediaType parameter;
+%mp_assertscope(SNAPSHOT)
+%mv_getviyafileextparms(ext=mp3, mediaTypeVar=viyaMediaType)
+%mp_assertscope(COMPARE
+  ,ignorelist=&mvarIgnoreList viyaMediaType
+)
+
+%mp_assert(
+  iftrue=(not %mf_isBlank(&viyaMediaType)),
+  desc=Check the requested macro variable viyaMediaType is not blank.
+)
+
+%put TEST 4 - Test with common extension, requesting all parameters;
+%mp_assertscope(SNAPSHOT)
+%mv_getviyafileextparms(
+  ext=css,
+  typeDefNameVar=cssViyaTypeDefName,
+  propertiesVar=cssViyaProperties,
+  mediaTypeVar=cssViyaMediaType
+  )
+%mp_assertscope(COMPARE
+  ,ignorelist=
+    &mvarIgnoreList cssViyaTypeDefName cssViyaProperties  cssViyaMediaType
+)
+
+%mp_assert(
+  iftrue=(not ( %mf_isBlank(&cssViyaTypeDefName) or
+                %mf_isBlank(%superq(cssViyaProperties)) or
+                %mf_isBlank(&cssViyaMediaType) ) ),
+  desc=Check a full set of requested macro variables are not blank.
+)
+
+
+%put TEST 5 - Test with invalid extension - requested parameters will be blank;
+%mp_assertscope(SNAPSHOT)
+%mv_getviyafileextparms(
+  ext=xxxINVALIDxxx,
+  typeDefNameVar=invalidTypeDefName,
+  propertiesVar=invalidProperties,
+  mediaTypeVar=invalidMediaType
+  )
+%mp_assertscope(COMPARE
+  ,ignorelist=
+    &mvarIgnoreList invalidTypeDefName invalidProperties invalidMediaType
+)
+
+%mp_assert(
+  iftrue=(
+    %mf_isBlank(&invalidTypeDefName) and
+    %mf_isBlank(%superq(invalidProperties)) and
+    %mf_isBlank(&invalidMediaType)
+    ),
+  desc=Check the requested macro variables are all blank.
+)

viya/mfv_existfolder.sas

@@ -33,16 +33,23 @@
     msg=Cannot enter mfv_existfolder.sas with syscc=&syscc
   )
 
-  %local fref rc;
+  %local fref rc var;
   %let fref=%mf_getuniquefileref();
 
   %if %sysfunc(filename(fref,,filesrvc,folderPath="&path"))=0 %then %do;
     1
+    %let var=_FILESRVC_&fref._URI;
     %let rc=%sysfunc(filename(fref));
+    %symdel &var;
   %end;
   %else %do;
     0
     %let syscc=0;
   %end;
 
+  %mf_abort(
+    iftrue=(&syscc ne 0),
+    msg=Cannot leave mfv_existfolder.sas with syscc=&syscc
+  )
+
 %mend mfv_existfolder;

viya/mfv_getfolderpath.sas

@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Returns the path of a folder from the URI
+  @details Makes use of the SYSMSG() ER8OR response, which resolves the uri,
+  seemingly without entering an er8or state.
+
+  Usage:
+
+      %mv_createfolder(path=/public/demo)
+      %let uri=%mfv_getpathuri(/public/demo);
+      %put %mfv_getfolderpath(&uri);
+
+  Notice above the new path has an uppercase P - the correct path.
+
+  @param [in] uri The uri of the folder -eg /folders/folders/xxxx)
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+
+  <h4> Related Macros </h4>
+  @li mfv_getpathuri.sas
+
+  @version 4
+  @author [Allan Bowe](https://www.linkedin.com/in/allanbowe/)
+**/
+%macro mfv_getfolderpath(uri
+)/*/STORE SOURCE*/;
+
+  %local fref rc path msg var /* var used to avoid delete timing issue */;
+  %let fref=%mf_getuniquefileref();
+  %if %quote(%substr(%str(&uri),1,17)) ne %quote(/folders/folders/)
+  %then %do;
+    %put &sysmacroname: Invalid URI: &uri;
+  %end;
+  %else %if %sysfunc(filename(fref,,filesrvc,folderuri="&uri" ))=0
+  %then %do;
+    %let var=_FILESRVC_&fref._URI;
+    %local fid ;
+    %let fid= %sysfunc(fopen(&fref,I));
+    %let msg=%quote(%sysfunc(sysmsg()));
+
+    %unquote(%scan(&msg,2,%str(,.)))
+
+    %let rc=%sysfunc(fclose(&fid));
+    %let rc=%sysfunc(filename(fref));
+    %symdel &var;
+  %end;
+  %else %do;
+    %put &sysmacroname: Not Found: &uri;
+    %let syscc=0;
+  %end;
+
+%mend mfv_getfolderpath ;

viya/mfv_getpathuri.sas

@@ -0,0 +1,56 @@
+/**
+  @file
+  @brief Returns the uri of a file or folder
+  @details The automatic variable `_FILESRVC_[fref]_URI` is used after assigning
+  a fileref using the filesrvc engine.
+
+  Usage:
+
+      %put %mfv_getpathuri(/Public/folder/file.txt);
+      %put %mfv_getpathuri(/Public/folder);
+
+  @param [in] filepath The full path to the file on SAS drive
+    (eg /Public/myfile.txt)
+
+  <h4> SAS Macros </h4>
+  @li mf_abort.sas
+  @li mf_getuniquefileref.sas
+
+  <h4> Related Macros </h4>
+  @li mfv_existfile.sas
+  @li mfv_existfolder.sas
+
+  @version 3.5
+  @author [Allan Bowe](https://www.linkedin.com/in/allanbowe/)
+**/
+
+%macro mfv_getpathuri(filepath
+)/*/STORE SOURCE*/;
+
+  %mf_abort(
+    iftrue=(&syscc ne 0),
+    msg=Cannot enter &sysmacroname with syscc=&syscc
+  )
+
+  %local fref rc path name var /* var is used to avoid delete timing issue */;
+  %let fref=%mf_getuniquefileref();
+  %let name=%scan(&filepath,-1,/);
+  %let path=%substr(&filepath,1,%length(&filepath)-%length(&name)-1);
+
+  %if %sysfunc(filename(fref,,filesrvc,folderPath="&path" filename="&name"))=0
+  %then %do;
+    %let var=_FILESRVC_&fref._URI;
+    %str(&&&var)
+    %let rc=%sysfunc(filename(fref));
+    %symdel &var;
+  %end;
+  %else %do;
+    %put &sysmacroname: did not find &filepath;
+    %let syscc=0;
+  %end;
+
+  %mf_abort(
+    iftrue=(&syscc ne 0),
+    msg=Cannot leave &sysmacroname with syscc=&syscc
+  )
+%mend mfv_getpathuri;

viya/mv_createfile.sas

@@ -1,8 +1,12 @@
 /**
   @file
-  @brief Creates a file in SAS Drive
-  @details Creates a file in SAS Drive and adds the appropriate content type.
+  @brief Creates a file in SAS Drive using the API method
+  @details Creates a file in SAS Drive using the API interface.
   If the parent folder does not exist, it is created.
+  The API approach is more flexible than using the filesrvc engine of the
+  filename statement, as it provides more options.
+
+  SAS docs:  https://developer.sas.com/rest-apis/files/createNewFile
 
   Usage:
 
@@ -13,38 +17,62 @@
       run;
       %mv_createfile(path=/Public/temp,name=newfile.txt,inref=myfile)
 
+  The macro also supports find & replace (used by the SASjs Streaming App
+  build program).  This allows one string to be replaced by another at the
+  point at which the file is created.  This is done by passing in the NAMES of
+  the macro variables containing the values to be swapped, eg:
 
-  @param [in] path= The parent folder in which to create the file
+      filename fref temp;
+      data _null_;
+        file fref;
+        put 'whenever life gets you down, Mrs Brown..';
+      run;
+      %let f=Mrs Brown;
+      %let r=just remember that you're standing on a planet that's evolving;
+      %mv_createfile(path=/Public,name=life.md,inref=fref,fin,swap=f r)
+
+
+  @param [in] path= The parent (SAS Drive) folder in which to create the file
   @param [in] name= The name of the file to be created
   @param [in] inref= The fileref pointing to the file to be uploaded
   @param [in] intype= (BINARY) The type of the input data.  Valid values:
     @li BINARY File is copied byte for byte using the mp_binarycopy.sas macro.
     @li BASE64 File will be first decoded using the mp_base64.sas macro, then
       loaded byte by byte to SAS Drive.
-  @param [in] contentdisp= (inline) Content Disposition. Example values:
+  @param [in] contentdisp= (attchment) Content Disposition. Example values:
     @li inline
     @li attachment
-  @param [in] ctype= (0) Set a default HTTP Content-Type header to be returned
-    with the file when the content is retrieved from the Files service.
+  @param [in] ctype= (0) The actual MIME type of the file (if blank will be
+    determined based on file extension))
   @param [in] access_token_var= The global macro variable to contain the access
     token, if using authorization_code grant type.
   @param [in] grant_type= (sas_services) Valid values are:
     @li password
     @li authorization_code
     @li sas_services
+  @param [in] force= (YES) Will overwrite (delete / recreate) files by default.
+    Set to NO to abort if a file already exists in that location.
+  @param pin] swap= (0) Provide two macro variable NAMES that contain the values
+    to be swapped, eg swap=find replace (see also the example above)
+  @param [out] outds= (_null_) Output dataset with the uri of the new file
 
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
 
-  @version VIYA V.03.05
-  @author Allan Bowe, source: https://github.com/sasjs/core
-
   <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
   @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
   @li mf_isblank.sas
+  @li mf_mimetype.sas
+  @li mfv_getpathuri.sas
   @li mp_abort.sas
   @li mp_base64copy.sas
-  @li mp_binarycopy.sas
+  @li mp_replace.sas
   @li mv_createfolder.sas
+  @li mv_getviyafileextparms.sas
+
+  <h4> Related Macros</h4>
+  @li mv_createfile.sas
 
 **/
 
@@ -52,11 +80,14 @@
     ,name=
     ,inref=
     ,intype=BINARY
-    ,contentdisp=inline
+    ,contentdisp=attachment
     ,ctype=0
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
     ,mdebug=0
+    ,outds=_null_
+    ,force=YES
+    ,swap=0
   );
 %local dbg;
 %if &mdebug=1 %then %do;
@@ -65,6 +96,11 @@
 %end;
 %else %let dbg=*;
 
+%mp_abort(
+  iftrue=(&syscc ne 0),
+  msg=Cannot enter &sysmacroname with syscc=&syscc
+)
+
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
@@ -78,52 +114,223 @@
 %mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
-  ,mac=&sysmacroname
+  ,mac=MV_CREATEFILE
   ,msg=%str(Invalid value for grant_type: &grant_type)
 )
 
 %mp_abort(iftrue=(%mf_isblank(&path)=1 or %length(&path)=1)
-  ,mac=&sysmacroname
+  ,mac=MV_CREATEFILE
   ,msg=%str(path value must be provided)
 )
 %mp_abort(iftrue=(%mf_isblank(&name)=1 or %length(&name)=1)
-  ,mac=&sysmacroname
+  ,mac=MV_CREATEFILE
   ,msg=%str(name value with length >1 must be provided)
 )
 
+/* prep the source file */
+%local fref;
+%let fref=%mf_getuniquefileref();
+
+%if %upcase(&intype)=BINARY %then %let fref=&inref;
+%else %if %upcase(&intype)=BASE64 %then %do;
+  %mp_base64copy(inref=&inref, outref=&fref, action=DECODE)
+%end;
+%else %put %str(ERR)OR: invalid value for intype: &intype;
+
+%if "&swap" ne "0" %then %do;
+  %mp_replace("%sysfunc(pathname(&fref))"
+    ,findvar=%scan(&swap,1,%str( ))
+    ,replacevar=%scan(&swap,2,%str( ))
+  )
+%end;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fref lrecl=32767;
+    input;
+    put _infile_;
+  run;
+%end;
+
+options noquotelenmax;
+%local base_uri; /* location of rest apis */
+%let base_uri=%trim(%mf_getplatform(VIYARESTAPI));
+
 /* create folder if it does not already exist */
+%local folderds self_uri;
+%let folderds=%mf_getuniquename(prefix=folderds);
 %mv_createfolder(path=&path
   ,access_token_var=&access_token_var
   ,grant_type=&grant_type
   ,mdebug=&mdebug
+  ,outds=&folderds
+)
+data _null_;
+  set &folderds;
+  call symputx('self_uri',self_uri,'l');
+run;
+
+/* abort or delete if file already exists */
+%let force=%upcase(&force);
+%local fileuri ;
+%let fileuri=%trim(%mfv_getpathuri(&path/&name));
+%mp_abort(iftrue=(%mf_isblank(&fileuri)=0 and &force ne YES)
+  ,mac=MV_CREATEFILE
+  ,msg=%str(File &path/&name already exists and force=&force)
+)
+%mp_abort(
+  iftrue=(&syscc ne 0),
+  mac=MV_CREATEFILE182
+  msg=syscc=&syscc after mfv_getpathuri
 )
 
-/* create file with relevant options */
-%local fref;
+%if %mf_isblank(&fileuri)=0 and &force=YES %then %do;
+  proc http method="DELETE" url="&base_uri&fileuri" &oauth_bearer;
+    headers
+  %if &grant_type=authorization_code %then %do;
+        "Authorization"="Bearer &&&access_token_var"
+  %end;
+        "Accept"="*/*";
+  run;
+  %put &sysmacroname DELETE &base_uri&fileuri;
+  %if &SYS_PROCHTTP_STATUS_CODE ne 204 %then %do;
+    %put &=SYS_PROCHTTP_STATUS_CODE &=SYS_PROCHTTP_STATUS_PHRASE;
+  %end;
+%end;
+
+%local url mimetype ext;
+%let url=&base_uri/files/files?parentFolderUri=&self_uri;
+%let ext=%upcase(%trim(%scan(&name,-1,.)));
+
+/* Get Viya file-extension details into some macro variables */
+%mv_getViyaFileExtParms(&ext
+                        ,propertiesVar=viyaProperties
+                        ,typeDefNameVar=viyaTypeDefName
+                        ,mdebug=&mdebug);
+
+/* fetch job info */
+%local fname1;
+%let fname1=%mf_getuniquefileref();
+proc http method='POST' out=&fname1 &oauth_bearer in=&fref
+  %if "&ctype" = "0" %then %do;
+    %let mimetype=%mf_mimetype(&ext);
+    ct="&mimetype"
+  %end;
+  %else %do;
+    ct="&ctype"
+  %end;
+
+  /* typeDefName */
+  %if not %mf_isBlank(&viyaTypeDefName) %then %do;
+    url="&url%str(&)typeDefName=&viyaTypeDefName";
+  %end;
+  %else %do;
+    %if "&ext"="HTM" or "&ext"="HTML" or "&ext"="XHTML" %then %do;
+      url="&url%str(&)typeDefName=file_html";
+    %end;
+    %else %do;
+      %if "&ext"="CSS" or "&ext"="JS" or "&ext"="PNG" or "&ext"="SVG" %then %do;
+        url="&url%str(&)typeDefName=file_%lowcase(&ext)";
+      %end;
+      %else %do;
+        url="&url";
+      %end;
+    %end;
+  %end;
+
+  headers "Accept"="application/json"
+  %if &grant_type=authorization_code %then %do;
+    "Authorization"="Bearer &&&access_token_var"
+  %end;
+  "Content-Disposition"=
+  %if "&ext"="SVG" or "&ext"="HTML" %then %do;
+    "filename=""&name"";"
+  %end;
+  %else %do;
+    "&contentdisp filename=""&name""; name=""&name"";"
+  %end;
+  ;
+run;
+%if &mdebug=1 %then %put &sysmacroname POST &=url
+  &=SYS_PROCHTTP_STATUS_CODE &=SYS_PROCHTTP_STATUS_PHRASE;
+%mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 201)
+  ,mac=MV_CREATEFILE
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+%local libref2;
+%let libref2=%mf_getuniquelibref();
+libname &libref2 JSON fileref=&fname1;
+/* Grab the follow on link */
+data &outds;
+  set &libref2..links end=last;
+  if rel='createChild' then do;
+    &dbg put (_all_)(=);
+  end;
+run;
+
+/* URI of the created file */
+%let fileuri=%trim(%mfv_getpathuri(&path/&name));
+
+/* If properties were found then patch the file to include them */
+%if not %mf_isBlank(%superq(viyaProperties)) %then %do;
+  /* Wrap the properties object in a root object also containing the file name */
+  %local viyapatch;
+  %let viyapatch = %sysfunc(pathname(work))/%mf_getuniquename(prefix=patch_json_);
+  data _null_;
+    length line $32767;
+    file "&viyapatch" lrecl=32767;
+    put '{ "name": "' "&name" '",';
+    line = cat('"properties": ',symget("viyaProperties"));
+    put line;
+    put '}';
+    stop;
+  run;
+
+  %if &mdebug=1 %then %do;
+    data _null_;
+      if (_n_ eq 1) then put 'DEBUG: ** PATCH JSON **';
+      infile "&viyapatch" end=last;
+      input;
+      put _infile_;
+    run;
+  %end;
+
+  /* And apply the properties to the newly created file, using the PATCH method */
   %let fref=%mf_getuniquefileref();
-filename &fref filesrvc
-  folderPath="&path"
-  filename="&name"
-  cdisp="&contentdisp"
-%if "&ctype" ne "0" %then %do;
-  ctype="&ctype"
+  filename &fref "&viyapatch";
+  %let url=&base_uri&fileuri;
+
+  proc http method='PATCH' oauth_bearer=sas_services in=&fref
+    url="&url";
+    headers "Accept"="application/json"
+            "Content-Type"="application/json"
+            "If-Match"="*";
+    %if &mdebug=1 %then %do;
+      debug level=2;
     %end;
-  lrecl=1048544;
-%if &intype=BINARY %then %do;
-  %mp_binarycopy(inref=&inref, outref=&fref)
-%end;
-%else %if &intype=BASE64 %then %do;
-  %mp_base64copy(inref=&inref, outref=&fref, action=DECODE)
+  run;
+  %if &mdebug=1 %then %put &sysmacroname PATCH &=url
+    &=SYS_PROCHTTP_STATUS_CODE &=SYS_PROCHTTP_STATUS_PHRASE;
+  %mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200)
+    ,mac=MV_CREATEFILE
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
 %end;
 
+%put &sysmacroname: &base_uri&fileuri;
+%put /SASJobExecution?_file=&path/&name;%put;
+
+%if &mdebug=0 %then %do;
+  /* clear refs */
+  filename &fname1 clear;
   filename &fref clear;
+  libname &libref2 clear;
+%end;
 
-%local base_uri; /* location of rest apis */
-%let base_uri=%mf_getplatform(VIYARESTAPI);
-
-%put &sysmacroname: File &name successfully created in &path;
-%put &sysmacroname:;%put;
-%put    &base_uri/SASJobExecution?_file=&path/&name;%put;
-%put &sysmacroname:;
+%mp_abort(
+  iftrue=(&syscc ne 0),
+  msg=Cannot leave &sysmacroname with syscc=&syscc
+)
 
 %mend mv_createfile;

viya/mv_createfolder.sas

@@ -17,15 +17,16 @@
     @li sas_services
 
   @param [in] mdebug=(0) set to 1 to enable DEBUG messages
+  @param [out] outds=(_null_) Optionally create an output dataset which will
+    contain the uri (self_uri) of the created (and parent) folder.
 
-  @version VIYA V.03.04
-  @author Allan Bowe, source: https://github.com/sasjs/core
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas
   @li mf_getuniquefileref.sas
   @li mf_getuniquelibref.sas
   @li mf_isblank.sas
+  @li mfv_getpathuri.sas
   @li mf_getplatform.sas
   @li mfv_existfolder.sas
 
@@ -36,6 +37,7 @@
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
     ,mdebug=0
+    ,outds=_null_
   );
 %local dbg;
 %if &mdebug=1 %then %do;
@@ -44,10 +46,21 @@
 %end;
 %else %let dbg=*;
 
+%mp_abort(
+  iftrue=(&syscc ne 0),
+  msg=Cannot enter &sysmacroname with syscc=&syscc
+)
+
 %if %mfv_existfolder(&path)=1 %then %do;
-  %put &sysmacroname: &path already exists;
+  %&dbg.put &sysmacroname: &path already exists;
+  data &outds;
+    self_uri="%mfv_getpathuri(&path)";
+    output;
+    stop;
+  run;
   %return;
 %end;
+%mp_abort(iftrue=(&syscc ne 0),msg=syscc=&syscc when folder checking)
 
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
@@ -80,12 +93,12 @@ options noquotelenmax;
 %local subfolder_cnt; /* determine the number of subfolders */
 %let subfolder_cnt=%sysfunc(countw(&path,/));
 
-%local href; /* resource address (none for root) */
-%let href="/folders/folders?parentFolderUri=/folders/folders/none";
-
 %local base_uri; /* location of rest apis */
 %let base_uri=%mf_getplatform(VIYARESTAPI);
 
+%local href; /* resource address (none for root) */
+%let href="&base_uri/folders/folders?parentFolderUri=/folders/folders/none";
+
 %local x newpath subfolder;
 %do x=1 %to &subfolder_cnt;
   %let subfolder=%scan(&path,&x,%str(/));
@@ -101,6 +114,17 @@ options noquotelenmax;
       headers "Authorization"="Bearer &&&access_token_var";
   %end;
   run;
+  %if &SYS_PROCHTTP_STATUS_CODE=401 %then %do;
+    /* relates to: https://github.com/sasjs/core/issues/400 */
+    %put 401 thrown in &sysmacroname;
+    %put sleeping: %sysfunc(sleep(12,1)) secs - will try again;
+    proc http method='GET' out=&fname1 &oauth_bearer
+        url="&base_uri/folders/folders/@item?path=&newpath";
+    %if &grant_type=authorization_code %then %do;
+        headers "Authorization"="Bearer &&&access_token_var";
+    %end;
+    run;
+  %end;
   %local libref1;
   %let libref1=%mf_getuniquelibref();
   libname &libref1 JSON fileref=&fname1;
@@ -108,14 +132,14 @@ options noquotelenmax;
     iftrue=(
       &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 404
     )
-    ,mac=&sysmacroname
+    ,mac=mv_createfolder124
     ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
   )
   %if &mdebug=1 %then %do;
     %put &sysmacroname following check to see if &newpath exists:;
     %put _local_;
     data _null_;
-      set &fname1;
+      infile &fname1;
       input;
       putlog _infile_;
     run;
@@ -157,8 +181,9 @@ options noquotelenmax;
                 'Content-Type'='application/vnd.sas.content.folder+json'
                 'Accept'='application/vnd.sas.content.folder+json';
     run;
-    %put &=SYS_PROCHTTP_STATUS_CODE;
-    %put &=SYS_PROCHTTP_STATUS_PHRASE;
+    %if &SYS_PROCHTTP_STATUS_CODE ne 201 %then %do;
+      %put &=SYS_PROCHTTP_STATUS_CODE &=SYS_PROCHTTP_STATUS_PHRASE;
+    %end;
     %mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 201)
       ,mac=&sysmacroname
       ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
@@ -167,12 +192,17 @@ options noquotelenmax;
     %let libref2=%mf_getuniquelibref();
     libname &libref2 JSON fileref=&fname2;
     %put &sysmacroname &newpath now created. Grabbing the follow on link ;
-    data _null_;
+    data &outds;
       set &libref2..links;
       if rel='createChild' then do;
         call symputx('href',quote(cats("&base_uri",href)),'l');
         &dbg put (_all_)(=);
       end;
+      if method='GET' and rel='self' then do;
+        self_uri=uri;
+        output;
+      end;
+      keep self_uri ;
     run;
 
     libname &libref2 clear;
@@ -181,4 +211,8 @@ options noquotelenmax;
   filename &fname1 clear;
   libname &libref1 clear;
 %end;
+%mp_abort(
+  iftrue=(&syscc ne 0),
+  msg=Cannot leave &sysmacroname with syscc=&syscc
+)
 %mend mv_createfolder;

viya/mv_createwebservice.sas

@@ -122,7 +122,7 @@ options noquotelenmax;
   %let path=%substr(&path,1,%length(&path)-1);
 
 /* ensure folder exists */
-%put &sysmacroname: Path &path being checked / created;
+%&dbg.put &sysmacroname: Path &path being checked / created;
 %mv_createfolder(path=&path)
 
 %local base_uri; /* location of rest apis */
@@ -955,12 +955,7 @@ run;
   libname &libref1 clear;
 %end;
 
-%put &sysmacroname: Job &name successfully created in &path;
-%put &sysmacroname:;
-%put &sysmacroname: Check it out here:;
-%put &sysmacroname:;%put;
-%put    &url/SASJobExecution?_PROGRAM=&path/&name;%put;
-%put &sysmacroname:;
-%put &sysmacroname:;
+%put &sysmacroname: Job &name created!  Check it out:;
+%put &url/SASJobExecution?_PROGRAM=&path/&name;
 
 %mend mv_createwebservice;

viya/mv_deletefoldermember.sas

@@ -111,13 +111,15 @@ proc http method='GET' out=&fname1a &oauth_bearer
   headers "Authorization"="Bearer &&&access_token_var";
 %end;
 run;
-%put &=SYS_PROCHTTP_STATUS_CODE;
+%if &SYS_PROCHTTP_STATUS_CODE ne 200 %then %do;
+  %put &=sysmacroname &=SYS_PROCHTTP_STATUS_CODE &=SYS_PROCHTTP_STATUS_PHRASE;
+%end;
 %local libref1a;
 %let libref1a=%mf_getuniquelibref();
 libname &libref1a JSON fileref=&fname1a;
 %local uri found;
 %let found=0;
-%put Getting object uri from &libref1a..items;
+/* %put Getting object uri from &libref1a..items; */
 data _null_;
   length contenttype name $1000;
   set &libref1a..items;

viya/mv_deletejes.sas

@@ -69,8 +69,7 @@
 options noquotelenmax;
 %local base_uri; /* location of rest apis */
 %let base_uri=%mf_getplatform(VIYARESTAPI);
-
-%put &sysmacroname: fetching details for &path ;
+/* fetch the members of the folder to get the uri */
 %local fname1;
 %let fname1=%mf_getuniquefileref();
 proc http method='GET' out=&fname1 &oauth_bearer
@@ -90,7 +89,7 @@ run;
   )
 %end;
 
-%put &sysmacroname: grab the follow on link ;
+/* grab the follow on link */
 %local libref1;
 %let libref1=%mf_getuniquelibref();
 libname &libref1 JSON fileref=&fname1;
@@ -108,13 +107,15 @@ proc http method='GET' out=&fname1a &oauth_bearer
   headers "Authorization"="Bearer &&&access_token_var";
 %end;
 run;
-%put &=SYS_PROCHTTP_STATUS_CODE;
+%if &SYS_PROCHTTP_STATUS_CODE ne 200 %then %do;
+  %put &=sysmacroname &=SYS_PROCHTTP_STATUS_CODE &=SYS_PROCHTTP_STATUS_PHRASE;
+%end;
 %local libref1a;
 %let libref1a=%mf_getuniquelibref();
 libname &libref1a JSON fileref=&fname1a;
 %local uri found;
 %let found=0;
-%put Getting object uri from &libref1a..items;
+/* %put Getting object uri from &libref1a..items; */
 data _null_;
   length contenttype name $1000;
   set &libref1a..items;
@@ -140,7 +141,7 @@ run;
     ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
   )
 %end;
-%else %put &sysmacroname: &path/&name successfully deleted;
+%else %put &sysmacroname: &path/&name deleted;
 
 /* clear refs */
 filename &fname1 clear;

viya/mv_deleteviyafolder.sas

@@ -1,6 +1,6 @@
 /**
-  @file mv_deleteviyafolder.sas
-  @brief Creates a viya folder if that folder does not already exist
+  @file
+  @brief Deletes a viya folder
   @details If not running in Studo 5 +, will expect an oauth token in a global
   macro variable (default ACCESS_TOKEN).
 

viya/mv_getjoblog.sas

@@ -126,7 +126,7 @@ data _null_;
   uri=symget('uri');
   if length(uri)<12 then do;
     call symputx('errflg',1);
-    call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
+    call symputx('errmsg',"URI is too short - "!!uri,'l');
   end;
   if scan(uri,-1)='state' or scan(uri,1) ne 'jobExecution' then do;
     call symputx('errflg',1);
@@ -191,7 +191,7 @@ data _null_;
   uri=symget('loglocation');
   if length(uri)<12 then do;
     call symputx('errflg',1);
-    call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
+    call symputx('errmsg',"URI is too short - "!!uri,'l');
   end;
   else if (scan(uri,1,'/') ne 'compute' or scan(uri,2,'/') ne 'sessions')
     and (scan(uri,1,'/') ne 'files' or scan(uri,2,'/') ne 'files')

viya/mv_getviyafileextparms.sas

@@ -0,0 +1,248 @@
+/**
+  @file mv_getviyafileextparms.sas
+  @brief Reads the VIYA file-extension type definition and returns selected
+    values in SAS macro variables
+
+  @details Content is derived from the following endpoint:
+    "https://${serverUrl}/types/types?limit=999999"
+
+  @param [in] ext File extension to retrieve property info for.
+  @param [out] propertiesVar= SAS macro variable name that will contain
+    the 'properties' object json, if found, else blank.
+  @param [out] typeDefNameVar= SAS macro variable name that will contain
+    the 'typeDefName' property value, if found, else blank.
+  @param [out] mediaTypeVar= SAS macro variable name that will contain
+    the 'mediaType' property value, if found, else blank.
+  @param [out] viyaFileExtRespLibDs (work.mv_getViyaFileExtParmsResponse)
+    Library.name of the dataset to receive the local working copy of the initial
+    response that requests all file extension details. Created once per session
+    to avoid multiple api calls.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_getvalue.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
+  @li mf_isblank.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+**/
+
+%macro mv_getViyaFileExtParms(
+  ext,
+  typeDefNameVar=,
+  propertiesVar=,
+  mediaTypeVar=,
+  viyaFileExtRespLibDs=work.mv_getViyaFileExtParmsResponse,
+  mdebug=0
+  );
+  %local base_uri; /* location of rest apis */
+  %local url; /* File extension info end-point */
+
+  %mp_abort(
+    iftrue=(%mf_isBlank(&ext))
+    ,msg=%str(No file extension provided.)
+    ,mac=MV_GETVIYAFILEEXTPARMS
+  );
+
+  %mp_abort(
+    iftrue=(%mf_isBlank(&typeDefNameVar) and
+            %mf_isBlank(&propertiesVar) and
+            %mf_isBlank(&mediaTypeVar))
+    ,msg=%str(MV_GETVIYAFILEEXTPARMS - No parameter was requested.)
+    ,mac=MV_GETVIYAFILEEXTPARMS
+  );
+
+  %mp_abort(
+    iftrue=(%mf_isBlank(&viyaFileExtRespLibDs))
+    ,msg=%str(No <libname.>dataset name provided to cache inital response.)
+    ,mac=MV_GETVIYAFILEEXTPARMS
+  );
+
+  /* Declare requested parameters as global macro vars and initialize blank */
+  %if not %mf_isBlank(&typeDefNameVar) %then %do;
+    %global &typeDefNameVar;
+    %let &typeDefNameVar = %str();
+  %end;
+  %if not %mf_isBlank(&propertiesVar) %then %do;
+    %global &propertiesVar;
+    %let &propertiesVar = %str();
+  %end;
+  %if not %mf_isBlank(&mediaTypeVar) %then %do;
+    %global &mediaTypeVar;
+    %let &mediaTypeVar = %str();
+  %end;
+
+  %let base_uri=%mf_getplatform(VIYARESTAPI);
+  %if &mdebug=1 %then %do;
+    %put DEBUG: &=base_uri;
+  %end;
+
+  %let ext=%lowcase(&ext);
+
+  /* Create a local copy of the Viya response containing all file type info, if
+  it does not already exist. */
+  %if not %mf_existds(&viyaFileExtRespLibDs) %then %do;
+    /* Create a temp file and fill with JSON that declares */
+    /* VIYA file-type details for the given file extension */
+    %local viyatypedefs;
+    %let viyatypedefs=%mf_getuniquefileref();
+    filename &viyatypedefs temp;
+
+    %let url = &base_uri/types/types?limit=999999;
+
+    proc http oauth_bearer=sas_services out=&viyatypedefs
+      url="&url";
+    run;
+
+    %if &mdebug=1 %then %put DEBUG: &sysmacroname &=url
+      &=SYS_PROCHTTP_STATUS_CODE &=SYS_PROCHTTP_STATUS_PHRASE;
+
+    %if (&SYS_PROCHTTP_STATUS_CODE ne 200) %then %do;
+      /* To avoid a breaking change, exit early if the request failed.
+        The calling process will proceed with empty requested macro variables. */
+      %put INFO: &sysmacroname File extension details were not retrieved.;
+      filename &viyatypedefs clear;
+      %return;
+    %end;
+
+    %if &mdebug=1 %then %do;
+      /* Dump the response to the log */
+      data _null_;
+        length line $120;
+        null=byte(0);
+        infile &viyatypedefs dlm=null lrecl=120 recfm=n;
+        input line $120.;
+        if _n_ = 1 then put "DEBUG:";
+        put line;
+      run;
+    %end;
+
+    /* Convert the content of that JSON into SAS datasets */
+      /* First prepare a new WORK-based folder to receive the datasets */
+    %local jsonworkfolder jsonlib opt_dlcreatedir;
+    %let jsonworkfolder=%sysfunc(pathname(work))/%mf_getuniquename(prefix=json_);
+    %let jsonlib=%mf_getuniquelibref(prefix=json);
+      /* And point a libname at it */
+    %let opt_dlcreatedir = %sysfunc(getoption(dlcreatedir));
+    options dlcreatedir; libname &jsonlib "&jsonworkfolder"; options &opt_dlcreatedir;
+
+    /* Read the json output once and copy datasets to its work folder */
+    %local libref1;
+    %let libref1=%mf_getuniquelibref();
+    libname &libref1 JSON fileref=&viyatypedefs automap=create;
+    proc copy in=&libref1 out=&jsonlib; run;
+
+    libname &libref1 clear;
+
+    /* Now give all rows belonging to the same items array a grouping value */
+    data &viyaFileExtRespLibDs;
+      length _viyaItemIdx 8;
+      set &jsonlib..alldata;
+      retain _viyaItemIdx 0;
+      /* Increment the row group index when a new 'items' group is observed */
+      if P=1 and P1='items' then _viyaItemIdx + 1;
+    run;
+
+    %if &mdebug=0 %then %do;
+      /* Tidy up, unless debug=1 */
+      proc datasets library=&jsonlib nolist kill; quit;
+      libname &jsonlib clear;
+    %end;
+
+    filename &viyatypedefs clear;
+
+  %end; /* If initial filetype query response didn't exist */
+
+  /* Find the row-group for the current file extension */
+  %local itemRowGroup;
+  %let itemRowGroup =
+    %mf_getValue(
+      &viyaFileExtRespLibDs
+      ,_viyaItemIdx
+      ,filter=%quote(p1='items' and p2='extensions' and value="&ext")
+    );
+
+  %if &mdebug %then %put DEBUG: &=itemRowGroup;
+
+  %if %mf_isBlank(&itemRowGroup) %then %do;
+    /* extension was not found */
+    %if(&mdebug=1) %then %put DEBUG: No type details found for extension "&ext".;
+    %return;
+  %end;
+
+  /* Filter the cached response data down to the required file extension */
+  %local dsItems;
+  %let dsItems = %mf_getuniquename(prefix=dsItems_);
+  data work.&dsItems;
+    set &viyaFileExtRespLibDs;
+    where _viyaItemIdx = &itemRowGroup;
+  run;
+
+  /* Populate typeDefName, if requested */
+  %if (not %mf_isBlank(&typeDefNameVar)) %then %do;
+    %let &typeDefNameVar = %mf_getvalue(&dsItems,value,filter=%quote(p1="items" and p2="name"));
+    %if &mdebug=1 %then %put DEBUG: &=typeDefNameVar &typeDefNameVar=&&&typeDefNameVar;
+  %end;
+
+  /* Populate mediaType, if requested */
+  %if (not %mf_isBlank(&mediaTypeVar)) %then %do;
+    %let &mediaTypeVar = %mf_getvalue(&dsItems,value,filter=%quote(p1="items" and p2="mediaType"));
+    %if &mdebug=1 %then %put DEBUG: &=mediaTypeVar &mediaTypeVar=&&&mediaTypeVar;
+  %end;
+
+  /* Populate properties macro variable, if requested */
+  %if not %mf_isBlank(&propertiesVar) %then %do;
+
+    /* Filter dsItems down to the properties */
+    %local dsProperties;
+    %let dsProperties = %mf_getuniquename(prefix=dsProperties_);
+    data work.&dsProperties ( rename=(p3 = propertyName) );
+        set work.&dsItems;
+        where p2="properties" and v=1;
+    run;
+
+    /* Check for 1+ properties */
+    %if ( %mf_nobs(&dsProperties) = 0 ) %then %do;
+      %let &propertiesVar = %str();
+      %if &mdebug=1 %then %put DEBUG: &SYSMACRONAME - No Viya properties found for file suffix %str(%')&ext%str(%');
+    %end;
+    %else %do;
+      /* Properties potentially span multiple rows in the input table */
+      data _null_;
+        length
+          line $32767
+          properties $32767
+        ;
+        retain properties;
+        set &dsProperties end=last;
+        if _n_ = 1 then properties = '{';
+
+        line = cats(quote(trim(propertyName)),':');
+        /* Only strings and bools appear in properties */
+        if value not in ("true","false") then value = quote(trim(value));
+        line = catx(' ',line,value);
+        /* Add a comma separator to all except the last line */
+        if not last then line = cats(line,',');
+
+        /* Add this line to the output value */
+        properties = catx(' ',properties,line);
+
+        if last then do;
+          /* Close off the properties object and output to the macro variable */
+          properties=catx(' ',properties,'}');
+          call symputx("&propertiesVar",properties);
+        end;
+      run;
+
+      %if &mdebug=1 %then %put DEBUG: &=propertiesVar &propertiesVar=&&&propertiesVar;
+    %end;
+
+  %end;
+
+%mend mv_getViyaFileExtParms;

viya/mv_jobflow.sas

@@ -188,6 +188,8 @@
     %if %mf_existvarList(&inds,FLOW_ID)=0 %then %do;
       retain FLOW_ID 0;
     %end;
+    /* https://github.com/sasjs/adapter/pull/845#issuecomment-2956589644 */
+    retain _omitSessionResults "false";
     set &inds;
     &dbg. putlog (_all_)(=);
   run;