Merge pull request #306 from sasjs/mp_dsmeta

feat: new mp_dsmeta macro

.all-contributorsrc

@@ -126,6 +126,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "eltociear",
+      "name": "Ikko Ashimine",
+      "avatar_url": "https://avatars.githubusercontent.com/u/22633385?v=4",
+      "profile": "https://bandism.net/",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.devcontainer/Dockerfile

@@ -0,0 +1,8 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/javascript-node/.devcontainer/base.Dockerfile
+
+# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
+ARG VARIANT="18-bullseye"
+FROM mcr.microsoft.com/vscode/devcontainers/javascript-node:0-${VARIANT}
+
+RUN apt-get update \
+    && apt-get install -y doxygen

.devcontainer/devcontainer.json

@@ -0,0 +1,26 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/typescript-node
+{
+	"name": "Node.js & TypeScript",
+	"build": {
+		"dockerfile": "Dockerfile",
+		// Update 'VARIANT' to pick a Node version: 18, 16, 14.
+		// Append -bullseye or -buster to pin to an OS version.
+		// Use -bullseye variants on local on arm64/Apple Silicon.
+		"args": {
+			"VARIANT": "16-bullseye"
+		}
+	},
+	// Set *default* container specific settings.json values on container create.
+	"settings": {},
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"SASjs.sasjs-for-vscode"
+	],
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "npm i && npm i -g @sasjs/cli",
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "node"
+}

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,17 @@
+## Issue
+
+Link any related issue(s) in this section.
+
+## Intent
+
+What this PR intends to achieve.
+
+## Implementation
+
+What code changes have been made to achieve the intent.
+
+## Checks
+
+- [ ] Code is formatted correctly (`sasjs lint`).
+- [ ] Any new functionality has been unit tested.
+- [ ] All unit tests are passing (`sasjs test`).

.github/workflows/run-tests.yml

@@ -78,7 +78,5 @@ jobs:
           SECRET: ${{secrets.SECRET}}
           SAS_USERNAME: ${{secrets.SAS_USERNAME}}
           SAS_PASSWORD: ${{secrets.SAS_PASSWORD}}
-          SERVER_URL: ${{secrets.SERVER_URL}}
-          SERVER_TYPE: ${{secrets.SERVER_TYPE}}
           ACCESS_TOKEN: ${{secrets.ACCESS_TOKEN}}
           REFRESH_TOKEN: ${{secrets.REFRESH_TOKEN}}

.npmignore

@@ -6,7 +6,6 @@ sasjs/
 .github/
 .git-hooks/
 .vscode/
-main.dox
 make_singlefile.sh
 *.md
 .all-contributorsrc

README.md

@@ -18,7 +18,7 @@
 [dependency-url]:https://github.com/sasjs/core/blob/main/package.json
 
 
-Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
+Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/.github/CONTRIBUTING.md) are welcome.
 
 You can download and compile them all in just two lines of SAS code:
 
@@ -31,62 +31,30 @@ Documentation: https://core.sasjs.io
 
 ## Components
 
-### BASE library (All Platforms)
+### BASE folder (All Platforms)
 
 - OS independent
 - Works on all SAS Platforms
 - No X command
-- Prefixes: _mf_, _mp_
+- Prefixes: `mf_`, `mp_`
 
-### DDL library (All Platforms)
+### DDL folder (All Platforms)
 
 - OS independent
 - Works on all SAS Platforms
 - No X command
-- Prefixes: _mddl_(lib)_ -> where lib can be "SAS" (in relation to a SAS component) or "DC" (in relation to a Data Controller component)
+- Prefixes: `mddl_(lib)_` -> where lib can be "SAS" (in relation to a SAS component) or "DC" (in relation to a Data Controller component)
 
 This library will not be used for storing data entries (such as formats or datalines).  Where this becomes necessary in the future, a new repo will be created, in order to keep the NPM bundle size down (for the benefit of those looking to embed purely macros in their applications).
 
-### FCMP library (All Platforms)
+### FCMP folder (All Platforms)
 
 - Function and macro names are identical, except for special cases
-- Prefixes: _mcf_
+- Prefixes: `mcf_`
 
 The fcmp macros are used to generate fcmp functions, and can be used with or without the `proc fcmp` wrapper.
 
-### META library (SAS9 only)
-
-Macros used in SAS EBI, which connect to the metadata server.
-
-- OS independent
-- Metadata aware
-- No X command
-- Prefixes: _mm_
-
-### SERVER library (@sasjs/server only)
-These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
-
-- OS independent
-- @sasjs/server aware
-- No X command
-- Prefixes: _ms_
-
-### VIYA library (Viya only)
-
-Macros used for interfacing with SAS Viya.
-
-- OS independent
-- No X command
-- Prefixes: _mv_, _mvf_
-
-### METAX library (SAS9 only)
-
-- OS specific
-- Metadata aware
-- X command enabled
-- Prefixes: _mmw_,_mmu_,_mmx_
-
-### LUA library
+### LUA folder
 
 Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
 
@@ -104,15 +72,63 @@ endsubmit;
 run;
 ```
 
-- Prefixes: _ml_
+- Prefixes: `ml_`
+
+### META folder (SAS9 only)
+
+Macros used in SAS EBI, which connect to the metadata server.
+
+- OS independent
+- Metadata aware
+- No X command
+- Prefixes: `mm_`
+
+### METAX folder (SAS9 only)
+
+- OS specific
+- Metadata aware
+- X command enabled
+- Prefixes: `mmx_`
+
+### SERVER folder (@sasjs/server only)
+These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
+
+- OS independent
+- @sasjs/server aware
+- No X command
+- Prefixes: `ms_`
+
+### VIYA folder (Viya only)
+
+Macros used for interfacing with SAS Viya.
+
+- OS independent
+- No X command
+- Prefixes: `mv_`, `mvf_`
+
+### XPLATFORM folder (Viya, Meta, and Server)
+
+Sometimes it is helpful to use a macro that can be used interchangeably regardless of the server type on which is is running (SASVIYA, SAS9, SASJS).
+
+- OS independent
+- No X command
+- Prefixes: `mx_`
 
 ## Installation
 
 First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available, eg:
 
 ```sas
-options insert=(sasautos="/your/path/macrocore/base");
-options insert=(sasautos="/your/path/macrocore/meta");
+%let repoloc=/your/path/core;
+options insert=(sasautos="&repoloc/base");
+options insert=(sasautos="&repoloc/ddl");
+options insert=(sasautos="&repoloc/fcmp");
+options insert=(sasautos="&repoloc/lua");
+options insert=(sasautos="&repoloc/meta");
+options insert=(sasautos="&repoloc/metax");
+options insert=(sasautos="&repoloc/server");
+options insert=(sasautos="&repoloc/viya");
+options insert=(sasautos="&repoloc/xplatform");
 ```
 
 The above can be done directly in your sas program, via an autoexec, or an initialisation program.
@@ -142,7 +158,7 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
   - _mp_ for macro procedures (which generate sas code)
   - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
   - _mv_ for macro procedures that will only work in Viya
-  - _mx_ for macros that are XCMD enabled (working on both windows and unix)
+  - _mx_ for macros that work on Viya, SAS 9 EBI and SASjs Server
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
@@ -221,15 +237,17 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 The following repositories are also worth checking out:
 
+* [SASJedi/sas-macros](https://github.com/SASJedi/sas-macros)
 * [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
 * [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)
 * [KatjaGlassConsulting/SMILE-SmartSASMacros](https://github.com/KatjaGlassConsulting/SMILE-SmartSASMacros)
+* [rogerjdeangelis](https://github.com/rogerjdeangelis)
 * [scottbass/sas](https://github.com/scottbass/SAS)
 * [yabwon/sas_packages](https://github.com/yabwon/SAS_PACKAGES)
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-11-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-12-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -251,6 +269,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
     <td align="center"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
     <td align="center"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
+    <td align="center"><a href="https://bandism.net/"><img src="https://avatars.githubusercontent.com/u/22633385?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Ikko Ashimine</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=eltociear" title="Code">💻</a></td>
   </tr>
 </table>
 

base/mf_existfeature.sas

@@ -30,7 +30,7 @@
     %put No feature was requested for detection;
   %end;
   %else %if &feature=COLCONSTRAINTS %then %do;
-    %if %substr(&sysver,1,1)=4 %then 0;
+    %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 0;
     %else 1;
   %end;
   %else %if &feature=PROCLUA %then %do;
@@ -40,6 +40,17 @@
     %else %if "&SYSVLONG" < "9.04.01M3" %then 0;
     %else 1;
   %end;
+  %else %if &feature=DBMS_MEMTYPE %then %do;
+    /* does dbms_memtype exist in dictionary.tables? */
+    %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 0;
+    %else 1;
+  %end;
+  %else %if &feature=EXPORTXLS %then %do;
+    /* is it possible to PROC EXPORT an excel file? */
+    %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 1;
+    %else %if %sysfunc(sysprod(SAS/ACCESS Interface to PC Files)) = 1 %then 1;
+    %else 0;
+  %end;
   %else %do;
     -1
     %put &sysmacroname: &feature not found;

base/mf_existvarlist.sas

@@ -6,9 +6,6 @@
 
       %put %mf_existVarList(sashelp.class, age sex name dummyvar);
 
-  <h4> SAS Macros </h4>
-  @li mf_abort.sas
-
   @param libds 2 part dataset or view reference
   @param varlist space separated variable names
 

base/mf_fmtdttm.sas

@@ -0,0 +1,42 @@
+/**
+  @file
+  @brief Returns E8601DT26.6 if compatible else DATETIME19.3
+  @details From our experience in [Data Controller for SAS]
+  (https://datacontroller.io) deployments, the E8601DT26.6 datetime format has
+  the widest support when it comes to pass-through SQL queries.
+
+  However, it is not supported in WPS or early versions of SAS 9 (M3 and below)
+  when used as a datetime literal, eg:
+
+      data _null_;
+        demo="%sysfunc(datetime(),E8601DT26.6)"dt;
+        demo=;
+      run;
+
+  This macro will therefore return DATEITME19.3 as an alternative format
+  based on the runtime environment so that it can be used in such cases, eg:
+
+      data _null_;
+        demo="%sysfunc(datetime(),%mf_fmtdttm())"dt;
+        demo=;
+      run;
+
+  <h4> Related Macros </h4>
+  @li mf_fmtdttm.test.sas
+
+  @author Allan Bowe
+**/
+
+%macro mf_fmtdttm(
+)/*/STORE SOURCE*/;
+
+%if "&sysver"="9.2" or "&sysver"="9.3"
+  or ("&sysver"="9.4" and "%substr(&SYSVLONG,9,1)" le "3")
+  or "%substr(&sysver,1,1)"="4"
+  or "%substr(&sysver,1,1)"="5"
+%then %do;DATETIME19.3%end;
+%else %do;E8601DT26.6%end;
+
+%mend mf_fmtdttm;
+
+

base/mf_getplatform.sas

@@ -5,8 +5,12 @@
 
       %put %mf_getplatform();
 
-    returns:
-      SASMETA  (or SASVIYA)
+  returns one of:
+
+  @li SASMETA
+  @li SASVIYA
+  @li SASJS
+  @li BASESAS
 
   @param switch the param for which to return a platform specific variable
 
@@ -68,4 +72,4 @@
 %else %if &switch=VIYARESTAPI %then %do;
   %mf_trimstr(%sysfunc(getoption(servicesbaseurl)),/)
 %end;
-%mend mf_getplatform;
+%mend mf_getplatform;

base/mf_getuniquelibref.sas

@@ -3,12 +3,12 @@
   @brief Returns an unused libref
   @details Use as follows:
 
-    libname mclib0 (work);
-    libname mclib1 (work);
-    libname mclib2 (work);
+      libname mclib0 (work);
+      libname mclib1 (work);
+      libname mclib2 (work);
 
-    %let libref=%mf_getuniquelibref();
-    %put &=libref;
+      %let libref=%mf_getuniquelibref();
+      %put &=libref;
 
   which returns:
 

base/mf_getuser.sas

@@ -23,18 +23,19 @@
   @author Allan Bowe
 **/
 
-%macro mf_getuser(type=META
+%macro mf_getuser(
 )/*/STORE SOURCE*/;
-  %local user metavar;
-  %if &type=OS %then %let metavar=_secureusername;
-  %else %let metavar=_metaperson;
+  %local user;
 
-  %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %let user=&SYS_COMPUTE_SESSION_OWNER;
-  %else %if %symexist(&metavar) %then %do;
-    %if %length(&&&metavar)=0 %then %let user=&sysuserid;
+  %if %symexist(_sasjs_username) %then %let user=&_sasjs_username;
+  %else %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %do;
+    %let user=&SYS_COMPUTE_SESSION_OWNER;
+  %end;
+  %else %if %symexist(_metaperson) %then %do;
+    %if %length(&_metaperson)=0 %then %let user=&sysuserid;
     /* sometimes SAS will add @domain extension - remove for consistency */
     /* but be sure to quote in case of usernames with commas */
-    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@));
+    %else %let user=%unquote(%scan(%quote(&_metaperson),1,@));
   %end;
   %else %let user=&sysuserid;
 

base/mf_getvarcount.sas

@@ -2,31 +2,48 @@
   @file
   @brief Returns number of variables in a dataset
   @details Useful to identify those renagade datasets that have no columns!
+  Can also be used to count for numeric, or character columns
 
-        %put Number of Variables=%mf_getvarcount(sashelp.class);
+      %put Number of Variables=%mf_getvarcount(sashelp.class);
+      %put Character Variables=%mf_getvarcount(sashelp.class,typefilter=C);
+      %put Numeric Variables = %mf_getvarcount(sashelp.class,typefilter=N);
 
   returns:
   > Number of Variables=4
 
-  @param libds Two part dataset (or view) reference.
+
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] typefilter= (A) Filter for certain types of column.  Valid values:
+    @li A Count All columns
+    @li C Count Character columns only
+    @li N Count Numeric columns only
 
   @version 9.2
   @author Allan Bowe
 
 **/
 
-%macro mf_getvarcount(libds
+%macro mf_getvarcount(libds,typefilter=A
 )/*/STORE SOURCE*/;
-  %local dsid nvars rc ;
+  %local dsid nvars rc outcnt x;
   %let dsid=%sysfunc(open(&libds));
   %let nvars=.;
+  %let outcnt=0;
+  %let typefilter=%upcase(&typefilter);
   %if &dsid %then %do;
     %let nvars=%sysfunc(attrn(&dsid,NVARS));
+    %if &typefilter=A %then %let outcnt=&nvars;
+    %else %if &nvars>0 %then %do x=1 %to &nvars;
+      /* increment based on variable type */
+      %if %sysfunc(vartype(&dsid,&x))=&typefilter %then %do;
+        %let outcnt=%eval(&outcnt+1);
+      %end;
+    %end;
     %let rc=%sysfunc(close(&dsid));
   %end;
   %else %do;
     %put unable to open &libds (rc=&dsid);
     %let rc=%sysfunc(close(&dsid));
   %end;
-  &nvars
+  &outcnt
 %mend mf_getvarcount;

base/mf_increment.sas

@@ -0,0 +1,29 @@
+/**
+  @file
+  @brief Increments a macro variable
+  @details Useful outside of do-loops - will increment a macro variable every
+  time it is called.
+
+  Example:
+
+      %let cnt=1;
+      %put We have run %mf_increment(cnt) lines;
+      %put Now we have run %mf_increment(cnt) lines;
+      %put There are %mf_increment(cnt) lines in total;
+
+  @param [in] MACRO_NAME the name of the macro variable to increment
+  @param [in] ITER= The amount to add or subtract to the macro
+
+  <h4> Related Files </h4>
+  @li mf_increment.test.sas
+
+**/
+
+%macro mf_increment(macro_name,incr=1);
+
+  /* iterate the value */
+  %let &macro_name=%eval(&&&macro_name+&incr);
+  /* return the value */
+  &&&macro_name
+
+%mend mf_increment;

base/mf_isint.sas

@@ -33,4 +33,4 @@
   %if %sysfunc(findc(%str(&val),,kd)) %then %do;0%end;
   %else %do;1%end;
 
-%mend mf_isint;
+%mend mf_isint;

base/mp_abort.sas

@@ -8,23 +8,29 @@
 
   The method used varies according to the context.  Important points:
 
-  @li should not use endsas or abort cancel in 9.4m3 environments as this can
-    cause hung multibridge sessions and result in a frozen STP server
+  @li should not use endsas or abort cancel in 9.4m3 WIN environments as this
+    can cause hung multibridge sessions and result in a frozen STP server
+  @li The use of endsas in 9.4m6+ windows environments for POST requests to the
+    STP server can result in an empty response body
   @li should not use endsas in viya 3.5 as this destroys the session and cannot
     fetch results (although both mv_getjoblog.sas and the @sasjs/adapter will
     recognise this and fetch the log of the parent session instead)
   @li STP environments must finish cleanly to avoid the log being sent to
     _webout.  To assist with this, we also run stpsrvset('program error', 0)
-    and set SYSCC=0.  We take a unique "soft abort" approach - we open a macro
+    and set SYSCC=0.
+    Where possible, we take a unique "soft abort" approach - we open a macro
     but don't close it!  This works everywhere EXCEPT inside a \%include inside
     a macro.  For that, we recommend you use mp_include.sas to perform the
     include, and then call \%mp_abort(mode=INCLUDE) from the source program (ie,
     OUTSIDE of the top-parent macro).
+    The soft abort has become ineffective in 9.4m6 WINDOWS environments.  We are
+    currently investigating approaches to deal with this.
 
 
-  @param mac= to contain the name of the calling macro
+  @param mac= (mp_abort.sas) To contain the name of the calling macro. Do not
+    use &sysmacroname as this will always resolve to MP_ABORT.
   @param msg= message to be returned
-  @param iftrue= supply a condition under which the macro should be executed.
+  @param iftrue= (1=1) Supply a condition for which the macro should be executed
   @param errds= (work.mp_abort_errds) There is no clean way to end a process
     within a %include called within a macro.  Furthermore, there is no way to
     test if a macro is called within a %include.  To handle this particular
@@ -45,11 +51,12 @@
     @li REGULAR (default)
     @li INCLUDE
 
+  @version 9.4
+  @author Allan Bowe
+
   <h4> Related Macros </h4>
   @li mp_include.sas
 
-  @version 9.4
-  @author Allan Bowe
   @cond
 **/
 
@@ -58,169 +65,197 @@
   , mode=REGULAR
 )/*/STORE SOURCE*/;
 
-  %global sysprocessmode sysprocessname;
+%global sysprocessmode sysprocessname sasjs_stpsrv_header_loc sasjsprocessmode;
+%local fref fid i;
 
-  %if not(%eval(%unquote(&iftrue))) %then %return;
+%if not(%eval(%unquote(&iftrue))) %then %return;
 
-  %put NOTE: ///  mp_abort macro executing //;
-  %if %length(&mac)>0 %then %put NOTE- called by &mac;
-  %put NOTE - &msg;
+%put NOTE: ///  mp_abort macro executing //;
+%if %length(&mac)>0 %then %put NOTE- called by &mac;
+%put NOTE - &msg;
 
-  %if %symexist(_SYSINCLUDEFILEDEVICE)
-  /* abort cancel FILE does not restart outside the INCLUDE on Viya 3.5 */
-  and "&SYSPROCESSNAME " ne "Compute Server "
+%if %symexist(_SYSINCLUDEFILEDEVICE)
+/* abort cancel FILE does not restart outside the INCLUDE on Viya 3.5 */
+and %superq(SYSPROCESSNAME) ne %str(Compute Server)
+%then %do;
+  %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do;
+    data &errds;
+      iftrue='1=1';
+      length mac $100 msg $5000;
+      mac=symget('mac');
+      msg=symget('msg');
+    run;
+    data _null_;
+      abort cancel FILE;
+    run;
+    %return;
+  %end;
+%end;
+
+/* Web App Context */
+%if %symexist(_PROGRAM)
+  or %superq(SYSPROCESSNAME) = %str(Compute Server)
+  or &mode=INCLUDE
+%then %do;
+  options obs=max replace mprint;
+  %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5"
   %then %do;
-    %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do;
-      data &errds;
-        iftrue='1=1';
-        length mac $100 msg $5000;
-        mac=symget('mac');
-        msg=symget('msg');
-      run;
+    options nosyntaxcheck;
+  %end;
+
+  %if &mode=INCLUDE %then %do;
+    %if %sysfunc(exist(&errds))=1 %then %do;
       data _null_;
-        abort cancel FILE;
+        set &errds;
+        call symputx('iftrue',iftrue,'l');
+        call symputx('mac',mac,'l');
+        call symputx('msg',msg,'l');
+        putlog (_all_)(=);
       run;
+      %if (&iftrue)=0 %then %return;
+    %end;
+    %else %do;
+      %put &sysmacroname: No include errors found;
       %return;
     %end;
   %end;
 
-  /* Stored Process Server web app context */
-  %if %symexist(_METAFOLDER)
-    or "&SYSPROCESSNAME "="Compute Server "
-    or &mode=INCLUDE
-  %then %do;
-    options obs=max replace nosyntaxcheck mprint;
-    %if &mode=INCLUDE %then %do;
-      %if %sysfunc(exist(&errds))=1 %then %do;
-        data _null_;
-          set &errds;
-          call symputx('iftrue',iftrue,'l');
-          call symputx('mac',mac,'l');
-          call symputx('msg',msg,'l');
-          putlog (_all_)(=);
-        run;
-        %if (&iftrue)=0 %then %return;
-      %end;
-      %else %do;
-        %put &sysmacroname: No include errors found;
-        %return;
-      %end;
-    %end;
-
-    /* extract log errs / warns, if exist */
-    %local logloc logline;
-    %global logmsg; /* capture global messages */
-    %if %symexist(SYSPRINTTOLOG) %then %let logloc=&SYSPRINTTOLOG;
-    %else %let logloc=%qsysfunc(getoption(LOG));
-    proc printto log=log;run;
-    %let logline=0;
-    %if %length(&logloc)>0 %then %do;
+  /* extract log errs / warns, if exist */
+  %local logloc logline;
+  %global logmsg; /* capture global messages */
+  %if %symexist(SYSPRINTTOLOG) %then %let logloc=&SYSPRINTTOLOG;
+  %else %let logloc=%qsysfunc(getoption(LOG));
+  proc printto log=log;run;
+  %let logline=0;
+  %if %length(&logloc)>0 %then %do;
+    data _null_;
+      infile &logloc lrecl=5000;
+      input; putlog _infile_;
+      i=1;
+      retain logonce 0;
+      if (
+          _infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR"
+        ) and logonce=0 then
+      do;
+        call symputx('logline',_n_);
+        logonce+1;
+      end;
+    run;
+    /* capture log including lines BEFORE the err */
+    %if &logline>0 %then %do;
       data _null_;
         infile &logloc lrecl=5000;
-        input; putlog _infile_;
+        input;
         i=1;
-        retain logonce 0;
-        if (
-            _infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR"
-          ) and logonce=0 then
-        do;
-          call symputx('logline',_n_);
-          logonce+1;
-        end;
-      run;
-      /* capture log including lines BEFORE the err */
-      %if &logline>0 %then %do;
-        data _null_;
-          infile &logloc lrecl=5000;
+        stoploop=0;
+        if _n_ ge &logline-15 and stoploop=0 then do until (i>22);
+          call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
           input;
-          i=1;
-          stoploop=0;
-          if _n_ ge &logline-15 and stoploop=0 then do until (i>22);
-            call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
-            input;
-            i+1;
-            stoploop=1;
-          end;
-          if stoploop=1 then stop;
-        run;
-      %end;
-    %end;
-
-    %if %symexist(SYS_JES_JOB_URI) %then %do;
-      /* setup webout */
-      OPTIONS NOBOMFILE;
-      %if "X&SYS_JES_JOB_URI.X"="XX" %then %do;
-          filename _webout temp lrecl=999999 mod;
-      %end;
-      %else %do;
-        filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
-          name="_webout.json" lrecl=999999 mod;
-      %end;
-    %end;
-
-    /* send response in SASjs JSON format */
-    data _null_;
-      file _webout mod lrecl=32000 encoding='utf-8';
-      length msg syswarningtext syserrortext $32767 ;
-      sasdatetime=datetime();
-      msg=symget('msg');
-    %if &logline>0 %then %do;
-      msg=cats(msg,'\n\nLog Extract:\n',symget('logmsg'));
-    %end;
-      /* escape the escapes */
-      msg=tranwrd(msg,'\','\\');
-      /* escape the quotes */
-      msg=tranwrd(msg,'"','\"');
-      /* ditch the CRLFs as chrome complains */
-      msg=compress(msg,,'kw');
-      /* quote without quoting the quotes (which are escaped instead) */
-      msg=cats('"',msg,'"');
-      if symexist('_debug') then debug=quote(trim(symget('_debug')));
-      else debug='""';
-      put '>>weboutBEGIN<<';
-      put '{"SYSDATE" : "' "&SYSDATE" '"';
-      put ',"SYSTIME" : "' "&SYSTIME" '"';
-      put ',"sasjsAbort" : [{';
-      put ' "MSG":' msg ;
-      put ' ,"MAC": "' "&mac" '"}]';
-      put ",""SYSUSERID"" : ""&sysuserid"" ";
-      put ',"_DEBUG":' debug ;
-      if symexist('_metauser') then do;
-        _METAUSER=quote(trim(symget('_METAUSER')));
-        put ",""_METAUSER"": " _METAUSER;
-        _METAPERSON=quote(trim(symget('_METAPERSON')));
-        put ',"_METAPERSON": ' _METAPERSON;
-      end;
-      if symexist('SYS_JES_JOB_URI') then do;
-        SYS_JES_JOB_URI=quote(trim(symget('SYS_JES_JOB_URI')));
-        put ',"SYS_JES_JOB_URI": ' SYS_JES_JOB_URI;
-      end;
-      _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
-      put ',"_PROGRAM" : ' _PROGRAM ;
-      put ",""SYSCC"" : ""&syscc"" ";
-      syserrortext=quote(trim(symget('syserrortext')));
-      put ",""SYSERRORTEXT"" : " syserrortext;
-      put ",""SYSHOSTNAME"" : ""&syshostname"" ";
-      put ",""SYSJOBID"" : ""&sysjobid"" ";
-      put ",""SYSSCPL"" : ""&sysscpl"" ";
-      put ",""SYSSITE"" : ""&syssite"" ";
-      sysvlong=quote(trim(symget('sysvlong')));
-      put ',"SYSVLONG" : ' sysvlong;
-      syswarningtext=quote(trim(symget('syswarningtext')));
-      put ",""SYSWARNINGTEXT"" : " syswarningtext;
-      put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
-      put "}" @;
-      put '>>weboutEND<<';
-    run;
-
-    %put _all_;
-
-    %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
-      data _null_;
-        putlog 'stpsrvset program err and syscc';
-        rc=stpsrvset('program error', 0);
-        call symputx("syscc",0,"g");
+          i+1;
+          stoploop=1;
+        end;
+        if stoploop=1 then stop;
       run;
+    %end;
+  %end;
+
+  %if %symexist(SYS_JES_JOB_URI) %then %do;
+    /* setup webout for Viya */
+    options nobomfile;
+    %if "X&SYS_JES_JOB_URI.X"="XX" %then %do;
+        filename _webout temp lrecl=999999 mod;
+    %end;
+    %else %do;
+      filename _webout filesrvc parenturi="&SYS_JES_JOB_URI"
+        name="_webout.json" lrecl=999999 mod;
+    %end;
+  %end;
+  %else %if %sysfunc(filename(fref,&sasjs_stpsrv_header_loc))=0 %then %do;
+    options nobomfile;
+    /* set up http header for SASjs Server */
+    %let fid=%sysfunc(fopen(&fref,A));
+    %if &fid=0 %then %do;
+      %put %str(ERR)OR: %sysfunc(sysmsg());
+      %return;
+    %end;
+    %let rc=%sysfunc(fput(&fid,%str(Content-Type: application/json)));
+    %let rc=%sysfunc(fwrite(&fid));
+    %let rc=%sysfunc(fclose(&fid));
+    %let rc=%sysfunc(filename(&fref));
+  %end;
+
+  /* send response in SASjs JSON format */
+  data _null_;
+    file _webout mod lrecl=32000 encoding='utf-8';
+    length msg syswarningtext syserrortext $32767 mode $10 ;
+    sasdatetime=datetime();
+    msg=symget('msg');
+  %if &logline>0 %then %do;
+    msg=cats(msg,'\n\nLog Extract:\n',symget('logmsg'));
+  %end;
+    /* escape the escapes */
+    msg=tranwrd(msg,'\','\\');
+    /* escape the quotes */
+    msg=tranwrd(msg,'"','\"');
+    /* ditch the CRLFs as chrome complains */
+    msg=compress(msg,,'kw');
+    /* quote without quoting the quotes (which are escaped instead) */
+    msg=cats('"',msg,'"');
+    if symexist('_debug') then debug=quote(trim(symget('_debug')));
+    else debug='""';
+    if symget('sasjsprocessmode')='Stored Program' then mode='SASJS';
+    if mode ne 'SASJS' then put '>>weboutBEGIN<<';
+    put '{"SYSDATE" : "' "&SYSDATE" '"';
+    put ',"SYSTIME" : "' "&SYSTIME" '"';
+    put ',"sasjsAbort" : [{';
+    put ' "MSG":' msg ;
+    put ' ,"MAC": "' "&mac" '"}]';
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
+    put ',"_DEBUG":' debug ;
+    if symexist('_metauser') then do;
+      _METAUSER=quote(trim(symget('_METAUSER')));
+      put ",""_METAUSER"": " _METAUSER;
+      _METAPERSON=quote(trim(symget('_METAPERSON')));
+      put ',"_METAPERSON": ' _METAPERSON;
+    end;
+    if symexist('SYS_JES_JOB_URI') then do;
+      SYS_JES_JOB_URI=quote(trim(symget('SYS_JES_JOB_URI')));
+      put ',"SYS_JES_JOB_URI": ' SYS_JES_JOB_URI;
+    end;
+    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
+    put ',"_PROGRAM" : ' _PROGRAM ;
+    put ",""SYSCC"" : ""&syscc"" ";
+    syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
+    put ",""SYSERRORTEXT"" : " syserrortext;
+    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSJOBID"" : ""&sysjobid"" ";
+    put ",""SYSSCPL"" : ""&sysscpl"" ";
+    put ",""SYSSITE"" : ""&syssite"" ";
+    sysvlong=quote(trim(symget('sysvlong')));
+    put ',"SYSVLONG" : ' sysvlong;
+    syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');
+    put ",""SYSWARNINGTEXT"" : " syswarningtext;
+    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
+    put "}" ;
+    if mode ne 'SASJS' then put '>>weboutEND<<';
+  run;
+
+  %put _all_;
+
+  %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
+    data _null_;
+      putlog 'stpsrvset program err and syscc';
+      rc=stpsrvset('program error', 0);
+      call symputx("syscc",0,"g");
+    run;
+    %if &sysscp=WIN
+    and 1=0 /* deprecating this logic until we figure out a consistent abort */
+    and "%substr(%str(&sysvlong         ),1,8)"="9.04.01M"
+    and "%substr(%str(&sysvlong         ),9,1)">"5" %then %do;
+      /* skip approach (below) does not work in windows m6+ envs */
+      endsas;
+    %end;
+    %else %do;
       /**
         * endsas kills 9.4m3 deployments by orphaning multibridges.
         * Abort variants are ungraceful (non zero return code)
@@ -238,28 +273,29 @@
       run;
       %inc skip;
     %end;
-    %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
-      /* endsas kills the session making it harder to fetch results */
-      data _null_;
-        syswarningtext=symget('syswarningtext');
-        syserrortext=symget('syserrortext');
-        abort_msg=symget('msg');
-        syscc=symget('syscc');
-        sysuserid=symget('sysuserid');
-        iftrue=symget('iftrue');
-        put (_all_)(/=);
-        call symputx('syscc',0);
-        abort cancel nolist;
-      run;
-    %end;
-    %else %do;
-      %abort cancel;
-    %end;
+  %end;
+  %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
+    /* endsas kills the session making it harder to fetch results */
+    data _null_;
+      syswarningtext=symget('syswarningtext');
+      syserrortext=symget('syserrortext');
+      abort_msg=symget('msg');
+      syscc=symget('syscc');
+      sysuserid=symget('sysuserid');
+      iftrue=symget('iftrue');
+      put (_all_)(/=);
+      call symputx('syscc',0);
+      abort cancel nolist;
+    run;
   %end;
   %else %do;
-    %put _all_;
     %abort cancel;
   %end;
+%end;
+%else %do;
+  %put _all_;
+  %abort cancel;
+%end;
 %mend mp_abort;
 
 /** @endcond */

base/mp_binarycopy.sas

@@ -4,8 +4,8 @@
   @details Reads in a file byte by byte and writes it back out.  Is an
   os-independent method to copy files.  In case of naming collision, the
   default filerefs can be modified.
-  Based on:
-  https://stackoverflow.com/questions/13046116/using-sas-to-copy-a-text-file
+  Note that if you have a new enough version of SAS, and you don't need features
+  such as APPEND, you may be better of using the fcopy() function instead.
 
         %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
 
@@ -31,6 +31,7 @@
   @param [in] mode (CREATE) Valid values:
     @li CREATE - Create the file (even if it already exists)
     @li APPEND - Append to the file (don't overwrite)
+  @param iftrue= (1=1) Supply a condition for which the macro should be executed
 
   @returns nothing
 
@@ -44,15 +45,14 @@
     ,inref=____in   /* override default to use own filerefs */
     ,outref=____out /* override default to use own filerefs */
     ,mode=CREATE
+    ,iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
-  %local mod outmode;
-  %if &mode=APPEND %then %do;
-    %let mod=mod;
-    %let outmode='a';
-  %end;
-  %else %do;
-    %let outmode='o';
-  %end;
+  %local mod;
+
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
+  %if &mode=APPEND %then %let mod=mod;
+
   /* these IN and OUT filerefs can point to anything */
   %if &inref = ____in %then %do;
     filename &inref &inloc lrecl=1048576 ;
@@ -63,22 +63,17 @@
 
   /* copy the file byte-for-byte  */
   data _null_;
-    length filein 8 fileid 8;
-    filein = fopen("&inref",'I',1,'B');
-    fileid = fopen("&outref",&outmode,1,'B');
-    rec = '20'x;
-    do while(fread(filein)=0);
-      rc = fget(filein,rec,1);
-      rc = fput(fileid, rec);
-      rc =fwrite(fileid);
-    end;
-    rc = fclose(filein);
-    rc = fclose(fileid);
+    infile &inref lrecl=1 recfm=n;
+    file &outref &mod recfm=n;
+    input sourcechar $char1. @@;
+    format sourcechar hex2.;
+    put sourcechar char1. @@;
   run;
+
   %if &inref = ____in %then %do;
     filename &inref clear;
   %end;
   %if &outref=____out %then %do;
     filename &outref clear;
   %end;
-%mend mp_binarycopy;
+%mend mp_binarycopy;

base/mp_cleancsv.sas

@@ -1,5 +1,5 @@
 /**
-  @file mp_cleancsv.sas
+  @file
   @brief Fixes embedded cr / lf / crlf in CSV
   @details CSVs will sometimes contain lf or crlf within quotes (eg when
   saved by excel).  When the termstr is ALSO lf or crlf that can be tricky
@@ -7,14 +7,16 @@
   This macro converts any csv to follow the convention of a windows excel file,
   applying CRLF line endings and converting embedded cr and crlf to lf.
 
-  usage:
+  Usage:
 
       fileref mycsv "/path/your/csv";
       %mp_cleancsv(in=mycsv,out=/path/new.csv)
 
-  @param in= provide path or fileref to input csv
-  @param out= output path or fileref to output csv
-  @param qchar= quote char - hex code 22 is the double quote.
+  @param in= (NOTPROVIDED) Provide path or fileref to input csv. If a period is
+    found, it is assumed to be a file.
+  @param out= (NOTPROVIDED) Output path or fileref to output csv.  If a period
+    is found, it is assumed to be a file.
+  @param qchar= ('22'x) Quote char - hex code 22 is the double quote.
 
   @version 9.2
   @author Allan Bowe
@@ -56,9 +58,14 @@
     else do;
       /* outside a quote, change cr and lf to crlf */
       if inchar='0D'x then do;
+        crblank:
         put '0D0A'x;
         input inchar $char1.;
-        if inchar ne '0A'x then do;
+        if inchar='0D'x then do;
+          /* multiple CR indicates CR formatted file with blank lines */
+          goto crblank;
+        end;
+        else if inchar ne '0A'x then do;
           put inchar $char1.;
           if inchar=qchar then isq = mod(isq+1,2);
         end;

base/mp_copyfolder.sas

@@ -16,8 +16,11 @@
 
       %mp_copyfolder(&rootdir,&copydir)
 
-  @param source Unquoted path to the folder to copy from.
-  @param target Unquoted path to the folder to copy to.
+  @param [in] source Unquoted path to the folder to copy from.
+  @param [out] target Unquoted path to the folder to copy to.
+  @param [in] copymax=(MAX) Set to a positive integer to indicate the level of
+    subdirectory copy recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas
@@ -31,7 +34,7 @@
 
 **/
 
-%macro mp_copyfolder(source,target);
+%macro mp_copyfolder(source,target,copymax=MAX);
 
   %mp_abort(iftrue=(%mf_isdir(&source)=0)
     ,mac=&sysmacroname
@@ -50,7 +53,7 @@
   %let tempds=%mf_getuniquename();
 
   /* recursive directory listing */
-  %mp_dirlist(path=&source,outds=work.&tempds, maxdepth=MAX)
+  %mp_dirlist(path=&source,outds=work.&tempds,maxdepth=&copymax)
 
   /* create folders and copy content */
   data _null_;
@@ -78,4 +81,4 @@
   proc sql;
   drop table work.&tempds;
 
-%mend mp_copyfolder;
+%mend mp_copyfolder;

base/mp_coretable.sas

@@ -30,6 +30,7 @@
   @li mddl_dc_filtersummary.sas
   @li mddl_dc_locktable.sas
   @li mddl_dc_maxkeytable.sas
+  @li mf_getuniquename.sas
 
   <h4> Related Macros </h4>
   @li mp_filterstore.sas
@@ -46,7 +47,7 @@
 %macro mp_coretable(table_ref,libds=0
 )/*/STORE SOURCE*/;
 %local outds ;
-%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
+%let outds=%sysfunc(ifc(&libds=0,%mf_getuniquename(),&libds));
 proc sql;
 %if &table_ref=DIFFTABLE %then %do;
   %mddl_dc_difftable(libds=&outds)
@@ -65,7 +66,8 @@ proc sql;
 %end;
 
 %if &libds=0 %then %do;
+  proc sql;
   describe table &syslast;
   drop table &syslast;
 %end;
-%mend mp_coretable;
+%mend mp_coretable;

base/mp_createwebservice.sas

@@ -1,50 +1,13 @@
 /**
   @file mp_createwebservice.sas
-  @brief Create a web service in SAS 9 or Viya
-  @details Creates a SASJS ready Stored Process in SAS 9 or Job Execution
-  Service in SAS Viya
+  @brief Create a web service in SAS 9, Viya or SASjs Server
+  @details This is actually a wrapper for mx_createwebservice.sas, remaining
+  for legacy purposes.  For new apps, use mx_createwebservice.sas.
 
-Usage:
-
-    %* compile macros ;
-    filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
-    %inc mc;
-
-    %* write some code;
-    filename ft15f001 temp;
-    parmcards4;
-        %* fetch any data from frontend ;
-        %webout(FETCH)
-        data example1 example2;
-          set sashelp.class;
-        run;
-        %* send data back;
-        %webout(OPEN)
-        %webout(ARR,example1) * Array format, fast, suitable for large tables ;
-        %webout(OBJ,example2) * Object format, easier to work with ;
-        %webout(CLOSE)
-    ;;;;
-    %mp_createwebservice(path=/Public/app/common,name=appInit,replace=YES)
 
   <h4> SAS Macros </h4>
-  @li mf_getplatform.sas
-  @li mm_createwebservice.sas
-  @li ms_createwebservice.sas
-  @li mv_createwebservice.sas
+  @li mx_createwebservice.sas
 
-  @param [in,out] path= The full folder path where the service will be created
-  @param [in,out] name= Service name.  Avoid spaces.
-  @param [in] desc= The description of the service (optional)
-  @param [in] precode= Space separated list of filerefs, pointing to the code
-    that needs to be attached to the beginning of the service (optional)
-  @param [in] code= (ft15f001) Space seperated fileref(s) of the actual code to
-    be added
-  @param [in] replace= (YES) Select YES to replace any existing service in that
-    location
-  @param [in] mDebug= (0) set to 1 to show debug messages in the log
-
-  @version 9.2
-  @author Allan Bowe
 
 **/
 
@@ -57,40 +20,13 @@ Usage:
     ,mdebug=0
 )/*/STORE SOURCE*/;
 
-%if &syscc ge 4 %then %do;
-  %put syscc=&syscc - &sysmacroname will not execute in this state;
-  %return;
-%end;
-
-%local platform; %let platform=%mf_getplatform();
-%if &platform=SASVIYA %then %do;
-  %if "&path"="HOME" %then %let path=/Users/&sysuserid/My Folder;
-  %mv_createwebservice(path=&path
+  %mx_createwebservice(path=&path
     ,name=&name
-    ,code=&code
     ,precode=&precode
+    ,code=&code
     ,desc=&desc
     ,replace=&replace
-  )
-%end;
-%else %if &platform=SASJS %then %do;
-  %if "&path"="HOME" %then %let path=/Users/&_sasjs_username/My Folder;
-  %ms_createwebservice(path=&path
-    ,name=&name
-    ,code=&code
-    ,precode=&precode
     ,mdebug=&mdebug
   )
-%end;
-%else %do;
-  %if "&path"="HOME" %then %let path=/User Folders/&_METAPERSON/My Folder;
-  %mm_createwebservice(path=&path
-    ,name=&name
-    ,code=&code
-    ,precode=&precode
-    ,desc=&desc
-    ,replace=&replace
-  )
-%end;
 
 %mend mp_createwebservice;

base/mp_ds2cards.sas

@@ -11,7 +11,7 @@
 
   Usage:
 
-      %mp_ds2cards(base_ds=sashelp.class
+      %mp_ds2cards(sashelp.class
         , tgt_ds=work.class
         , cards_file= "C:\temp\class.sas"
         , showlog=NO
@@ -23,7 +23,7 @@
     - explicity setting a unix LF
     - constraints / indexes etc
 
-  @param [in] base_ds= Should be two level - eg work.blah.  This is the table
+  @param [in] base_ds Should be two level - eg work.blah.  This is the table
                   that is converted to a cards file.
   @param [in] tgt_ds= Table that the generated cards file would create.
     Optional - if omitted, will be same as BASE_DS.
@@ -51,7 +51,7 @@
   @cond
 **/
 
-%macro mp_ds2cards(base_ds=, tgt_ds=
+%macro mp_ds2cards(base_ds, tgt_ds=
     ,cards_file="%sysfunc(pathname(work))/cardgen.sas"
     ,maxobs=max
     ,random_sample=NO
@@ -254,6 +254,7 @@ data _null_;
         ;
       %end;
       put ";";
+      put 'missing a b c d e f g h i j k l m n o p q r s t u v w x y z _;';
       put "datalines4;";
     end;
   end;

base/mp_ds2fmtds.sas

@@ -17,7 +17,7 @@
   <h4> SAS Macros </h4>
   @li mf_existds.sas
 
-  <h4> Related Macros <h4>
+  <h4> Related Macros </h4>
   @li mp_jsonout.sas
 
   @version 9.2

base/mp_ds2md.sas

@@ -57,6 +57,11 @@
 %local vars;
 %let vars=%upcase(%mf_getvarlist(&libds));
 
+%if %trim(X&vars)=X %then %do;
+  %put &sysmacroname: Table &libds has no columns!!;
+  %return;
+%end;
+
 /* create the header row */
 data _null_;
   file &outref;

base/mp_dsmeta.sas

@@ -0,0 +1,99 @@
+/**
+  @file
+  @brief Export dataset metadata to a single output table
+  @details Exports the dataset attributes and enginehost information, then
+  converts the datasets into a single output table in the following format:
+
+|ODS_TABLE:$10.|NAME:$100.|VALUE:$1000.|
+|---|---|---|
+|`ATTRIBUTES `|`Data Set Name `|`SASHELP.CLASS `|
+|`ATTRIBUTES `|`Observations `|`19 `|
+|`ATTRIBUTES `|`Member Type `|`DATA `|
+|`ATTRIBUTES `|`Variables `|`5 `|
+|`ATTRIBUTES `|`Engine `|`V9 `|
+|`ATTRIBUTES `|`Indexes `|`0 `|
+|`ATTRIBUTES `|`Created `|`06/08/2020 00:59:14 `|
+|`ATTRIBUTES `|`Observation Length `|`40 `|
+|`ATTRIBUTES `|`Last Modified `|`06/08/2020 00:59:14 `|
+|`ATTRIBUTES `|`Deleted Observations `|`0 `|
+|`ATTRIBUTES `|`Protection `|`. `|
+|`ATTRIBUTES `|`Compressed `|`NO `|
+|`ATTRIBUTES `|`Data Set Type `|`. `|
+|`ATTRIBUTES `|`Sorted `|`NO `|
+|`ATTRIBUTES `|`Label `|`Student Data `|
+|`ATTRIBUTES `|`Data Representation `|`SOLARIS_X86_64, LINUX_X86_64, ALPHA_TRU64, LINUX_IA64 `|
+|`ATTRIBUTES `|`Encoding `|`us-ascii  ASCII (ANSI) `|
+|`ENGINEHOST `|`Data Set Page Size `|`65536 `|
+|`ENGINEHOST `|`Number of Data Set Pages `|`1 `|
+|`ENGINEHOST `|`First Data Page `|`1 `|
+|`ENGINEHOST `|`Max Obs per Page `|`1632 `|
+|`ENGINEHOST `|`Obs in First Data Page `|`19 `|
+|`ENGINEHOST `|`Number of Data Set Repairs `|`0 `|
+|`ENGINEHOST `|`Filename `|`/opt/sas/sas9/SASHome/SASFoundation/9.4/sashelp/class.sas7bdat `|
+|`ENGINEHOST `|`Release Created `|`9.0401M7 `|
+|`ENGINEHOST `|`Host Created `|`Linux `|
+|`ENGINEHOST `|`Inode Number `|`28314616 `|
+|`ENGINEHOST `|`Access Permission `|`rw-r--r-- `|
+|`ENGINEHOST `|`Owner Name `|`sas `|
+|`ENGINEHOST `|`File Size `|`128KB `|
+|`ENGINEHOST `|`File Size (bytes) `|`131072 `|
+
+  Example usage:
+
+      %mp_dsmeta(work.sashelp,outds=work.mymeta)
+      proc print data=work.mymeta;
+      run;
+
+  @param libds The library.dataset to export the metadata for
+  @param outds= (work.dsmeta) The output table to contain the metadata
+
+  <h4> Related Files </h4>
+  @li mp_dsmeta.test.sas
+
+**/
+
+%macro mp_dsmeta(libds,outds=work.dsmeta);
+
+%local ds1 ds2;
+data;run; %let ds1=&syslast;
+data;run; %let ds2=&syslast;
+
+/* setup the ODS capture */
+ods output attributes=&ds1 enginehost=&ds2;
+
+/* export the metadata */
+proc contents data=&libds;
+run;
+
+/* load it into a single table */
+data &outds (keep=ods_table name value);
+  length ods_table $10 name label2 label1 label $100
+    value cvalue cvalue1 cvalue2 $1000
+    nvalue nvalue1 nvalue2 8;
+  if _n_=1 then call missing (of _all_);
+  * putlog (_all_)(=);
+  set &ds1 (in=atrs) &ds2 (in=eng);
+  if atrs then do;
+    ods_table='ATTRIBUTES';
+    name=coalescec(label1,label);
+    value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.));
+    output;
+    if label2 ne '' then do;
+    name=label2;
+    value=coalescec(cvalue2,put(nvalue2,best.));
+    output;
+    end;
+  end;
+  else if eng then do;
+    ods_table='ENGINEHOST';
+    name=coalescec(label1,label);
+    value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.));
+    output;
+  end;
+run;
+
+proc sql;
+drop table &ds1, &ds2;
+
+%mend mp_dsmeta;
+

base/mp_filtercheck.sas

@@ -92,7 +92,37 @@ data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
-  length reason_cd $4032;
+  length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000;
+  drop tmp;
+
+  /* quick check to ensure column exists */
+  if upcase(VARIABLE_NM) not in
+  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
+  then do;
+    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
+    putlog REASON_CD= VARIABLE_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
+    output;
+    return;
+  end;
+
+  /* need to open the dataset to get the column type */
+  dsid=open("&targetds","i");
+  if dsid>0 then do;
+    vnum=varnum(dsid,VARIABLE_NM);
+    if vnum<1 then do;
+      /* should not happen as was also tested for above */
+      REASON_CD=cats("Variable (",VARIABLE_NM,") not found in &targetds");
+      putlog REASON_CD= dsid=;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
+      output;
+      return;
+    end;
+    /* now we can get the type */
+    else vtype=vartype(dsid,vnum);
+  end;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
@@ -116,17 +146,8 @@ data &outds;
     call symputx('nobs',_n_,'l');
     output;
   end;
-  if upcase(VARIABLE_NM) not in
-  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
-  then do;
-    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
-    putlog REASON_CD= VARIABLE_NM=;
-    call symputx('reason_cd',reason_cd,'l');
-    call symputx('nobs',_n_,'l');
-    output;
-  end;
   if OPERATOR_NM not in
-  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE')
+  ('=','>','<','<=','>=','NE','GE','LE','BETWEEN','IN','NOT IN','CONTAINS')
   then do;
     REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
@@ -135,19 +156,45 @@ data &outds;
     output;
   end;
 
+  /* special missing logic */
+  if vtype='N'
+    and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE')
+    and cats(upcase(raw_value)) in (
+      '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N'
+      '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._'
+    )
+  then do;
+    /* valid numeric - exit data step loop */
+    return;
+  end;
+
   /* special logic */
-  if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ','');
-  else if OPERATOR_NM in ('IN','NOT IN') then do;
-    if substr(raw_value,1,1) ne '('
-    or substr(cats(reverse(raw_value)),1,1) ne ')'
-    then do;
-      REASON_CD='Missing start/end bracket in RAW_VALUE';
-      putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
-      call symputx('reason_cd',reason_cd,'l');
-      call symputx('nobs',_n_,'l');
-      output;
+  if OPERATOR_NM in ('IN','NOT IN','BETWEEN') then do;
+    if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ',',');
+    else do;
+      if substr(raw_value,1,1) ne '('
+      or substr(cats(reverse(raw_value)),1,1) ne ')'
+      then do;
+        REASON_CD='Missing start/end bracket in RAW_VALUE';
+        putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+        call symputx('reason_cd',reason_cd,'l');
+        call symputx('nobs',_n_,'l');
+        output;
+      end;
+      else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
+    end;
+    /* we now have a comma seperated list of values */
+    if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
+      tmp=scan(raw_value1,i,',');
+      if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
+        REASON_CD='Non Numeric value provided';
+        putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+        call symputx('reason_cd',reason_cd,'l');
+        call symputx('nobs',_n_,'l');
+        output;
+      end;
+      return;
     end;
-    else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
   end;
   else raw_value1=raw_value;
 

base/mp_filtergenerate.sas

@@ -84,6 +84,9 @@ filename &outref temp;
   run;
 %end;
 %else %do;
+  proc sort data=&inds;
+    by SUBGROUP_ID;
+  run;
   data _null_;
     file &outref lrecl=32800;
     set &inds end=last;

base/mp_getconstraints.sas

@@ -40,6 +40,22 @@
 %let lib=%upcase(&lib);
 %let ds=%upcase(&ds);
 
+/**
+  * Cater for environments where sashelp.vcncolu is not available
+  */
+%if %sysfunc(exist(sashelp.vcncolu,view))=0 %then %do;
+  proc sql;
+  create table &outds(
+      libref char(8)
+    ,TABLE_NAME char(32)
+    ,constraint_type char(8)     label='Constraint Type'
+    ,constraint_name char(32)     label='Constraint Name'
+    ,column_name char(32)     label='Column'
+    ,constraint_order num
+  );
+  %return;
+%end;
+
 /**
   * Neither dictionary tables nor sashelp provides a constraint order column,
   * however they DO arrive in the correct order.  So, create the col.
@@ -78,8 +94,11 @@ create table &outds as
 /**
   * We cannot apply this clause to the underlying dictionary table.  See:
   * https://communities.sas.com/t5/SAS-Programming/Unexpected-Where-Clause-behaviour-in-dictionary-TABLE/m-p/771554#M244867
+  * cannot use`where calculated libref="&lib"` either as it will STILL execute
+  * all the underlying constraint queries, causing exception errors in some
+  * cases: https://github.com/sasjs/core/issues/283
   */
-  where calculated libref="&lib"
+  where a.TABLE_CATALOG="&lib"
   %if "&ds" ne "" %then %do;
     and upcase(a.TABLE_NAME)="&ds"
     and upcase(b.TABLE_NAME)="&ds"

base/mp_getddl.sas

@@ -130,13 +130,13 @@ run;
 
 %local x curds;
 %if &flavour=SAS %then %do;
-  data _null_;
-    file &fref mod;
-    put "/* SAS Flavour DDL for %upcase(&libref).&curds */";
-    put "proc sql;";
-  run;
   %do x=1 %to %sysfunc(countw(&dsnlist));
     %let curds=%scan(&dsnlist,&x);
+    data _null_;
+      file &fref mod;
+      put "/* SAS Flavour DDL for %upcase(&libref).&curds */";
+      put "proc sql;";
+    run;
     data _null_;
       file &fref mod;
       length lab $1024 typ $20;

base/mp_getmaxvarlengths.sas

@@ -41,6 +41,7 @@
   <h4> SAS Macros </h4>
   @li mcf_length.sas
   @li mf_getuniquename.sas
+  @li mf_getvarcount.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
   @li mf_getvarformat.sas
@@ -60,7 +61,7 @@
   ,outds=work.mp_getmaxvarlengths
 )/*/STORE SOURCE*/;
 
-%local vars prefix x var fmt;
+%local vars prefix x var fmt srcds;
 %let vars=%mf_getvarlist(libds=&libds);
 %let prefix=%substr(%mf_getuniquename(),1,25);
 %let num2char=%upcase(&num2char);
@@ -70,6 +71,24 @@
   %mcf_length(wrap=YES, insert_cmplib=YES)
 %end;
 
+%if &num2char=NO
+  and ("%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5")
+  and %mf_getvarcount(&libds,typefilter=N) gt 0
+%then %do;
+  /* custom functions not supported in summary operations */
+  %let srcds=%mf_getuniquename();
+  data &srcds/view=&srcds;
+    set &libds;
+  %do x=1 %to %sysfunc(countw(&vars,%str( )));
+    %let var=%scan(&vars,&x);
+    %if %mf_getvartype(&libds,&var)=N %then %do;
+      &prefix.&x=mcf_length(&var);
+    %end;
+  %end;
+  run;
+%end;
+%else %let srcds=&libds;
+
 proc sql;
 create table &outds (rename=(
     %do x=1 %to %sysfunc(countw(&vars,%str( )));
@@ -94,10 +113,15 @@ create table &outds (rename=(
         %end;
       %end;
       %else %do;
-        max(mcf_length(&var)) as &prefix.&x
+        %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do;
+          max(&prefix.&x) as &prefix.&x
+        %end;
+        %else %do;
+          max(mcf_length(&var)) as &prefix.&x
+        %end;
       %end;
     %end;
-  from &libds;
+  from &srcds;
 
   proc transpose data=&outds
     out=&outds(rename=(_name_=NAME COL1=MAXLEN));

base/mp_getpk.sas

@@ -34,6 +34,7 @@
   @param [out] outds= (work.mp_getpk) The name of the output table to create.
 
   <h4> SAS Macros </h4>
+  @li mf_existfeature.sas
   @li mf_getengine.sas
   @li mf_getschema.sas
   @li mp_dropmembers.sas
@@ -229,7 +230,12 @@ create table work.&tabs1 as select
   libname as libref
   ,upcase(memname) as dsn
   ,memtype
+%if %mf_existfeature(DBMS_MEMTYPE)=1 %then %do;
   ,dbms_memtype
+%end;
+%else %do;
+  ,'n/a' as dbms_memtype format=$32.
+%end;
   ,typemem
   ,memlabel
   ,nvar

base/mp_hashdataset.sas

@@ -58,7 +58,7 @@
 %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
   data &outds;
     length hashkey $32;
-    retain hashkey "%sysfunc(md5(%str(&salt)),$hex32.)";
+    hashkey=put(md5("&salt"),$hex32.);
     output;
     stop;
   run;
@@ -69,9 +69,14 @@
   %put %str(ERR)OR: Dataset &libds is not a dataset;
 %end;
 %else %do;
-  data &outds(rename=(&keyvar=hashkey) keep=&keyvar)/nonote2err;
+  data &outds(rename=(&keyvar=hashkey) keep=&keyvar)
+  %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+    /nonote2err
+  %end;
+  ;
     length &prevkeyvar &keyvar $32;
-    retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
+    retain &prevkeyvar;
+    if _n_=1 then &prevkeyvar=put(md5("&salt"),$hex32.);
     set &libds end=&lastvar;
     /* hash should include previous row */
     &keyvar=%mp_md5(

base/mp_init.sas

@@ -67,7 +67,7 @@ options
   validvarname=V7         /* avoid special characters etc in variable names */
   varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
   varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
-%if %substr(&sysver,1,1) ne 4 %then %do;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
   noautocorrect           /* disallow misspelled procedure names            */
   dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
 %end;

base/mp_jsonout.sas

@@ -1,7 +1,12 @@
 /**
   @file mp_jsonout.sas
   @brief Writes JSON in SASjs format to a fileref
-  @details PROC JSON is faster but will produce errs like the ones below if
+  @details This macro can be used to OPEN a JSON stream and send one or more
+  tables as arrays of rows, where each row can be an object or a nested array.
+
+  There are two engines available - DATASTEP or PROCJSON.
+
+  PROC JSON is fast but will produce errs like the ones below if
   special chars are encountered.
 
   > (ERR)OR: Some code points did not transcode.
@@ -12,6 +17,10 @@
 
   If this happens, try running with ENGINE=DATASTEP.
 
+  The DATASTEP engine is used to handle special SAS missing numerics, and
+  can also convert entire datasets to formatted values.  Output JSON is always
+  in UTF-8.
+
   Usage:
 
         filename tmp temp;
@@ -19,7 +28,7 @@
 
         %mp_jsonout(OPEN,jref=tmp)
         %mp_jsonout(OBJ,class,jref=tmp)
-        %mp_jsonout(OBJ,class,dslabel=class2,jref=tmp,showmeta=YES)
+        %mp_jsonout(OBJ,class,dslabel=class2,jref=tmp,showmeta=Y)
         %mp_jsonout(CLOSE,jref=tmp)
 
         data _null_;
@@ -46,12 +55,14 @@
     @li DATASTEP (more reliable when data has non standard characters)
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
-  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+  @param [in] showmeta= (N) Set to Y to output metadata alongside each table,
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to JSON
 
-  <h4> Related Macros <h4>
+  <h4> Related Files </h4>
   @li mp_ds2fmtds.sas
 
   @version 9.2
@@ -59,14 +70,16 @@
   @source https://github.com/sasjs/core
 
 **/
-
 %macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y
   ,engine=DATASTEP
   ,missing=NULL
-  ,showmeta=NO
+  ,showmeta=N
+  ,maxobs=MAX
 )/*/STORE SOURCE*/;
-%local tempds colinfo fmtds i numcols;
+%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval
+  tmpds1 tmpds2 tmpds3 tmpds4;
 %let numcols=0;
+%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;);
 
 %if &action=OPEN %then %do;
   options nobomfile;
@@ -75,9 +88,23 @@
   run;
 %end;
 %else %if (&action=ARR or &action=OBJ) %then %do;
+  /* force variable names to always be uppercase in the JSON */
   options validvarname=upcase;
-  data _null_; file &jref encoding='utf-8' mod;
+  /* To avoid issues with _webout on EBI - such as encoding diffs and truncation
+    (https://support.sas.com/kb/49/325.html) we use temporary files */
+  filename _sjs1 temp lrecl=200 ;
+  data _null_; file _sjs1 encoding='utf-8';
     put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":";
+  run;
+  /* now write to _webout 1 char at a time */
+  data _null_;
+    infile _sjs1 lrecl=1 recfm=n;
+    file &jref mod lrecl=1 recfm=n;
+    input sourcechar $char1. @@;
+    format sourcechar hex2.;
+    put sourcechar char1. @@;
+  run;
+  filename _sjs1 clear;
 
   /* grab col defs */
   proc contents noprint data=&ds
@@ -88,7 +115,7 @@
     by varnum;
   run;
   /* move meta to mac vars */
-  data _null_;
+  data &colinfo;
     if _n_=1 then call symputx('numcols',nobs,'l');
     set &colinfo end=last nobs=nobs;
     name=upcase(name);
@@ -99,7 +126,6 @@
       if format='' then fmt=cats('$',length,'.');
       else if formatl=0 then fmt=cats(format,'.');
       else fmt=cats(format,formatl,'.');
-      newlen=max(formatl,length);
     end;
     else do;
       typelong='num';
@@ -107,23 +133,26 @@
       else if formatl=0 then fmt=cats(format,'.');
       else if formatd=0 then fmt=cats(format,formatl,'.');
       else fmt=cats(format,formatl,'.',formatd);
-      /* needs to be wide, for datetimes etc */
-      newlen=max(length,formatl,24);
     end;
     /* 32 char unique name */
     newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
 
     call symputx(cats('name',_n_),name,'l');
     call symputx(cats('newname',_n_),newname,'l');
-    call symputx(cats('len',_n_),newlen,'l');
     call symputx(cats('length',_n_),length,'l');
     call symputx(cats('fmt',_n_),fmt,'l');
     call symputx(cats('type',_n_),type,'l');
     call symputx(cats('typelong',_n_),typelong,'l');
     call symputx(cats('label',_n_),coalescec(label,name),'l');
+    /* overwritten when fmt=Y and a custom format exists in catalog */
+    if typelong='num' then call symputx(cats('fmtlen',_n_),200,'l');
+    else call symputx(cats('fmtlen',_n_),min(32767,ceil((length+3)*1.5)),'l');
   run;
 
   %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+  proc sql;
+  select count(*) into: lastobs from &ds;
+  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs));
 
   %if &engine=PROCJSON %then %do;
     %if &missing=STRING %then %do;
@@ -131,13 +160,25 @@
       %put &sysmacroname: Switching to DATASTEP engine;
       %goto datastep;
     %end;
-    data &tempds;set &ds;
+    data &tempds;
+      set &ds;
+      &stmt_obs;
     %if &fmt=N %then format _numeric_ best32.;;
     /* PRETTY is necessary to avoid line truncation in large files */
-    proc json out=&jref pretty
+    filename _sjs2 temp lrecl=131068 encoding='utf-8';
+    proc json out=_sjs2 pretty
         %if &action=ARR %then nokeys ;
         ;export &tempds / nosastags fmtnumeric;
     run;
+    /* send back to webout */
+    data _null_;
+      infile _sjs2 lrecl=1 recfm=n;
+      file &jref mod lrecl=1 recfm=n;
+      input sourcechar $char1. @@;
+      format sourcechar hex2.;
+      put sourcechar char1. @@;
+    run;
+    filename _sjs2 clear;
   %end;
   %else %if &engine=DATASTEP %then %do;
     %datastep:
@@ -148,26 +189,99 @@
     %end;
 
     %if &fmt=Y %then %do;
-      data _data_;
+      /**
+        * Extract format definitions
+        * First, by getting library locations from dictionary.formats
+        * Then, by exporting the width using proc format
+        * Cannot use maxw from sashelp.vformat as not always populated
+        * Cannot use fmtinfo() as not supported in all flavours
+        */
+      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32);
+      proc sql noprint;
+      create table &tmpds1 as
+          select cats(libname,'.',memname) as fmtcat,
+          fmtname
+        from dictionary.formats
+        where fmttype='F' and libname is not null
+          and fmtname in (select format from &colinfo where format is not null)
+        order by 1;
+      create table &tmpds2(
+          FMTNAME char(32),
+          LENGTH num
+      );
+      %local catlist cat fmtlist i;
+      select distinct fmtcat into: catlist separated by ' ' from &tmpds1;
+      %do i=1 %to %sysfunc(countw(&catlist,%str( )));
+        %let cat=%scan(&catlist,&i,%str( ));
+        proc sql;
+        select distinct fmtname into: fmtlist separated by ' '
+          from &tmpds1 where fmtcat="&cat";
+        proc format lib=&cat cntlout=&tmpds3(keep=fmtname length);
+          select &fmtlist;
+        run;
+        proc sql;
+        insert into &tmpds2 select distinct fmtname,length from &tmpds3;
+      %end;
+
+      proc sql;
+      create table &tmpds4 as
+        select a.*, b.length as maxw
+        from &colinfo a
+        left join &tmpds2 b
+        on cats(a.format)=cats(upcase(b.fmtname))
+        order by a.varnum;
+      data _null_;
+        set &tmpds4;
+        if not missing(maxw);
+        call symputx(
+          cats('fmtlen',_n_),
+          /* vars need extra padding due to JSON escaping of special chars */
+          min(32767,ceil((max(length,maxw)+3)*1.5))
+          ,'l'
+        );
+      run;
+
+      /* configure varlenchk - as we are explicitly shortening the variables */
+      %let optval=%sysfunc(getoption(varlenchk));
+      options varlenchk=NOWARN;
+      data _data_(compress=char);
+        /* shorten the new vars */
+        length
+      %do i=1 %to &numcols;
+          &&name&i $&&fmtlen&i
+      %end;
+          ;
         /* rename on entry */
         set &ds(rename=(
       %do i=1 %to &numcols;
-        &&name&i=&&newname&i
+          &&name&i=&&newname&i
       %end;
         ));
+      &stmt_obs;
+
+      drop
+      %do i=1 %to &numcols;
+        &&newname&i
+      %end;
+        ;
       %do i=1 %to &numcols;
-        length &&name&i $&&len&i;
         %if &&typelong&i=num %then %do;
-          &&name&i=left(put(&&newname&i,&&fmt&i));
+          &&name&i=cats(put(&&newname&i,&&fmt&i));
         %end;
         %else %do;
           &&name&i=put(&&newname&i,&&fmt&i);
         %end;
-        drop &&newname&i;
       %end;
-        if _error_ then call symputx('syscc',1012);
+        if _error_ then do;
+          call symputx('syscc',1012);
+          stop;
+        end;
       run;
       %let fmtds=&syslast;
+      options varlenchk=&optval;
     %end;
 
     proc format; /* credit yabwon for special null removal */
@@ -186,8 +300,8 @@
       attrib _all_ label='';
       %do i=1 %to &numcols;
         %if &&typelong&i=char or &fmt=Y %then %do;
-          length &&name&i $32767;
-          format &&name&i $32767.;
+          length &&name&i $&&fmtlen&i...;
+          format &&name&i $&&fmtlen&i...;
         %end;
       %end;
       %if &fmt=Y %then %do;
@@ -196,6 +310,7 @@
       %else %do;
         set &ds;
       %end;
+      &stmt_obs;
       format _numeric_ bart.;
     %do i=1 %to &numcols;
       %if &&typelong&i=char or &fmt=Y %then %do;
@@ -220,10 +335,9 @@
     %end;
     run;
 
-    /* write to temp loc to avoid _webout truncation
-      - https://support.sas.com/kb/49/325.html */
-    filename _sjs temp lrecl=131068 encoding='utf-8';
-    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod ;
+    filename _sjs3 temp lrecl=131068 ;
+    data _null_;
+      file _sjs3 encoding='utf-8';
       if _n_=1 then put "[";
       set &tempds;
       if _n_>1 then put "," @; put
@@ -231,39 +345,38 @@
       %do i=1 %to &numcols;
         %if &i>1 %then  "," ;
         %if &action=OBJ %then """&&name&i"":" ;
-        &&name&i
+        "&&name&i"n /* name literal for reserved variable names */
       %end;
       %if &action=ARR %then "]" ; %else "}" ; ;
-    /* now write the long strings to _webout 1 byte at a time */
+
+    /* close out the table */
     data _null_;
-      length filein 8 fileid 8;
-      filein=fopen("_sjs",'I',1,'B');
-      fileid=fopen("&jref",'A',1,'B');
-      rec='20'x;
-      do while(fread(filein)=0);
-        rc=fget(filein,rec,1);
-        rc=fput(fileid, rec);
-        rc=fwrite(fileid);
-      end;
-      /* close out the table */
-      rc=fput(fileid, "]");
-      rc=fwrite(fileid);
-      rc=fclose(filein);
-      rc=fclose(fileid);
+      file _sjs3 mod encoding='utf-8';
+      put ']';
     run;
-    filename _sjs clear;
+    data _null_;
+      infile _sjs3 lrecl=1 recfm=n;
+      file &jref mod lrecl=1 recfm=n;
+      input sourcechar $char1. @@;
+      format sourcechar hex2.;
+      put sourcechar char1. @@;
+    run;
+    filename _sjs3 clear;
   %end;
 
   proc sql;
   drop table &colinfo, &tempds;
 
-  %if &showmeta=YES %then %do;
-    data _null_; file &jref encoding='utf-8' mod;
+  %if %substr(&showmeta,1,1)=Y %then %do;
+    filename _sjs4 temp lrecl=131068 encoding='utf-8';
+    data _null_;
+      file _sjs4;
+      length label $350;
       put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{";
       do i=1 to &numcols;
         name=quote(trim(symget(cats('name',i))));
         format=quote(trim(symget(cats('fmt',i))));
-        label=quote(trim(symget(cats('label',i))));
+        label=quote(prxchange('s/\\/\\\\/',-1,trim(symget(cats('label',i)))));
         length=quote(trim(symget(cats('length',i))));
         type=quote(trim(symget(cats('typelong',i))));
         if i>1 then put "," @@;
@@ -272,6 +385,15 @@
       end;
       put '}}';
     run;
+    /* send back to webout */
+    data _null_;
+      infile _sjs4 lrecl=1 recfm=n;
+      file &jref mod lrecl=1 recfm=n;
+      input sourcechar $char1. @@;
+      format sourcechar hex2.;
+      put sourcechar char1. @@;
+    run;
+    filename _sjs4 clear;
   %end;
 %end;
 

base/mp_lockanytable.sas

@@ -22,6 +22,7 @@
   @param [in] loop_secs= (1) Seconds to wait between each lock attempt
 
   <h4> SAS Macros </h4>
+  @li mf_fmtdttm.sas
   @li mp_abort.sas
   @li mp_lockfilecheck.sas
   @li mf_getuser.sas
@@ -111,7 +112,7 @@ run;
       LOCK_LIB ="&lib";
       LOCK_DS="&ds";
       LOCK_STATUS_CD='LOCKED';
-      LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt;
+      LOCK_START_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt;
       LOCK_USER_NM="&user";
       LOCK_PID="&sysjobid";
       LOCK_REF="&ref";
@@ -131,7 +132,7 @@ run;
     proc sql;
     update &ctl_ds
       set LOCK_STATUS_CD='LOCKED'
-        , LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_START_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
         , LOCK_USER_NM="&user"
         , LOCK_PID="&sysjobid"
         , LOCK_REF="&ref"
@@ -206,7 +207,7 @@ run;
     proc sql;
     update &ctl_ds
       set LOCK_STATUS_CD='UNLOCKED'
-        , LOCK_END_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_END_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
         , LOCK_USER_NM="&user"
         , LOCK_PID="&sysjobid"
         , LOCK_REF="&ref"

base/mp_perflog.sas

@@ -1,13 +1,14 @@
 /**
   @file
-  @brief Logs the time the macro was executed in a control dataset.
-  @details If the dataset does not exist, it is created.  Usage:
+  @brief Logs a message in a dataset every time it is invoked
+  @details If the dataset does not exist, it is created.
+  Usage:
 
-    %mp_perflog(started)
-    %mp_perflog()
-    %mp_perflog(startanew,libds=work.newdataset)
-    %mp_perflog(finished,libds=work.newdataset)
-    %mp_perflog(finished)
+      %mp_perflog(started)
+      %mp_perflog()
+      %mp_perflog(startanew,libds=work.newdataset)
+      %mp_perflog(finished,libds=work.newdataset)
+      %mp_perflog(finished)
 
 
   @param label Provide label to go into the control dataset

base/mp_resetoption.sas

@@ -21,15 +21,19 @@ https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-s
 %macro mp_resetoption(option /* the option to reset */
 )/*/STORE SOURCE*/;
 
-data _null_;
-  length code  $1500;
-  startup=getoption("&option",'startupvalue');
-  current=getoption("&option");
-  if startup ne current then do;
-    code =cat('OPTIONS ',getoption("&option",'keyword','startupvalue'),';');
-    putlog "NOTE: Resetting system option: " code ;
-    call execute(code );
-  end;
-run;
-
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  data _null_;
+    length code  $1500;
+    startup=getoption("&option",'startupvalue');
+    current=getoption("&option");
+    if startup ne current then do;
+      code =cat('OPTIONS ',getoption("&option",'keyword','startupvalue'),';');
+      putlog "NOTE: Resetting system option: " code ;
+      call execute(code );
+    end;
+  run;
+%end;
+%else %do;
+  %put &sysmacroname: reset option feature unavailable on &sysvlong;
+%end;
 %mend mp_resetoption;

base/mp_retainedkey.sas

@@ -58,6 +58,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_existvar.sas
+  @li mf_fmtdttm.sas
   @li mf_getquotedstr.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
@@ -217,12 +218,12 @@ quit;
     set keytable="&base_libds"
       ,keycolumn="&retained_key"
       ,max_key=%eval(&maxkey+&newkey_cnt)
-      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt;
+      ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt;
   %end;
   %else %do;
   update &maxkeytable
     set max_key=%eval(&maxkey+&newkey_cnt)
-      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt
+      ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt
     where keytable="&base_libds";
   %end;
   %mp_lockanytable(UNLOCK

base/mp_storediffs.sas

@@ -125,7 +125,7 @@
 data &ds1;
   set &dslist indsname=&inds_auto;
   &hashkey=put(md5(catx('|',%mf_getquotedstr(&key,quote=N))),$hex32.);
-  &inds_keep=&inds_auto;
+  &inds_keep=upcase(&inds_auto);
 proc sort;
   by &inds_keep &hashkey;
 run;
@@ -154,14 +154,16 @@ run;
 %else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
 
 data &ds4;
-  length &inds_keep $41 tgtvar_nm $32;
+  length &inds_keep $41 tgtvar_nm $32 _label_ $256;
+  if _n_=1 then call missing(_label_);
+  drop _label_;
   set &ds2 &ds3 indsname=&inds_auto;
 
   tgtvar_nm=upcase(tgtvar_nm);
   if tgtvar_nm in (%upcase(&vlist));
 
-  if &inds_auto="&ds2" then tgtvar_type='N';
-  else if &inds_auto="&ds3" then tgtvar_type='C';
+  if upcase(&inds_auto)="&ds2" then tgtvar_type='N';
+  else if upcase(&inds_auto)="&ds3" then tgtvar_type='C';
   else do;
     putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
     call symputx('syscc',98);

base/mp_streamfile.sas

@@ -1,8 +1,9 @@
 /**
   @file
   @brief Streams a file to _webout according to content type
-  @details Will set headers using appropriate functions (SAS 9 vs Viya) and send
-  content as a binary stream.
+  @details Will set headers using appropriate functions per the server type
+  (Viya, EBI, [SASjs Server](https://github.com/sasjs/server)) and stream
+  content using mp_binarycopy().
 
   Usage:
 
@@ -12,7 +13,14 @@
 
       %mp_streamfile(contenttype=csv,inloc=/some/where.txt,outname=myfile.txt)
 
-  @param [in] contenttype= (TEXT) Either TEXT, ZIP, CSV, EXCEL
+  @param [in] contenttype= (TEXT) Supported:
+    @li CSV
+    @li EXCEL
+    @li MARKDOWN
+    @li TEXT
+    @li ZIP
+    Feel free to submit PRs to support more mime types!  The official list is
+    here:  https://www.iana.org/assignments/media-types/media-types.xhtml
   @param [in] inloc= /path/to/file.ext to be sent
   @param [in] inref= fileref of file to be sent (if provided, overrides `inloc`)
   @param [in] iftrue= (1=1) Provide a condition under which to execute.
@@ -58,7 +66,7 @@ run;
 %if &contentype=CSV %then %do;
   %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/csv');
+      rc=stpsrv_header('Content-Type','application/csv');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
@@ -68,7 +76,7 @@ run;
       contentdisp="attachment; filename=&outname";
   %end;
   %else %if &platform=SASJS %then %do;
-    %mfs_httpheader(Content-type,application/csv)
+    %mfs_httpheader(Content-Type,application/csv)
     %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
   %end;
 %end;
@@ -76,7 +84,7 @@ run;
   /* suitable for XLS format */
   %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/vnd.ms-excel');
+      rc=stpsrv_header('Content-Type','application/vnd.ms-excel');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
@@ -86,14 +94,14 @@ run;
       contentdisp="attachment; filename=&outname";
   %end;
   %else %if &platform=SASJS %then %do;
-    %mfs_httpheader(Content-type,application/vnd.ms-excel)
+    %mfs_httpheader(Content-Type,application/vnd.ms-excel)
     %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
   %end;
 %end;
 %else %if &contentype=GIF or &contentype=JPEG or &contentype=PNG %then %do;
   %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type',"image/%lowcase(&contenttype)");
+      rc=stpsrv_header('Content-Type',"image/%lowcase(&contenttype)");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
@@ -101,30 +109,30 @@ run;
       contenttype="image/%lowcase(&contenttype)";
   %end;
   %else %if &platform=SASJS %then %do;
-    %mfs_httpheader(Content-type,image/%lowcase(&contenttype))
+    %mfs_httpheader(Content-Type,image/%lowcase(&contenttype))
   %end;
 %end;
-%else %if &contentype=HTML %then %do;
+%else %if &contentype=HTML or &contenttype=MARKDOWN %then %do;
   %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','text/html');
+      rc=stpsrv_header('Content-Type',"text/%lowcase(&contenttype)");
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
     filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
-      contenttype="text/html"
+      contenttype="text/%lowcase(&contenttype)"
       contentdisp="attachment; filename=&outname";
   %end;
   %else %if &platform=SASJS %then %do;
-    %mfs_httpheader(Content-type,text/html)
+    %mfs_httpheader(Content-Type,text/%lowcase(&contenttype))
     %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
   %end;
 %end;
 %else %if &contentype=TEXT %then %do;
   %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/text');
+      rc=stpsrv_header('Content-Type','application/text');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
@@ -134,14 +142,14 @@ run;
       contentdisp="attachment; filename=&outname";
   %end;
   %else %if &platform=SASJS %then %do;
-    %mfs_httpheader(Content-type,application/text)
+    %mfs_httpheader(Content-Type,application/text)
     %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
   %end;
 %end;
 %else %if &contentype=WOFF or &contentype=WOFF2 or &contentype=TTF %then %do;
   %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type',"font/%lowcase(&contenttype)");
+      rc=stpsrv_header('Content-Type',"font/%lowcase(&contenttype)");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
@@ -149,13 +157,13 @@ run;
       contenttype="font/%lowcase(&contenttype)";
   %end;
   %else %if &platform=SASJS %then %do;
-    %mfs_httpheader(Content-type,font/%lowcase(&contenttype))
+    %mfs_httpheader(Content-Type,font/%lowcase(&contenttype))
   %end;
 %end;
 %else %if &contentype=XLSX %then %do;
   %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type',
+      rc=stpsrv_header('Content-Type',
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
@@ -167,7 +175,7 @@ run;
       contentdisp="attachment; filename=&outname";
   %end;
   %else %if &platform=SASJS %then %do;
-    %mfs_httpheader(Content-type
+    %mfs_httpheader(Content-Type
       ,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
     )
     %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
@@ -176,7 +184,7 @@ run;
 %else %if &contentype=ZIP %then %do;
   %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/zip');
+      rc=stpsrv_header('Content-Type','application/zip');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
@@ -186,7 +194,7 @@ run;
       contentdisp="attachment; filename=&outname";
   %end;
   %else %if &platform=SASJS %then %do;
-    %mfs_httpheader(Content-type,application/zip)
+    %mfs_httpheader(Content-Type,application/zip)
     %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
   %end;
 %end;

base/mp_testservice.sas

@@ -1,53 +1,11 @@
 /**
   @file
-  @brief Will execute a SASjs web service on SAS 9 or Viya
-  @details Prepares the input files and retrieves the resulting datasets from
-  the response JSON.
-
-
-
-  Note - the _webout fileref should NOT be assigned prior to running this macro.
-
-  @param [in] program The _PROGRAM endpoint to test
-  @param [in] inputfiles=(0) A list of space seperated fileref:filename pairs as
-    follows:
-        inputfiles=inref:filename inref2:filename2
-  @param [in] inputdatasets= (0) All datasets in this space seperated list are
-    converted into SASJS-formatted CSVs (see mp_ds2csv.sas) files and added to
-    the list of `inputfiles` for ingestion.  The dataset will be sent with the
-    same name (no need for a colon modifier).
-  @param [in] inputparams=(0) A dataset containing name/value pairs in the
-    following format:
-    |name:$32|value:$1000|
-    |---|---|
-    |stpmacname|some value|
-    |mustbevalidname|can be anything, oops, %abort!!|
-
-  @param [in] debug= (log) Provide the _debug value
-  @param [in] mdebug= (0) Set to 1 to provide macro debugging
-  @param [in] viyaresult= (WEBOUT_JSON) The Viya result type to return.  For
-    more info, see mv_getjobresult.sas
-  @param [in] viyacontext= (SAS Job Execution compute context) The Viya compute
-    context on which to run the service
-  @param [out] outlib= (0) Output libref to contain the final tables.  Set to
-    0 if the service output is not in JSON format.
-  @param [out] outref= (0) Output fileref to create, to contain the full _webout
-    response.
+  @brief To be deprecated.  Will execute a SASjs web service on SAS 9 or Viya
+  @details Use the mx_testservice.sas macro instead (documentation can be
+  found there)
 
   <h4> SAS Macros </h4>
-  @li mf_getplatform.sas
-  @li mf_getuniquefileref.sas
-  @li mf_getuniquename.sas
-  @li mp_abort.sas
-  @li mp_binarycopy.sas
-  @li mp_chop.sas
-  @li mp_ds2csv.sas
-  @li ms_testservice.sas
-  @li mv_getjobresult.sas
-  @li mv_jobflow.sas
-
-  <h4> Related Programs </h4>
-  @li mp_testservice.test.sas
+  @li mx_testservice.sas
 
   @version 9.4
   @author Allan Bowe
@@ -65,237 +23,17 @@
   viyaresult=WEBOUT_JSON,
   viyacontext=SAS Job Execution compute context
 )/*/STORE SOURCE*/;
-%local dbg pcnt fref1 fref2 webref webrefpath i webcount var platform;
-%if &mdebug=1 %then %do;
-  %put &sysmacroname entry vars:;
-  %put _local_;
-%end;
-%else %let dbg=*;
 
-/* sanitise inputparams */
-%let pcnt=0;
-%if &inputparams ne 0 %then %do;
-  data _null_;
-    set &inputparams;
-    if not nvalid(name,'v7') then putlog (_all_)(=);
-    else if name in (
-      'program','inputfiles','inputparams','debug','outlib','outref'
-    ) then putlog (_all_)(=);
-    else do;
-      x+1;
-      call symputx(name,quote(cats(value)),'l');
-      call symputx(cats('pval',x),name,'l');
-      call symputx('pcnt',x,'l');
-    end;
-  run;
-  %mp_abort(iftrue= (%mf_nobs(&inputparams) ne &pcnt)
-    ,mac=&sysmacroname
-    ,msg=%str(Invalid values in &inputparams)
-  )
-%end;
+%mx_testservice(&program,
+  inputfiles=&inputfiles,
+  inputdatasets=&inputdatasets,
+  inputparams=&inputparams,
+  debug=&debug,
+  mdebug=&mdebug,
+  outlib=&outlib,
+  outref=&outref,
+  viyaresult=&viyaresult,
+  viyacontext=&viyacontext
+)
 
-/* convert inputdatasets to filerefs */
-%if "&inputdatasets" ne "0" %then %do;
-  %if %quote(&inputfiles)=0 %then %let inputfiles=;
-  %do i=1 %to %sysfunc(countw(&inputdatasets,%str( )));
-    %let var=%scan(&inputdatasets,&i,%str( ));
-    %local dsref&i;
-    %let dsref&i=%mf_getuniquefileref();
-    %mp_ds2csv(&var,outref=&&dsref&i,headerformat=SASJS)
-    %let inputfiles=&inputfiles &&dsref&i:%scan(&var,-1,.);
-  %end;
-%end;
-
-%let platform=%mf_getplatform();
-%let fref1=%mf_getuniquefileref();
-%let fref2=%mf_getuniquefileref();
-%let webref=%mf_getuniquefileref();
-%let webrefpath=%sysfunc(pathname(work))/%mf_getuniquename();
-/* mp_chop requires a physical path as input */
-filename &webref "&webrefpath";
-
-%if &platform=SASMETA %then %do;
-
-  /* parse the input files */
-  %if %quote(&inputfiles) ne 0 %then %do;
-    %let webcount=%sysfunc(countw(&inputfiles));
-    %put &=webcount;
-    %do i=1 %to &webcount;
-      %let var=%scan(&inputfiles,&i,%str( ));
-      %local webfref&i webname&i;
-      %let webref&i=%scan(&var,1,%str(:));
-      %let webname&i=%scan(&var,2,%str(:));
-      %put webref&i=&&webref&i;
-      %put webname&i=&&webname&i;
-    %end;
-  %end;
-  %else %let webcount=0;
-
-  proc stp program="&program";
-    inputparam _program="&program"
-  %do i=1 %to &webcount;
-    %if &webcount=1 %then %do;
-      _webin_fileref="&&webref&i"
-      _webin_name="&&webname&i"
-    %end;
-    %else %do;
-      _webin_fileref&i="&&webref&i"
-      _webin_name&i="&&webname&i"
-    %end;
-  %end;
-    _webin_file_count="&webcount"
-    _debug="&debug"
-  %do i=1 %to &pcnt;
-    /* resolve name only, proc stp fetches value */
-    &&pval&i=&&&&&&pval&i
-  %end;
-    ;
-  %do i=1 %to &webcount;
-    inputfile &&webref&i;
-  %end;
-    outputfile _webout=&webref;
-  run;
-
-  data _null_;
-    infile &webref;
-    file &fref1;
-    input;
-    length line $10000;
-    if index(_infile_,'>>weboutBEGIN<<') then do;
-        line=tranwrd(_infile_,'>>weboutBEGIN<<','');
-        put line;
-    end;
-    else if index(_infile_,'>>weboutEND<<') then do;
-        line=tranwrd(_infile_,'>>weboutEND<<','');
-        put line;
-        stop;
-    end;
-    else put _infile_;
-  run;
-  data _null_;
-    infile &fref1;
-    input;
-    put _infile_;
-  run;
-  %if &outlib ne 0 %then %do;
-    libname &outlib json (&fref1);
-  %end;
-  %if &outref ne 0 %then %do;
-    filename &outref temp;
-    %mp_binarycopy(inref=&webref,outref=&outref)
-  %end;
-
-%end;
-%else %if &platform=SASVIYA %then %do;
-
-  /* prepare inputparams */
-  %local ds1;
-  %let ds1=%mf_getuniquename();
-  %if "&inputparams" ne "0" %then %do;
-    proc transpose data=&inputparams out=&ds1;
-      id name;
-      var value;
-    run;
-  %end;
-  %else %do;
-    data &ds1;run;
-  %end;
-
-  /* parse the input files - convert to sasjs params */
-  %local webcount i var sasjs_tables;
-  %if %quote(&inputfiles) ne 0 %then %do;
-    %let webcount=%sysfunc(countw(&inputfiles));
-    %put &=webcount;
-    %do i=1 %to &webcount;
-      %let var=%scan(&inputfiles,&i,%str( ));
-      %local webfref&i webname&i sasjs&i.data;
-      %let webref&i=%scan(&var,1,%str(:));
-      %let webname&i=%scan(&var,2,%str(:));
-      %put webref&i=&&webref&i;
-      %put webname&i=&&webname&i;
-
-      %let sasjs_tables=&sasjs_tables &&webname&i;
-      data _null_;
-        infile &&webref&i lrecl=32767;
-        input;
-        if _n_=1 then call symputx("sasjs&i.data",_infile_);
-        else call symputx(
-          "sasjs&i.data",cats(symget("sasjs&i.data"),'0D0A'x,_infile_)
-        );
-        putlog "&sysmacroname infile: " _infile_;
-      run;
-      data &ds1;
-        set &ds1;
-        length sasjs&i.data $32767 sasjs_tables $1000;
-        sasjs&i.data=symget("sasjs&i.data");
-        sasjs_tables=symget("sasjs_tables");
-      run;
-    %end;
-  %end;
-  %else %let webcount=0;
-
-  data &ds1;
-    retain _program "&program";
-    retain _contextname "&viyacontext";
-    set &ds1;
-    putlog "&sysmacroname inputparams:";
-    putlog (_all_)(=);
-  run;
-
-  %mv_jobflow(inds=&ds1
-    ,maxconcurrency=1
-    ,outds=work.results
-    ,outref=&fref1
-    ,mdebug=&mdebug
-  )
-  /* show the log */
-  data _null_;
-    infile &fref1;
-    input;
-    putlog _infile_;
-  run;
-  /* get the uri to fetch results */
-  data _null_;
-    set work.results;
-    call symputx('uri',uri);
-    putlog "&sysmacroname: fetching results for " uri;
-  run;
-  /* fetch results from webout.json */
-  %mv_getjobresult(uri=&uri,
-    result=&viyaresult,
-    outref=&outref,
-    outlib=&outlib,
-    mdebug=&mdebug
-  )
-
-%end;
-%else %if &platform=SASJS %then %do;
-
-  %ms_testservice(&program
-    ,inputfiles=&inputfiles
-    ,inputdatasets=&inputdatasets
-    ,inputparams=&inputparams
-    ,debug=&debug
-    ,mdebug=&mdebug
-    ,outlib=&outlib
-    ,outref=&outref
-  )
-
-%end;
-%else %do;
-  %put %str(ERR)OR: Unrecognised platform:  &platform;
-%end;
-
-%if &mdebug=0 %then %do;
-  filename &fref1 clear;
-  %if &platform ne SASJS %then %do;
-    filename &fref2 clear;
-    filename &webref clear;
-  %end;
-%end;
-%else %do;
-  %put &sysmacroname exit vars:;
-  %put _local_;
-%end;
-
-%mend mp_testservice;
+%mend mp_testservice;

build.py

@@ -102,7 +102,7 @@ options noquotelenmax;
 """
 f = open('all.sas', "w")             # r / r+ / rb / rb+ / w / wb
 f.write(header)
-folders = ['base', 'ddl', 'meta', 'metax', 'server', 'viya', 'lua', 'fcmp']
+folders = ['base', 'ddl', 'meta', 'metax', 'server', 'viya', 'lua', 'fcmp', 'xplatform']
 for folder in folders:
     filenames = [fn for fn in Path(
         './' + folder).iterdir() if fn.match("*.sas")]

ddl/mddl_dc_difftable.sas

@@ -26,9 +26,19 @@
       oldval_num num format=best32. label='Old (numeric) value',
       newval_num num format=best32. label='New (numeric) value',
       oldval_char char(32765) label='Old (character) value',
-      newval_char char(32765) label='New (character) value',
-    constraint pk_mpe_audit
-      primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
+      newval_char char(32765) label='New (character) value'
   );
 
+  %local lib;
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create
+      pk_mpe_audit=(load_ref libref dsn key_hash tgtvar_nm)
+      /nomiss unique;
+  quit;
+
 %mend mddl_dc_difftable;

ddl/mddl_dc_filterdetail.sas

@@ -9,19 +9,32 @@
 
 %macro mddl_dc_filterdetail(libds=WORK.FILTER_DETAIL);
 
+%local nn lib;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  %let nn=not null;
+%end;
+%else %let nn=;
+
   proc sql;
   create table &libds(
-      filter_hash char(32) not null,
-      filter_line num not null,
-      group_logic char(3) not null,
-      subgroup_logic char(3) not null,
-      subgroup_id num not null,
-      variable_nm varchar(32) not null,
-      operator_nm varchar(12) not null,
-      raw_value varchar(4000) not null,
-      processed_dttm num not null format=E8601DT26.6,
-    constraint pk_mpe_filteranytable
-      primary key(filter_hash,filter_line)
+      filter_hash char(32) &nn,
+      filter_line num &nn,
+      group_logic char(3) &nn,
+      subgroup_logic char(3) &nn,
+      subgroup_id num &nn,
+      variable_nm varchar(32) &nn,
+      operator_nm varchar(12) &nn,
+      raw_value varchar(4000) &nn,
+      processed_dttm num &nn format=E8601DT26.6
   );
 
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create pk_mpe_filterdetail=(filter_hash filter_line)/nomiss unique;
+  quit;
+
 %mend mddl_dc_filterdetail;

ddl/mddl_dc_filtersummary.sas

@@ -9,14 +9,27 @@
 
 %macro mddl_dc_filtersummary(libds=WORK.FILTER_SUMMARY);
 
+%local nn lib;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  %let nn=not null;
+%end;
+%else %let nn=;
+
   proc sql;
   create table &libds(
-      filter_rk num not null,
-      filter_hash char(32) not null,
-      filter_table char(41) not null,
-      processed_dttm num not null format=E8601DT26.6,
-    constraint pk_mpe_filteranytable
-      primary key(filter_rk)
+      filter_rk num &nn,
+      filter_hash char(32) &nn,
+      filter_table char(41) &nn,
+      processed_dttm num &nn format=E8601DT26.6
   );
 
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create filter_rk /nomiss unique;
+  quit;
+
 %mend mddl_dc_filtersummary;

ddl/mddl_dc_locktable.sas

@@ -9,17 +9,33 @@
 
 %macro mddl_dc_locktable(libds=WORK.LOCKTABLE);
 
+%local nn lib;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  %let nn=not null;
+%end;
+%else %let nn=;
+
   proc sql;
   create table &libds(
       lock_lib char(8),
       lock_ds char(32),
-      lock_status_cd char(10) not null,
-      lock_user_nm char(100) not null ,
+      lock_status_cd char(10) &nn,
+      lock_user_nm char(100) &nn ,
       lock_ref char(200),
       lock_pid char(10),
       lock_start_dttm num format=E8601DT26.6,
-      lock_end_dttm num format=E8601DT26.6,
-    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds)
+      lock_end_dttm num format=E8601DT26.6
   );
 
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create
+      pk_mp_lockanytable=(lock_lib lock_ds)
+      /nomiss unique;
+  quit;
+
 %mend mddl_dc_locktable;

ddl/mddl_dc_maxkeytable.sas

@@ -17,8 +17,17 @@
       max_key num label=
         'Integer representing current max RK or SK value in the KEYTABLE',
       processed_dttm num format=E8601DT26.6
-        label='Datetime this value was last updated',
-    constraint pk_mpe_maxkeyvalues
-        primary key(keytable));
+        label='Datetime this value was last updated'
+  );
+
+  %local lib;
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create keytable /nomiss unique;
+  quit;
 
 %mend mddl_dc_maxkeytable;

ddl/mddl_sas_cntlout.sas

@@ -14,8 +14,8 @@ proc sql;
 create table &libds(
     FMTNAME char(32)      label='Format name'
     /*
-      to accomodate larger START values, mp_loadformat.sas will need the
-      SQL dependency removed (proc sql needs to accomodate 3 index values in
+      to accommodate larger START values, mp_loadformat.sas will need the
+      SQL dependency removed (proc sql needs to accommodate 3 index values in
       a 32767 ibufsize limit)
     */
     ,START char(10000)    label='Starting value for format'
@@ -40,4 +40,4 @@ create table &libds(
     ,LANGUAGE char(8)     label='Language for date strings'
 );
 
-%mend mddl_sas_cntlout;
+%mend mddl_sas_cntlout;

main.dox

@@ -18,7 +18,17 @@
     statements.  Those starting `mp_` are macro _procedures_, which generate
     SAS statements, and must therefore be applied accordingly.
 
- */
+*/
+
+/*! \dir ddl
+ *  \brief Data Definition Language files
+ *  \details Provides templates for commonly used tables in sasjs/core.
+    Attributes:
+
+        * OS independent
+        * No X command
+        * Prefixes: _mddl_
+*/
 
 /*! \dir fcmp
  *  \brief Macros for generating FCMP functions
@@ -92,4 +102,39 @@
         * Auto-generated from the plain source `.lua` files in the same directory
         * Prefixes: _ml_
 
- */
+*/
+
+/*! \dir tests/base
+ *  \brief Tests for Base macros
+ */
+
+/*! \dir tests/ddlonly
+ *  \brief Tests for DDL macros
+ */
+
+/*! \dir tests/sas9only
+ *  \brief Tests for SAS Metadata macros
+ */
+
+/*! \dir tests/serveronly
+ *  \brief Tests for SASjs Server macros
+ */
+
+/*! \dir tests/viyaonly
+ *  \brief Tests for Viya macros
+ */
+
+/*! \dir tests/x-platform
+ *  \brief Tests for cross-platform macros
+ */
+
+/*! \dir xplatform
+ *  \brief Cross Platform, works on all SAS servers (Viya, EBI, SASjs)
+ *  \details Useful when you need to run a single piece of code against Viya,
+     SAS 9 with metadata, or SASjs on Base SAS.
+
+        * OS independent
+        * No X command
+        * Prefixes: _mx_
+
+ */

meta/mm_adduser2group.sas

@@ -2,7 +2,10 @@
   @file mm_adduser2group.sas
   @brief Adds a user to a group
   @details Adds a user to a metadata group.  The macro first checks whether the
-    user is in that group, and if not, the user is added.
+  user is in that group, and if not, the user is added.
+
+  Note that the macro does not check inherited group memberships - it looks at
+    direct members only.
 
   Usage:
 
@@ -12,10 +15,10 @@
 
   @param user= the user name (not displayname)
   @param group= the group to which to add the user
-  @param mdebug= set to 1 to show debug info in log
+  @param mdebug= (0) set to 1 to show debug info in log
 
-  @warning the macro does not check inherited group memberships - it looks at
-    direct members only
+  <h4> Related Files </h4>
+  @li ms_adduser2group.sas
 
   @version 9.3
   @author Allan Bowe

meta/mm_createdataset.sas

@@ -1,22 +1,21 @@
 /**
-  @file mm_createdataset.sas
-  @brief Create a dataset from a metadata definition
+  @file
+  @brief Create an empty dataset from a metadata definition
   @details This macro was built to support viewing empty tables in
-    https://datacontroller.io - a free evaluation copy is available by
-    contacting the author (Allan Bowe).
+  https://datacontroller.io
 
-    The table can be retrieved using LIBRARY.DATASET reference, or directly
-    using the metadata URI.
+  The table can be retrieved using LIBRARY.DATASET reference, or directly
+  using the metadata URI.
 
-    The dataset is written to the WORK library.
+  The dataset is written to the WORK library.
 
-  usage:
+  Usage:
 
-    %mm_createdataset(libds=metlib.some_dataset)
+      %mm_createdataset(libds=metlib.some_dataset)
 
-    or
+  or
 
-    %mm_createdataset(tableuri=G5X8AFW1.BE00015Y)
+      %mm_createdataset(tableuri=G5X8AFW1.BE00015Y)
 
   <h4> SAS Macros </h4>
   @li mm_getlibs.sas
@@ -26,9 +25,9 @@
   @param libds= library.dataset metadata source.  Note - table names in metadata
     can be longer than 32 chars (just fyi, not an issue here)
   @param tableuri= Metadata URI of the table to be created
-  @param outds= The dataset to create, default is `work.mm_createdataset`.
-    The table name needs to be 32 chars or less as per SAS naming rules.
-  @param mdebug= set DBG to 1 to disable DEBUG messages
+  @param outds= (work.mm_createdataset) The dataset to create.  The table name
+    needs to be 32 chars or less as per SAS naming rules.
+  @param mdebug= (0) Set to 1 to enable DEBUG messages
 
   @version 9.4
   @author Allan Bowe
@@ -54,14 +53,23 @@
   %mm_gettables(uri=&liburi,outds=&tempds2)
   data _null_;
     set &tempds2;
-    if upcase(tablename)="%upcase(%scan(&libds,2,.))";
+    where upcase(tablename)="%upcase(%scan(&libds,2,.))";
+    &dbg putlog tableuri=;
     call symputx('tableuri',tableuri);
   run;
 %end;
 
-data;run;%let tempds3=&syslast;
+data;run;
+%let tempds3=&syslast;
 %mm_getcols(tableuri=&tableuri,outds=&tempds3)
 
+%if %mf_nobs(&tempds3)=0 %then %do;
+  %put &libds (&tableuri) has no columns defined!!;
+  data &outds;
+  run;
+  %return;
+%end;
+
 data _null_;
   set &tempds3 end=last;
   if _n_=1 then call execute('data &outds;');

meta/mm_createfolder.sas

@@ -17,7 +17,7 @@
       %mm_createfolder(path=/some/meta/folder)
 
   @param [in] path= Name of the folder to create.
-  @param [in] mdebug= set DBG to 1 to disable DEBUG messages
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
 
 
   @version 9.4

meta/mm_createwebservice.sas

@@ -2,7 +2,8 @@
   @file mm_createwebservice.sas
   @brief Create a Web Ready Stored Process
   @details This macro creates a Type 2 Stored Process with the mm_webout macro
-    included as pre-code.
+  (and dependencies) included as pre-code.
+
 Usage:
 
     %* compile macros ;
@@ -92,14 +93,16 @@ data _null_;
   file sasjs lrecl=3000 ;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
-  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
-  put '  ,showmeta=NO ';
+  put '  ,showmeta=N ';
+  put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols; ';
+  put '%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval ';
+  put '  tmpds1 tmpds2 tmpds3 tmpds4; ';
   put '%let numcols=0; ';
+  put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
@@ -108,9 +111,23 @@ data _null_;
   put '  run; ';
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
+  put '  /* force variable names to always be uppercase in the JSON */ ';
   put '  options validvarname=upcase; ';
-  put '  data _null_; file &jref encoding=''utf-8'' mod; ';
+  put '  /* To avoid issues with _webout on EBI - such as encoding diffs and truncation ';
+  put '    (https://support.sas.com/kb/49/325.html) we use temporary files */ ';
+  put '  filename _sjs1 temp lrecl=200 ; ';
+  put '  data _null_; file _sjs1 encoding=''utf-8''; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
+  put '  run; ';
+  put '  /* now write to _webout 1 char at a time */ ';
+  put '  data _null_; ';
+  put '    infile _sjs1 lrecl=1 recfm=n; ';
+  put '    file &jref mod lrecl=1 recfm=n; ';
+  put '    input sourcechar $char1. @@; ';
+  put '    format sourcechar hex2.; ';
+  put '    put sourcechar char1. @@; ';
+  put '  run; ';
+  put '  filename _sjs1 clear; ';
   put ' ';
   put '  /* grab col defs */ ';
   put '  proc contents noprint data=&ds ';
@@ -121,7 +138,7 @@ data _null_;
   put '    by varnum; ';
   put '  run; ';
   put '  /* move meta to mac vars */ ';
-  put '  data _null_; ';
+  put '  data &colinfo; ';
   put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
   put '    set &colinfo end=last nobs=nobs; ';
   put '    name=upcase(name); ';
@@ -132,7 +149,6 @@ data _null_;
   put '      if format='''' then fmt=cats(''$'',length,''.''); ';
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else fmt=cats(format,formatl,''.''); ';
-  put '      newlen=max(formatl,length); ';
   put '    end; ';
   put '    else do; ';
   put '      typelong=''num''; ';
@@ -140,23 +156,26 @@ data _null_;
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else if formatd=0 then fmt=cats(format,formatl,''.''); ';
   put '      else fmt=cats(format,formatl,''.'',formatd); ';
-  put '      /* needs to be wide, for datetimes etc */ ';
-  put '      newlen=max(length,formatl,24); ';
   put '    end; ';
   put '    /* 32 char unique name */ ';
   put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
   put ' ';
   put '    call symputx(cats(''name'',_n_),name,''l''); ';
   put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
-  put '    call symputx(cats(''len'',_n_),newlen,''l''); ';
   put '    call symputx(cats(''length'',_n_),length,''l''); ';
   put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
+  put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
+  put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
+  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '  proc sql; ';
+  put '  select count(*) into: lastobs from &ds; ';
+  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -164,13 +183,25 @@ data _null_;
   put '      %put &sysmacroname: Switching to DATASTEP engine; ';
   put '      %goto datastep; ';
   put '    %end; ';
-  put '    data &tempds;set &ds; ';
+  put '    data &tempds; ';
+  put '      set &ds; ';
+  put '      &stmt_obs; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
   put '    /* PRETTY is necessary to avoid line truncation in large files */ ';
-  put '    proc json out=&jref pretty ';
+  put '    filename _sjs2 temp lrecl=131068 encoding=''utf-8''; ';
+  put '    proc json out=_sjs2 pretty ';
   put '        %if &action=ARR %then nokeys ; ';
   put '        ;export &tempds / nosastags fmtnumeric; ';
   put '    run; ';
+  put '    /* send back to webout */ ';
+  put '    data _null_; ';
+  put '      infile _sjs2 lrecl=1 recfm=n; ';
+  put '      file &jref mod lrecl=1 recfm=n; ';
+  put '      input sourcechar $char1. @@; ';
+  put '      format sourcechar hex2.; ';
+  put '      put sourcechar char1. @@; ';
+  put '    run; ';
+  put '    filename _sjs2 clear; ';
   put '  %end; ';
   put '  %else %if &engine=DATASTEP %then %do; ';
   put '    %datastep: ';
@@ -181,26 +212,99 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      data _data_; ';
+  put '      /** ';
+  put '        * Extract format definitions ';
+  put '        * First, by getting library locations from dictionary.formats ';
+  put '        * Then, by exporting the width using proc format ';
+  put '        * Cannot use maxw from sashelp.vformat as not always populated ';
+  put '        * Cannot use fmtinfo() as not supported in all flavours ';
+  put '        */ ';
+  put '      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      proc sql noprint; ';
+  put '      create table &tmpds1 as ';
+  put '          select cats(libname,''.'',memname) as fmtcat, ';
+  put '          fmtname ';
+  put '        from dictionary.formats ';
+  put '        where fmttype=''F'' and libname is not null ';
+  put '          and fmtname in (select format from &colinfo where format is not null) ';
+  put '        order by 1; ';
+  put '      create table &tmpds2( ';
+  put '          FMTNAME char(32), ';
+  put '          LENGTH num ';
+  put '      ); ';
+  put '      %local catlist cat fmtlist i; ';
+  put '      select distinct fmtcat into: catlist separated by '' '' from &tmpds1; ';
+  put '      %do i=1 %to %sysfunc(countw(&catlist,%str( ))); ';
+  put '        %let cat=%scan(&catlist,&i,%str( )); ';
+  put '        proc sql; ';
+  put '        select distinct fmtname into: fmtlist separated by '' '' ';
+  put '          from &tmpds1 where fmtcat="&cat"; ';
+  put '        proc format lib=&cat cntlout=&tmpds3(keep=fmtname length); ';
+  put '          select &fmtlist; ';
+  put '        run; ';
+  put '        proc sql; ';
+  put '        insert into &tmpds2 select distinct fmtname,length from &tmpds3; ';
+  put '      %end; ';
+  put ' ';
+  put '      proc sql; ';
+  put '      create table &tmpds4 as ';
+  put '        select a.*, b.length as maxw ';
+  put '        from &colinfo a ';
+  put '        left join &tmpds2 b ';
+  put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
+  put '        order by a.varnum; ';
+  put '      data _null_; ';
+  put '        set &tmpds4; ';
+  put '        if not missing(maxw); ';
+  put '        call symputx( ';
+  put '          cats(''fmtlen'',_n_), ';
+  put '          /* vars need extra padding due to JSON escaping of special chars */ ';
+  put '          min(32767,ceil((max(length,maxw)+3)*1.5)) ';
+  put '          ,''l'' ';
+  put '        ); ';
+  put '      run; ';
+  put ' ';
+  put '      /* configure varlenchk - as we are explicitly shortening the variables */ ';
+  put '      %let optval=%sysfunc(getoption(varlenchk)); ';
+  put '      options varlenchk=NOWARN; ';
+  put '      data _data_(compress=char); ';
+  put '        /* shorten the new vars */ ';
+  put '        length ';
+  put '      %do i=1 %to &numcols; ';
+  put '          &&name&i $&&fmtlen&i ';
+  put '      %end; ';
+  put '          ; ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
-  put '        &&name&i=&&newname&i ';
+  put '          &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
+  put '      &stmt_obs; ';
+  put ' ';
+  put '      drop ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&newname&i ';
+  put '      %end; ';
+  put '        ; ';
   put '      %do i=1 %to &numcols; ';
-  put '        length &&name&i $&&len&i; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
-  put '        drop &&newname&i; ';
   put '      %end; ';
-  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '        if _error_ then do; ';
+  put '          call symputx(''syscc'',1012); ';
+  put '          stop; ';
+  put '        end; ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
+  put '      options varlenchk=&optval; ';
   put '    %end; ';
   put ' ';
   put '    proc format; /* credit yabwon for special null removal */ ';
@@ -219,8 +323,8 @@ data _null_;
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $32767; ';
-  put '          format &&name&i $32767.; ';
+  put '          length &&name&i $&&fmtlen&i...; ';
+  put '          format &&name&i $&&fmtlen&i...; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -229,6 +333,7 @@ data _null_;
   put '      %else %do; ';
   put '        set &ds; ';
   put '      %end; ';
+  put '      &stmt_obs; ';
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
@@ -253,10 +358,9 @@ data _null_;
   put '    %end; ';
   put '    run; ';
   put ' ';
-  put '    /* write to temp loc to avoid _webout truncation ';
-  put '      - https://support.sas.com/kb/49/325.html */ ';
-  put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
-  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
+  put '    filename _sjs3 temp lrecl=131068 ; ';
+  put '    data _null_; ';
+  put '      file _sjs3 encoding=''utf-8''; ';
   put '      if _n_=1 then put "["; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
@@ -264,39 +368,38 @@ data _null_;
   put '      %do i=1 %to &numcols; ';
   put '        %if &i>1 %then  "," ; ';
   put '        %if &action=OBJ %then """&&name&i"":" ; ';
-  put '        &&name&i ';
+  put '        "&&name&i"n /* name literal for reserved variable names */ ';
   put '      %end; ';
   put '      %if &action=ARR %then "]" ; %else "}" ; ; ';
-  put '    /* now write the long strings to _webout 1 byte at a time */ ';
+  put ' ';
+  put '    /* close out the table */ ';
   put '    data _null_; ';
-  put '      length filein 8 fileid 8; ';
-  put '      filein=fopen("_sjs",''I'',1,''B''); ';
-  put '      fileid=fopen("&jref",''A'',1,''B''); ';
-  put '      rec=''20''x; ';
-  put '      do while(fread(filein)=0); ';
-  put '        rc=fget(filein,rec,1); ';
-  put '        rc=fput(fileid, rec); ';
-  put '        rc=fwrite(fileid); ';
-  put '      end; ';
-  put '      /* close out the table */ ';
-  put '      rc=fput(fileid, "]"); ';
-  put '      rc=fwrite(fileid); ';
-  put '      rc=fclose(filein); ';
-  put '      rc=fclose(fileid); ';
+  put '      file _sjs3 mod encoding=''utf-8''; ';
+  put '      put '']''; ';
   put '    run; ';
-  put '    filename _sjs clear; ';
+  put '    data _null_; ';
+  put '      infile _sjs3 lrecl=1 recfm=n; ';
+  put '      file &jref mod lrecl=1 recfm=n; ';
+  put '      input sourcechar $char1. @@; ';
+  put '      format sourcechar hex2.; ';
+  put '      put sourcechar char1. @@; ';
+  put '    run; ';
+  put '    filename _sjs3 clear; ';
   put '  %end; ';
   put ' ';
   put '  proc sql; ';
   put '  drop table &colinfo, &tempds; ';
   put ' ';
-  put '  %if &showmeta=YES %then %do; ';
-  put '    data _null_; file &jref encoding=''utf-8'' mod; ';
+  put '  %if %substr(&showmeta,1,1)=Y %then %do; ';
+  put '    filename _sjs4 temp lrecl=131068 encoding=''utf-8''; ';
+  put '    data _null_; ';
+  put '      file _sjs4; ';
+  put '      length label $350; ';
   put '      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; ';
   put '      do i=1 to &numcols; ';
   put '        name=quote(trim(symget(cats(''name'',i)))); ';
   put '        format=quote(trim(symget(cats(''fmt'',i)))); ';
-  put '        label=quote(trim(symget(cats(''label'',i)))); ';
+  put '        label=quote(prxchange(''s/\\/\\\\/'',-1,trim(symget(cats(''label'',i))))); ';
   put '        length=quote(trim(symget(cats(''length'',i)))); ';
   put '        type=quote(trim(symget(cats(''typelong'',i)))); ';
   put '        if i>1 then put "," @@; ';
@@ -305,6 +408,15 @@ data _null_;
   put '      end; ';
   put '      put ''}}''; ';
   put '    run; ';
+  put '    /* send back to webout */ ';
+  put '    data _null_; ';
+  put '      infile _sjs4 lrecl=1 recfm=n; ';
+  put '      file &jref mod lrecl=1 recfm=n; ';
+  put '      input sourcechar $char1. @@; ';
+  put '      format sourcechar hex2.; ';
+  put '      put sourcechar char1. @@; ';
+  put '    run; ';
+  put '    filename _sjs4 clear; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
@@ -315,26 +427,27 @@ data _null_;
   put '%end; ';
   put '%mend mp_jsonout; ';
   put ' ';
-  put '%macro mf_getuser(type=META ';
+  put '%macro mf_getuser( ';
   put ')/*/STORE SOURCE*/; ';
-  put '  %local user metavar; ';
-  put '  %if &type=OS %then %let metavar=_secureusername; ';
-  put '  %else %let metavar=_metaperson; ';
+  put '  %local user; ';
   put ' ';
-  put '  %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %let user=&SYS_COMPUTE_SESSION_OWNER; ';
-  put '  %else %if %symexist(&metavar) %then %do; ';
-  put '    %if %length(&&&metavar)=0 %then %let user=&sysuserid; ';
+  put '  %if %symexist(_sasjs_username) %then %let user=&_sasjs_username; ';
+  put '  %else %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %do; ';
+  put '    %let user=&SYS_COMPUTE_SESSION_OWNER; ';
+  put '  %end; ';
+  put '  %else %if %symexist(_metaperson) %then %do; ';
+  put '    %if %length(&_metaperson)=0 %then %let user=&sysuserid; ';
   put '    /* sometimes SAS will add @domain extension - remove for consistency */ ';
   put '    /* but be sure to quote in case of usernames with commas */ ';
-  put '    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@)); ';
+  put '    %else %let user=%unquote(%scan(%quote(&_metaperson),1,@)); ';
   put '  %end; ';
   put '  %else %let user=&sysuserid; ';
   put ' ';
   put '  %quote(&user) ';
   put ' ';
   put '%mend mf_getuser; ';
-  put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
-  put '  ,showmeta=NO ';
+  put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL ';
+  put '  ,showmeta=N,maxobs=MAX,workobs=0 ';
   put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
@@ -401,13 +514,14 @@ data _null_;
   put ' ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta ';
+  put '    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
   put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  %if %str(&_debug) ge 131 %then %do; ';
-  put '    /* if debug mode, send back first 10 records of each work table also */ ';
-  put '    options obs=10; ';
+  put '  /* To avoid issues with _webout on EBI we use a temporary file */ ';
+  put '  filename _sjsref temp lrecl=131068; ';
+  put '  %if %str(&workobs) > 0 %then %do; ';
+  put '    /* if debug mode, send back first XX records of each work table also */ ';
   put '    data;run;%let tempds=%scan(&syslast,2,.); ';
   put '    ods output Members=&tempds; ';
   put '    proc datasets library=WORK memtype=data; ';
@@ -418,11 +532,11 @@ data _null_;
   put '      i+1; ';
   put '      call symputx(cats(''wt'',i),name,''l''); ';
   put '      call symputx(''wtcnt'',i,''l''); ';
-  put '    data _null_; file &fref mod encoding=''utf-8''; ';
+  put '    data _null_; file _sjsref mod encoding=''utf-8''; ';
   put '      put ",""WORK"":{"; ';
   put '    %do i=1 %to &wtcnt; ';
   put '      %let wt=&&wt&i; ';
-  put '      data _null_; file &fref mod encoding=''utf-8''; ';
+  put '      data _null_; file _sjsref mod encoding=''utf-8''; ';
   put '        dsid=open("WORK.&wt",''is''); ';
   put '        nlobs=attrn(dsid,''NLOBS''); ';
   put '        nvars=attrn(dsid,''NVARS''); ';
@@ -431,35 +545,45 @@ data _null_;
   put '        put " ""&wt"" : {"; ';
   put '        put ''"nlobs":'' nlobs; ';
   put '        put '',"nvars":'' nvars; ';
-  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) ';
-  put '      data _null_; file &fref mod encoding=''utf-8''; ';
+  put '      %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=Y,maxobs=10 ';
+  put '        ,maxobs=&workobs ';
+  put '      ) ';
+  put '      data _null_; file _sjsref mod encoding=''utf-8''; ';
   put '        put "}"; ';
   put '    %end; ';
-  put '    data _null_; file &fref mod encoding=''utf-8''; ';
+  put '    data _null_; file _sjsref mod encoding=''utf-8''; ';
   put '      put "}"; ';
   put '    run; ';
   put '  %end; ';
   put '  /* close off json */ ';
-  put '  data _null_;file &fref mod encoding=''utf-8''; ';
-  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
-  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
-  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
+  put '  data _null_;file _sjsref mod encoding=''utf-8''; ';
+  put '    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; ';
   put '    put ",""_DEBUG"" : ""&_debug"" "; ';
   put '    _METAUSER=quote(trim(symget(''_METAUSER''))); ';
   put '    put ",""_METAUSER"": " _METAUSER; ';
   put '    _METAPERSON=quote(trim(symget(''_METAPERSON''))); ';
   put '    put '',"_METAPERSON": '' _METAPERSON; ';
+  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
+  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
+  put '    put '',"AUTOEXEC" : '' autoexec; ';
+  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put ",""SYSENCODING"" : ""&sysencoding"" "; ';
+  put '    syserrortext=cats(''"'',tranwrd(symget(''syserrortext''),''"'',''\"''),''"''); ';
   put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
+  put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
+  put '    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; ';
+  put '    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); ';
+  put '    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
+  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    syswarningtext=cats(''"'',tranwrd(symget(''syswarningtext''),''"'',''\"''),''"''); ';
   put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';
@@ -471,6 +595,16 @@ data _null_;
   put '    put ''>>weboutEND<<''; ';
   put '  %end; ';
   put '  run; ';
+  put '  /* now write to _webout 1 char at a time */ ';
+  put '  data _null_; ';
+  put '    infile _sjsref lrecl=1 recfm=n; ';
+  put '    file &fref mod lrecl=1 recfm=n; ';
+  put '    input sourcechar $char1. @@; ';
+  put '    format sourcechar hex2.; ';
+  put '    put sourcechar char1. @@; ';
+  put '  run; ';
+  put '  filename _sjsref clear; ';
+  put ' ';
   put '%end; ';
   put ' ';
   put '%mend mm_webout; ';
@@ -492,7 +626,7 @@ run;
   %if &x>1 %then %let mod=mod;
 
   %let fref=%scan(&freflist,&x);
-  %put &sysmacroname: adding &fref;
+  %&mD.put &sysmacroname: adding &fref;
   data _null_;
     file "&work/&tmpfile" lrecl=3000 &mod;
     infile &fref;
@@ -528,12 +662,10 @@ data _null_;
   if rc=0 then call symputx('url',url,'l');
 run;
 
-%put ;%put ;%put ;%put ;%put ;%put ;
 %put &sysmacroname: STP &name successfully created in &path;
-%put ;%put ;%put ;
 %put Check it out here:;
 %put ;%put ;%put ;
 %put &url?_PROGRAM=&path/&name;
-%put ;%put ;%put ;%put ;%put ;%put ;
+%put ;%put ;%put ;
 
 %mend mm_createwebservice;

meta/mm_getfoldertree.sas

@@ -8,17 +8,15 @@
 
   Usage:
 
-    options ps=max nonotes nosource;
-    %mm_getfoldertree(root=/My/Meta/Path, outds=iwantthisdataset)
-    options notes source;
+      options ps=max nonotes nosource;
+      %mm_getfoldertree(root=/My/Meta/Path, outds=iwantthisdataset)
+      options notes source;
 
   @param [in] root= the parent folder under which to return all contents
   @param [out] outds= the dataset to create that contains the list of
     directories
   @param [in] mDebug= set to 1 to show debug messages in the log
 
-  <h4> SAS Macros </h4>
-
   @version 9.4
   @author Allan Bowe
 

meta/mm_getstpcode.sas

@@ -21,6 +21,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
+  @li mp_abort.sas
 
   @author Allan Bowe
 
@@ -57,7 +58,7 @@ data _null_;
     cnt=1;
     do while (metadata_getnasn(uri,"Notes",cnt,tsuri)>0);
       rc=metadata_getattr(tsuri,"Name",value);
-      put tsuri= value=;
+      &mD.put tsuri= value=;
       if value="SourceCode" then do;
         /* found it! */
         rc=metadata_getattr(tsuri,"Id",value);
@@ -70,11 +71,10 @@ data _null_;
   else put (_all_)(=);
 run;
 
-%if &tsuri=stopifempty %then %do;
-  %put %str(WARN)ING:  &tree&name.(StoredProcess) not found!;
-  %return;
-%end;
-
+%mp_abort(iftrue= (&tsuri=stopifempty)
+  ,mac=mm_getstpcode
+  ,msg=%str(&tree&name.(StoredProcess) not found!)
+)
 
 /**
   * Now we can extract the textstore

meta/mm_getstpinfo.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Get the properties of a Stored Process
+  @details Extracts various properties and creates an output table in the
+  structure below:
+
+|STP_URI:$200.|SERVERCONTEXT:$200.|STOREDPROCESSCONFIGURATION:$1000.|SOURCECODE_FIRST32K:$32767.|PATH:$76.|
+|---|---|---|---|---|
+|`A5DN9TDQ.BH0000C8 `|`SASApp `|`<?xml version="1.0" encoding="UTF-8"?><StoredProcess><ServerContext LogicalServerType="Sps" OtherAllowed="false"/><ResultCapabilities Package="false" Streaming="true"/><OutputParameters/></StoredProcess> `|`%put first 32767 bytes of code; `|`/path/to/my/stp`|
+
+  @param [in] pgm The metadata path of the Stored Process
+  @param [out] outds= (work.mm_getstpinfo) The output table to create
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> Related Files </h4>
+  @li mm_getstpcode.sas
+  @li mm_getstps.sas
+  @li mm_createstp.sas
+  @li mm_deletestp.sas
+
+**/
+
+%macro mm_getstpinfo(pgm
+  ,outds=work.mm_getstpinfo
+  ,mDebug=0
+);
+
+%local mD;
+%if &mDebug=1 %then %let mD=;
+%else %let mD=%str(*);
+%&mD.put Executing &sysmacroname..sas;
+%&mD.put _local_;
+
+data &outds;
+  length type stp_uri tsuri servercontext value $200
+    StoredProcessConfiguration $1000 sourcecode_first32k $32767;
+  keep path stp_uri sourcecode_first32k StoredProcessConfiguration
+    servercontext;
+  call missing (of _all_);
+  path="&pgm(StoredProcess)";
+  /* first, find the STP ID */
+  if metadata_pathobj("",path,"StoredProcess",type,stp_uri)>0 then do;
+    /* get attributes */
+    cnt=1;
+    do while (metadata_getnasn(stp_uri,"Notes",cnt,tsuri)>0);
+      rc1=metadata_getattr(tsuri,"Name",value);
+      &mD.put tsuri= value=;
+      if value="SourceCode" then do;
+        rc2=metadata_getattr(tsuri,"StoredText",sourcecode_first32k);
+      end;
+      else if value="Stored Process" then do;
+        rc3=metadata_getattr(tsuri,"StoredText",StoredProcessConfiguration);
+      end;
+      cnt+1;
+    end;
+    /* get context (should only be one) */
+    rc4=metadata_getnasn(stp_uri,"ComputeLocations",1,tsuri);
+    rc5=metadata_getattr(tsuri,"Name",servercontext);
+  end;
+  else do;
+    put "%str(ERR)OR: could not find " pgm;
+    put (_all_)(=);
+  end;
+  &md.put (_all_)(=);
+run;
+
+%mend mm_getstpinfo ;

meta/mm_getusers.sas

@@ -8,17 +8,18 @@
 
       %mm_getusers()
 
+  Optionally, filter for a user (useful to get the uri):
+
+      %mm_getusers(user=&_metaperson)
+
   @param outds the dataset to create that contains the list of libraries
 
   @returns outds  dataset containing all users, with the following columns:
     - uri
     - name
 
-  @warning The following filenames are created and then de-assigned:
-
-      filename sxlemap clear;
-      filename response clear;
-      libname _XML_ clear;
+  @param user= (0) Set to a metadata user to filter on that user
+  @param outds= (work.mm_getusers) The output table to create
 
   @version 9.3
   @author Allan Bowe
@@ -26,23 +27,49 @@
 **/
 
 %macro mm_getusers(
-    outds=work.mm_getusers
+    outds=work.mm_getusers,
+    user=0
 )/*/STORE SOURCE*/;
 
 filename response temp;
-proc metadata in= '<GetMetadataObjects>
-  <Reposid>$METAREPOSITORY</Reposid>
-  <Type>Person</Type>
-  <NS>SAS</NS>
-  <Flags>0</Flags>
-  <Options>
-  <Templates>
-  <Person Name=""/>
-  </Templates>
-  </Options>
-  </GetMetadataObjects>'
-  out=response;
-run;
+%if %superq(user)=0 %then %do;
+  proc metadata in= '<GetMetadataObjects>
+    <Reposid>$METAREPOSITORY</Reposid>
+    <Type>Person</Type>
+    <NS>SAS</NS>
+    <Flags>0</Flags>
+    <Options>
+    <Templates>
+    <Person Name=""/>
+    </Templates>
+    </Options>
+    </GetMetadataObjects>'
+    out=response;
+  run;
+%end;
+%else %do;
+  filename inref temp;
+  data _null_;
+    file inref;
+    put "<GetMetadataObjects>";
+    put "<Reposid>$METAREPOSITORY</Reposid>";
+    put "<Type>Person</Type>";
+    put "<NS>SAS</NS>";
+    put "<!-- Specify the OMI_XMLSELECT (128) flag  -->";
+    put "<Flags>128</Flags>";
+    put "<Options>";
+    put "<Templates>";
+    put '<Person Name=""/>';
+    put "</Templates>";
+    length string $10000;
+    string=cats('<XMLSELECT search="Person[@Name=',"'&user'",']"/>');
+    put string;
+    put "</Options>";
+    put "</GetMetadataObjects>";
+  run;
+  proc metadata in=inref out=response;
+  run;
+%end;
 
 filename sxlemap temp;
 data _null_;

meta/mm_updatestpsourcecode.sas

@@ -43,7 +43,9 @@ data _null_;
     cnt=1;
     do while (metadata_getnasn(uri,"Notes",cnt,tsuri)>0);
       rc=metadata_getattr(tsuri,"Name",value);
+    %if &mdebug=1 %then %do;
       put tsuri= value=;
+    %end;
       if value="SourceCode" then do;
         /* found it! */
         rc=metadata_getattr(tsuri,"Id",value);
@@ -126,4 +128,4 @@ run;
   filename &frefout clear;
 %end;
 
-%mend mm_updatestpsourcecode;
+%mend mm_updatestpsourcecode;

meta/mm_webout.sas

@@ -4,43 +4,55 @@
   @details This macro should be added to the start of each Stored Process,
   **immediately** followed by a call to:
 
-        %mm_webout(FETCH)
+      %mm_webout(FETCH)
 
-    This will read all the input data and create same-named SAS datasets in the
-    WORK library.  You can then insert your code, and send data back using the
-    following syntax:
+  This will read all the input data and create same-named SAS datasets in the
+  WORK library.  You can then insert your code, and send data back using the
+  following syntax:
 
-        data some datasets; * make some data ;
-        retain some columns;
-        run;
+      data some datasets; * make some data ;
+      retain some columns;
+      run;
 
-        %mm_webout(OPEN)
-        %mm_webout(ARR,some)  * Array format, fast, suitable for large tables ;
-        %mm_webout(OBJ,datasets) * Object format, easier to work with ;
+      %mm_webout(OPEN)
+      %mm_webout(ARR,some)  * Array format, fast, suitable for large tables ;
+      %mm_webout(OBJ,datasets) * Object format, easier to work with ;
 
-    Finally, wrap everything up send some helpful system variables too
+  Finally, wrap everything up send some helpful system variables too
 
-        %mm_webout(CLOSE)
+      %mm_webout(CLOSE)
 
 
   @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE
   @param [in] ds The dataset to send back to the frontend
   @param [out] dslabel= Value to use instead of table name for sending to JSON
-  @param [in] fmt=(Y) Set to N to send back unformatted values
+  @param [in] fmt= (N) Setting Y converts all vars to their formatted values
   @param [out] fref= (_webout) The fileref to which to write the JSON
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
-  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+  @param [in] showmeta= (N) Set to Y to output metadata alongside each table,
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
+  @param [in] workobs= (0) When set to a positive integer, will create a new
+    output object (WORK) which contains this number of observations from all
+    tables in the WORK library.
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to output JSON
+
+  <h4> SAS Macros </h4>
+  @li mp_jsonout.sas
+
+  <h4> Related Macros </h4>
+  @li ms_webout.sas
+  @li mv_webout.sas
 
   @version 9.3
   @author Allan Bowe
 
 **/
-%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
-  ,showmeta=NO
+%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL
+  ,showmeta=N,maxobs=MAX,workobs=0
 );
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
@@ -107,13 +119,14 @@
 
 %else %if &action=ARR or &action=OBJ %then %do;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta
+    ,engine=&jsonengine,missing=&missing,showmeta=&showmeta,maxobs=&maxobs
   )
 %end;
 %else %if &action=CLOSE %then %do;
-  %if %str(&_debug) ge 131 %then %do;
-    /* if debug mode, send back first 10 records of each work table also */
-    options obs=10;
+  /* To avoid issues with _webout on EBI we use a temporary file */
+  filename _sjsref temp lrecl=131068;
+  %if %str(&workobs) > 0 %then %do;
+    /* if debug mode, send back first XX records of each work table also */
     data;run;%let tempds=%scan(&syslast,2,.);
     ods output Members=&tempds;
     proc datasets library=WORK memtype=data;
@@ -124,11 +137,11 @@
       i+1;
       call symputx(cats('wt',i),name,'l');
       call symputx('wtcnt',i,'l');
-    data _null_; file &fref mod encoding='utf-8';
+    data _null_; file _sjsref mod encoding='utf-8';
       put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
-      data _null_; file &fref mod encoding='utf-8';
+      data _null_; file _sjsref mod encoding='utf-8';
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
         nvars=attrn(dsid,'NVARS');
@@ -137,35 +150,45 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES)
-      data _null_; file &fref mod encoding='utf-8';
+      %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=Y,maxobs=10
+        ,maxobs=&workobs
+      )
+      data _null_; file _sjsref mod encoding='utf-8';
         put "}";
     %end;
-    data _null_; file &fref mod encoding='utf-8';
+    data _null_; file _sjsref mod encoding='utf-8';
       put "}";
     run;
   %end;
   /* close off json */
-  data _null_;file &fref mod encoding='utf-8';
-    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
-    put ",""SYSUSERID"" : ""&sysuserid"" ";
-    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
+  data _null_;file _sjsref mod encoding='utf-8';
+    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512;
     put ",""_DEBUG"" : ""&_debug"" ";
     _METAUSER=quote(trim(symget('_METAUSER')));
     put ",""_METAUSER"": " _METAUSER;
     _METAPERSON=quote(trim(symget('_METAPERSON')));
     put ',"_METAPERSON": ' _METAPERSON;
+    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ',"_PROGRAM" : ' _PROGRAM ;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
+    put ',"AUTOEXEC" : ' autoexec;
+    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""SYSCC"" : ""&syscc"" ";
-    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ",""SYSENCODING"" : ""&sysencoding"" ";
+    syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
     put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
+    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
+    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
+    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');
     put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;
@@ -177,6 +200,16 @@
     put '>>weboutEND<<';
   %end;
   run;
+  /* now write to _webout 1 char at a time */
+  data _null_;
+    infile _sjsref lrecl=1 recfm=n;
+    file &fref mod lrecl=1 recfm=n;
+    input sourcechar $char1. @@;
+    format sourcechar hex2.;
+    put sourcechar char1. @@;
+  run;
+  filename _sjsref clear;
+
 %end;
 
 %mend mm_webout;

metax/mmx_createmetafolder.sas

@@ -0,0 +1,50 @@
+/**
+  @file
+  @brief Creates a metadata folder
+  @details Creates a metadata folder using the batch tools
+
+  Usage:
+
+      %mmx_createmetafolder(loc=/some/meta/folder,user=sasdemo,pass=mars345)
+
+  <h4> SAS Macros </h4>
+  @li mf_loc.sas
+  @li mp_abort.sas
+
+  @param loc= the metadata folder to delete
+  @param user= username
+  @param pass= password
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mmx_createmetafolder(loc=,user=,pass=);
+
+%local host port path connx_string msg;
+%let host=%sysfunc(getoption(metaserver));
+%let port=%sysfunc(getoption(metaport));
+%let path=%mf_loc(POF)/tools;
+
+%let connx_string= -host &host -port &port -user '&user' -password '&pass';
+/* remove directory */
+data _null_;
+  infile " &path/sas-make-folder &connx_string ""&loc"" -makeFullPath 2>&1"
+    pipe lrecl=10000;
+  input;
+  putlog _infile_;
+run;
+
+data _null_; /* check tree exists */
+  length type uri $256;
+  rc=metadata_pathobj("","&loc","Folder",type,uri);
+  call symputx('foldertype',type,'l');
+run;
+%let msg=Location (&loc) was not created!!;
+%mp_abort(iftrue= (&foldertype ne Tree)
+  ,mac=&_program..sas
+  ,msg=%superq(msg)
+)
+
+%mend mmx_createmetafolder;

metax/mmx_deletemetafolder.sas

@@ -7,7 +7,7 @@
 
   Usage:
 
-    %mmx_deletemetafolder(loc=/some/meta/folder,user=sasdemo,pass=mars345)
+      %mmx_deletemetafolder(loc=/some/meta/folder,user=sasdemo,pass=mars345)
 
   <h4> SAS Macros </h4>
   @li mf_loc.sas
@@ -37,4 +37,4 @@ data _null_;
   putlog _infile_;
 run;
 
-%mend mmx_deletemetafolder;
+%mend mmx_deletemetafolder;

sasjs/doxy/new_header.html

@@ -25,6 +25,21 @@
   <link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
   <link rel="shortcut icon" href="$relpath^favicon.ico" type="image/x-icon" />
   $extrastylesheet
+  <!-- Matomo -->
+  <script>
+    var _paq = window._paq = window._paq || [];
+    /* tracker methods like "setCustomDimension" should be called before "trackPageView" */
+    _paq.push(['trackPageView']);
+    _paq.push(['enableLinkTracking']);
+    (function () {
+      var u = "https://analytics.4gl.io/";
+      _paq.push(['setTrackerUrl', u + 'matomo.php']);
+      _paq.push(['setSiteId', '6']);
+      var d = document, g = d.createElement('script'), s = d.getElementsByTagName('script')[0];
+      g.async = true; g.src = u + 'matomo.js'; s.parentNode.insertBefore(g, s);
+    })();
+  </script>
+  <!-- End Matomo Code -->
 </head>
 
 <body>
@@ -68,4 +83,4 @@
   </div>
 </body>
 
-</html>
+</html>

sasjs/sasjsconfig.json

@@ -6,8 +6,10 @@
     "fcmp",
     "lua",
     "server",
-    "tests/crossplatform",
-    "tests/ddl"
+    "xplatform",
+    "tests/base",
+    "tests/ddlonly",
+    "tests/x-platform"
   ],
   "docConfig": {
     "displayMacroCore": false,
@@ -43,7 +45,7 @@
     },
     {
       "name": "sas9",
-      "serverUrl": "https://sas.analytium.co.uk:8343",
+      "serverUrl": "",
       "serverType": "SAS9",
       "httpsAgentOptions": {
         "allowInsecureRequests": false
@@ -65,7 +67,7 @@
     },
     {
       "name": "server",
-      "serverUrl": "https://sas.analytium.co.uk:5007",
+      "serverUrl": "https://sas.4gl.io",
       "serverType": "SASJS",
       "httpsAgentOptions": {
         "allowInsecureRequests": false
@@ -78,7 +80,7 @@
     },
     {
       "name": "docsonly",
-      "serverType": "SAS9",
+      "serverType": "SASJS",
       "appLoc": "dummy",
       "macroFolders": [
         "meta",

server/ms_adduser2group.sas

@@ -0,0 +1,122 @@
+/**
+  @file
+  @brief Adds a user to a group on SASjs Server
+  @details Adds a user to a group based on userid and groupid.  Both user and
+  group must already exist.
+
+  Examples:
+
+      %ms_adduser2group(uid=1,gid=1)
+
+
+  @param [in] uid= (0) The User ID to be added
+  @param [in] gid= (0) The Group ID to contain the new user
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outds= (work.ms_adduser2group) This output dataset will contain
+    the new list of group members, eg:
+|DISPLAYNAME:$18.|USERNAME:$10.|ID:best.|
+|---|---|---|
+|`Super Admin `|`secretuser `|`1`|
+|`Sabir Hassan`|`sabir`|`2`|
+|`Mihajlo Medjedovic `|`mihajlo `|`3`|
+|`Ivor Townsend `|`ivor `|`4`|
+|`New User `|`newuser `|`5`|
+
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mp_abort.sas
+
+  <h4> Related Files </h4>
+  @li ms_creategroup.sas
+  @li ms_createuser.sas
+
+**/
+
+%macro ms_adduser2group(uid=0
+    ,gid=0
+    ,outds=work.ms_adduser2group
+    ,mdebug=0
+  );
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_adduser2group.sas
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+
+%local fref0 fref1 fref2 libref optval rc msg;
+%let fref0=%mf_getuniquefileref();
+%let fref1=%mf_getuniquefileref();
+%let libref=%mf_getuniquelibref();
+
+/* avoid sending bom marker to API */
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
+data _null_;
+  file &fref0 lrecl=1000;
+  infile "&_sasjs_tokenfile" lrecl=1000;
+  input;
+  if _n_=1 then put "accept: application/json";
+  put _infile_;
+run;
+
+%if &mdebug=1 %then %do;
+  %put _local_;
+  data _null_;
+    infile &fref0;
+    input;
+    put _infile_;
+  run;
+%end;
+
+proc http method='POST' headerin=&fref0 out=&fref1
+  url="&_sasjs_apiserverurl/SASjsApi/group/&gid/&uid";
+%if &mdebug=1 %then %do;
+  debug level=1;
+%end;
+run;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_adduser2group.sas
+  ,msg=%str(Issue submitting query to SASjsApi/group)
+)
+
+libname &libref JSON fileref=&fref1;
+
+data &outds;
+  set &libref..users;
+  drop ordinal_root ordinal_users;
+%if &mdebug=1 %then %do;
+  putlog _all_;
+%end;
+run;
+
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_creategroup.sas
+  ,msg=%str(Issue reading response JSON)
+)
+
+/* reset options */
+options &optval;
+
+%if &mdebug=0 %then %do;
+  filename &fref0 clear;
+  filename &fref1 clear;
+  libname &libref clear;
+%end;
+%else %do;
+  data _null_;
+    infile &fref1;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend ms_adduser2group;

server/ms_createfile.sas

@@ -68,8 +68,8 @@ data _null_;
   file &fname1 lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put "Content-Type: multipart/form-data; boundary=&boundary";
-  put "Authorization: Bearer " _infile_;
+  if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary";
+  put _infile_;
 run;
 
 %if &mdebug=1 %then %do;

server/ms_creategroup.sas

@@ -0,0 +1,149 @@
+/**
+  @file
+  @brief Creates a group on SASjs Server
+  @details Creates a group on SASjs Server with the following attributes:
+
+  @li name
+  @li description
+  @li isActive
+
+  Examples:
+
+      %ms_creategroup(mynewgroup)
+
+      %ms_creategroup(mynewergroup, desc=The group description)
+
+  @param [in] groupname The group name to create.  No spaces or special chars.
+  @param [in] desc= (0) If no description provided, group name will be used.
+  @param [in] isactive= (true) Set to false to create an inactive group.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outds= (work.ms_creategroup) This output dataset will contain the
+    values from the JSON response (such as the id of the new group)
+|DESCRIPTION:$1.|GROUPID:best.|ISACTIVE:best.|NAME:$11.|
+|---|---|---|---|
+|`The group description`|`2 `|`1 `|`mynewergroup `|
+
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mp_abort.sas
+
+  <h4> Related Files </h4>
+  @li ms_creategroup.test.sas
+  @li ms_getgroups.sas
+
+**/
+
+%macro ms_creategroup(groupname
+    ,desc=0
+    ,isactive=true
+    ,outds=work.ms_creategroup
+    ,mdebug=0
+  );
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_creategroup.sas
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+
+%local fref0 fref1 fref2 libref optval rc msg;
+%let fref0=%mf_getuniquefileref();
+%let fref1=%mf_getuniquefileref();
+%let fref2=%mf_getuniquefileref();
+%let libref=%mf_getuniquelibref();
+
+/* avoid sending bom marker to API */
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
+data _null_;
+  file &fref0 termstr=crlf;
+  name=quote(cats(symget('groupname')));
+  description=quote(cats(symget('desc')));
+  if cats(description)='"0"' then description=name;
+  isactive=symget('isactive');
+%if &mdebug=1 %then %do;
+  putlog _all_;
+%end;
+
+  put '{'@;
+  put '"name":' name @;
+  put ',"description":' description @;
+  put ',"isActive":' isactive @;
+  put '}';
+run;
+
+data _null_;
+  file &fref1 lrecl=1000;
+  infile "&_sasjs_tokenfile" lrecl=1000;
+  input;
+  if _n_=1 then do;
+    put "Content-Type: application/json";
+    put "accept: application/json";
+  end;
+  put _infile_;
+run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fref0;
+    input;
+    put _infile_;
+  data _null_;
+    infile &fref1;
+    input;
+    put _infile_;
+  run;
+%end;
+
+proc http method='POST' in=&fref0 headerin=&fref1 out=&fref2
+  url="&_sasjs_apiserverurl/SASjsApi/group";
+%if &mdebug=1 %then %do;
+  debug level=1;
+%end;
+run;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_creategroup.sas
+  ,msg=%str(Issue submitting query to SASjsApi/group)
+)
+
+libname &libref JSON fileref=&fref2;
+
+data &outds;
+  set &libref..root;
+  drop ordinal_root;
+%if &mdebug=1 %then %do;
+  putlog _all_;
+%end;
+run;
+
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_creategroup.sas
+  ,msg=%str(Issue reading response JSON)
+)
+
+/* reset options */
+options &optval;
+
+%if &mdebug=0 %then %do;
+  filename &fref0 clear;
+  filename &fref1 clear;
+  filename &fref2 clear;
+  libname &libref clear;
+%end;
+%else %do;
+  data _null_;
+    infile &fref2;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend ms_creategroup;

server/ms_createuser.sas

@@ -87,9 +87,11 @@ data _null_;
   file &fref1 lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put "Authorization: Bearer " _infile_;
-  put "Content-Type: application/json";
-  put "accept: application/json";
+  if _n_=1 then do;
+    put "Content-Type: application/json";
+    put "accept: application/json";
+  end;
+  put _infile_;
 run;
 
 %if &mdebug=1 %then %do;

server/ms_createwebservice.sas

@@ -94,14 +94,16 @@ data _null_;
   file &sasjsref termstr=crlf lrecl=512;
   put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */";
 /* WEBOUT BEGIN */
-  put ' ';
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
-  put '  ,showmeta=NO ';
+  put '  ,showmeta=N ';
+  put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols; ';
+  put '%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval ';
+  put '  tmpds1 tmpds2 tmpds3 tmpds4; ';
   put '%let numcols=0; ';
+  put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
@@ -110,9 +112,23 @@ data _null_;
   put '  run; ';
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
+  put '  /* force variable names to always be uppercase in the JSON */ ';
   put '  options validvarname=upcase; ';
-  put '  data _null_; file &jref encoding=''utf-8'' mod; ';
+  put '  /* To avoid issues with _webout on EBI - such as encoding diffs and truncation ';
+  put '    (https://support.sas.com/kb/49/325.html) we use temporary files */ ';
+  put '  filename _sjs1 temp lrecl=200 ; ';
+  put '  data _null_; file _sjs1 encoding=''utf-8''; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
+  put '  run; ';
+  put '  /* now write to _webout 1 char at a time */ ';
+  put '  data _null_; ';
+  put '    infile _sjs1 lrecl=1 recfm=n; ';
+  put '    file &jref mod lrecl=1 recfm=n; ';
+  put '    input sourcechar $char1. @@; ';
+  put '    format sourcechar hex2.; ';
+  put '    put sourcechar char1. @@; ';
+  put '  run; ';
+  put '  filename _sjs1 clear; ';
   put ' ';
   put '  /* grab col defs */ ';
   put '  proc contents noprint data=&ds ';
@@ -123,7 +139,7 @@ data _null_;
   put '    by varnum; ';
   put '  run; ';
   put '  /* move meta to mac vars */ ';
-  put '  data _null_; ';
+  put '  data &colinfo; ';
   put '    if _n_=1 then call symputx(''numcols'',nobs,''l''); ';
   put '    set &colinfo end=last nobs=nobs; ';
   put '    name=upcase(name); ';
@@ -134,7 +150,6 @@ data _null_;
   put '      if format='''' then fmt=cats(''$'',length,''.''); ';
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else fmt=cats(format,formatl,''.''); ';
-  put '      newlen=max(formatl,length); ';
   put '    end; ';
   put '    else do; ';
   put '      typelong=''num''; ';
@@ -142,23 +157,26 @@ data _null_;
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else if formatd=0 then fmt=cats(format,formatl,''.''); ';
   put '      else fmt=cats(format,formatl,''.'',formatd); ';
-  put '      /* needs to be wide, for datetimes etc */ ';
-  put '      newlen=max(length,formatl,24); ';
   put '    end; ';
   put '    /* 32 char unique name */ ';
   put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
   put ' ';
   put '    call symputx(cats(''name'',_n_),name,''l''); ';
   put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
-  put '    call symputx(cats(''len'',_n_),newlen,''l''); ';
   put '    call symputx(cats(''length'',_n_),length,''l''); ';
   put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
   put '    call symputx(cats(''typelong'',_n_),typelong,''l''); ';
   put '    call symputx(cats(''label'',_n_),coalescec(label,name),''l''); ';
+  put '    /* overwritten when fmt=Y and a custom format exists in catalog */ ';
+  put '    if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); ';
+  put '    else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); ';
   put '  run; ';
   put ' ';
   put '  %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '  proc sql; ';
+  put '  select count(*) into: lastobs from &ds; ';
+  put '  %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
   put '    %if &missing=STRING %then %do; ';
@@ -166,13 +184,25 @@ data _null_;
   put '      %put &sysmacroname: Switching to DATASTEP engine; ';
   put '      %goto datastep; ';
   put '    %end; ';
-  put '    data &tempds;set &ds; ';
+  put '    data &tempds; ';
+  put '      set &ds; ';
+  put '      &stmt_obs; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
   put '    /* PRETTY is necessary to avoid line truncation in large files */ ';
-  put '    proc json out=&jref pretty ';
+  put '    filename _sjs2 temp lrecl=131068 encoding=''utf-8''; ';
+  put '    proc json out=_sjs2 pretty ';
   put '        %if &action=ARR %then nokeys ; ';
   put '        ;export &tempds / nosastags fmtnumeric; ';
   put '    run; ';
+  put '    /* send back to webout */ ';
+  put '    data _null_; ';
+  put '      infile _sjs2 lrecl=1 recfm=n; ';
+  put '      file &jref mod lrecl=1 recfm=n; ';
+  put '      input sourcechar $char1. @@; ';
+  put '      format sourcechar hex2.; ';
+  put '      put sourcechar char1. @@; ';
+  put '    run; ';
+  put '    filename _sjs2 clear; ';
   put '  %end; ';
   put '  %else %if &engine=DATASTEP %then %do; ';
   put '    %datastep: ';
@@ -183,26 +213,99 @@ data _null_;
   put '    %end; ';
   put ' ';
   put '    %if &fmt=Y %then %do; ';
-  put '      data _data_; ';
+  put '      /** ';
+  put '        * Extract format definitions ';
+  put '        * First, by getting library locations from dictionary.formats ';
+  put '        * Then, by exporting the width using proc format ';
+  put '        * Cannot use maxw from sashelp.vformat as not always populated ';
+  put '        * Cannot use fmtinfo() as not supported in all flavours ';
+  put '        */ ';
+  put '      %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); ';
+  put '      proc sql noprint; ';
+  put '      create table &tmpds1 as ';
+  put '          select cats(libname,''.'',memname) as fmtcat, ';
+  put '          fmtname ';
+  put '        from dictionary.formats ';
+  put '        where fmttype=''F'' and libname is not null ';
+  put '          and fmtname in (select format from &colinfo where format is not null) ';
+  put '        order by 1; ';
+  put '      create table &tmpds2( ';
+  put '          FMTNAME char(32), ';
+  put '          LENGTH num ';
+  put '      ); ';
+  put '      %local catlist cat fmtlist i; ';
+  put '      select distinct fmtcat into: catlist separated by '' '' from &tmpds1; ';
+  put '      %do i=1 %to %sysfunc(countw(&catlist,%str( ))); ';
+  put '        %let cat=%scan(&catlist,&i,%str( )); ';
+  put '        proc sql; ';
+  put '        select distinct fmtname into: fmtlist separated by '' '' ';
+  put '          from &tmpds1 where fmtcat="&cat"; ';
+  put '        proc format lib=&cat cntlout=&tmpds3(keep=fmtname length); ';
+  put '          select &fmtlist; ';
+  put '        run; ';
+  put '        proc sql; ';
+  put '        insert into &tmpds2 select distinct fmtname,length from &tmpds3; ';
+  put '      %end; ';
+  put ' ';
+  put '      proc sql; ';
+  put '      create table &tmpds4 as ';
+  put '        select a.*, b.length as maxw ';
+  put '        from &colinfo a ';
+  put '        left join &tmpds2 b ';
+  put '        on cats(a.format)=cats(upcase(b.fmtname)) ';
+  put '        order by a.varnum; ';
+  put '      data _null_; ';
+  put '        set &tmpds4; ';
+  put '        if not missing(maxw); ';
+  put '        call symputx( ';
+  put '          cats(''fmtlen'',_n_), ';
+  put '          /* vars need extra padding due to JSON escaping of special chars */ ';
+  put '          min(32767,ceil((max(length,maxw)+3)*1.5)) ';
+  put '          ,''l'' ';
+  put '        ); ';
+  put '      run; ';
+  put ' ';
+  put '      /* configure varlenchk - as we are explicitly shortening the variables */ ';
+  put '      %let optval=%sysfunc(getoption(varlenchk)); ';
+  put '      options varlenchk=NOWARN; ';
+  put '      data _data_(compress=char); ';
+  put '        /* shorten the new vars */ ';
+  put '        length ';
+  put '      %do i=1 %to &numcols; ';
+  put '          &&name&i $&&fmtlen&i ';
+  put '      %end; ';
+  put '          ; ';
   put '        /* rename on entry */ ';
   put '        set &ds(rename=( ';
   put '      %do i=1 %to &numcols; ';
-  put '        &&name&i=&&newname&i ';
+  put '          &&name&i=&&newname&i ';
   put '      %end; ';
   put '        )); ';
+  put '      &stmt_obs; ';
+  put ' ';
+  put '      drop ';
+  put '      %do i=1 %to &numcols; ';
+  put '        &&newname&i ';
+  put '      %end; ';
+  put '        ; ';
   put '      %do i=1 %to &numcols; ';
-  put '        length &&name&i $&&len&i; ';
   put '        %if &&typelong&i=num %then %do; ';
-  put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
+  put '          &&name&i=cats(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
   put '        %else %do; ';
   put '          &&name&i=put(&&newname&i,&&fmt&i); ';
   put '        %end; ';
-  put '        drop &&newname&i; ';
   put '      %end; ';
-  put '        if _error_ then call symputx(''syscc'',1012); ';
+  put '        if _error_ then do; ';
+  put '          call symputx(''syscc'',1012); ';
+  put '          stop; ';
+  put '        end; ';
   put '      run; ';
   put '      %let fmtds=&syslast; ';
+  put '      options varlenchk=&optval; ';
   put '    %end; ';
   put ' ';
   put '    proc format; /* credit yabwon for special null removal */ ';
@@ -221,8 +324,8 @@ data _null_;
   put '      attrib _all_ label=''''; ';
   put '      %do i=1 %to &numcols; ';
   put '        %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '          length &&name&i $32767; ';
-  put '          format &&name&i $32767.; ';
+  put '          length &&name&i $&&fmtlen&i...; ';
+  put '          format &&name&i $&&fmtlen&i...; ';
   put '        %end; ';
   put '      %end; ';
   put '      %if &fmt=Y %then %do; ';
@@ -231,6 +334,7 @@ data _null_;
   put '      %else %do; ';
   put '        set &ds; ';
   put '      %end; ';
+  put '      &stmt_obs; ';
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
@@ -255,10 +359,9 @@ data _null_;
   put '    %end; ';
   put '    run; ';
   put ' ';
-  put '    /* write to temp loc to avoid _webout truncation ';
-  put '      - https://support.sas.com/kb/49/325.html */ ';
-  put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
-  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
+  put '    filename _sjs3 temp lrecl=131068 ; ';
+  put '    data _null_; ';
+  put '      file _sjs3 encoding=''utf-8''; ';
   put '      if _n_=1 then put "["; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
@@ -266,39 +369,38 @@ data _null_;
   put '      %do i=1 %to &numcols; ';
   put '        %if &i>1 %then  "," ; ';
   put '        %if &action=OBJ %then """&&name&i"":" ; ';
-  put '        &&name&i ';
+  put '        "&&name&i"n /* name literal for reserved variable names */ ';
   put '      %end; ';
   put '      %if &action=ARR %then "]" ; %else "}" ; ; ';
-  put '    /* now write the long strings to _webout 1 byte at a time */ ';
+  put ' ';
+  put '    /* close out the table */ ';
   put '    data _null_; ';
-  put '      length filein 8 fileid 8; ';
-  put '      filein=fopen("_sjs",''I'',1,''B''); ';
-  put '      fileid=fopen("&jref",''A'',1,''B''); ';
-  put '      rec=''20''x; ';
-  put '      do while(fread(filein)=0); ';
-  put '        rc=fget(filein,rec,1); ';
-  put '        rc=fput(fileid, rec); ';
-  put '        rc=fwrite(fileid); ';
-  put '      end; ';
-  put '      /* close out the table */ ';
-  put '      rc=fput(fileid, "]"); ';
-  put '      rc=fwrite(fileid); ';
-  put '      rc=fclose(filein); ';
-  put '      rc=fclose(fileid); ';
+  put '      file _sjs3 mod encoding=''utf-8''; ';
+  put '      put '']''; ';
   put '    run; ';
-  put '    filename _sjs clear; ';
+  put '    data _null_; ';
+  put '      infile _sjs3 lrecl=1 recfm=n; ';
+  put '      file &jref mod lrecl=1 recfm=n; ';
+  put '      input sourcechar $char1. @@; ';
+  put '      format sourcechar hex2.; ';
+  put '      put sourcechar char1. @@; ';
+  put '    run; ';
+  put '    filename _sjs3 clear; ';
   put '  %end; ';
   put ' ';
   put '  proc sql; ';
   put '  drop table &colinfo, &tempds; ';
   put ' ';
-  put '  %if &showmeta=YES %then %do; ';
-  put '    data _null_; file &jref encoding=''utf-8'' mod; ';
+  put '  %if %substr(&showmeta,1,1)=Y %then %do; ';
+  put '    filename _sjs4 temp lrecl=131068 encoding=''utf-8''; ';
+  put '    data _null_; ';
+  put '      file _sjs4; ';
+  put '      length label $350; ';
   put '      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; ';
   put '      do i=1 to &numcols; ';
   put '        name=quote(trim(symget(cats(''name'',i)))); ';
   put '        format=quote(trim(symget(cats(''fmt'',i)))); ';
-  put '        label=quote(trim(symget(cats(''label'',i)))); ';
+  put '        label=quote(prxchange(''s/\\/\\\\/'',-1,trim(symget(cats(''label'',i))))); ';
   put '        length=quote(trim(symget(cats(''length'',i)))); ';
   put '        type=quote(trim(symget(cats(''typelong'',i)))); ';
   put '        if i>1 then put "," @@; ';
@@ -307,6 +409,15 @@ data _null_;
   put '      end; ';
   put '      put ''}}''; ';
   put '    run; ';
+  put '    /* send back to webout */ ';
+  put '    data _null_; ';
+  put '      infile _sjs4 lrecl=1 recfm=n; ';
+  put '      file &jref mod lrecl=1 recfm=n; ';
+  put '      input sourcechar $char1. @@; ';
+  put '      format sourcechar hex2.; ';
+  put '      put sourcechar char1. @@; ';
+  put '    run; ';
+  put '    filename _sjs4 clear; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
@@ -317,18 +428,19 @@ data _null_;
   put '%end; ';
   put '%mend mp_jsonout; ';
   put ' ';
-  put '%macro mf_getuser(type=META ';
+  put '%macro mf_getuser( ';
   put ')/*/STORE SOURCE*/; ';
-  put '  %local user metavar; ';
-  put '  %if &type=OS %then %let metavar=_secureusername; ';
-  put '  %else %let metavar=_metaperson; ';
+  put '  %local user; ';
   put ' ';
-  put '  %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %let user=&SYS_COMPUTE_SESSION_OWNER; ';
-  put '  %else %if %symexist(&metavar) %then %do; ';
-  put '    %if %length(&&&metavar)=0 %then %let user=&sysuserid; ';
+  put '  %if %symexist(_sasjs_username) %then %let user=&_sasjs_username; ';
+  put '  %else %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %do; ';
+  put '    %let user=&SYS_COMPUTE_SESSION_OWNER; ';
+  put '  %end; ';
+  put '  %else %if %symexist(_metaperson) %then %do; ';
+  put '    %if %length(&_metaperson)=0 %then %let user=&sysuserid; ';
   put '    /* sometimes SAS will add @domain extension - remove for consistency */ ';
   put '    /* but be sure to quote in case of usernames with commas */ ';
-  put '    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@)); ';
+  put '    %else %let user=%unquote(%scan(%quote(&_metaperson),1,@)); ';
   put '  %end; ';
   put '  %else %let user=&sysuserid; ';
   put ' ';
@@ -336,8 +448,8 @@ data _null_;
   put ' ';
   put '%mend mf_getuser; ';
   put ' ';
-  put '%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
-  put '  ,showmeta=NO ';
+  put '%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL ';
+  put '  ,showmeta=N,maxobs=MAX,workobs=0 ';
   put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
@@ -390,14 +502,18 @@ data _null_;
   put '%end; ';
   put ' ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
+  put '  %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do; ';
+  put '    /* functions in formats unsupported */ ';
+  put '    %put &sysmacroname: forcing missing back to NULL as feature not supported; ';
+  put '    %let missing=NULL; ';
+  put '  %end; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
-  put '    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta ';
+  put '    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs ';
   put '  ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  %if %str(&_debug) ge 131 %then %do; ';
-  put '    /* if debug mode, send back first 10 records of each work table also */ ';
-  put '    options obs=10; ';
+  put '  %if %str(&workobs) > 0 %then %do; ';
+  put '    /* if debug mode, send back first XX records of each work table also */ ';
   put '    data;run;%let tempds=%scan(&syslast,2,.); ';
   put '    ods output Members=&tempds; ';
   put '    proc datasets library=WORK memtype=data; ';
@@ -422,7 +538,9 @@ data _null_;
   put '        put " ""&wt"" : {"; ';
   put '        put ''"nlobs":'' nlobs; ';
   put '        put '',"nvars":'' nvars; ';
-  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) ';
+  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=Y,maxobs=10 ';
+  put '        ,maxobs=&workobs ';
+  put '      ) ';
   put '      data _null_; file &fref mod encoding=''utf-8'' termstr=lf; ';
   put '        put "}"; ';
   put '    %end; ';
@@ -432,34 +550,34 @@ data _null_;
   put '  %end; ';
   put '  /* close off json */ ';
   put '  data _null_;file &fref mod encoding=''utf-8'' termstr=lf lrecl=32767; ';
-  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
-  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
-  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
+  put '    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; ';
   put '    put ",""_DEBUG"" : ""&_debug"" "; ';
+  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
+  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
+  put '    put '',"AUTOEXEC" : '' autoexec; ';
+  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put ",""SYSENCODING"" : ""&sysencoding"" "; ';
+  put '    syserrortext=cats(''"'',tranwrd(symget(''syserrortext''),''"'',''\"''),''"''); ';
   put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    SYSHOSTINFOLONG=quote(trim(symget(''SYSHOSTINFOLONG''))); ';
   put '    put '',"SYSHOSTINFOLONG" : '' SYSHOSTINFOLONG; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
   put '    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; ';
-  put '    length SYSPROCESSNAME $512; ';
   put '    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); ';
   put '    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" "; ';
+  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    syswarningtext=cats(''"'',tranwrd(symget(''syswarningtext''),''"'',''\"''),''"''); ';
   put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
-  put '    length autoexec $512; ';
-  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
-  put '    put '',"AUTOEXEC" : '' autoexec; ';
   put '    length memsize $32; ';
   put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
   put '    memsize=quote(cats(memsize)); ';

server/ms_deletefile.sas

@@ -33,7 +33,7 @@ data _null_;
   file &headref lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put "Authorization: Bearer " _infile_;
+  put _infile_;
 run;
 
 proc http method='DELETE' headerin=&headref

server/ms_getfile.sas

@@ -34,7 +34,7 @@ data _null_;
   file &headref lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put "Authorization: Bearer " _infile_;
+  put _infile_;
 run;
 
 proc http method='GET' out=&binaryfref headerin=&headref

server/ms_getgroups.sas

@@ -0,0 +1,147 @@
+/**
+  @file
+  @brief Fetches the list of groups from SASjs Server
+  @details Fetches the list of groups from SASjs Server and writes them to an
+  output dataset.  Provide a username to filter for the groups for a particular
+  user.
+
+  Example:
+
+      %ms_getgroups(outds=userlist)
+
+  With filter on username:
+
+      %ms_getgroups(outds=userlist, user=James)
+
+  With filter on userid:
+
+      %ms_getgroups(outds=userlist, uid=1)
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] user= (0) Provide the username on which to filter
+  @param [in] uid= (0) Provide the userid on which to filter
+  @param [out] outds= (work.ms_getgroups) This output dataset will contain the
+    list of groups. Format:
+|NAME:$32.|DESCRIPTION:$64.|GROUPID:best.|
+|---|---|---|
+|`SomeGroup `|`A group `|`1`|
+|`Another Group`|`this is a different group`|`2`|
+|`admin`|`Administrators `|`3`|
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mp_abort.sas
+
+  <h4> Related Files </h4>
+  @li ms_creategroup.sas
+  @li ms_getgroups.test.sas
+
+**/
+
+%macro ms_getgroups(
+  user=0,
+  uid=0,
+  outds=work.ms_getgroups,
+  mdebug=0
+);
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_getgroups.sas
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+
+%local fref0 fref1 libref optval rc msg url;
+
+%if %sysget(MODE)=desktop %then %do;
+  /* groups api does not exist in desktop mode */
+  data &outds;
+    length NAME $32 DESCRIPTION $64. GROUPID 8;
+    name="&sysuserid";
+    description="&sysuserid (group - desktop mode)";
+    groupid=1;
+    output;
+    stop;
+  run;
+  %return;
+%end;
+
+%let fref0=%mf_getuniquefileref();
+%let fref1=%mf_getuniquefileref();
+%let libref=%mf_getuniquelibref();
+
+/* avoid sending bom marker to API */
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
+data _null_;
+  file &fref0 lrecl=1000;
+  infile "&_sasjs_tokenfile" lrecl=1000;
+  input;
+  if _n_=1 then put "accept: application/json";
+  put _infile_;
+run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fref0;
+    input;
+    put _infile_;
+  run;
+%end;
+
+%if "&user" ne "0" %then %let url=/SASjsApi/user/by/username/&user;
+%else %if "&uid" ne "0" %then %let url=/SASjsApi/user/&uid;
+%else %let url=/SASjsApi/group;
+
+
+proc http method='GET' headerin=&fref0 out=&fref1
+  url="&_sasjs_apiserverurl.&url";
+%if &mdebug=1 %then %do;
+  debug level=1;
+%end;
+run;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_getgroups.sas
+  ,msg=%str(Issue submitting GET query to SASjsApi)
+)
+
+libname &libref JSON fileref=&fref1;
+
+%if "&user"="0" and "&uid"="0" %then %do;
+  data &outds;
+    length NAME $32 DESCRIPTION $64. GROUPID 8;
+    if _n_=1 then call missing(of _all_);
+    set &libref..root;
+    drop ordinal_root;
+  run;
+%end;
+%else %do;
+  data &outds;
+    length NAME $32 DESCRIPTION $64. GROUPID 8;
+    if _n_=1 then call missing(of _all_);
+    set &libref..groups;
+    drop ordinal_:;
+  run;
+%end;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_getgroups.sas
+  ,msg=%str(Issue reading response JSON)
+)
+
+/* reset options */
+options &optval;
+
+%if &mdebug=1 %then %do;
+  filename &fref0 clear;
+  filename &fref1 clear;
+  libname &libref clear;
+%end;
+
+%mend ms_getgroups;

server/ms_getusers.sas

@@ -2,16 +2,26 @@
   @file
   @brief Fetches the list of users from SASjs Server
   @details Fetches the list of users from SASjs Server and writes them to an
-  output dataset.
+  output dataset.  Can also be filtered, for a particular group.
 
   Example:
 
       %ms_getusers(outds=userlist)
 
+  Filtering for a group by group name:
+
+      %ms_getusers(outds=work.groupmembers, group=GROUPNAME)
+
+  Filtering for a group by group id:
+
+      %ms_getusers(outds=work.groupmembers, gid=1)
+
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] group= (0) Set to a group name to filter members for that group
+  @param [in] gid= (0) Set to a group id to filter members for that group
   @param [out] outds= (work.ms_getusers) This output dataset will contain the
     list of user accounts. Format:
-|DISPLAYNAME:$18.|USERNAME:$10.|ID:best.|
+|DISPLAYNAME:$60.|USERNAME:$30.|ID:best.|
 |---|---|---|
 |`Super Admin `|`secretuser `|`1`|
 |`Sabir Hassan`|`sabir`|`2`|
@@ -20,7 +30,6 @@
 |`New User `|`newuser `|`5`|
 
 
-
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
   @li mf_getuniquelibref.sas
@@ -28,14 +37,17 @@
 
   <h4> Related Files </h4>
   @li ms_createuser.sas
+  @li ms_getgroups.sas
   @li ms_getusers.test.sas
 
 **/
 
 %macro ms_getusers(
-    outds=work.ms_getusers
-    ,mdebug=0
-  );
+  outds=work.ms_getusers,
+  group=0,
+  gid=0,
+  mdebug=0
+);
 
 %mp_abort(
   iftrue=(&syscc ne 0)
@@ -43,11 +55,24 @@
   ,msg=%str(syscc=&syscc on macro entry)
 )
 
-%local fref0 fref1 libref optval rc msg;
+%local fref0 fref1 libref optval rc msg url;
 %let fref0=%mf_getuniquefileref();
 %let fref1=%mf_getuniquefileref();
 %let libref=%mf_getuniquelibref();
 
+%if %sysget(MODE)=desktop %then %do;
+  /* users api does not exist in desktop mode */
+  data &outds;
+    length DISPLAYNAME $60 USERNAME:$30 ID 8;
+    USERNAME="&sysuserid";
+    DISPLAYNAME="&sysuserid (desktop mode)";
+    ID=1;
+    output;
+    stop;
+  run;
+  %return;
+%end;
+
 /* avoid sending bom marker to API */
 %let optval=%sysfunc(getoption(bomfile));
 options nobomfile;
@@ -56,8 +81,8 @@ data _null_;
   file &fref0 lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put "Authorization: Bearer " _infile_;
-  put "accept: application/json";
+  if _n_=1 then put "accept: application/json";
+  put _infile_;
 run;
 
 %if &mdebug=1 %then %do;
@@ -68,26 +93,42 @@ run;
   run;
 %end;
 
+%if "&group" ne "0" %then %let url=/SASjsApi/group/by/groupname/&group;
+%else %if "&gid" ne "0" %then %let url=/SASjsApi/group/&gid;
+%else %let url=/SASjsApi/user;
+
 proc http method='GET' headerin=&fref0 out=&fref1
-  url="&_sasjs_apiserverurl/SASjsApi/user";
+  url="&_sasjs_apiserverurl.&url";
 %if &mdebug=1 %then %do;
   debug level=1;
 %end;
 run;
 
+
 %mp_abort(
   iftrue=(&syscc ne 0)
   ,mac=ms_getusers.sas
-  ,msg=%str(Issue submitting GET query to SASjsApi/user)
+  ,msg=%str(Issue submitting API query)
 )
 
 libname &libref JSON fileref=&fref1;
 
-data &outds;
-  set &libref..root;
-  drop ordinal_root;
-run;
-
+%if "&group"="0" and "&gid"="0" %then %do;
+  data &outds;
+    length DISPLAYNAME $60 USERNAME:$30 ID 8;
+    if nobs=0 then call missing(of _all_);
+    set &libref..root nobs=nobs;
+    drop ordinal_root;
+  run;
+%end;
+%else %do;
+  data &outds;
+    length DISPLAYNAME $60 USERNAME:$30 ID 8;
+    if nobs=0 then call missing(of _all_);
+    set &libref..users nobs=nobs;
+    drop ordinal_root ordinal_users;
+  run;
+%end;
 
 %mp_abort(
   iftrue=(&syscc ne 0)

server/ms_runstp.sas

@@ -137,8 +137,8 @@ data _null_;
   file &authref lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put 'Authorization: Bearer ' _infile_;
-  put "Content-Type: multipart/form-data; boundary=&boundary";
+  if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary";
+  put _infile_;
 run;
 
 %if &mdebug=1 %then %do;
@@ -162,6 +162,7 @@ proc http method='POST' headerin=&authref in=&mainref out=&outref
 %end;
 run;
 %if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+or &mdebug=1
 %then %do;
   data _null_;infile &outref;input;putlog _infile_;run;
 %end;
@@ -177,7 +178,7 @@ options &optval;
 %if &outlogds ne _null_ or &mdebug=1 %then %do;
   %local dumplib;
   %let dumplib=%mf_getuniquelibref();
-  libname &dumplib json (&outref);
+  libname &dumplib json fileref=&outref;
   data &outlogds;
     set &dumplib..log;
   %if &mdebug=1 %then %do;
@@ -195,4 +196,4 @@ options &optval;
   filename &authref;
   filename &mainref;
 %end;
-%mend ms_runstp;
+%mend ms_runstp;

server/ms_testservice.sas

@@ -59,7 +59,7 @@
   outref=0,
   outlogds=_null_
 )/*/STORE SOURCE*/;
-%local dbg fref1 chopout1 chopout2;
+%local dbg i var ds1 fref1 chopout1 chopout2;
 %if &mdebug=1 %then %do;
   %put &sysmacroname entry vars:;
   %put _local_;
@@ -82,13 +82,15 @@
 %let ds1=%mf_getuniquename();
 data &ds1;
   length fileref $8 name $32 filename $256 var $300;
-  webcount=countw("&inputfiles");
-  do i=1 to webcount;
-    var=scan("&inputfiles",i,' ');
-    fileref=scan(var,1,':');
-    name=scan(var,2,':');
-    filename=cats(name,'.csv');
-    output;
+  if "&inputfiles" ne "0" then do;
+    webcount=countw("&inputfiles");
+    do i=1 to webcount;
+      var=scan("&inputfiles",i,' ');
+      fileref=scan(var,1,':');
+      name=scan(var,2,':');
+      filename=cats(name,'.csv');
+      output;
+    end;
   end;
 run;
 

server/ms_webout.sas

@@ -23,14 +23,19 @@
   @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE
   @param [in] ds The dataset to send back to the frontend
   @param [out] dslabel= value to use instead of table name for sending to JSON
-  @param [in] fmt= (Y) Set to N to send back unformatted values
+  @param [in] fmt= (N) Setting Y converts all vars to their formatted values
   @param [out] fref= (_webout) The fileref to which to write the JSON
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
-  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+  @param [in] showmeta= (N) Set to Y to output metadata alongside each table,
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
+  @param [in] workobs= (0) When set to a positive integer, will create a new
+    output object (WORK) which contains this number of observations from all
+    tables in the WORK library.
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to output JSON
 
   <h4> SAS Macros </h4>
   @li mf_getuser.sas
@@ -46,8 +51,8 @@
 
 **/
 
-%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
-  ,showmeta=NO
+%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL
+  ,showmeta=N,maxobs=MAX,workobs=0
 );
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
@@ -100,14 +105,18 @@
 %end;
 
 %else %if &action=ARR or &action=OBJ %then %do;
+  %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do;
+    /* functions in formats unsupported */
+    %put &sysmacroname: forcing missing back to NULL as feature not supported;
+    %let missing=NULL;
+  %end;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
-    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta
+    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs
   )
 %end;
 %else %if &action=CLOSE %then %do;
-  %if %str(&_debug) ge 131 %then %do;
-    /* if debug mode, send back first 10 records of each work table also */
-    options obs=10;
+  %if %str(&workobs) > 0 %then %do;
+    /* if debug mode, send back first XX records of each work table also */
     data;run;%let tempds=%scan(&syslast,2,.);
     ods output Members=&tempds;
     proc datasets library=WORK memtype=data;
@@ -132,7 +141,9 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES)
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=Y,maxobs=10
+        ,maxobs=&workobs
+      )
       data _null_; file &fref mod encoding='utf-8' termstr=lf;
         put "}";
     %end;
@@ -142,34 +153,34 @@
   %end;
   /* close off json */
   data _null_;file &fref mod encoding='utf-8' termstr=lf lrecl=32767;
-    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
-    put ",""SYSUSERID"" : ""&sysuserid"" ";
-    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
+    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512;
     put ",""_DEBUG"" : ""&_debug"" ";
+    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ',"_PROGRAM" : ' _PROGRAM ;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
+    put ',"AUTOEXEC" : ' autoexec;
+    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""SYSCC"" : ""&syscc"" ";
-    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ",""SYSENCODING"" : ""&sysencoding"" ";
+    syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
     put ',"SYSERRORTEXT" : ' syserrortext;
     SYSHOSTINFOLONG=quote(trim(symget('SYSHOSTINFOLONG')));
     put ',"SYSHOSTINFOLONG" : ' SYSHOSTINFOLONG;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
     put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
-    length SYSPROCESSNAME $512;
     SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
     put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" ";
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');
     put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
-    length autoexec $512;
-    autoexec=quote(urlencode(trim(getoption('autoexec'))));
-    put ',"AUTOEXEC" : ' autoexec;
     length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
     memsize=quote(cats(memsize));

tests/base/mf_fmtdttm.test.sas

@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Testing mf_fmtdttm macro
+
+  <h4> SAS Macros </h4>
+  @li mf_fmtdttm.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%global test1;
+
+%mp_assertscope(SNAPSHOT)
+%let test1=%mf_fmtdttm();
+%mp_assertscope(COMPARE,ignorelist=test1)
+
+%mp_assert(
+  iftrue=("&test1"="DATETIME19.3" or "&test1"="E8601DT26.6"),
+  desc=Basic test,
+  outds=work.test_results
+)

tests/base/mf_getvarcount.test.sas

@@ -0,0 +1,46 @@
+/**
+  @file
+  @brief Testing mf_getvarlist macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getvarcount.sas
+  @li mp_assertscope.sas
+
+**/
+
+data work.all work.nums(keep=num1 num2) work.chars(keep=char1 char2);
+  length num1 num2 8 char1 char2 char3 $4;
+  call missing (of _all_);
+  output;
+run;
+
+%mp_assertscope(SNAPSHOT)
+%put scope check:%mf_getvarcount(work.all);
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(%mf_getvarcount(work.all)=5),
+  desc=%str(Checking for mixed vars),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_getvarcount(work.all,typefilter=C)=3),
+  desc=%str(Checking for char in mixed vars),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_getvarcount(work.all,typefilter=N)=2),
+  desc=%str(Checking for num in mixed vars),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_getvarcount(work.nums,typefilter=c)=0),
+  desc=%str(Checking for char in num vars),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_getvarcount(work.chars,typefilter=N)=0),
+  desc=%str(Checking for num in char vars),
+  outds=work.test_results
+)
+

tests/base/mf_increment.test.sas

@@ -0,0 +1,35 @@
+/**
+  @file
+  @brief Testing mf_increment macro
+
+  <h4> SAS Macros </h4>
+  @li mf_increment.sas
+  @li mp_assert.sas
+
+**/
+
+%let var=0;
+
+%mp_assert(
+  iftrue=(
+    "%mf_increment(var)"="1"
+  ),
+  desc=Checking basic mf_increment usage 1,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_increment(var)"="2"
+  ),
+  desc=Checking basic mf_increment usage 2,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_increment(var,incr=2)"="4"
+  ),
+  desc=Checking incr option,
+  outds=work.test_results
+)