fix: mp_abort on windows m6+

fix: enable reserved names in mp_jsonout.  Closes #249

.devcontainer/Dockerfile

@@ -0,0 +1,8 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/javascript-node/.devcontainer/base.Dockerfile
+
+# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
+ARG VARIANT="18-bullseye"
+FROM mcr.microsoft.com/vscode/devcontainers/javascript-node:0-${VARIANT}
+
+RUN apt-get update \
+    && apt-get install -y doxygen

.devcontainer/devcontainer.json

@@ -0,0 +1,26 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/typescript-node
+{
+	"name": "Node.js & TypeScript",
+	"build": {
+		"dockerfile": "Dockerfile",
+		// Update 'VARIANT' to pick a Node version: 18, 16, 14.
+		// Append -bullseye or -buster to pin to an OS version.
+		// Use -bullseye variants on local on arm64/Apple Silicon.
+		"args": {
+			"VARIANT": "16-bullseye"
+		}
+	},
+	// Set *default* container specific settings.json values on container create.
+	"settings": {},
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"SASjs.sasjs-for-vscode"
+	],
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "npm i && npm i -g @sasjs/cli",
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "node"
+}

README.md

@@ -31,14 +31,14 @@ Documentation: https://core.sasjs.io
 
 ## Components
 
-### BASE library (All Platforms)
+### BASE folder (All Platforms)
 
 - OS independent
 - Works on all SAS Platforms
 - No X command
 - Prefixes: _mf_, _mp_
 
-### DDL library (All Platforms)
+### DDL folder (All Platforms)
 
 - OS independent
 - Works on all SAS Platforms
@@ -47,46 +47,14 @@ Documentation: https://core.sasjs.io
 
 This library will not be used for storing data entries (such as formats or datalines).  Where this becomes necessary in the future, a new repo will be created, in order to keep the NPM bundle size down (for the benefit of those looking to embed purely macros in their applications).
 
-### FCMP library (All Platforms)
+### FCMP folder (All Platforms)
 
 - Function and macro names are identical, except for special cases
 - Prefixes: _mcf_
 
 The fcmp macros are used to generate fcmp functions, and can be used with or without the `proc fcmp` wrapper.
 
-### META library (SAS9 only)
+### LUA folder
-
-Macros used in SAS EBI, which connect to the metadata server.
-
-- OS independent
-- Metadata aware
-- No X command
-- Prefixes: _mm_
-
-### SERVER library (@sasjs/server only)
-These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
-
-- OS independent
-- @sasjs/server aware
-- No X command
-- Prefixes: _ms_
-
-### VIYA library (Viya only)
-
-Macros used for interfacing with SAS Viya.
-
-- OS independent
-- No X command
-- Prefixes: _mv_, _mvf_
-
-### METAX library (SAS9 only)
-
-- OS specific
-- Metadata aware
-- X command enabled
-- Prefixes: _mmw_,_mmu_,_mmx_
-
-### LUA library
 
 Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
 
@@ -106,13 +74,61 @@ run;
 
 - Prefixes: _ml_
 
+### META folder (SAS9 only)
+
+Macros used in SAS EBI, which connect to the metadata server.
+
+- OS independent
+- Metadata aware
+- No X command
+- Prefixes: _mm_
+
+### METAX folder (SAS9 only)
+
+- OS specific
+- Metadata aware
+- X command enabled
+- Prefixes: _mmw_,_mmu_,_mmx_
+
+### SERVER folder (@sasjs/server only)
+These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
+
+- OS independent
+- @sasjs/server aware
+- No X command
+- Prefixes: _ms_
+
+### VIYA folder (Viya only)
+
+Macros used for interfacing with SAS Viya.
+
+- OS independent
+- No X command
+- Prefixes: _mv_, _mvf_
+
+### XPLATFORM folder (Viya, Meta, and Server)
+
+Sometimes it is helpful to use a macro that can be used interchangeably regardless of the server type on which is is running (SASVIYA, SAS9, SASJS).
+
+- OS independent
+- No X command
+- Prefixes: _mx_
+
 ## Installation
 
 First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available, eg:
 
 ```sas
-options insert=(sasautos="/your/path/macrocore/base");
+%let repoloc=/your/path/core;
-options insert=(sasautos="/your/path/macrocore/meta");
+options insert=(sasautos="&repoloc/base");
+options insert=(sasautos="&repoloc/ddl");
+options insert=(sasautos="&repoloc/fcmp");
+options insert=(sasautos="&repoloc/lua");
+options insert=(sasautos="&repoloc/meta");
+options insert=(sasautos="&repoloc/metax");
+options insert=(sasautos="&repoloc/server");
+options insert=(sasautos="&repoloc/viya");
+options insert=(sasautos="&repoloc/xplatform");
 ```
 
 The above can be done directly in your sas program, via an autoexec, or an initialisation program.
@@ -142,7 +158,7 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
   - _mp_ for macro procedures (which generate sas code)
   - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
   - _mv_ for macro procedures that will only work in Viya
-  - _mx_ for macros that are XCMD enabled (working on both windows and unix)
+  - _mx_ for macros that work on Viya, SAS 9 EBI and SASjs Server
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long

base/mf_existfeature.sas

@@ -40,6 +40,12 @@
     %else %if "&SYSVLONG" < "9.04.01M3" %then 0;
     %else 1;
   %end;
+  %else %if &feature=EXPORTXLS %then %do;
+    /* is it possible to PROC EXPORT an excel file? */
+    %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 1;
+    %else %if %sysfunc(sysprod(SAS/ACCESS Interface to PC Files)) = 1 %then 1;
+    %else 0;
+  %end;
   %else %do;
     -1
     %put &sysmacroname: &feature not found;

base/mf_getplatform.sas

@@ -5,8 +5,12 @@
 
       %put %mf_getplatform();
 
-    returns:
+  returns one of:
-      SASMETA  (or SASVIYA)
+
+  @li SASMETA
+  @li SASVIYA
+  @li SASJS
+  @li BASESAS
 
   @param switch the param for which to return a platform specific variable
 
@@ -68,4 +72,4 @@
 %else %if &switch=VIYARESTAPI %then %do;
   %mf_trimstr(%sysfunc(getoption(servicesbaseurl)),/)
 %end;
-%mend mf_getplatform;
+%mend mf_getplatform;

base/mf_isint.sas

@@ -33,4 +33,4 @@
   %if %sysfunc(findc(%str(&val),,kd)) %then %do;0%end;
   %else %do;1%end;
 
-%mend mf_isint;
+%mend mf_isint;

base/mp_abort.sas

@@ -15,16 +15,20 @@
     recognise this and fetch the log of the parent session instead)
   @li STP environments must finish cleanly to avoid the log being sent to
     _webout.  To assist with this, we also run stpsrvset('program error', 0)
-    and set SYSCC=0.  We take a unique "soft abort" approach - we open a macro
+    and set SYSCC=0.
+    Where possible, we take a unique "soft abort" approach - we open a macro
     but don't close it!  This works everywhere EXCEPT inside a \%include inside
     a macro.  For that, we recommend you use mp_include.sas to perform the
     include, and then call \%mp_abort(mode=INCLUDE) from the source program (ie,
     OUTSIDE of the top-parent macro).
+    The soft abort has also been found to be ineffective in 9.4m6 windows
+    environments and above, so in these cases, endsas is used.
 
 
-  @param mac= to contain the name of the calling macro
+  @param mac= (mp_abort.sas) To contain the name of the calling macro. Do not
+    use &sysmacroname as this will always resolve to MP_ABORT.
   @param msg= message to be returned
-  @param iftrue= supply a condition under which the macro should be executed.
+  @param iftrue= (1=1) Supply a condition for which the macro should be executed
   @param errds= (work.mp_abort_errds) There is no clean way to end a process
     within a %include called within a macro.  Furthermore, there is no way to
     test if a macro is called within a %include.  To handle this particular
@@ -45,11 +49,12 @@
     @li REGULAR (default)
     @li INCLUDE
 
+  @version 9.4
+  @author Allan Bowe
+
   <h4> Related Macros </h4>
   @li mp_include.sas
 
-  @version 9.4
-  @author Allan Bowe
   @cond
 **/
 
@@ -84,8 +89,8 @@
     %end;
   %end;
 
-  /* Stored Process Server web app context */
+  /* Web App Context */
-  %if %symexist(_METAFOLDER)
+  %if %symexist(_PROGRAM)
     or "&SYSPROCESSNAME "="Compute Server "
     or &mode=INCLUDE
   %then %do;
@@ -226,22 +231,30 @@
         rc=stpsrvset('program error', 0);
         call symputx("syscc",0,"g");
       run;
-      /**
+      %if &sysscp=WIN
-        * endsas kills 9.4m3 deployments by orphaning multibridges.
+      and "%substr(%str(&sysvlong         ),1,8)"="9.04.01M"
-        * Abort variants are ungraceful (non zero return code)
+      and "%substr(%str(&sysvlong         ),9,1)">"5" %then %do;
-        * This approach lets SAS run silently until the end :-)
+        /* skip approach (below) does not work in windows m6+ envs */
-        * Caution - fails when called within a %include within a macro
+        endsas;
-        * Use mp_include() to handle this.
+      %end;
-        */
+      %else %do;
-      filename skip temp;
+        /**
-      data _null_;
+          * endsas kills 9.4m3 deployments by orphaning multibridges.
-        file skip;
+          * Abort variants are ungraceful (non zero return code)
-        put '%macro skip();';
+          * This approach lets SAS run silently until the end :-)
-        comment '%mend skip; -> fix lint ';
+          * Caution - fails when called within a %include within a macro
-        put '%macro skippy();';
+          * Use mp_include() to handle this.
-        comment '%mend skippy; -> fix lint ';
+          */
-      run;
+        filename skip temp;
-      %inc skip;
+        data _null_;
+          file skip;
+          put '%macro skip();';
+          comment '%mend skip; -> fix lint ';
+          put '%macro skippy();';
+          comment '%mend skippy; -> fix lint ';
+        run;
+        %inc skip;
+      %end;
     %end;
     %else %if "&sysprocessmode " = "SAS Compute Server " %then %do;
       /* endsas kills the session making it harder to fetch results */

base/mp_copyfolder.sas

@@ -16,8 +16,11 @@
 
       %mp_copyfolder(&rootdir,&copydir)
 
-  @param source Unquoted path to the folder to copy from.
+  @param [in] source Unquoted path to the folder to copy from.
-  @param target Unquoted path to the folder to copy to.
+  @param [out] target Unquoted path to the folder to copy to.
+  @param [in] copymax=(MAX) Set to a positive integer to indicate the level of
+    subdirectory copy recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas
@@ -31,7 +34,7 @@
 
 **/
 
-%macro mp_copyfolder(source,target);
+%macro mp_copyfolder(source,target,copymax=MAX);
 
   %mp_abort(iftrue=(%mf_isdir(&source)=0)
     ,mac=&sysmacroname
@@ -50,7 +53,7 @@
   %let tempds=%mf_getuniquename();
 
   /* recursive directory listing */
-  %mp_dirlist(path=&source,outds=work.&tempds, maxdepth=MAX)
+  %mp_dirlist(path=&source,outds=work.&tempds,maxdepth=&copymax)
 
   /* create folders and copy content */
   data _null_;
@@ -78,4 +81,4 @@
   proc sql;
   drop table work.&tempds;
 
-%mend mp_copyfolder;
+%mend mp_copyfolder;

base/mp_coretable.sas

@@ -30,6 +30,7 @@
   @li mddl_dc_filtersummary.sas
   @li mddl_dc_locktable.sas
   @li mddl_dc_maxkeytable.sas
+  @li mf_getuniquename.sas
 
   <h4> Related Macros </h4>
   @li mp_filterstore.sas
@@ -46,7 +47,7 @@
 %macro mp_coretable(table_ref,libds=0
 )/*/STORE SOURCE*/;
 %local outds ;
-%let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
+%let outds=%sysfunc(ifc(&libds=0,%mf_getuniquename(),&libds));
 proc sql;
 %if &table_ref=DIFFTABLE %then %do;
   %mddl_dc_difftable(libds=&outds)
@@ -65,7 +66,8 @@ proc sql;
 %end;
 
 %if &libds=0 %then %do;
+  proc sql;
   describe table &syslast;
   drop table &syslast;
 %end;
-%mend mp_coretable;
+%mend mp_coretable;

base/mp_createwebservice.sas

@@ -1,50 +1,13 @@
 /**
   @file mp_createwebservice.sas
-  @brief Create a web service in SAS 9 or Viya
+  @brief Create a web service in SAS 9, Viya or SASjs Server
-  @details Creates a SASJS ready Stored Process in SAS 9 or Job Execution
+  @details This is actually a wrapper for mx_createwebservice.sas, remaining
-  Service in SAS Viya
+  for legacy purposes.  For new apps, use mx_createwebservice.sas.
 
-Usage:
-
-    %* compile macros ;
-    filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
-    %inc mc;
-
-    %* write some code;
-    filename ft15f001 temp;
-    parmcards4;
-        %* fetch any data from frontend ;
-        %webout(FETCH)
-        data example1 example2;
-          set sashelp.class;
-        run;
-        %* send data back;
-        %webout(OPEN)
-        %webout(ARR,example1) * Array format, fast, suitable for large tables ;
-        %webout(OBJ,example2) * Object format, easier to work with ;
-        %webout(CLOSE)
-    ;;;;
-    %mp_createwebservice(path=/Public/app/common,name=appInit,replace=YES)
 
   <h4> SAS Macros </h4>
-  @li mf_getplatform.sas
+  @li mx_createwebservice.sas
-  @li mm_createwebservice.sas
-  @li ms_createwebservice.sas
-  @li mv_createwebservice.sas
 
-  @param [in,out] path= The full folder path where the service will be created
-  @param [in,out] name= Service name.  Avoid spaces.
-  @param [in] desc= The description of the service (optional)
-  @param [in] precode= Space separated list of filerefs, pointing to the code
-    that needs to be attached to the beginning of the service (optional)
-  @param [in] code= (ft15f001) Space seperated fileref(s) of the actual code to
-    be added
-  @param [in] replace= (YES) Select YES to replace any existing service in that
-    location
-  @param [in] mDebug= (0) set to 1 to show debug messages in the log
-
-  @version 9.2
-  @author Allan Bowe
 
 **/
 
@@ -57,40 +20,13 @@ Usage:
     ,mdebug=0
 )/*/STORE SOURCE*/;
 
-%if &syscc ge 4 %then %do;
+  %mx_createwebservice(path=&path
-  %put syscc=&syscc - &sysmacroname will not execute in this state;
-  %return;
-%end;
-
-%local platform; %let platform=%mf_getplatform();
-%if &platform=SASVIYA %then %do;
-  %if "&path"="HOME" %then %let path=/Users/&sysuserid/My Folder;
-  %mv_createwebservice(path=&path
     ,name=&name
-    ,code=&code
     ,precode=&precode
+    ,code=&code
     ,desc=&desc
     ,replace=&replace
-  )
-%end;
-%else %if &platform=SASJS %then %do;
-  %if "&path"="HOME" %then %let path=/Users/&_sasjs_username/My Folder;
-  %ms_createwebservice(path=&path
-    ,name=&name
-    ,code=&code
-    ,precode=&precode
     ,mdebug=&mdebug
   )
-%end;
-%else %do;
-  %if "&path"="HOME" %then %let path=/User Folders/&_METAPERSON/My Folder;
-  %mm_createwebservice(path=&path
-    ,name=&name
-    ,code=&code
-    ,precode=&precode
-    ,desc=&desc
-    ,replace=&replace
-  )
-%end;
 
 %mend mp_createwebservice;

base/mp_ds2cards.sas

@@ -11,7 +11,7 @@
 
   Usage:
 
-      %mp_ds2cards(base_ds=sashelp.class
+      %mp_ds2cards(sashelp.class
         , tgt_ds=work.class
         , cards_file= "C:\temp\class.sas"
         , showlog=NO
@@ -23,7 +23,7 @@
     - explicity setting a unix LF
     - constraints / indexes etc
 
-  @param [in] base_ds= Should be two level - eg work.blah.  This is the table
+  @param [in] base_ds Should be two level - eg work.blah.  This is the table
                   that is converted to a cards file.
   @param [in] tgt_ds= Table that the generated cards file would create.
     Optional - if omitted, will be same as BASE_DS.

base/mp_ds2md.sas

@@ -57,6 +57,11 @@
 %local vars;
 %let vars=%upcase(%mf_getvarlist(&libds));
 
+%if %trim(X&vars)=X %then %do;
+  %put &sysmacroname: Table &libds has no columns!!;
+  %return;
+%end;
+
 /* create the header row */
 data _null_;
   file &outref;

base/mp_filtercheck.sas

@@ -92,7 +92,36 @@ data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
-  length reason_cd $4032;
+  length reason_cd $4032 vtype $1 vnum dsid 8;
+
+  /* quick check to ensure column exists */
+  if upcase(VARIABLE_NM) not in
+  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
+  then do;
+    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
+    putlog REASON_CD= VARIABLE_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
+    output;
+    return;
+  end;
+
+  /* need to open the dataset to get the column type */
+  dsid=open("&targetds","i");
+  if dsid>0 then do;
+    vnum=varnum(dsid,VARIABLE_NM);
+    if vnum<1 then do;
+      /* should not happen as was also tested for above */
+      REASON_CD=cats("Variable (",VARIABLE_NM,") not found in &targetds");
+      putlog REASON_CD= dsid=;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
+      output;
+      return;
+    end;
+    /* now we can get the type */
+    else vtype=vartype(dsid,vnum);
+  end;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
@@ -116,17 +145,8 @@ data &outds;
     call symputx('nobs',_n_,'l');
     output;
   end;
-  if upcase(VARIABLE_NM) not in
-  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
-  then do;
-    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
-    putlog REASON_CD= VARIABLE_NM=;
-    call symputx('reason_cd',reason_cd,'l');
-    call symputx('nobs',_n_,'l');
-    output;
-  end;
   if OPERATOR_NM not in
-  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE')
+  ('=','>','<','<=','>=','NE','GE','LE','BETWEEN','IN','NOT IN','CONTAINS')
   then do;
     REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
@@ -135,6 +155,18 @@ data &outds;
     output;
   end;
 
+  /* special missing logic */
+  if vtype='N'
+    and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE')
+    and cats(upcase(raw_value)) in (
+      '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N'
+      '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._'
+    )
+  then do;
+    /* valid numeric - exit data step loop */
+    return;
+  end;
+
   /* special logic */
   if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ','');
   else if OPERATOR_NM in ('IN','NOT IN') then do;

base/mp_filtergenerate.sas

@@ -84,6 +84,9 @@ filename &outref temp;
   run;
 %end;
 %else %do;
+  proc sort data=&inds;
+    by SUBGROUP_ID;
+  run;
   data _null_;
     file &outref lrecl=32800;
     set &inds end=last;

base/mp_hashdataset.sas

@@ -58,7 +58,7 @@
 %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
   data &outds;
     length hashkey $32;
-    retain hashkey "%sysfunc(md5(%str(&salt)),$hex32.)";
+    hashkey=put(md5("&salt"),$hex32.);
     output;
     stop;
   run;
@@ -69,9 +69,14 @@
   %put %str(ERR)OR: Dataset &libds is not a dataset;
 %end;
 %else %do;
-  data &outds(rename=(&keyvar=hashkey) keep=&keyvar)/nonote2err;
+  data &outds(rename=(&keyvar=hashkey) keep=&keyvar)
+  %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+    /nonote2err
+  %end;
+  ;
     length &prevkeyvar &keyvar $32;
-    retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
+    retain &prevkeyvar;
+    if _n_=1 then &prevkeyvar=put(md5("&salt"),$hex32.);
     set &libds end=&lastvar;
     /* hash should include previous row */
     &keyvar=%mp_md5(

base/mp_jsonout.sas

@@ -231,7 +231,7 @@
       %do i=1 %to &numcols;
         %if &i>1 %then  "," ;
         %if &action=OBJ %then """&&name&i"":" ;
-        &&name&i
+        "&&name&i"n /* name literal for reserved variable names */
       %end;
       %if &action=ARR %then "]" ; %else "}" ; ;
     /* now write the long strings to _webout 1 byte at a time */

base/mp_resetoption.sas

@@ -21,15 +21,19 @@ https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-s
 %macro mp_resetoption(option /* the option to reset */
 )/*/STORE SOURCE*/;
 
-data _null_;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
-  length code  $1500;
+  data _null_;
-  startup=getoption("&option",'startupvalue');
+    length code  $1500;
-  current=getoption("&option");
+    startup=getoption("&option",'startupvalue');
-  if startup ne current then do;
+    current=getoption("&option");
-    code =cat('OPTIONS ',getoption("&option",'keyword','startupvalue'),';');
+    if startup ne current then do;
-    putlog "NOTE: Resetting system option: " code ;
+      code =cat('OPTIONS ',getoption("&option",'keyword','startupvalue'),';');
-    call execute(code );
+      putlog "NOTE: Resetting system option: " code ;
-  end;
+      call execute(code );
-run;
+    end;
-
+  run;
+%end;
+%else %do;
+  %put &sysmacroname: reset option feature unavailable on &sysvlong;
+%end;
 %mend mp_resetoption;

base/mp_testservice.sas

@@ -1,53 +1,11 @@
 /**
   @file
-  @brief Will execute a SASjs web service on SAS 9 or Viya
+  @brief To be deprecated.  Will execute a SASjs web service on SAS 9 or Viya
-  @details Prepares the input files and retrieves the resulting datasets from
+  @details Use the mx_testservice.sas macro instead (documentation can be
-  the response JSON.
+  found there)
-
-
-
-  Note - the _webout fileref should NOT be assigned prior to running this macro.
-
-  @param [in] program The _PROGRAM endpoint to test
-  @param [in] inputfiles=(0) A list of space seperated fileref:filename pairs as
-    follows:
-        inputfiles=inref:filename inref2:filename2
-  @param [in] inputdatasets= (0) All datasets in this space seperated list are
-    converted into SASJS-formatted CSVs (see mp_ds2csv.sas) files and added to
-    the list of `inputfiles` for ingestion.  The dataset will be sent with the
-    same name (no need for a colon modifier).
-  @param [in] inputparams=(0) A dataset containing name/value pairs in the
-    following format:
-    |name:$32|value:$1000|
-    |---|---|
-    |stpmacname|some value|
-    |mustbevalidname|can be anything, oops, %abort!!|
-
-  @param [in] debug= (log) Provide the _debug value
-  @param [in] mdebug= (0) Set to 1 to provide macro debugging
-  @param [in] viyaresult= (WEBOUT_JSON) The Viya result type to return.  For
-    more info, see mv_getjobresult.sas
-  @param [in] viyacontext= (SAS Job Execution compute context) The Viya compute
-    context on which to run the service
-  @param [out] outlib= (0) Output libref to contain the final tables.  Set to
-    0 if the service output is not in JSON format.
-  @param [out] outref= (0) Output fileref to create, to contain the full _webout
-    response.
 
   <h4> SAS Macros </h4>
-  @li mf_getplatform.sas
+  @li mx_testservice.sas
-  @li mf_getuniquefileref.sas
-  @li mf_getuniquename.sas
-  @li mp_abort.sas
-  @li mp_binarycopy.sas
-  @li mp_chop.sas
-  @li mp_ds2csv.sas
-  @li ms_testservice.sas
-  @li mv_getjobresult.sas
-  @li mv_jobflow.sas
-
-  <h4> Related Programs </h4>
-  @li mp_testservice.test.sas
 
   @version 9.4
   @author Allan Bowe
@@ -65,237 +23,17 @@
   viyaresult=WEBOUT_JSON,
   viyacontext=SAS Job Execution compute context
 )/*/STORE SOURCE*/;
-%local dbg pcnt fref1 fref2 webref webrefpath i webcount var platform;
-%if &mdebug=1 %then %do;
-  %put &sysmacroname entry vars:;
-  %put _local_;
-%end;
-%else %let dbg=*;
 
-/* sanitise inputparams */
+%mx_testservice(&program,
-%let pcnt=0;
+  inputfiles=&inputfiles,
-%if &inputparams ne 0 %then %do;
+  inputdatasets=&inputdatasets,
-  data _null_;
+  inputparams=&inputparams,
-    set &inputparams;
+  debug=&debug,
-    if not nvalid(name,'v7') then putlog (_all_)(=);
+  mdebug=&mdebug,
-    else if name in (
+  outlib=&outlib,
-      'program','inputfiles','inputparams','debug','outlib','outref'
+  outref=&outref,
-    ) then putlog (_all_)(=);
+  viyaresult=&viyaresult,
-    else do;
+  viyacontext=&viyacontext
-      x+1;
+)
-      call symputx(name,quote(cats(value)),'l');
-      call symputx(cats('pval',x),name,'l');
-      call symputx('pcnt',x,'l');
-    end;
-  run;
-  %mp_abort(iftrue= (%mf_nobs(&inputparams) ne &pcnt)
-    ,mac=&sysmacroname
-    ,msg=%str(Invalid values in &inputparams)
-  )
-%end;
 
-/* convert inputdatasets to filerefs */
+%mend mp_testservice;
-%if "&inputdatasets" ne "0" %then %do;
-  %if %quote(&inputfiles)=0 %then %let inputfiles=;
-  %do i=1 %to %sysfunc(countw(&inputdatasets,%str( )));
-    %let var=%scan(&inputdatasets,&i,%str( ));
-    %local dsref&i;
-    %let dsref&i=%mf_getuniquefileref();
-    %mp_ds2csv(&var,outref=&&dsref&i,headerformat=SASJS)
-    %let inputfiles=&inputfiles &&dsref&i:%scan(&var,-1,.);
-  %end;
-%end;
-
-%let platform=%mf_getplatform();
-%let fref1=%mf_getuniquefileref();
-%let fref2=%mf_getuniquefileref();
-%let webref=%mf_getuniquefileref();
-%let webrefpath=%sysfunc(pathname(work))/%mf_getuniquename();
-/* mp_chop requires a physical path as input */
-filename &webref "&webrefpath";
-
-%if &platform=SASMETA %then %do;
-
-  /* parse the input files */
-  %if %quote(&inputfiles) ne 0 %then %do;
-    %let webcount=%sysfunc(countw(&inputfiles));
-    %put &=webcount;
-    %do i=1 %to &webcount;
-      %let var=%scan(&inputfiles,&i,%str( ));
-      %local webfref&i webname&i;
-      %let webref&i=%scan(&var,1,%str(:));
-      %let webname&i=%scan(&var,2,%str(:));
-      %put webref&i=&&webref&i;
-      %put webname&i=&&webname&i;
-    %end;
-  %end;
-  %else %let webcount=0;
-
-  proc stp program="&program";
-    inputparam _program="&program"
-  %do i=1 %to &webcount;
-    %if &webcount=1 %then %do;
-      _webin_fileref="&&webref&i"
-      _webin_name="&&webname&i"
-    %end;
-    %else %do;
-      _webin_fileref&i="&&webref&i"
-      _webin_name&i="&&webname&i"
-    %end;
-  %end;
-    _webin_file_count="&webcount"
-    _debug="&debug"
-  %do i=1 %to &pcnt;
-    /* resolve name only, proc stp fetches value */
-    &&pval&i=&&&&&&pval&i
-  %end;
-    ;
-  %do i=1 %to &webcount;
-    inputfile &&webref&i;
-  %end;
-    outputfile _webout=&webref;
-  run;
-
-  data _null_;
-    infile &webref;
-    file &fref1;
-    input;
-    length line $10000;
-    if index(_infile_,'>>weboutBEGIN<<') then do;
-        line=tranwrd(_infile_,'>>weboutBEGIN<<','');
-        put line;
-    end;
-    else if index(_infile_,'>>weboutEND<<') then do;
-        line=tranwrd(_infile_,'>>weboutEND<<','');
-        put line;
-        stop;
-    end;
-    else put _infile_;
-  run;
-  data _null_;
-    infile &fref1;
-    input;
-    put _infile_;
-  run;
-  %if &outlib ne 0 %then %do;
-    libname &outlib json (&fref1);
-  %end;
-  %if &outref ne 0 %then %do;
-    filename &outref temp;
-    %mp_binarycopy(inref=&webref,outref=&outref)
-  %end;
-
-%end;
-%else %if &platform=SASVIYA %then %do;
-
-  /* prepare inputparams */
-  %local ds1;
-  %let ds1=%mf_getuniquename();
-  %if "&inputparams" ne "0" %then %do;
-    proc transpose data=&inputparams out=&ds1;
-      id name;
-      var value;
-    run;
-  %end;
-  %else %do;
-    data &ds1;run;
-  %end;
-
-  /* parse the input files - convert to sasjs params */
-  %local webcount i var sasjs_tables;
-  %if %quote(&inputfiles) ne 0 %then %do;
-    %let webcount=%sysfunc(countw(&inputfiles));
-    %put &=webcount;
-    %do i=1 %to &webcount;
-      %let var=%scan(&inputfiles,&i,%str( ));
-      %local webfref&i webname&i sasjs&i.data;
-      %let webref&i=%scan(&var,1,%str(:));
-      %let webname&i=%scan(&var,2,%str(:));
-      %put webref&i=&&webref&i;
-      %put webname&i=&&webname&i;
-
-      %let sasjs_tables=&sasjs_tables &&webname&i;
-      data _null_;
-        infile &&webref&i lrecl=32767;
-        input;
-        if _n_=1 then call symputx("sasjs&i.data",_infile_);
-        else call symputx(
-          "sasjs&i.data",cats(symget("sasjs&i.data"),'0D0A'x,_infile_)
-        );
-        putlog "&sysmacroname infile: " _infile_;
-      run;
-      data &ds1;
-        set &ds1;
-        length sasjs&i.data $32767 sasjs_tables $1000;
-        sasjs&i.data=symget("sasjs&i.data");
-        sasjs_tables=symget("sasjs_tables");
-      run;
-    %end;
-  %end;
-  %else %let webcount=0;
-
-  data &ds1;
-    retain _program "&program";
-    retain _contextname "&viyacontext";
-    set &ds1;
-    putlog "&sysmacroname inputparams:";
-    putlog (_all_)(=);
-  run;
-
-  %mv_jobflow(inds=&ds1
-    ,maxconcurrency=1
-    ,outds=work.results
-    ,outref=&fref1
-    ,mdebug=&mdebug
-  )
-  /* show the log */
-  data _null_;
-    infile &fref1;
-    input;
-    putlog _infile_;
-  run;
-  /* get the uri to fetch results */
-  data _null_;
-    set work.results;
-    call symputx('uri',uri);
-    putlog "&sysmacroname: fetching results for " uri;
-  run;
-  /* fetch results from webout.json */
-  %mv_getjobresult(uri=&uri,
-    result=&viyaresult,
-    outref=&outref,
-    outlib=&outlib,
-    mdebug=&mdebug
-  )
-
-%end;
-%else %if &platform=SASJS %then %do;
-
-  %ms_testservice(&program
-    ,inputfiles=&inputfiles
-    ,inputdatasets=&inputdatasets
-    ,inputparams=&inputparams
-    ,debug=&debug
-    ,mdebug=&mdebug
-    ,outlib=&outlib
-    ,outref=&outref
-  )
-
-%end;
-%else %do;
-  %put %str(ERR)OR: Unrecognised platform:  &platform;
-%end;
-
-%if &mdebug=0 %then %do;
-  filename &fref1 clear;
-  %if &platform ne SASJS %then %do;
-    filename &fref2 clear;
-    filename &webref clear;
-  %end;
-%end;
-%else %do;
-  %put &sysmacroname exit vars:;
-  %put _local_;
-%end;
-
-%mend mp_testservice;

build.py

@@ -102,7 +102,7 @@ options noquotelenmax;
 """
 f = open('all.sas', "w")             # r / r+ / rb / rb+ / w / wb
 f.write(header)
-folders = ['base', 'ddl', 'meta', 'metax', 'server', 'viya', 'lua', 'fcmp']
+folders = ['base', 'ddl', 'meta', 'metax', 'server', 'viya', 'lua', 'fcmp', 'xplatform']
 for folder in folders:
     filenames = [fn for fn in Path(
         './' + folder).iterdir() if fn.match("*.sas")]

ddl/mddl_dc_difftable.sas

@@ -26,9 +26,19 @@
       oldval_num num format=best32. label='Old (numeric) value',
       newval_num num format=best32. label='New (numeric) value',
       oldval_char char(32765) label='Old (character) value',
-      newval_char char(32765) label='New (character) value',
+      newval_char char(32765) label='New (character) value'
-    constraint pk_mpe_audit
-      primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
   );
 
+  %local lib;
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create
+      pk_mpe_audit=(load_ref libref dsn key_hash tgtvar_nm)
+      /nomiss unique;
+  quit;
+
 %mend mddl_dc_difftable;

ddl/mddl_dc_filterdetail.sas

@@ -9,19 +9,32 @@
 
 %macro mddl_dc_filterdetail(libds=WORK.FILTER_DETAIL);
 
+%local nn lib;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  %let nn=not null;
+%end;
+%else %let nn=;
+
   proc sql;
   create table &libds(
-      filter_hash char(32) not null,
+      filter_hash char(32) &nn,
-      filter_line num not null,
+      filter_line num &nn,
-      group_logic char(3) not null,
+      group_logic char(3) &nn,
-      subgroup_logic char(3) not null,
+      subgroup_logic char(3) &nn,
-      subgroup_id num not null,
+      subgroup_id num &nn,
-      variable_nm varchar(32) not null,
+      variable_nm varchar(32) &nn,
-      operator_nm varchar(12) not null,
+      operator_nm varchar(12) &nn,
-      raw_value varchar(4000) not null,
+      raw_value varchar(4000) &nn,
-      processed_dttm num not null format=E8601DT26.6,
+      processed_dttm num &nn format=E8601DT26.6
-    constraint pk_mpe_filteranytable
-      primary key(filter_hash,filter_line)
   );
 
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create pk_mpe_filterdetail=(filter_hash filter_line)/nomiss unique;
+  quit;
+
 %mend mddl_dc_filterdetail;

ddl/mddl_dc_filtersummary.sas

@@ -9,14 +9,27 @@
 
 %macro mddl_dc_filtersummary(libds=WORK.FILTER_SUMMARY);
 
+%local nn lib;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  %let nn=not null;
+%end;
+%else %let nn=;
+
   proc sql;
   create table &libds(
-      filter_rk num not null,
+      filter_rk num &nn,
-      filter_hash char(32) not null,
+      filter_hash char(32) &nn,
-      filter_table char(41) not null,
+      filter_table char(41) &nn,
-      processed_dttm num not null format=E8601DT26.6,
+      processed_dttm num &nn format=E8601DT26.6
-    constraint pk_mpe_filteranytable
-      primary key(filter_rk)
   );
 
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create filter_rk /nomiss unique;
+  quit;
+
 %mend mddl_dc_filtersummary;

ddl/mddl_dc_locktable.sas

@@ -9,17 +9,33 @@
 
 %macro mddl_dc_locktable(libds=WORK.LOCKTABLE);
 
+%local nn lib;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  %let nn=not null;
+%end;
+%else %let nn=;
+
   proc sql;
   create table &libds(
       lock_lib char(8),
       lock_ds char(32),
-      lock_status_cd char(10) not null,
+      lock_status_cd char(10) &nn,
-      lock_user_nm char(100) not null ,
+      lock_user_nm char(100) &nn ,
       lock_ref char(200),
       lock_pid char(10),
       lock_start_dttm num format=E8601DT26.6,
-      lock_end_dttm num format=E8601DT26.6,
+      lock_end_dttm num format=E8601DT26.6
-    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds)
   );
 
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create
+      pk_mp_lockanytable=(lock_lib lock_ds)
+      /nomiss unique;
+  quit;
+
 %mend mddl_dc_locktable;

ddl/mddl_dc_maxkeytable.sas

@@ -17,8 +17,17 @@
       max_key num label=
         'Integer representing current max RK or SK value in the KEYTABLE',
       processed_dttm num format=E8601DT26.6
-        label='Datetime this value was last updated',
+        label='Datetime this value was last updated'
-    constraint pk_mpe_maxkeyvalues
+  );
-        primary key(keytable));
+
+  %local lib;
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create keytable /nomiss unique;
+  quit;
 
 %mend mddl_dc_maxkeytable;

main.dox

@@ -18,7 +18,17 @@
     statements.  Those starting `mp_` are macro _procedures_, which generate
     SAS statements, and must therefore be applied accordingly.
 
- */
+*/
+
+/*! \dir ddl
+ *  \brief Data Definition Language files
+ *  \details Provides templates for commonly used tables in sasjs/core.
+    Attributes:
+
+        * OS independent
+        * No X command
+        * Prefixes: _mddl_
+*/
 
 /*! \dir fcmp
  *  \brief Macros for generating FCMP functions
@@ -92,4 +102,39 @@
         * Auto-generated from the plain source `.lua` files in the same directory
         * Prefixes: _ml_
 
- */
+*/
+
+/*! \dir tests/base
+ *  \brief Tests for Base macros
+ */
+
+/*! \dir tests/ddlonly
+ *  \brief Tests for DDL macros
+ */
+
+/*! \dir tests/sas9only
+ *  \brief Tests for SAS Metadata macros
+ */
+
+/*! \dir tests/serveronly
+ *  \brief Tests for SASjs Server macros
+ */
+
+/*! \dir tests/viyaonly
+ *  \brief Tests for Viya macros
+ */
+
+/*! \dir tests/x-platform
+ *  \brief Tests for cross-platform macros
+ */
+
+/*! \dir xplatform
+ *  \brief Cross Platform, works on all SAS servers (Viya, EBI, SASjs)
+ *  \details Useful when you need to run a single piece of code against Viya,
+     SAS 9 with metadata, or SASjs on Base SAS.
+
+        * OS independent
+        * No X command
+        * Prefixes: _mx_
+
+ */

meta/mm_createdataset.sas

@@ -1,22 +1,21 @@
 /**
-  @file mm_createdataset.sas
+  @file
-  @brief Create a dataset from a metadata definition
+  @brief Create an empty dataset from a metadata definition
   @details This macro was built to support viewing empty tables in
-    https://datacontroller.io - a free evaluation copy is available by
+  https://datacontroller.io
-    contacting the author (Allan Bowe).
 
-    The table can be retrieved using LIBRARY.DATASET reference, or directly
+  The table can be retrieved using LIBRARY.DATASET reference, or directly
-    using the metadata URI.
+  using the metadata URI.
 
-    The dataset is written to the WORK library.
+  The dataset is written to the WORK library.
 
-  usage:
+  Usage:
 
-    %mm_createdataset(libds=metlib.some_dataset)
+      %mm_createdataset(libds=metlib.some_dataset)
 
-    or
+  or
 
-    %mm_createdataset(tableuri=G5X8AFW1.BE00015Y)
+      %mm_createdataset(tableuri=G5X8AFW1.BE00015Y)
 
   <h4> SAS Macros </h4>
   @li mm_getlibs.sas
@@ -26,9 +25,9 @@
   @param libds= library.dataset metadata source.  Note - table names in metadata
     can be longer than 32 chars (just fyi, not an issue here)
   @param tableuri= Metadata URI of the table to be created
-  @param outds= The dataset to create, default is `work.mm_createdataset`.
+  @param outds= (work.mm_createdataset) The dataset to create.  The table name
-    The table name needs to be 32 chars or less as per SAS naming rules.
+    needs to be 32 chars or less as per SAS naming rules.
-  @param mdebug= set DBG to 1 to disable DEBUG messages
+  @param mdebug= (0) Set to 1 to enable DEBUG messages
 
   @version 9.4
   @author Allan Bowe
@@ -54,14 +53,23 @@
   %mm_gettables(uri=&liburi,outds=&tempds2)
   data _null_;
     set &tempds2;
-    if upcase(tablename)="%upcase(%scan(&libds,2,.))";
+    where upcase(tablename)="%upcase(%scan(&libds,2,.))";
+    &dbg putlog tableuri=;
     call symputx('tableuri',tableuri);
   run;
 %end;
 
-data;run;%let tempds3=&syslast;
+data;run;
+%let tempds3=&syslast;
 %mm_getcols(tableuri=&tableuri,outds=&tempds3)
 
+%if %mf_nobs(&tempds3)=0 %then %do;
+  %put &libds (&tableuri) has no columns defined!!;
+  data &outds;
+  run;
+  %return;
+%end;
+
 data _null_;
   set &tempds3 end=last;
   if _n_=1 then call execute('data &outds;');

meta/mm_createwebservice.sas

@@ -264,7 +264,7 @@ data _null_;
   put '      %do i=1 %to &numcols; ';
   put '        %if &i>1 %then  "," ; ';
   put '        %if &action=OBJ %then """&&name&i"":" ; ';
-  put '        &&name&i ';
+  put '        "&&name&i"n /* name literal for reserved variable names */ ';
   put '      %end; ';
   put '      %if &action=ARR %then "]" ; %else "}" ; ; ';
   put '    /* now write the long strings to _webout 1 byte at a time */ ';

meta/mm_getfoldertree.sas

@@ -8,17 +8,15 @@
 
   Usage:
 
-    options ps=max nonotes nosource;
+      options ps=max nonotes nosource;
-    %mm_getfoldertree(root=/My/Meta/Path, outds=iwantthisdataset)
+      %mm_getfoldertree(root=/My/Meta/Path, outds=iwantthisdataset)
-    options notes source;
+      options notes source;
 
   @param [in] root= the parent folder under which to return all contents
   @param [out] outds= the dataset to create that contains the list of
     directories
   @param [in] mDebug= set to 1 to show debug messages in the log
 
-  <h4> SAS Macros </h4>
-
   @version 9.4
   @author Allan Bowe
 

meta/mm_getstpcode.sas

@@ -21,6 +21,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
+  @li mp_abort.sas
 
   @author Allan Bowe
 
@@ -70,11 +71,10 @@ data _null_;
   else put (_all_)(=);
 run;
 
-%if &tsuri=stopifempty %then %do;
+%mp_abort(iftrue= (&tsuri=stopifempty)
-  %put %str(WARN)ING:  &tree&name.(StoredProcess) not found!;
+  ,mac=mm_getstpcode
-  %return;
+  ,msg=%str(&tree&name.(StoredProcess) not found!)
-%end;
+)
-
 
 /**
   * Now we can extract the textstore

sasjs/sasjsconfig.json

@@ -6,8 +6,10 @@
     "fcmp",
     "lua",
     "server",
-    "tests/crossplatform",
+    "xplatform",
-    "tests/ddl"
+    "tests/base",
+    "tests/ddlonly",
+    "tests/x-platform"
   ],
   "docConfig": {
     "displayMacroCore": false,
@@ -65,7 +67,7 @@
     },
     {
       "name": "server",
-      "serverUrl": "",
+      "serverUrl": "https://sas.analytium.co.uk:5007",
       "serverType": "SASJS",
       "httpsAgentOptions": {
         "allowInsecureRequests": false
@@ -105,4 +107,4 @@
       "contextName": "SAS Job Execution compute context"
     }
   ]
-}
+}

server/ms_creategroup.sas

@@ -0,0 +1,149 @@
+/**
+  @file
+  @brief Creates a group on SASjs Server
+  @details Creates a group on SASjs Server with the following attributes:
+
+  @li name
+  @li description
+  @li isActive
+
+  Examples:
+
+      %ms_creategroup(mynewgroup)
+
+      %ms_creategroup(mynewergroup, desc=The group description)
+
+  @param [in] groupname The group name to create.  No spaces or special chars.
+  @param [in] desc= (0) If no description provided, group name will be used.
+  @param [in] isactive= (true) Set to false to create an inactive group.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outds= (work.ms_creategroup) This output dataset will contain the
+    values from the JSON response (such as the id of the new group)
+|DESCRIPTION:$1.|GROUPID:best.|ISACTIVE:best.|NAME:$11.|
+|---|---|---|---|
+|`The group description`|`2 `|`1 `|`mynewergroup `|
+
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mp_abort.sas
+
+  <h4> Related Files </h4>
+  @li ms_creategroup.test.sas
+  @li ms_getgroups.sas
+
+**/
+
+%macro ms_creategroup(groupname
+    ,desc=0
+    ,isactive=true
+    ,outds=work.ms_creategroup
+    ,mdebug=0
+  );
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_creategroup.sas
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+
+%local fref0 fref1 fref2 libref optval rc msg;
+%let fref0=%mf_getuniquefileref();
+%let fref1=%mf_getuniquefileref();
+%let fref2=%mf_getuniquefileref();
+%let libref=%mf_getuniquelibref();
+
+/* avoid sending bom marker to API */
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
+data _null_;
+  file &fref0 termstr=crlf;
+  name=quote(cats(symget('groupname')));
+  description=quote(cats(symget('desc')));
+  if cats(description)='"0"' then description=name;
+  isactive=symget('isactive');
+%if &mdebug=1 %then %do;
+  putlog _all_;
+%end;
+
+  put '{'@;
+  put '"name":' name @;
+  put ',"description":' description @;
+  put ',"isActive":' isactive @;
+  put '}';
+run;
+
+data _null_;
+  file &fref1 lrecl=1000;
+  infile "&_sasjs_tokenfile" lrecl=1000;
+  input;
+  if _n_=1 then do;
+    put "Content-Type: application/json";
+    put "accept: application/json";
+  end;
+  put _infile_;
+run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fref0;
+    input;
+    put _infile_;
+  data _null_;
+    infile &fref1;
+    input;
+    put _infile_;
+  run;
+%end;
+
+proc http method='POST' in=&fref0 headerin=&fref1 out=&fref2
+  url="&_sasjs_apiserverurl/SASjsApi/group";
+%if &mdebug=1 %then %do;
+  debug level=1;
+%end;
+run;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_creategroup.sas
+  ,msg=%str(Issue submitting query to SASjsApi/group)
+)
+
+libname &libref JSON fileref=&fref2;
+
+data &outds;
+  set &libref..root;
+  drop ordinal_root;
+%if &mdebug=1 %then %do;
+  putlog _all_;
+%end;
+run;
+
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_creategroup.sas
+  ,msg=%str(Issue reading response JSON)
+)
+
+/* reset options */
+options &optval;
+
+%if &mdebug=0 %then %do;
+  filename &fref0 clear;
+  filename &fref1 clear;
+  filename &fref2 clear;
+  libname &libref clear;
+%end;
+%else %do;
+  data _null_;
+    infile &fref2;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend ms_creategroup;

server/ms_createwebservice.sas

@@ -266,7 +266,7 @@ data _null_;
   put '      %do i=1 %to &numcols; ';
   put '        %if &i>1 %then  "," ; ';
   put '        %if &action=OBJ %then """&&name&i"":" ; ';
-  put '        &&name&i ';
+  put '        "&&name&i"n /* name literal for reserved variable names */ ';
   put '      %end; ';
   put '      %if &action=ARR %then "]" ; %else "}" ; ; ';
   put '    /* now write the long strings to _webout 1 byte at a time */ ';

server/ms_getgroups.sas

@@ -0,0 +1,105 @@
+/**
+  @file
+  @brief Fetches the list of groups from SASjs Server
+  @details Fetches the list of groups from SASjs Server and writes them to an
+  output dataset.
+
+  Example:
+
+      %ms_getgroups(outds=userlist)
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outds= (work.ms_getgroups) This output dataset will contain the
+    list of groups. Format:
+|NAME:$32.|DESCRIPTION:$64.|GROUPID:best.|
+|---|---|---|
+|`SomeGroup `|`A group `|`1`|
+|`Another Group`|`this is a different group`|`2`|
+|`admin`|`Administrators `|`3`|
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mp_abort.sas
+
+  <h4> Related Files </h4>
+  @li ms_creategroup.sas
+  @li ms_getusers.test.sas
+
+**/
+
+%macro ms_getgroups(
+    outds=work.ms_getgroups
+    ,mdebug=0
+  );
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_getusers.sas
+  ,msg=%str(syscc=&syscc on macro entry)
+)
+
+%local fref0 fref1 libref optval rc msg;
+%let fref0=%mf_getuniquefileref();
+%let fref1=%mf_getuniquefileref();
+%let libref=%mf_getuniquelibref();
+
+/* avoid sending bom marker to API */
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
+data _null_;
+  file &fref0 lrecl=1000;
+  infile "&_sasjs_tokenfile" lrecl=1000;
+  input;
+  if _n_=1 then put "accept: application/json";
+  put _infile_;
+run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fref0;
+    input;
+    put _infile_;
+  run;
+%end;
+
+proc http method='GET' headerin=&fref0 out=&fref1
+  url="&_sasjs_apiserverurl/SASjsApi/group";
+%if &mdebug=1 %then %do;
+  debug level=1;
+%end;
+run;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_getgroups.sas
+  ,msg=%str(Issue submitting GET query to SASjsApi/group)
+)
+
+libname &libref JSON fileref=&fref1;
+
+data &outds;
+  length NAME $32 DESCRIPTION $64. GROUPID 8;
+  if _n_=1 then call missing(of _all_);
+  set &libref..root;
+  drop ordinal_root;
+run;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=ms_getusers.sas
+  ,msg=%str(Issue reading response JSON)
+)
+
+/* reset options */
+options &optval;
+
+%if &mdebug=1 %then %do;
+  filename &fref0 clear;
+  filename &fref1 clear;
+  libname &libref clear;
+%end;
+
+%mend ms_getgroups;

server/ms_runstp.sas

@@ -162,6 +162,7 @@ proc http method='POST' headerin=&authref in=&mainref out=&outref
 %end;
 run;
 %if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+or &mdebug=1
 %then %do;
   data _null_;infile &outref;input;putlog _infile_;run;
 %end;
@@ -177,7 +178,7 @@ options &optval;
 %if &outlogds ne _null_ or &mdebug=1 %then %do;
   %local dumplib;
   %let dumplib=%mf_getuniquelibref();
-  libname &dumplib json (&outref);
+  libname &dumplib json fileref=&outref;
   data &outlogds;
     set &dumplib..log;
   %if &mdebug=1 %then %do;
@@ -195,4 +196,4 @@ options &optval;
   filename &authref;
   filename &mainref;
 %end;
-%mend ms_runstp;
+%mend ms_runstp;

tests/base/mp_dirlist.test.sas

@@ -21,7 +21,8 @@
 %mf_mkdir(&root/a/d)
 %mf_mkdir(&root/a/e)
 %mf_mkdir(&root/a/e/f)
-data "&root/a/e/f/ds1.sas7bdat";
+libname test "&root/a/e/f";
+data test.ds1;
   x=1;
 run;
 

tests/base/mp_filtercheck.test.sas

@@ -6,11 +6,39 @@
   @li mp_filtercheck.sas
   @li mp_assertdsobs.sas
   @li mp_assert.sas
+  @li mp_assertscope.sas
 
 **/
 
+/* set up test data */
+data work.class ;
+length name $8 sex $1 age height weight 8;
+infile cards dsd;
+input Name:$char. Sex :$char. Age Height Weight;
+datalines4;
+Alfred,M,14,69,112.5
+Alice,F,13,56.5,84
+Barbara,F,13,65.3,98
+Carol,F,14,62.8,102.5
+Henry,M,14,63.5,102.5
+James,M,12,57.3,83
+Jane,F,12,59.8,84.5
+Janet,F,15,62.5,112.5
+Jeffrey,M,13,62.5,84
+John,M,12,59,99.5
+Joyce,F,11,51.3,50.5
+Judy,F,14,64.3,90
+Louise,F,12,56.3,77
+Mary,F,15,66.5,112
+Philip,M,16,72,150
+Robert,M,12,64.8,128
+Ronald,M,15,67,133
+Thomas,M,11,57.5,85
+William,M,15,66.5,112
+;;;;
+run;
 
-/* valid filter */
+/* valid filter conditions */
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
@@ -21,14 +49,19 @@ AND,AND,1,SEX,<=,"'M'"
 AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
 AND,OR,2,Weight,>=,77.7
 AND,OR,2,Weight,NE,77.7
+AND,AND,1,age,=,.A
+AND,AND,1,height,<,.B
 ;;;;
 run;
 
+%mp_assertscope(SNAPSHOT)
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )
+%mp_assertscope(COMPARE)
+
 %let syscc=0;
 %mp_assertdsobs(work.badrecords,
   desc=Valid filter query,
@@ -49,7 +82,7 @@ AND,OR,2,Weight,>=,7
 ;;;;
 run;
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )
@@ -71,7 +104,7 @@ AND,OR,2,Name,NOT IN,"(''''Jane','Alfred')"
 run;
 
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )
@@ -94,7 +127,7 @@ AND,OR,2,Weight,>=,7
 run;
 
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )
@@ -115,7 +148,7 @@ AND,AND,1,age,=,;;%abort
 ;;;;
 run;
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )
@@ -137,7 +170,7 @@ AND,AND,1,age,=,0
 run;
 %let syscc=0;
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )

tests/base/mp_filtergenerate.test.sas

@@ -11,6 +11,34 @@
 
 options source2;
 
+/* set up test data */
+data work.class ;
+length name $8 sex $1 age height weight 8;
+infile cards dsd;
+input Name:$char. Sex :$char. Age Height Weight;
+datalines4;
+Alfred,M,14,69,112.5
+Alice,F,13,56.5,84
+Barbara,F,13,65.3,98
+Carol,F,14,62.8,102.5
+Henry,M,14,63.5,102.5
+James,M,12,57.3,83
+Jane,F,12,59.8,84.5
+Janet,F,15,62.5,112.5
+Jeffrey,M,13,62.5,84
+John,M,12,59,99.5
+Joyce,F,11,51.3,50.5
+Judy,F,14,64.3,90
+Louise,F,12,56.3,77
+Mary,F,15,66.5,112
+Philip,M,16,72,150
+Robert,M,12,64.8,128
+Ronald,M,15,67,133
+Thomas,M,11,57.5,85
+William,M,15,66.5,112
+;;;;
+run;
+
 /* valid filter */
 data work.inds;
   infile datalines4 dsd;
@@ -23,10 +51,10 @@ AND,OR,2,Name,NOT IN,"('Jane','Janet')"
 AND,OR,2,Weight,>=,84.6
 ;;;;
 run;
-%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtercheck(work.inds,targetds=work.class)
 %mp_filtergenerate(work.inds,outref=myfilter)
 data work.test;
-  set sashelp.class;
+  set work.class;
   where %inc myfilter;;
 run;
 %mp_assertdsobs(work.test,
@@ -43,10 +71,10 @@ data work.inds;
 datalines4;
 ;;;;
 run;
-%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtercheck(work.inds,targetds=work.class)
 %mp_filtergenerate(work.inds,outref=myfilter)
 data work.test;
-  set sashelp.class;
+  set work.class;
   where %inc myfilter;;
 run;
 %mp_assertdsobs(work.test,
@@ -64,10 +92,10 @@ datalines4;
 AND,OR,2,Name,IN,"('Jane','Janet')"
 ;;;;
 run;
-%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtercheck(work.inds,targetds=work.class)
 %mp_filtergenerate(work.inds,outref=myfilter)
 data work.test;
-  set sashelp.class;
+  set work.class;
   where %inc myfilter;;
 run;
 %mp_assertdsobs(work.test,
@@ -86,10 +114,10 @@ OR,OR,2,Name,IN,"('Jane','Janet')"
 OR,OR,3,Name,IN,"('James')"
 ;;;;
 run;
-%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtercheck(work.inds,targetds=work.class)
 %mp_filtergenerate(work.inds,outref=myfilter)
 data work.test;
-  set sashelp.class;
+  set work.class;
   where %inc myfilter;;
 run;
 %mp_assertdsobs(work.test,
@@ -108,10 +136,10 @@ AND,OR,2,Name,IN,"('Jane','Janet')"
 AND,OR,3,Name,IN,"('James')"
 ;;;;
 run;
-%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtercheck(work.inds,targetds=work.class)
 %mp_filtergenerate(work.inds,outref=myfilter)
 data work.test;
-  set sashelp.class;
+  set work.class;
   where %inc myfilter;;
 run;
 %mp_assertdsobs(work.test,

tests/base/mp_filterstore.test.1.sas

@@ -10,6 +10,34 @@
 
 **/
 
+/* set up test data */
+data work.class ;
+length name $8 sex $1 age height weight 8;
+infile cards dsd;
+input Name:$char. Sex :$char. Age Height Weight;
+datalines4;
+Alfred,M,14,69,112.5
+Alice,F,13,56.5,84
+Barbara,F,13,65.3,98
+Carol,F,14,62.8,102.5
+Henry,M,14,63.5,102.5
+James,M,12,57.3,83
+Jane,F,12,59.8,84.5
+Janet,F,15,62.5,112.5
+Jeffrey,M,13,62.5,84
+John,M,12,59,99.5
+Joyce,F,11,51.3,50.5
+Judy,F,14,64.3,90
+Louise,F,12,56.3,77
+Mary,F,15,66.5,112
+Philip,M,16,72,150
+Robert,M,12,64.8,128
+Ronald,M,15,67,133
+Thomas,M,11,57.5,85
+William,M,15,66.5,112
+;;;;
+run;
+
 libname permlib (work);
 
 %mp_coretable(LOCKTABLE,libds=permlib.locktable)
@@ -31,7 +59,7 @@ AND,OR,2,Weight,NE,77.7
 ;;;;
 run;
 
-%mp_filterstore(libds=sashelp.class,
+%mp_filterstore(libds=work.class,
   queryds=work.inds,
   filter_summary=permlib.filtsum,
   filter_detail=permlib.filtdet,
@@ -60,7 +88,7 @@ select max(filter_rk) into: test1 from work.result;
 )
 
 /* Test 2 - load same table again and ensure we get the same RK */
-%mp_filterstore(libds=sashelp.class,
+%mp_filterstore(libds=work.class,
   queryds=work.inds,
   filter_summary=permlib.filtsum,
   filter_detail=permlib.filtdet,

tests/base/mp_filtervalidate.test.sas

@@ -7,9 +7,36 @@
   @li mp_filtervalidate.sas
   @li mp_assertdsobs.sas
   @li mp_assert.sas
+  @li mp_assertscope.sas
 
 **/
 
+data work.class ;
+length name $8 sex $1 age height weight 8;
+infile cards dsd;
+input Name:$char. Sex :$char. Age Height Weight;
+datalines4;
+Alfred,M,14,69,112.5
+Alice,F,13,56.5,84
+Barbara,F,13,65.3,98
+Carol,F,14,62.8,102.5
+Henry,M,14,63.5,102.5
+James,M,12,57.3,83
+Jane,F,12,59.8,84.5
+Janet,F,15,62.5,112.5
+Jeffrey,M,13,62.5,84
+John,M,12,59,99.5
+Joyce,F,11,51.3,50.5
+Judy,F,14,64.3,90
+Louise,F,12,56.3,77
+Mary,F,15,66.5,112
+Philip,M,16,72,150
+Robert,M,12,64.8,128
+Ronald,M,15,67,133
+Thomas,M,11,57.5,85
+William,M,15,66.5,112
+;;;;
+run;
 
 /* valid filter */
 data work.inds;
@@ -21,10 +48,16 @@ AND,AND,1,AGE,>,5
 AND,AND,1,SEX,NE,"'M'"
 AND,OR,2,Name,NOT IN,"('Jane','Janet')"
 AND,OR,2,Weight,>=,84.6
+AND,AND,3,age,=,.a
+AND,AND,4,weight,NE,._
 ;;;;
 run;
 %mp_filtergenerate(work.inds,outref=myfilter)
-%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+
+%mp_assertscope(SNAPSHOT)
+%mp_filtervalidate(myfilter,work.class,outds=work.results,abort=NO)
+%mp_assertscope(COMPARE)
+
 %mp_assertdsobs(work.results,
   desc=Valid filter,
   test=EMPTY,
@@ -40,7 +73,7 @@ datalines4;
 ;;;;
 run;
 %mp_filtergenerate(work.inds,outref=myfilter)
-%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%mp_filtervalidate(myfilter,work.class,outds=work.results,abort=NO)
 %mp_assertdsobs(work.results,
   desc=Empty filter,
   test=EMPTY,
@@ -59,7 +92,7 @@ AND,AND,1,SEX,NE,2
 ;;;;
 run;
 %mp_filtergenerate(work.inds,outref=myfilter)
-%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%mp_filtervalidate(myfilter,work.class,outds=work.results,abort=NO)
 %let syscc=0;
 %let test3=0;
 data _null_;
@@ -84,7 +117,7 @@ AND,AND,1,age,NE,"'M'"
 ;;;;
 run;
 %mp_filtergenerate(work.inds,outref=myfilter)
-%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%mp_filtervalidate(myfilter,work.class,outds=work.results,abort=NO)
 %let syscc=0;
 %let test4=0;
 data _null_;