fix: missing end comment in mp_init.sas

feat: new macros

.all-contributorsrc

@@ -108,6 +108,15 @@
         "test",
         "review"
       ]
+    },
+    {
+      "login": "vznesh",
+      "name": "Vignesh T.",
+      "avatar_url": "https://avatars.githubusercontent.com/u/28916792?v=4",
+      "profile": "https://github.com/vznesh",
+      "contributions": [
+        "bug"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.editorconfig

@@ -1,12 +0,0 @@
-# http://editorconfig.org
-root = true
-
-[*]
-charset = utf-8
-indent_style = space
-indent_size = 2
-end_of_line = lf
-insert_final_newline = false
-trim_trailing_whitespace = true
-
-

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2020 (Allan Bowe)
+Copyright 2021 (Allan Bowe)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

README.md

@@ -1,8 +1,7 @@
 # Macro Core
 [![npm package][npm-image]][npm-url]
 [![Github Workflow][githubworkflow-image]][githubworkflow-url]
-[![Dependency Status][dependency-image]][dependency-url]
-[![npm](https://img.shields.io/npm/dt/@sasjs/core)]()
+![npm](https://img.shields.io/npm/dt/@sasjs/core)
 ![Snyk Vulnerabilities for npm package](https://img.shields.io/snyk/vulnerabilities/npm/@sasjs/core)
 [![License](https://img.shields.io/apm/l/atomic-design-ui.svg)](/LICENSE)
 ![GitHub top language](https://img.shields.io/github/languages/top/sasjs/core)
@@ -16,11 +15,9 @@
 [npm-url]:http://npmjs.org/package/@sasjs/core
 [githubworkflow-image]:https://github.com/sasjs/core/actions/workflows/main.yml/badge.svg
 [githubworkflow-url]:https://github.com/sasjs/core/blob/main/.github/workflows/main.yml
-[dependency-image]:https://david-dm.org/sasjs/core.svg
 [dependency-url]:https://github.com/sasjs/core/blob/main/package.json
 
 
-
 Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
 
 You can download and compile them all in just two lines of SAS code:
@@ -32,49 +29,61 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 
 Documentation: https://core.sasjs.io
 
-# Components
+## Components
 
-**base** library (SAS9/Viya)
+### BASE library (SAS9/Viya)
 
 - OS independent
 - Not metadata aware
 - No X command
 - Prefixes: _mf_, _mp_
 
-**fcmp** library (SAS9/Viya)
+#### FCMP library (SAS9/Viya)
 - Function and macro names are identical, except for special cases
 - Prefixes: _mcf_
 
 The fcmp macros are used to generate fcmp functions, and can be used with or
 without the `proc fcmp` wrapper.
 
-**meta** library (SAS9 only)
+### META library (SAS9 only)
+
+Macros used in SAS EBI, which connect to the metadata server.
 
 - OS independent
 - Metadata aware
 - No X command
 - Prefixes: _mm_
 
-**viya** library (Viya only)
+### SERVER library (@sasjs/server only)
+These macros are used for building applications using [@sasjs/server](https://server.sasjs.io) - an open source REST API for Desktop SAS.
+
+- OS independent
+- @sasjs/server aware
+- No X command
+- Prefixes: _ms_
+
+### VIYA library (Viya only)
+
+Macros used for interfacing with SAS Viya.
 
 - OS independent
 - No X command
-- Prefixes: _mv_
+- Prefixes: _mv_, _mvf_
 
-**metax** library (SAS9 only)
+### METAX library (SAS9 only)
 
 - OS specific
 - Metadata aware
 - X command enabled
 - Prefixes: _mmw_,_mmu_,_mmx_
 
-**lua** library
+### LUA library
 
 Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
 
 To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert your LUA into a data step with put statements, and create the macro wrapper with a `ml_` prefix.  You can then use your module in any program by running:
 
-```
+```sas
 /* compile the lua module */
 %ml_yourmodule()
 
@@ -89,7 +98,7 @@ run;
 - X command enabled
 - Prefixes: _mmw_,_mmu_,_mmx_
 
-# Installation
+## Installation
 
 First, download the repo to a location your SAS system can access. Then update your sasautos path to include the components you wish to have available, eg:
 
@@ -107,9 +116,9 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 %inc mc;
 ```
 
-# Standards
+## Standards
 
-## File Properties
+### File Properties
 
 - filenames much match macro names
 - filenames must be lowercase, without spaces
@@ -117,19 +126,20 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 - one macro per file
 - prefixes:
   - _mf_ for macro functions (can be used in open code).
-  - _mp_ for macro procedures (which generate sas code)
+  - _ml_ for macros that are used to compile LUA modules
   - _mm_ for metadata macros (interface with the metadata server).
   - _mmx_ for macros that use metadata and are XCMD enabled
+  - _mp_ for macro procedures (which generate sas code)
+  - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
+  - _mv_ for macro procedures that will only work in Viya
   - _mx_ for macros that are XCMD enabled
-  - _ml_ for macros that are used to compile LUA modules
-  - _mv_ for macros that will only work in Viya
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
 - UTF-8
 
 
-## Header Properties
+### Header Properties
 
 The **Macro Core** documentation is created using [doxygen](http://www.doxygen.nl). A full list of attributes can be found [here](http://www.doxygen.nl/manual/commands.html) but the following are most relevant:
 
@@ -143,7 +153,7 @@ The **Macro Core** documentation is created using [doxygen](http://www.doxygen.n
 
 All macros must be commented in the doxygen format, to enable the [online documentation](https://core.sasjs.io).
 
-### Dependencies
+#### Dependencies
 SAS code can contain one of two types of dependency - SAS Macros, and SAS Includes.  When compiling projects using the [SASjs CLI](https://cli.sasjs.io) the doxygen header is scanned for `  @li` items under the following headers:
 
 ```sas
@@ -161,7 +171,7 @@ The CLI can then extract all the dependencies and insert as precode (SAS Macros)
 When contributing to this library, it is therefore important to ensure that all dependencies are listed in the header in this format.
 
 
-## Coding Standards
+### Coding Standards
 
 - Indentation = 2 spaces. No tabs!
 - no trailing white space
@@ -174,7 +184,7 @@ When contributing to this library, it is therefore important to ensure that all
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
 - The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
 
-# General Notes
+## General Notes
 
 - All macros should be compatible with SAS versions from support level B and above (so currently 9.2 and later). If an earlier version is not supported, then the macro should say as such in the header documentation, and exit gracefully (eg `%if %sysevalf(&sysver<9.3) %then %return`).
 
@@ -186,10 +196,9 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 
 
-
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-9-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-10-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -209,6 +218,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
   <tr>
     <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
     <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
+    <td align="center"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
   </tr>
 </table>
 

base/mf_existds.sas

@@ -12,6 +12,10 @@
 
   @param libds library.dataset
   @return output returns 1 or 0
+
+  <h4> Related Macros </h4>
+  @li mf_existds.test.sas
+
   @warning Untested on tables registered in metadata but not physically present
   @version 9.2
   @author Allan Bowe

base/mf_getquotedstr.sas

@@ -15,12 +15,12 @@
   for:
 > "these","words","are","double","quoted"
 
-  @param in_str the unquoted, spaced delimited string to transform
-  @param dlm= the delimeter to be applied to the output (default comma)
-  @param indlm= the delimeter used for the input (default is space)
-  @param quote= the quote mark to apply (S=Single, D=Double). If any other value
-    than uppercase S or D is supplied, then that value will be used as the
-    quoting character.
+  @param [in] in_str The unquoted, spaced delimited string to transform
+  @param [in] dlm= The delimeter to be applied to the output (default comma)
+  @param [in] indlm= (,) The delimeter used for the input (default is space)
+  @param [in] quote= (S) The quote mark to apply (S=Single, D=Double, N=None).
+    If any other value than uppercase S or D is supplied, then that value will
+    be used as the quoting character.
   @return output returns a string with the newly quoted / delimited output.
 
   @version 9.2
@@ -30,9 +30,10 @@
 
 %macro mf_getquotedstr(IN_STR,DLM=%str(,),QUOTE=S,indlm=%str( )
 )/*/STORE SOURCE*/;
-  %if &quote=S %then %let quote=%str(%');
-  %else %if &quote=D %then %let quote=%str(%");
-  %else %let quote=%str();
+  /* credit Rowland Hale  - byte34 is double quote, 39 is single quote */
+  %if &quote=S %then %let quote=%qsysfunc(byte(39));
+  %else %if &quote=D %then %let quote=%qsysfunc(byte(34));
+  %else %if &quote=N %then %let quote=;
   %local i item buffer;
   %let i=1;
   %do %while (%qscan(&IN_STR,&i,%str(&indlm)) ne %str() ) ;

base/mf_getvalue.sas

@@ -10,6 +10,9 @@
   <h4> SAS Macros </h4>
   @li mf_getattrn.sas
 
+  <h4> Related Macros </h4>
+  @li mp_setkeyvalue.sas
+
   @param libds dataset to query
   @param variable the variable which contains the value to return.
   @param filter contents of where clause

base/mp_assertdsobs.sas

@@ -8,6 +8,10 @@
 
       %mp_assertdsobs(sashelp.class) %* tests if any observations are present;
 
+      %mp_assertdsobs(sashelp.class,test=ATLEAST 10) %* pass if >9 obs present;
+
+      %mp_assertdsobs(sashelp.class,test=ATMOST 20) %* pass if <21 obs present;
+
   <h4> SAS Macros </h4>
   @li mf_nobs.sas
   @li mp_abort.sas
@@ -19,9 +23,9 @@
     @li HASOBS - Test is a PASS if the input dataset has any observations
     @li EMPTY - Test is a PASS if input dataset is empty
     @li EQUALS [integer] - Test passes if row count matches the provided integer
-    @LI ATLEAST [integer] - Test passes if row count is more than or equal to
+    @li ATLEAST [integer] - Test passes if row count is more than or equal to
       the provided integer
-    @LI ATMOST [integer] - Test passes if row count is less than or equal to
+    @li ATMOST [integer] - Test passes if row count is less than or equal to
       the provided integer
   @param [out] outds= (work.test_results) The output dataset to contain the
   results.  If it does not exist, it will be created, with the following format:

base/mp_copyfolder.sas

@@ -0,0 +1,79 @@
+/**
+  @file
+  @brief A macro to recursively copy a directory
+  @details Performs a recursive directory listing then works from top to bottom
+    copying files and creating subdirectories.
+
+  Usage:
+
+      %let rootdir=%sysfunc(pathname(work))/demo;
+      %let copydir=%sysfunc(pathname(work))/demo_copy;
+      %mf_mkdir(&rootdir)
+      %mf_mkdir(&rootdir/subdir)
+      %mf_mkdir(&rootdir/subdir/subsubdir)
+      data "&rootdir/subdir/example.sas7bdat";
+      run;
+
+      %mp_copyfolder(&rootdir,&copydir)
+
+  @param source Unquoted path to the folder to copy from.
+  @param target Unquoted path to the folder to copy to.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_isdir.sas
+  @li mf_mkdir.sas
+  @li mp_abort.sas
+  @li mp_dirlist.sas
+
+  <h4> Related Macros </h4>
+  @li mp_copyfolder.test.sas
+
+**/
+
+%macro mp_copyfolder(source,target);
+
+  %mp_abort(iftrue=(%mf_isdir(&source)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Source dir does not exist (&source))
+  )
+
+  %mf_mkdir(&target)
+
+  %mp_abort(iftrue=(%mf_isdir(&target)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Target dir could not be created (&target))
+  )
+
+  /* prep temp table */
+  %local tempds;
+  %let tempds=%mf_getuniquename();
+
+  /* recursive directory listing */
+  %mp_dirlist(path=&source,outds=work.&tempds, maxdepth=MAX)
+
+  /* create folders and copy content */
+  data _null_;
+    set work.&tempds;
+    if _n_ = 1 then dpos+sum(length(directory),2);
+    filepath2="&target/"!!substr(filepath,dpos);
+    if file_or_folder='folder' then call execute('%mf_mkdir('!!filepath2!!')');
+    else do;
+      length fref1 fref2 $8;
+      rc1=filename(fref1,filepath,'disk','recfm=n');
+      rc2=filename(fref2,filepath2,'disk','recfm=n');
+      if fcopy(fref1,fref2) ne 0 then do;
+        sysmsg=sysmsg();
+        putlog "%str(ERR)OR: Unable to copy " filepath " to " filepath2;
+        putlog sysmg=;
+      end;
+    end;
+    rc=filename(fref1);
+    rc=filename(fref2);
+  run;
+
+  /* tidy up */
+  proc sql;
+  drop table work.&tempds;
+
+%mend mp_copyfolder;

base/mp_deletefolder.sas

@@ -0,0 +1,69 @@
+/**
+  @file
+  @brief A macro to delete a directory
+  @details Will delete all folder content (including subfolder content) and
+    finally, the folder itself.
+
+  Usage:
+
+      %let rootdir=%sysfunc(pathname(work))/demo;
+      %mf_mkdir(&rootdir)
+      %mf_mkdir(&rootdir/subdir)
+      %mf_mkdir(&rootdir/subdir/subsubdir)
+      data "&rootdir/subdir/example.sas7bdat";
+      run;
+
+      %mp_deletefolder(&rootdir)
+
+  @param path Unquoted path to the folder to delete.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_isdir.sas
+  @li mp_dirlist.sas
+
+  <h4> Related Macros </h4>
+  @li mp_deletefolder.test.sas
+
+**/
+
+%macro mp_deletefolder(folder);
+  /* proceed if valid directory */
+  %if %mf_isdir(&folder)=1 %then %do;
+
+    /* prep temp table */
+    %local tempds;
+    %let tempds=%mf_getuniquename();
+
+    /* recursive directory listing */
+    %mp_dirlist(path=&folder,outds=work.&tempds, maxdepth=MAX)
+
+    /* sort descending level so can delete folder contents before folders */
+    proc sort data=work.&tempds;
+      by descending level;
+    run;
+
+    /* ensure top level folder is removed at the end */
+    proc sql;
+    insert into work.&tempds set filepath="&folder";
+
+    /* delete everything */
+    data _null_;
+      set work.&tempds end=last;
+      length fref $8;
+      rc=filename(fref,filepath);
+      rc=fdelete(fref);
+      if rc then do;
+        msg=sysmsg();
+        put "&sysmacroname:" / rc= / msg= / filepath=;
+      end;
+      rc=filename(fref);
+    run;
+
+    /* tidy up */
+    proc sql;
+    drop table work.&tempds;
+
+  %end;
+  %else %put &sysmacroname: &folder: is not a valid / accessible folder. ;
+%mend mp_deletefolder;

base/mp_dirlist.sas

@@ -3,20 +3,13 @@
   @brief Returns all files and subdirectories within a specified parent
   @details When used with getattrs=NO, is not OS specific (uses dopen / dread).
 
-  If getattrs=YES then the doptname / foptname functions are used to scan all
-  properties - any characters that are not valid in a SAS name (v7) are simply
-  stripped, and the table is transposed so theat each property is a column
-  and there is one file per row.  An attempt is made to get all properties
-  whether a file or folder, but some files/folders cannot be accessed, and so
-  not all properties can / will be populated.
-
   Credit for the rename approach:
   https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003
 
 
   usage:
 
-      %mp_dirlist(path=/some/location,outds=myTable)
+      %mp_dirlist(path=/some/location, outds=myTable, maxdepth=MAX)
 
       %mp_dirlist(outds=cwdfileprops, getattrs=YES)
 
@@ -30,11 +23,19 @@
   X CMD) do please raise an issue!
 
 
-  @param path= for which to return contents
-  @param fref= Provide a DISK engine fileref as an alternative to PATH
-  @param outds= the output dataset to create
-  @param getattrs= YES/NO (default=NO).  Uses doptname and foptname to return
-  all attributes for each file / folder.
+  @param [in] path= for which to return contents
+  @param [in] fref= Provide a DISK engine fileref as an alternative to PATH
+  @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
+    subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
+  @param [out] outds= the output dataset to create
+  @param [out] getattrs= (NO)  If getattrs=YES then the doptname / foptname
+    functions are used to scan all properties - any characters that are not
+    valid in a SAS name (v7) are simply stripped, and the table is transposed
+    so theat each property is a column and there is one file per row.  An
+    attempt is made to get all properties whether a file or folder, but some
+    files/folders cannot be accessed, and so not all properties can / will be
+    populated.
 
 
   @returns outds contains the following variables:
@@ -44,8 +45,15 @@
     - filename (just the file name)
     - ext (.extension)
     - msg (system message if any issues)
+    - level (depth of folder)
     - OS SPECIFIC variables, if <code>getattrs=</code> is used.
 
+  <h4> SAS Macros </h4>
+  @li mp_dropmembers.sas
+
+  <h4> Related Macros </h4>
+  @li mp_dirlist.test.sas
+
   @version 9.2
   @author Allan Bowe
 **/
@@ -54,14 +62,27 @@
     , fref=0
     , outds=work.mp_dirlist
     , getattrs=NO
+    , maxdepth=0
+    , level=0 /* The level of recursion to perform.  For internal use only. */
 )/*/STORE SOURCE*/;
 %let getattrs=%upcase(&getattrs)XX;
 
-data &outds(compress=no
-    keep=file_or_folder filepath filename ext msg directory
+/* temp table */
+%local out_ds;
+data;run;
+%let out_ds=%str(&syslast);
+
+/* drop main (top) table if it exists */
+%if &level=0 %then %do;
+  %mp_dropmembers(%scan(&outds,-1,.), libref=WORK)
+%end;
+
+data &out_ds(compress=no
+    keep=file_or_folder filepath filename ext msg directory level
   );
   length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
     ext $20 msg $200;
+  retain level &level;
   %if &fref=0 %then %do;
     rc = filename(fref, "&path");
   %end;
@@ -119,8 +140,8 @@ data &outds(compress=no
 run;
 
 %if %substr(&getattrs,1,1)=Y %then %do;
-  data &outds;
-    set &outds;
+  data &out_ds;
+    set &out_ds;
     length infoname infoval $60 fref $8;
     rc=filename(fref,filepath);
     drop rc infoname fid i close fref;
@@ -161,10 +182,36 @@ run;
   run;
   proc sort;
     by filepath sasname;
-  proc transpose data=&outds out=&outds(drop=_:);
+  proc transpose data=&out_ds out=&out_ds(drop=_:);
     id sasname;
     var infoval;
     by filepath file_or_folder filename ext ;
   run;
 %end;
+
+data &out_ds;
+  set &out_ds(where=(filepath ne ''));
+run;
+
+/* update main table */
+proc append base=&outds data=&out_ds;
+run;
+
+/* recursive call */
+%if &maxdepth>&level or &maxdepth=MAX %then %do;
+  data _null_;
+    set &out_ds;
+    where file_or_folder='folder';
+    length code $10000;
+    code=cats('%nrstr(%mp_dirlist(path=',filepath,",outds=&outds"
+      ,",getattrs=&getattrs,level=%eval(&level+1),maxdepth=&maxdepth))");
+    put code=;
+    call execute(code);
+  run;
+%end;
+
+/* tidy up */
+proc sql;
+drop table &out_ds;
+
 %mend mp_dirlist;

base/mp_dropmembers.sas

@@ -17,8 +17,9 @@
   @li mf_isblank.sas
 
 
-  @param list space separated list of datasets / views, WITHOUT libref
-  @param libref= (WORK) Note - you can only drop from a single library at a time
+  @param [in] list space separated list of datasets / views, WITHOUT libref
+  @param [in] libref= (WORK) Note - you can only drop from one library at a time
+  @param [in] iftrue= (1=1) Conditionally drop tables, eg if &debug=N
 
   @version 9.2
   @author Allan Bowe
@@ -28,8 +29,11 @@
 %macro mp_dropmembers(
     list /* space separated list of datasets / views */
     ,libref=WORK  /* can only drop from a single library at a time */
+    ,iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
   %if %mf_isblank(&list) %then %do;
     %put NOTE: nothing to drop!;
     %return;

base/mp_getcols.sas

@@ -51,7 +51,7 @@ data &outds(keep=name type length varnum format label ddtype);
     else if formatd=0 then format=cats(format2,formatl,'.');
     else format=cats(format2,formatl,'.',formatd);
     type='N';
-    if format=:'DATETIME' then ddtype='DATETIME';
+    if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
     else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
       or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
       or format=:'MONYY'

base/mp_getconstraints.sas

@@ -4,24 +4,27 @@
   @details Useful for capturing constraints before they are dropped / reapplied
   during an update.
 
-        proc sql;
-        create table work.example(
-          TX_FROM float format=datetime19.,
-          DD_TYPE char(16),
-          DD_SOURCE char(2048),
-          DD_SHORTDESC char(256),
-          constraint pk primary key(tx_from, dd_type,dd_source),
-          constraint unq unique(tx_from, dd_type),
-          constraint nnn not null(DD_SHORTDESC)
-        );
+      proc sql;
+      create table work.example(
+        TX_FROM float format=datetime19.,
+        DD_TYPE char(16),
+        DD_SOURCE char(2048),
+        DD_SHORTDESC char(256),
+        constraint pk primary key(tx_from, dd_type,dd_source),
+        constraint unq unique(tx_from, dd_type),
+        constraint nnn not null(DD_SHORTDESC)
+      );
 
       %mp_getconstraints(lib=work,ds=example,outds=work.constraints)
 
-  @param lib= The target library (default=WORK)
-  @param ds= The target dataset.  Leave blank (default) for all datasets.
-  @param outds the output dataset
+  @param [in] lib= (WORK) The target library
+  @param [in] ds= The target dataset.  Leave blank (default) for all datasets.
+  @param [in] mdebug= (0) Set to 1 to preserve temp tables, print var values etc
+  @param [out] outds= (mp_getconstraints) the output dataset
 
   <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mp_dropmembers.sas
 
   @version 9.2
   @author Allan Bowe
@@ -31,11 +34,33 @@
 %macro mp_getconstraints(lib=WORK
   ,ds=
   ,outds=mp_getconstraints
+  ,mdebug=0
 )/*/STORE SOURCE*/;
 
 %let lib=%upcase(&lib);
 %let ds=%upcase(&ds);
 
+/**
+  * Neither dictionary tables nor sashelp provides a constraint order column,
+  * however they DO arrive in the correct order.  So, create the col.
+  **/
+%local vw;
+%let vw=%mf_getuniquename(prefix=mp_getconstraints_vw_);
+data &vw /view=&vw;
+  set sashelp.vcncolu;
+  where TABLE_CATALOG="&lib";
+
+  /* use retain approach to reset the constraint order with each constraint */
+  length tmp $1000;
+  retain tmp;
+  drop tmp;
+  if tmp ne catx('|',libref,table_name,constraint_type,constraint_name) then do;
+    constraint_order=1;
+  end;
+  else constraint_order+1;
+  tmp=catx('|',libref, table_name, constraint_type,constraint_name);
+run;
+
 /* must use SQL as proc datasets does not support length changes */
 proc sql noprint;
 create table &outds as
@@ -44,8 +69,9 @@ create table &outds as
     ,a.constraint_type
     ,a.constraint_name
     ,b.column_name
+    ,b.constraint_order
   from dictionary.TABLE_CONSTRAINTS a
-  left join dictionary.constraint_column_usage  b
+  left join &vw  b
   on upcase(a.TABLE_CATALOG)=upcase(b.TABLE_CATALOG)
     and upcase(a.TABLE_NAME)=upcase(b.TABLE_NAME)
     and a.constraint_name=b.constraint_name
@@ -58,6 +84,13 @@ create table &outds as
     and upcase(a.TABLE_NAME)="&ds"
     and upcase(b.TABLE_NAME)="&ds"
   %end;
+  order by libref, table_name, constraint_name, constraint_order
   ;
 
+/* tidy up */
+%mp_dropmembers(
+  &vw,
+  iftrue=(&mdebug=0)
+)
+
 %mend mp_getconstraints;

base/mp_getpk.sas

@@ -0,0 +1,259 @@
+/**
+  @file
+  @brief Extract the primary key fields from a table or library
+  @details Examines the constraints to identify primary key fields - indicated
+  by an explicit PK constraint, or a unique index that is also NOT NULL.
+
+  Can be executed at both table and library level.  Supports both BASE engine
+  libraries and SQL Server.
+
+  Usage:
+
+      proc sql;
+      create table work.example(
+        TX_FROM float format=datetime19.,
+        DD_TYPE char(16),
+        DD_SOURCE char(2048),
+        DD_SHORTDESC char(256),
+        constraint pk primary key(tx_from, dd_type,dd_source),
+        constraint unq unique(tx_from, dd_type),
+        constraint nnn not null(DD_SHORTDESC)
+      );
+      %mp_getpk(work,ds=example)
+
+  Returns:
+
+|libref:$8.|dsn:$32.|memtype:$8.|dbms_memtype:$32.|typemem:$8.|memlabel:$256.|nvar:best.|compress:$8.|pk_fields:$512.|
+|---|---|---|---|---|---|---|---|---|
+|WORK|EXAMPLE|DATA| |DATA| |4|NO|TX_FROM DD_TYPE DD_SOURCE|
+
+
+  @param [in] lib The libref to examine
+  @param [in] ds= (0) Select the dataset to examine, else use 0 for all tables
+  @param [in] mdebug= (0) Set to 1 to preserve temp tables, print var values etc
+  @param [out] outds= (work.mp_getpk) The name of the output table to create.
+
+  <h4> SAS Macros </h4>
+  @li mf_getengine.sas
+  @li mf_getschema.sas
+  @li mp_dropmembers.sas
+  @li mp_getconstraints.sas
+
+  <h4> Related Macros </h4>
+  @li mp_getpk.test.sas
+  @li mp_guesspk.sas
+
+  @version 9.3
+  @author Macro People Ltd
+**/
+
+%macro mp_getpk(
+  lib,
+  ds=0,
+  outds=work.mp_getpk,
+  mdebug=0
+)/*/STORE SOURCE*/;
+
+
+%local engine schema ds1 ds2 ds3 dsn tabs1 tabs2 sum pk4sure pkdefault finalpks;
+
+%let lib=%upcase(&lib);
+%let ds=%upcase(&ds);
+%let engine=%mf_getengine(&lib);
+%let schema=%mf_getschema(&lib);
+
+%let ds1=%mf_getuniquename(prefix=getpk_ds1);
+%let ds2=%mf_getuniquename(prefix=getpk_ds2);
+%let ds3=%mf_getuniquename(prefix=getpk_ds3);
+%let tabs1=%mf_getuniquename(prefix=getpk_tabs1);
+%let tabs2=%mf_getuniquename(prefix=getpk_tabs2);
+%let sum=%mf_getuniquename(prefix=getpk_sum);
+%let pk4sure=%mf_getuniquename(prefix=getpk_pk4sure);
+%let pkdefault=%mf_getuniquename(prefix=getpk_pkdefault);
+%let finalpks=%mf_getuniquename(prefix=getpk_finalpks);
+
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+proc sql;
+create table &ds1 as
+  select  libname as libref
+    ,upcase(memname) as dsn
+    ,memtype
+    ,upcase(name) as name
+    ,type
+    ,length
+    ,varnum
+    ,label
+    ,format
+    ,idxusage
+    ,notnull
+  from dictionary.columns
+  where upcase(libname)="&lib"
+%if &ds ne 0 %then %do;
+    and upcase(memname)="&ds"
+%end;
+  ;
+
+
+%if &engine=SQLSVR %then %do;
+  proc sql;
+  connect using &lib;
+  create table work.&ds2 as
+  select * from connection to &lib(
+  select
+      s.name as SchemaName,
+      t.name as memname,
+      tc.name as name,
+      ic.key_ordinal as KeyOrderNr
+  from
+      sys.schemas s
+      inner join sys.tables t   on s.schema_id=t.schema_id
+      inner join sys.indexes i  on t.object_id=i.object_id
+      inner join sys.index_columns ic on i.object_id=ic.object_id
+                                    and i.index_id=ic.index_id
+      inner join sys.columns tc on ic.object_id=tc.object_id
+                              and ic.column_id=tc.column_id
+  where i.is_primary_key=1
+    and s.name=%str(%')&schema%str(%')
+  order by t.name, ic.key_ordinal ;
+  );disconnect from &lib;
+  create table &ds3 as
+    select a.*
+      ,case when b.name is not null then 1 else 0 end as pk_ind
+    from work.&ds1 a
+    left join work.&ds2 b
+    on a.dsn=b.memname
+      and upcase(a.name)=upcase(b.name)
+    order by libref,dsn;
+%end;
+%else %do;
+
+  %if &ds = 0 %then %let dsn=;
+
+  /* get all constraints, in constraint order*/
+  %mp_getconstraints(lib=&lib,ds=&dsn,outds=work.&ds2)
+
+  /* extract cols that are clearly primary keys */
+  proc sql;
+  create table &pk4sure as
+    select libref
+      ,table_name
+      ,constraint_name
+      ,constraint_order
+      ,column_name as name
+    from work.&ds2
+    where constraint_type='PRIMARY'
+    order by 1,2,3,4;
+
+  /* extract unique constraints where every col is also NOT NULL */
+  proc sql;
+  create table &sum as
+    select a.libref
+      ,a.table_name
+      ,a.constraint_name
+      ,count(a.column_name) as unq_cnt
+      ,count(b.column_name) as nul_cnt
+    from work.&ds2(where=(constraint_type ='UNIQUE')) a
+    left join work.&ds2(where=(constraint_type ='NOT NULL')) b
+    on a.libref=b.libref
+      and a.table_name=b.table_name
+      and a.column_name=b.column_name
+    group by 1,2,3
+    having unq_cnt=nul_cnt;
+
+  /* extract cols from the relevant unique constraints */
+  create table &pkdefault as
+    select a.libref
+      ,a.table_name
+      ,a.constraint_name
+      ,b.constraint_order
+      ,b.column_name as name
+    from &sum a
+    left join &ds2(where=(constraint_type ='UNIQUE')) b
+    on a.libref=b.libref
+      and a.table_name=b.table_name
+      and a.constraint_name=b.constraint_name
+    order by 1,2,3,4;
+
+  /* create one table */
+  data &finalpks;
+    set &pkdefault &pk4sure ;
+    pk_ind=1;
+    /* if there are multiple unique constraints, take the first */
+    by libref table_name constraint_name;
+    retain keepme;
+    if first.table_name then keepme=1;
+    if first.constraint_name and not first.table_name then keepme=0;
+    if keepme=1;
+  run;
+
+  /* join back to starting table */
+  proc sql;
+  create table &ds3 as
+    select a.*
+      ,b.constraint_order
+      ,case when b.pk_ind=1 then 1 else 0 end as pk_ind
+    from work.&ds1 a
+    left join work.&finalpks b
+    on a.libref=b.libref
+      and a.dsn=b.table_name
+      and upcase(a.name)=upcase(b.name)
+    order by libref,dsn,constraint_order;
+%end;
+
+
+/* prepare tables */
+proc sql;
+create table work.&tabs1 as select
+  libname as libref
+  ,upcase(memname) as dsn
+  ,memtype
+  ,dbms_memtype
+  ,typemem
+  ,memlabel
+  ,nvar
+  ,compress
+from dictionary.tables
+  where upcase(libname)="&lib"
+%if &ds ne 0 %then %do;
+    and upcase(memname)="&ds"
+%end;
+  ;
+data &tabs2;
+  set &ds3;
+  length pk_fields $512;
+  retain pk_fields;
+  by libref dsn constraint_order;
+  if first.dsn then pk_fields='';
+  if pk_ind=1 then pk_fields=catx(' ',pk_fields,name);
+  if last.dsn then output;
+run;
+
+proc sql;
+create table &outds as
+  select a.libref
+    ,a.dsn
+    ,a.memtype
+    ,a.dbms_memtype
+    ,a.typemem
+    ,a.memlabel
+    ,a.nvar
+    ,a.compress
+    ,b.pk_fields
+  from work.&tabs1 a
+  left join work.&tabs2 b
+  on a.libref=b.libref
+    and a.dsn=b.dsn;
+
+/* tidy up */
+%mp_dropmembers(
+  &ds1 &ds2 &ds3 &dsn &tabs1 &tabs2 &sum &pk4sure &pkdefault &finalpks,
+  iftrue=(&mdebug=0)
+)
+
+%mend mp_getpk;

base/mp_guesspk.sas

@@ -1,32 +1,37 @@
 /**
-  @file mp_guesspk.sas
+  @file
   @brief Guess the primary key of a table
-  @details Tries to guess the primary key of a table based on the following logic:
+  @details Tries to guess the primary key of a table based on the following
+  logic:
 
       * Columns with nulls are ignored
       * Return only column combinations that provide unique results
-      * Start from one column, then move out to include composite keys of 2 to 6 columns
+      * Start from one column, then move out to composite keys of 2 to 6 columns
 
   The library of the target should be assigned before using this macro.
 
   Usage:
 
-      filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      filename mc url
+        "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
       %mp_guesspk(sashelp.class,outds=classpks)
 
   @param baseds The dataset to analyse
   @param outds= The output dataset to contain the possible PKs
-  @param max_guesses= The total number of possible primary keys to generate.  A
-    table is likely to have multiple unlikely PKs, so no need to list them all. Default=3.
-  @param min_rows= The minimum number of rows a table should have in order to try
-    and guess the PK.  Default=5.
+  @param max_guesses= (3) The total number of possible primary keys to generate.
+    A table may have multiple unlikely PKs, so no need to list them all.
+  @param min_rows= (5) The minimum number of rows a table should have in order
+    to try and guess the PK.
 
   <h4> SAS Macros </h4>
   @li mf_getvarlist.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
 
+  <h4> Related Macros </h4>
+  @li mp_getpk.sas
+
   @version 9.3
   @author Allan Bowe
 
@@ -196,7 +201,8 @@
           %let lev4=%scan(&posspks,&l);
           %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then %do;
             /* check for four level uniqueness */
-            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4) out=&tmpds noduprec;
+            proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4)
+                out=&tmpds noduprec;
               by _all_;
             run;
             %if %mf_nobs(&tmpds)=&rows %then %do;
@@ -233,7 +239,8 @@
             %let lev5=%scan(&posspks,&m);
             %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then %do;
               /* check for four level uniqueness */
-              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5) out=&tmpds noduprec;
+              proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5)
+                  out=&tmpds noduprec;
                 by _all_;
               run;
               %if %mf_nobs(&tmpds)=&rows %then %do;
@@ -282,7 +289,8 @@
                 run;
                 %if %mf_nobs(&tmpds)=&rows %then %do;
                   proc sql;
-                  insert into &outds values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
+                  insert into &outds
+                    values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6");
                   %if %mf_nobs(&outds) ge &max_guesses %then %do;
                     %put &sysmacroname: Max PKs reached at Level 6 for &baseds;
                     %return;

base/mp_init.sas

@@ -0,0 +1,54 @@
+/**
+  @file
+  @brief Initialise session with useful settings and variables
+  @details Implements a set of recommended options for general SAS use.  This
+    macro is NOT used elsewhere within the core library (other than in tests),
+    but it is used by the SASjs team when building web services for
+    SAS-Powered applications elsewhere.
+
+    If you have a good idea for an option, setting, or useful global variable -
+    feel free to [raise an issue](https://github.com/sasjs/core/issues/new)!
+
+    All global variables are prefixed with "SASJS_" (unless modfied with the
+    prefix parameter).
+
+  @param [in] prefix= (SASJS) The prefix to apply to the global macro variables
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_init(prefix=
+)/*/STORE SOURCE*/;
+
+  %global
+    &prefix._INIT_NUM   /* initialisation time as numeric             */
+    &prefix._INIT_DTTM  /* initialisation time in E8601DT26.6 format */
+  ;
+  %if %eval(&&&prefix._INIT_NUM>0) %then %return;  /* only run once */
+
+  data _null_;
+    dttm=datetime();
+    call symputx("&prefix._init_num",dttm);
+    call symputx("&prefix._init_dttm",put(dttm,E8601DT26.6));
+  run;
+
+  options
+    autocorrect             /* disallow mis-spelled procedure names           */
+    compress=CHAR           /* default is none so ensure we have something!   */
+    datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets     */
+    errorcheck=STRICT       /* catch errors in libname/filename statements    */
+    fmterr                  /* ensure error when a format cannot be found     */
+    mergenoby=ERROR         /* Throw error when a merge has no BY variables   */
+    missing=.    /* some sites change this which causes hard to detect errors */
+    noquotelenmax           /* avoid warnings for long strings                */
+    noreplace               /* avoid overwriting permanent datasets           */
+    ps=max                  /* reduce log size slightly                       */
+    validmemname=COMPATIBLE /* avoid special characters etc in table names    */
+    validvarname=V7         /* avoid special characters etc in variable names */
+    varlenchk=ERROR         /* fail hard if truncation (data loss) can result */
+  ;
+
+%mend mp_init;

base/mp_lockanytable.sas

@@ -0,0 +1,255 @@
+/**
+  @file
+  @brief Mechanism for locking tables to prevent parallel modifications
+  @details Uses a control table to enable ANY table to be locked for updates.
+  Only useful if every update uses the macro!   Used heavily within
+  [Data Controller for SAS](https://datacontroller.io).
+
+  The underlying table is structured as per the MAKETABLE action.
+
+  @param [in] action The action to be performed.  Valid values:
+    @li LOCK - Sets the lock flag, also confirms if a SAS lock is available
+    @li UNLOCK - Unlocks the table
+    @li MAKETABLE - creates the control table (ctl_ds)
+  @param [in] lib= (WORK) The libref of the table to lock.  Should already be
+    assigned.
+  @param [in] ds= The dataset to lock
+  @param [in] ref= A meaningful reference to enable the lock to be traced. Max
+    length is 200 characters.
+  @param [out] ctl_ds= (0) The control table which controls the actual locking.
+    Should already be assigned and available.
+  @param [in] loops= (25) Number of times to check for a lock.
+  @param [in] loop_secs= (1) Seconds to wait between each lock attempt
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_lockfilecheck.sas
+  @li mf_getuser.sas
+
+  <h4> Related Macros </h4>
+  @li mp_lockanytable.test.sas
+
+  @version 9.2
+
+**/
+
+%macro mp_lockanytable(
+  action
+  ,lib= WORK
+  ,ds=0
+  ,ref=
+  ,ctl_ds=0
+  ,loops=25
+  ,loop_secs=1
+  );
+data _null_;
+  if _n_=1 then putlog "&sysmacroname entry vars:";
+  set sashelp.vmacro;
+  where scope="&sysmacroname";
+  put name '=' value;
+run;
+
+%mp_abort(iftrue= (&ds=0 and &action ne MAKETABLE)
+  ,mac=&sysmacroname
+  ,msg=%str(dataset was not provided)
+)
+%mp_abort(iftrue= (&ctl_ds=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Control dataset was not provided)
+)
+
+/* set up lib & mac vars */
+%let lib=%upcase(&lib);
+%let ds=%upcase(&ds);
+%let action=%upcase(&action);
+%local user x trans msg abortme;
+%let user=%mf_getuser();
+%let abortme=0;
+
+%mp_abort(iftrue= (&action ne LOCK & &action ne UNLOCK & &action ne MAKETABLE)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid action (&action) provided)
+)
+
+/* if an err condition exists, exit before we even begin */
+%mp_abort(iftrue= (&syscc>0 and &action=LOCK)
+  ,mac=&sysmacroname
+  ,msg=%str(aborting due to syscc=&syscc on LOCK entry)
+)
+
+/* do not bother locking work tables (else may affect all WORK libraries) */
+%if (%upcase(&lib)=WORK or %str(&lib)=%str()) & &action ne MAKETABLE %then %do;
+  %put NOTE: WORK libraries will not be registered in the locking system.;
+  %return;
+%end;
+
+/* do not proceed if no observations can be processed */
+%mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)
+  ,mac=&sysmacroname
+  ,msg=%str(options obs = 0. syserrortext=&syserrortext)
+)
+
+%if &ACTION=LOCK %then %do;
+
+  /* abort if a SAS lock is already in place, or cannot be applied */
+  %mp_lockfilecheck(&lib..&ds)
+
+  /* next, check there is a record for this table */
+  %local record_exists_check;
+  proc sql noprint;
+  select count(*) into: record_exists_check from &ctl_ds
+    where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+  quit;
+  %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
+  %if &record_exists_check=0 %then %do;
+    data _null_;
+      putlog "&sysmacroname: adding record to lock table..";
+    run;
+
+    data ;
+      if 0 then set &ctl_ds;
+      LOCK_LIB ="&lib";
+      LOCK_DS="&ds";
+      LOCK_STATUS_CD='LOCKED';
+      LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt;
+      LOCK_USER_NM="&user";
+      LOCK_PID="&sysjobid";
+      LOCK_REF="&ref";
+      output;stop;
+    run;
+    %let trans=&syslast;
+    proc append base=&ctl_ds data=&trans;
+    run;
+  %end;
+  /* if record does exist, perform lock attempts */
+  %else %do x=1 %to &loops;
+    data _null_;
+      putlog "&sysmacroname: attempting lock (iteration &x) "@;
+      putlog "at %sysfunc(datetime(),datetime19.) ..";
+    run;
+
+    proc sql;
+    update &ctl_ds
+      set LOCK_STATUS_CD='LOCKED'
+        , LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_USER_NM="&user"
+        , LOCK_PID="&sysjobid"
+        , LOCK_REF="&ref"
+      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+    quit;
+    /**
+      * NOTE - occasionally SQL server will return an err code (deadlocked
+      * transaction).  If so, ignore it, keep calm, and carry on..
+      */
+    %if &syscc>0 %then %do;
+      data _null_;
+        putlog 'NOTE-' / 'NOTE-';
+        putlog "NOTE- &sysmacroname: Update failed. "@;
+        putlog "Resetting err conditions and re-attempting.";
+        putlog "NOTE- syscc=&syscc syserr=&syserr sqlrc=&sqlrc";
+        putlog 'NOTE-' / 'NOTE-';
+      run;
+      %let syscc=0;
+      %let sqlrc=0;
+    %end;
+
+    /* now check if the record was successfully updated */
+    %local success_check;
+    proc sql noprint;
+    select count(*) into: success_check from &ctl_ds
+      where LOCK_LIB ="&lib" and LOCK_DS="&ds"
+        and LOCK_PID="&sysjobid" and LOCK_STATUS_CD='LOCKED';
+    quit;
+    %if &success_check=0 %then %do;
+      %if &x < &loops %then %do;
+        /* pause before next check */
+        data _null_;
+          putlog 'NOTE-' / 'NOTE-';
+          putlog "NOTE- &sysmacroname: table locked, waiting "@;
+          putlog "%sysfunc(sleep(&loop_inc)) seconds.. ";
+          putlog "NOTE- (iteration &x of &loops)";
+          putlog 'NOTE-' / 'NOTE-';
+        run;
+      %end;
+      %else %do;
+        %let msg=Unable to lock &lib..&ds via &ctl_ds after &loops attempts.\n
+            Please ask your administrator to investigate!;
+        %let abortme=1;
+      %end;
+    %end;
+    %else %do;
+      data _null_;
+        putlog 'NOTE-' / 'NOTE-';
+        putlog "NOTE- &sysmacroname: Table &lib..&ds locked at "@
+        putlog " %sysfunc(datetime(),datetime19.) (iteration &x)"@;
+        putlog 'NOTE-' / 'NOTE-';
+      run;
+      %if &syscc>0 %then %do;
+        %put setting syscc(&syscc) back to 0;
+        %let syscc=0;
+      %end;
+      %let x=&loops;  /* no more iterations needed */
+    %end;
+  %end;
+%end;
+%else %if &ACTION=UNLOCK %then %do;
+  %local status;
+  proc sql noprint;
+  select LOCK_STATUS_CD into: status from &ctl_ds
+    where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+  quit;
+  %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
+  %if &status=LOCKED %then %do;
+    data _null_;
+      putlog "&sysmacroname: unlocking &lib..&ds:";
+    run;
+    proc sql;
+    update &ctl_ds
+      set LOCK_STATUS_CD='UNLOCKED'
+        , LOCK_END_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_USER_NM="&user"
+        , LOCK_PID="&sysjobid"
+        , LOCK_REF="&ref"
+      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+    quit;
+  %end;
+  %else %if &status=UNLOCKED %then %do;
+    %put %str(WAR)NING: &lib..&ds is already unlocked!;
+  %end;
+  %else %do;
+    %put NOTE: Unrecognised STATUS_CD (&status) in &ctl_ds;
+    %let abortme=1;
+  %end;
+%end;
+%else %if &action=MAKETABLE %then %do;
+  proc sql;
+  create table &ctl_ds(
+      lock_lib char(8),
+      lock_ds char(32),
+      lock_status_cd char(10) not null,
+      lock_user_nm char(100) not null ,
+      lock_ref char(200),
+      lock_pid char(10),
+      lock_start_dttm num format=E8601DT26.6,
+      lock_end_dttm num format=E8601DT26.6,
+    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+%end;
+%else %do;
+  %let msg=lock_anytable given unsupported action (&action);
+  %let abortme=1;
+%end;
+
+/* catch errors - mp_abort must be called outside of a logic block */
+%mp_abort(iftrue=(&abortme=1),
+  msg=%superq(msg),
+  mac=&sysmacroname
+)
+
+%exit_macro:
+data _null_;
+  put "&sysmacroname: Exit vars: action=&action lib=&lib ds=&ds";
+  put " syscc=&syscc sqlrc=&sqlrc syserr=&syserr";
+run;
+%mend mp_lockanytable;
+
+

base/mp_lockfilecheck.sas

@@ -0,0 +1,97 @@
+/**
+  @file
+  @brief Aborts if a SAS lock file is in place, or if one cannot be applied.
+  @details Used in conjuction with the mp_lockanytable macro.
+  More info here: https://sasensei.com/flash/24
+
+  Usage:
+
+      data work.test; a=1;run;
+      %mp_lockfilecheck(work.test)
+
+  @param [in] libds The libref.dataset for which to check the lock status
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_getattrc.sas
+
+  <h4> Related Macros </h4>
+  @li mp_lockanytable.sas
+  @li mp_lockfilecheck.test.sas
+
+  @version 9.2
+**/
+
+%macro mp_lockfilecheck(
+  libds
+)/*/STORE SOURCE*/;
+
+data _null_;
+  if _n_=1 then putlog "&sysmacroname entry vars:";
+  set sashelp.vmacro;
+  where scope="&sysmacroname";
+  put name '=' value;
+run;
+
+%mp_abort(iftrue= (&syscc>0)
+  ,mac=checklock.sas
+  ,msg=Aborting with syscc=&syscc on entry.
+)
+%mp_abort(iftrue= (&libds=0)
+  ,mac=&sysmacroname
+  ,msg=%str(libds not provided)
+)
+
+%local msg lib ds;
+%let lib=%upcase(%scan(&libds,1,.));
+%let ds=%upcase(%scan(&libds,2,.));
+
+/* do not proceed if no observations can be processed */
+%let msg=options obs = 0. syserrortext=%superq(syserrortext);
+%mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)
+  ,mac=checklock.sas
+  ,msg=%superq(msg)
+)
+
+data _null_;
+  putlog "Checking engine & member type";
+run;
+%local engine memtype;
+%let memtype=%mf_getattrc(&libds,MTYPE);
+%let engine=%mf_getattrc(&libds,ENGINE);
+
+%if &engine ne V9 and &engine ne BASE %then %do;
+  data _null_;
+    putlog "Lib &lib  is not assigned using BASE engine - uses &engine instead";
+    putlog "SAS lock check will not be performed";
+  run;
+  %return;
+%end;
+%else %if &memtype ne DATA %then %do;
+  %put NOTE: Cannot lock a VIEW!! Memtype=&memtype;
+  %return;
+%end;
+
+data _null_;
+  putlog "Engine = &engine, memtype=&memtype";
+  putlog "Attempting lock statement";
+run;
+
+lock &libds;
+
+%local abortme;
+%let abortme=0;
+%if &syscc>0 or &SYSLCKRC ne 0 %then %do;
+  %let msg=Unable to apply lock on &libds (SYSLCKRC=&SYSLCKRC syscc=&syscc);
+  %put %str(ERR)OR: &sysmacroname: &msg;
+  %let abortme=1;
+%end;
+
+lock &libds clear;
+
+%mp_abort(iftrue= (&abortme=1)
+  ,mac=&sysmacroname
+  ,msg=%superq(msg)
+)
+
+%mend mp_lockfilecheck;

base/mp_makedata.sas

@@ -0,0 +1,87 @@
+/**
+  @file
+  @brief Create sample data based on the structure of an empty table
+  @details Many SAS projects involve sensitive datasets.  One way to _ensure_
+    the data is anonymised, is never to receive it in the first place!  Often
+    consultants are provided with empty tables, and expected to create complex
+    ETL flows.
+
+    This macro can help by taking an empty table, and populating it with data
+    according to the variable types and formats.
+
+    TODO:
+      @li Respect PKs
+      @li Respect NOT NULLs
+      @li Consider dates, datetimes, times, integers etc
+
+  Usage:
+
+      proc sql;
+      create table work.example(
+        TX_FROM float format=datetime19.,
+        DD_TYPE char(16),
+        DD_SOURCE char(2048),
+        DD_SHORTDESC char(256),
+        constraint pk primary key(tx_from, dd_type,dd_source),
+        constraint nnn not null(DD_SHORTDESC)
+      );
+      %mp_makedata(work.example)
+
+  @param [in] libds The empty table in which to create data
+  @param [out] obs= (500) The number of records to create.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_getvarlen.sas
+  @li mf_nobs.sas
+  @li mp_getcols.sas
+  @li mp_getpk.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_makedata(libds
+  ,obs=500
+)/*/STORE SOURCE*/;
+
+%local ds1 c1 n1 i col charvars numvars;
+
+%if %mf_nobs(&libds)>0 %then %do;
+  %put &sysmacroname: &libds has data, it will not be recreated;
+  %return;
+%end;
+
+%local ds1 c1 n1;
+%let ds1=%mf_getuniquename(prefix=mp_makedata);
+%let c1=%mf_getuniquename(prefix=mp_makedatacol);
+%let n1=%mf_getuniquename(prefix=mp_makedatacol);
+data &ds1;
+  if 0 then set &libds;
+  do _n_=1 to &obs;
+    &c1=repeat(uuidgen(),10);
+    &n1=ranuni(1)*5000000;
+    drop &c1 &n1;
+    %let charvars=%mf_getvarlist(&libds,typefilter=C);
+    %do i=1 %to %sysfunc(countw(&charvars));
+      %let col=%scan(&charvars,&i);
+      &col=subpad(&c1,1,%mf_getvarlen(&libds,&col));
+    %end;
+
+    %let numvars=%mf_getvarlist(&libds,typefilter=N);
+    %do i=1 %to %sysfunc(countw(&numvars));
+      %let col=%scan(&numvars,&i);
+      &col=&n1;
+    %end;
+    output;
+  end;
+run;
+
+proc append base=&libds data=&ds1;
+run;
+
+proc sql;
+drop table &ds1;
+
+%mend mp_makedata;

base/mp_reseterror.sas

@@ -0,0 +1,27 @@
+/**
+  @file
+  @brief Reset when an err condition occurs
+  @details When building apps, sometimes an operation must be attempted that
+  can cause an err condition.  There is no try catch in SAS! So the err state
+  must be caught and reset.
+
+  This macro attempts to do that reset.
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_reseterror(
+)/*/STORE SOURCE*/;
+
+options obs=max replace nosyntaxcheck;
+%let syscc=0;
+
+%if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
+  data _null_;
+    rc=stpsrvset('program error', 0);
+  run;
+%end;
+
+%mend mp_reseterror;

base/mp_setkeyvalue.sas

@@ -9,11 +9,14 @@
   <h4> SAS Macros </h4>
   @li mf_existds.sas
 
-  @param key Provide a key on which to perform the lookup
-  @param value Provide a value
-  @param type= either C or N will populate valc and valn respectively.  C is
-              default.
-  @param libds= define the target table to hold the parameters
+  <h4> Related Macros </h4>
+  @li mf_getvalue.sas
+
+  @param [in] key Provide a key on which to perform the lookup
+  @param [in] value Provide a value
+  @param [in] type= either C or N will populate valc and valn respectively.
+    C is default.
+  @param [out] libds= define the target table to hold the parameters
 
   @version 9.2
   @author Allan Bowe

base/mp_sortinplace.sas

@@ -0,0 +1,118 @@
+/**
+  @file
+  @brief Sorts a SAS dataset in place, preserving constraints
+  @details Generally if a dataset contains indexes, then it is not necessary to
+  sort it before performing operations such as merges / joins etc.
+  That said, there are a few edge cases where it can be desirable:
+
+    @li To improve performance for particular scenarios
+    @li To allow adjacent records to be viewed directly in the dataset
+    @li To reduce dataset size (eg when there are deleted records)
+
+  This macro will only work for BASE (V9) engine libraries.  It works by
+  creating a copy of the dataset (without data, WITH constraints) in the same
+  library, appending a sorted view into it, and finally - renaming it.
+
+  Example usage:
+
+      proc sql;
+      create table work.example as
+        select * from sashelp.class;
+      alter table work.example
+        add constraint pk primary key(name);
+      %mp_sortinplace(work.example)
+
+  @param [in] libds The libref.datasetname that needs to be sorted
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_getengine.sas
+  @li mf_getquotedstr.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_getpk.sas
+
+  <h4> Related Macros </h4>
+  @li mp_sortinplace.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+  @source https://github.com/sasjs/core
+
+**/
+
+%macro mp_sortinplace(libds
+)/*/STORE SOURCE*/;
+
+%local lib ds tempds1 tempds2 tempvw sortkey;
+
+/* perform validations */
+%mp_abort(iftrue=(%sysfunc(countc(&libds,.)) ne 1)
+  ,mac=mp_sortinplace
+  ,msg=%str(LIBDS (&libds) should have LIBREF.DATASET format)
+)
+%mp_abort(iftrue=(%mf_existds(&libds)=0)
+  ,mac=mp_sortinplace
+  ,msg=%str(&libds does not exist)
+)
+
+%let lib=%scan(&libds,1,.);
+%let ds=%scan(&libds,2,.);
+%mp_abort(iftrue=(%mf_getengine(&lib) ne V9)
+  ,mac=mp_sortinplace
+  ,msg=%str(&lib is not a BASE engine library)
+)
+
+/* grab a copy of the constraints so we know what to sort by */
+%let tempds1=%mf_getuniquename(prefix=&sysmacroname);
+%mp_getpk(lib=&lib,ds=&ds,outds=work.&tempds1)
+
+%if %mf_nobs(work.&tempds1)=0 %then %do;
+  %put &sysmacroname: No PK found in &lib..&ds;
+  %put Sorting will not take place;
+  %return;
+%end;
+
+data _null_;
+  set work.&tempds1;
+  call symputx('sortkey',pk_fields);
+run;
+
+
+/* create empty copy, with ALL constraints, in the same library */
+%let tempds2=%mf_getuniquename(prefix=&sysmacroname);
+proc append base=&lib..&tempds2 data=&libds(obs=0);
+run;
+
+/* create sorted view */
+%let tempvw=%mf_getuniquename(prefix=&sysmacroname);
+proc sql;
+create view work.&tempvw as select * from &lib..&ds
+order by %mf_getquotedstr(&sortkey,quote=N);
+
+/* append sorted data */
+proc append base=&lib..&tempds2 data=work.&tempvw;
+run;
+
+/* do validations */
+%mp_abort(iftrue=(&syscc ne 0)
+  ,mac=mp_sortinplace
+  ,msg=%str(syscc=&syscc prior to replace operation)
+)
+%mp_abort(iftrue=(%mf_nobs(&lib..&tempds2) ne %mf_nobs(&lib..&ds))
+  ,mac=mp_sortinplace
+  ,msg=%str(new dataset has a different number of logical obs to the old)
+)
+
+/* drop old dataset */
+proc sql;
+drop table &lib..&ds;
+
+/* rename the new dataset */
+proc datasets library=&lib;
+  change &tempds2=&ds;
+run;
+
+
+%mend mp_sortinplace;

base/mp_unzip.sas

@@ -7,19 +7,23 @@
 
   Usage:
 
-      filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      filename mc url
+        "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
 
       %mp_unzip(ziploc="/some/file.zip",outdir=/some/folder)
 
-  <h4> SAS Macros </h4>
-  @li mf_mkdir.sas
-  @li mf_getuniquefileref.sas
+  More info:  https://blogs.sas.com/content/sasdummy/2015/05/11/using-filename-zip-to-unzip-and-read-data-files-in-sas/
 
   @param ziploc= Fileref or quoted full path to zip file ("/path/to/file.zip")
   @param outdir= (%sysfunc(pathname(work))) Directory in which to write the
     outputs (created if non existant)
 
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mf_getuniquefileref.sas
+  @li mp_binarycopy.sas
+
   @version 9.4
   @author Allan Bowe
   @source https://github.com/sasjs/core
@@ -31,18 +35,20 @@
   ,outdir=%sysfunc(pathname(work))
 )/*/STORE SOURCE*/;
 
-%local fname1 fname2 fname3;
-%let fname1=%mf_getuniquefileref();
-%let fname2=%mf_getuniquefileref();
-%let fname3=%mf_getuniquefileref();
+%local f1 f2 ;
+%let f1=%mf_getuniquefileref();
+%let f2=%mf_getuniquefileref();
 
 /* Macro variable &datazip would be read from the file */
-filename &fname1 ZIP &ziploc;
+filename &f1 ZIP &ziploc;
+
+/* create target folder */
+%mf_mkdir(&outdir)
 
 /* Read the "members" (files) from the ZIP file */
 data _data_(keep=memname isFolder);
   length memname $200 isFolder 8;
-  fid=dopen("&fname1");
+  fid=dopen("&f1");
   if fid=0 then stop;
   memcount=dnum(fid);
   do i=1 to memcount;
@@ -53,16 +59,32 @@ data _data_(keep=memname isFolder);
   end;
   rc=dclose(fid);
 run;
-filename &fname1 clear;
+
+filename &f2 temp;
 
 /* loop through each entry and either create the subfolder or extract member */
 data _null_;
   set &syslast;
+  file &f2;
   if isFolder then call execute('%mf_mkdir(&outdir/'!!memname!!')');
-  else call execute('filename &fname2 zip &ziploc member='
-    !!quote(trim(memname))!!';filename &fname3 "&outdir/'
-    !!trim(memname)!!'" recfm=n;data _null_; rc=fcopy("&fname2","&fname3");run;'
-    !!'filename &fname2 clear; filename &fname3 clear;');
+  else do;
+    qname=quote(cats("&outdir/",memname));
+    bname=cats('(',memname,')');
+    put '/* hat tip: "data _null_" on SAS-L */';
+    put 'data _null_;';
+    put '  infile &f1 ' bname ' lrecl=256 recfm=F length=length eof=eof unbuf;';
+    put '  file ' qname ' lrecl=256 recfm=N;';
+    put '  input;';
+    put '  put _infile_ $varying256. length;';
+    put '  return;';
+    put 'eof:';
+    put '  stop;';
+    put 'run;';
+  end;
 run;
 
-%mend mp_unzip;
+%inc &f2/source2;
+
+filename &f2 clear;
+
+%mend mp_unzip;

base/mp_zip.sas

@@ -16,11 +16,18 @@
   @li mp_dirlist.sas
 
   @param in= unquoted filepath, dataset of files or directory to zip
-  @param type= FILE, DATASET, DIRECTORY. (FILE / DATASET not ready yet)
-  @param outname= output file to create, without .zip extension
-  @param outpath= location for output zip file
+  @param type= (FILE) Valid values:
+    @li FILE - /full/path/and/filename.extension to a particular file
+    @li DATASET - a dataset containing a list of files to zip (see `incol`)
+    @li DIRECTORY - a directory to zip
+  @param outname= (FILE) Output file to create, _without_ .zip extension
+  @param outpath= (%sysfunc(pathname(WORK))) Parent folder for output zip file
   @param incol= if DATASET input, say which column contains the filepath
 
+  <h4> Related Macros </h4>
+  @li mp_unzip.sas
+  @li mp_zip.test.sas
+
   @version 9.2
   @author Allan Bowe
   @source https://github.com/sasjs/core
@@ -51,9 +58,9 @@ ods package open nopf;
     set &ds;
     length __command $4000;
     if file_or_folder='file';
-    command=cats('ods package add file="',filepath
+    __command=cats('ods package add file="',filepath
       ,'" mimetype="application/x-compress";');
-    call execute(command);
+    call execute(__command);
   run;
   /* tidy up */
   %if &debug=NO %then %do;
@@ -64,11 +71,10 @@ ods package open nopf;
   data _null_;
     set &in;
     length __command $4000;
-    command=cats('ods package add file="',&incol
+    __command=cats('ods package add file="',&incol
       ,'" mimetype="application/x-compress";');
-    call execute(command);
+    call execute(__command);
   run;
-  ods package add file="&in" mimetype="application/x-compress";
 %end;
 
 

build.py

@@ -84,7 +84,7 @@ options noquotelenmax;
 """
 f = open('all.sas', "w")             # r / r+ / rb / rb+ / w / wb
 f.write(header)
-folders=['base','meta','metax','viya','lua','fcmp']
+folders=['base','meta','metax','server','viya','lua','fcmp']
 for folder in folders:
     filenames = [fn for fn in Path('./' + folder).iterdir() if fn.match("*.sas")]
     filenames.sort()

fcmp/mcf_stpsrv_header.sas

@@ -54,6 +54,9 @@
   @param [out] pkg= (utils) The output package in which to create the function.
     Uses a 3 part format:  libref.catalog.package
 
+  <h4> SAS Macros </h4>
+  @li mf_existfunction.sas
+
 **/
 
 %macro mcf_stpsrv_header(wrap=NO
@@ -63,6 +66,8 @@
   ,pkg=UTILS
 )/*/STORE SOURCE*/;
 
+%if %mf_existfunction(stpsrv_header)=1 %then %return;
+
 %if &wrap=YES  %then %do;
   proc fcmp outcat=&lib..&cat..&pkg;
 %end;

main.dox

@@ -55,7 +55,18 @@
 
  */
 
- /*! \dir Tests
+/*! \dir server
+ *  \brief Macros used with [sasjs/server](https://server.sasjs.io)
+ *  \details These macros have the following attributes:
+
+        * OS independent
+        * sasjs/server aware
+        * No X command
+        * Prefixes: _ms_
+
+ */
+
+/*! \dir tests
  *  \brief SASjs Tests
  *  \details These folders contain the macro tests.  They are first compiled
    and deployed (sasjs cbd) then executed (sasjs test).
@@ -72,7 +83,7 @@
 
  */
 
- /*! \dir lua
+/*! \dir lua
  *  \brief Lua macros
  *  \details These macros have the following attributes:
 

make_singlefile.sh

@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-
-# Concatenate all macros into a single file
-
-OUTFILE='./macrocore.sas'
-
-cat > $OUTFILE <<'EOL'
-/**
-  @file
-  @brief Auto-generated file
-  @details
-    This file contains all the macros in a single file - which means it can be
-    'included' in SAS with just 2 lines of code:
-
-      filename mc url
-        "https://raw.githubusercontent.com/sasjs/core/main/macrocore.sas";
-      %inc mc;
-
-    The `build.sh` file in the https://github.com/sasjs/core repo
-    is used to create this file.
-
-  @author Allan Bowe
-**/
-EOL
-
-cat base/* >> $OUTFILE
-cat meta/* >> $OUTFILE
-cat metax/* >> $OUTFILE
-cat viya/* >> $OUTFILE

package.json

@@ -34,5 +34,8 @@
   },
   "devDependencies": {
     "@sasjs/cli": "^2.39.0"
+  },
+  "dependencies": {
+    "ts-loader": "^9.2.6"
   }
 }

sasjs/sasjsconfig.json

@@ -5,6 +5,7 @@
     "fcmp",
     "meta",
     "metax",
+    "server",
     "viya",
     "lua",
     "tests/crossplatform"
@@ -47,10 +48,29 @@
       "name": "sas9",
       "serverUrl": "https://sas.analytium.co.uk:8343",
       "serverType": "SAS9",
+      "httpsAgentOptions": {
+        "allowInsecureRequests": false
+      },
       "appLoc": "/Shared Data/temp/macrocore",
       "macroFolders": [
         "tests/sas9only"
       ],
+      "programFolders": [],
+      "deployConfig": {
+        "deployServicePack": true,
+        "deployScripts": []
+      },
+      "serverName": "SASApp",
+      "repositoryName": "Foundation"
+    },
+    {
+      "name": "server",
+      "serverUrl": "https://sas.analytium.co.uk:5001",
+      "serverType": "SASJS",
+      "appLoc": "/Shared Data/temp/macrocore",
+      "macroFolders": [
+        "tests/serveronly"
+      ],
       "deployConfig": {
         "deployServicePack": true
       }

server/ms_webout.sas

@@ -0,0 +1,171 @@
+/**
+  @file
+  @brief Send data to/from @sasjs/server
+  @details This macro should be added to the start of each web service,
+  **immediately** followed by a call to:
+
+        %ms_webout(FETCH)
+
+    This will read all the input data and create same-named SAS datasets in the
+    WORK library.  You can then insert your code, and send data back using the
+    following syntax:
+
+        data some datasets; * make some data ;
+          retain some columns;
+        run;
+
+        %ms_webout(OPEN)
+        %ms_webout(ARR,some)  * Array format, fast, suitable for large tables ;
+        %ms_webout(OBJ,datasets) * Object format, easier to work with ;
+        %ms_webout(CLOSE)
+
+
+  @param action Either FETCH, OPEN, ARR, OBJ or CLOSE
+  @param ds The dataset to send back to the frontend
+  @param dslabel= value to use instead of the real name for sending to JSON
+  @param fmt=(Y) Set to N to send back unformatted values
+  @param fref=(_webout) The fileref to which to write the JSON
+
+  <h4> SAS Macros </h4>
+  @li mp_jsonout.sas
+  @li mf_getuser.sas
+
+  <h4> Related Macros </h4>
+  @li mv_webout.sas
+  @li mm_webout.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y);
+%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
+  sasjs_tables;
+
+%local i tempds;
+%let action=%upcase(&action);
+
+%if &action=FETCH %then %do;
+  %if %str(&_debug) ge 131 %then %do;
+    options mprint notes mprintnest;
+  %end;
+  %let _webin_file_count=%eval(&_webin_file_count+0);
+  /* now read in the data */
+  %do i=1 %to &_webin_file_count;
+    %if &_webin_file_count=1 %then %do;
+      %let _webin_fileref1=&_webin_fileref;
+      %let _webin_name1=&_webin_name;
+    %end;
+    data _null_;
+      infile &&_webin_fileref&i termstr=crlf;
+      input;
+      call symputx('input_statement',_infile_);
+      putlog "&&_webin_name&i input statement: "  _infile_;
+      stop;
+    data &&_webin_name&i;
+      infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding='utf-8';
+      input &input_statement;
+      %if %str(&_debug) ge 131 %then %do;
+        if _n_<20 then putlog _infile_;
+      %end;
+    run;
+    %let sasjs_tables=&sasjs_tables &&_webin_name&i;
+  %end;
+%end;
+
+%else %if &action=OPEN %then %do;
+  /* fix encoding */
+  OPTIONS NOBOMFILE;
+
+  /* setup json */
+  data _null_;file &fref encoding='utf-8';
+  %if %str(&_debug) ge 131 %then %do;
+    put '>>weboutBEGIN<<';
+  %end;
+    put '{"SYSDATE" : "' "&SYSDATE" '"';
+    put ',"SYSTIME" : "' "&SYSTIME" '"';
+  run;
+
+%end;
+
+%else %if &action=ARR or &action=OBJ %then %do;
+  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
+    ,engine=DATASTEP,dbg=%str(&_debug)
+  )
+%end;
+%else %if &action=CLOSE %then %do;
+  %if %str(&_debug) ge 131 %then %do;
+    /* if debug mode, send back first 10 records of each work table also */
+    options obs=10;
+    data;run;%let tempds=%scan(&syslast,2,.);
+    ods output Members=&tempds;
+    proc datasets library=WORK memtype=data;
+    %local wtcnt;%let wtcnt=0;
+    data _null_;
+      set &tempds;
+      if not (upcase(name) =:"DATA"); /* ignore temp datasets */
+      i+1;
+      call symputx('wt'!!left(i),name,'l');
+      call symputx('wtcnt',i,'l');
+    data _null_; file &fref mod encoding='utf-8';
+      put ",""WORK"":{";
+    %do i=1 %to &wtcnt;
+      %let wt=&&wt&i;
+      proc contents noprint data=&wt
+        out=_data_ (keep=name type length format:);
+      run;%let tempds=%scan(&syslast,2,.);
+      data _null_; file &fref mod encoding='utf-8';
+        dsid=open("WORK.&wt",'is');
+        nlobs=attrn(dsid,'NLOBS');
+        nvars=attrn(dsid,'NVARS');
+        rc=close(dsid);
+        if &i>1 then put ','@;
+        put " ""&wt"" : {";
+        put '"nlobs":' nlobs;
+        put ',"nvars":' nvars;
+      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
+      data _null_; file &fref mod encoding='utf-8';
+        put "}";
+    %end;
+    data _null_; file &fref mod encoding='utf-8';
+      put "}";
+    run;
+  %end;
+  /* close off json */
+  data _null_;file &fref mod encoding='utf-8';
+    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
+    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
+    put ",""_DEBUG"" : ""&_debug"" ";
+    put ',"_PROGRAM" : ' _PROGRAM ;
+    put ",""SYSCC"" : ""&syscc"" ";
+    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    SYSHOSTINFOLONG=quote(trim(symget('SYSHOSTINFOLONG')));
+    put ',"SYSHOSTINFOLONG" : ' SYSHOSTINFOLONG;
+    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
+    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
+    put ",""SYSPROCESSNAME"" : ""&SYSPROCESSNAME"" ";
+    put ",""SYSJOBID"" : ""&sysjobid"" ";
+    put ",""SYSSCPL"" : ""&sysscpl"" ";
+    put ",""SYSSITE"" : ""&syssite"" ";
+    put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" ";
+    sysvlong=quote(trim(symget('sysvlong')));
+    put ',"SYSVLONG" : ' sysvlong;
+    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
+    autoexec=quote(trim(getoption('autoexec')));
+    put ',"AUTOEXEC" : ' autoexec;
+    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
+    memsize=quote(cats(memsize));
+    put ',"MEMSIZE" : ' memsize;
+    put "}" @;
+  %if %str(&_debug) ge 131 %then %do;
+    put '>>weboutEND<<';
+  %end;
+  run;
+%end;
+
+%mend ms_webout;

tests/crossplatform/mf_existds.test.sas

@@ -0,0 +1,25 @@
+/**
+  @file
+  @brief Testing mf_existfileref macro
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mp_assert.sas
+
+**/
+
+data work.testme;
+x=1;
+run;
+
+%mp_assert(
+  iftrue=(%mf_existds(work.testme)=1),
+  desc=Checking existing dataset exists,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_existds(work.try2testme)=0),
+  desc=Checking non existing dataset does not exist,
+  outds=work.test_results
+)

tests/crossplatform/mp_copyfolder.test.sas

@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief Testing mp_copyfolder.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_copyfolder.sas
+  @li mf_mkdir.sas
+  @li mf_nobs.sas
+  @li mp_assert.sas
+  @li mp_dirlist.sas
+
+**/
+
+/**
+  * make a directory structure
+  */
+
+%let root=%sysfunc(pathname(work))/top;
+%mf_mkdir(&root)
+%mf_mkdir(&root/a)
+%mf_mkdir(&root/b)
+%mf_mkdir(&root/a/d)
+%mf_mkdir(&root/a/e)
+%mf_mkdir(&root/a/e/f)
+data "&root/a/e/f/ds1.sas7bdat";x=1;
+data "&root/a/e/ds2.sas7bdat";x=1;
+data "&root/a/ds3.sas7bdat";x=1;
+run;
+
+%mp_dirlist(path=&root, outds=myTable, maxdepth=MAX)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable)=8),
+  desc=Temp data successfully created,
+  outds=work.test_results
+)
+
+/**
+  * copy it
+  */
+%let newfolder=%sysfunc(pathname(work))/new;
+%mp_copyfolder(&root,&newfolder)
+
+%mp_dirlist(path=&newfolder, outds=work.myTable2, maxdepth=MAX)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable2)=8),
+  desc=Folder successfully copied,
+  outds=work.test_results
+)
+
+

tests/crossplatform/mp_deletefolder.test.sas

@@ -0,0 +1,50 @@
+/**
+  @file
+  @brief Testing mp_deletefolder.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_deletefolder.sas
+  @li mf_mkdir.sas
+  @li mf_nobs.sas
+  @li mp_assert.sas
+  @li mp_dirlist.sas
+
+**/
+
+/**
+  * make a directory structure
+  */
+
+%let root=%sysfunc(pathname(work))/top;
+%mf_mkdir(&root)
+%mf_mkdir(&root/a)
+%mf_mkdir(&root/b)
+%mf_mkdir(&root/a/d)
+%mf_mkdir(&root/a/e)
+%mf_mkdir(&root/a/e/f)
+data "&root/a/e/f/ds1.sas7bdat";
+  x=1;
+run;
+
+%mp_dirlist(path=&root, outds=myTable, maxdepth=MAX)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable)=6),
+  desc=Temp data successfully created,
+  outds=work.test_results
+)
+
+%mp_deletefolder(&root/a)
+
+%mp_dirlist(path=&root, outds=work.myTable2, maxdepth=MAX)
+
+data _null_;
+  set work.mytable2;
+  putlog (_all_)(=);
+run;
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable2)=1),
+  desc=Subfolder and contents successfully deleted,
+  outds=work.test_results
+)

tests/crossplatform/mp_dirlist.test.sas

@@ -0,0 +1,50 @@
+/**
+  @file
+  @brief Testing mp_dirlist.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mf_mkdir.sas
+  @li mp_dirlist.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * make a directory structure
+  */
+
+%let root=%sysfunc(pathname(work))/top;
+%mf_mkdir(&root)
+%mf_mkdir(&root/a)
+%mf_mkdir(&root/b)
+%mf_mkdir(&root/a/d)
+%mf_mkdir(&root/a/e)
+%mf_mkdir(&root/a/e/f)
+data "&root/a/e/f/ds1.sas7bdat";
+  x=1;
+run;
+
+%mp_dirlist(path=&root, outds=myTable, maxdepth=MAX)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable)=6),
+  desc=All levels returned,
+  outds=work.test_results
+)
+
+%mp_dirlist(path=&root, outds=myTable2, maxdepth=2)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable2)=5),
+  desc=Top two levels returned,
+  outds=work.test_results
+)
+
+%mp_dirlist(path=&root, outds=work.myTable3, maxdepth=0)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable3)=2),
+  desc=Top level returned,
+  outds=work.test_results
+)

tests/crossplatform/mp_getpk.test.sas

@@ -0,0 +1,88 @@
+/**
+  @file
+  @brief Testing mp_getpk.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_getpk.sas
+  @li mp_assert.sas
+  @li mp_assertdsobs.sas
+
+  <h4> Related Macros </h4>
+  @li mp_getpk.sas
+
+**/
+
+/* ensure PK arrives in corrrect order */
+proc sql;
+create table work.example1(
+  TX_FROM float format=datetime19.,
+  DD_TYPE char(16),
+  DD_SOURCE char(2048),
+  DD_SHORTDESC char(256),
+  constraint pk primary key(tx_from, dd_type,dd_source),
+  constraint unq unique(tx_from, dd_type),
+  constraint nnn not null(DD_SHORTDESC)
+);
+%mp_getpk(work,ds=example1,outds=test1)
+
+data _null_;
+  set work.test1;
+  call symputx('test1',pk_fields);
+run;
+
+%mp_assert(
+  iftrue=("&test1"="TX_FROM DD_TYPE DD_SOURCE"),
+  desc=mp_getpk gets regular PK values in correct order,
+  outds=work.test_results
+)
+
+/* unique key with NOT NULL captured */
+proc sql;
+create table work.example2(
+  TX_FROM float format=datetime19.,
+  DD_TYPE char(16),
+  DD_SOURCE char(2048),
+  DD_SHORTDESC char(256),
+  constraint unq1 unique(tx_from, dd_type),
+  constraint unq2 unique(tx_from, dd_type, dd_source),
+  constraint nnn not null(tx_from),
+  constraint nnnn not null(dd_type)
+);
+%mp_getpk(work,ds=example2,outds=test2)
+
+data _null_;
+  set work.test1;
+  call symputx('test2',pk_fields);
+run;
+
+%mp_assert(
+  iftrue=("&test2"="TX_FROM DD_TYPE"),
+  desc=mp_getpk gets unique constraint with NOT NULL in correct order,
+  outds=work.test_results
+)
+
+/* unique key without NOT NULL NOT captured */
+proc sql;
+create table work.example3(
+  TX_FROM float format=datetime19.,
+  DD_TYPE char(16),
+  DD_SOURCE char(2048),
+  DD_SHORTDESC char(256),
+  constraint unq1 unique(tx_from, dd_type),
+  constraint unq2 unique(tx_from, dd_type, dd_source),
+  constraint nnn not null(tx_from),
+  constraint nnnn not null(dd_type)
+);
+%mp_getpk(work,ds=example3,outds=test3)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.test3)=0),
+  desc=mp_getpk does not capture unique constraint without NOT NULL,
+  outds=work.test_results
+)
+
+/* constraint capture at library level is functional - uses first 2 tests */
+%mp_getpk(work,outds=test4)
+
+%mp_assertdsobs(work.test4,test=ATLEAST 2)

tests/crossplatform/mp_init.test.sas

@@ -0,0 +1,24 @@
+/**
+  @file
+  @brief Testing mp_gsubfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_init.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * Test 1 - mp_init.sas actually already ran as part of testinit
+  * So lets test to make sure it will not run again
+  */
+
+%let initial_value=&sasjs_init_num;
+
+%mp_init();
+
+%mp_assert(
+  iftrue=("&initial_value"="&sasjs_init_num"),
+  desc=Check that mp_init() did not run twice,
+  outds=work.test_results
+)

tests/crossplatform/mp_lib2inserts.test.sas

@@ -11,9 +11,10 @@
 **/
 
 /* grab 20 datasets from SASHELP */
-%let path=%sysfunc(pathname(work));
+%let work=%sysfunc(pathname(work));
+%let path=&work/new;
 %mf_mkdir(&path)
-libname sashlp "&path";
+libname sashlp "&work";
 proc sql noprint;
 create table members as
   select distinct lowcase(memname) as memname
@@ -31,6 +32,7 @@ run;
 %mp_lib2inserts(sashlp, schema=work, outref=tempref,maxobs=50)
 
 /* check if it actually runs */
+libname sashlp "&path";
 options source2;
 %inc tempref;
 

tests/crossplatform/mp_lockanytable.test.sas

@@ -0,0 +1,62 @@
+/**
+  @file
+  @brief Testing mp_lockfilecheck macro
+
+  <h4> SAS Macros </h4>
+  @li mp_lockanytable.sas
+  @li mp_assertcols.sas
+  @li mp_assertcolvals.sas
+
+**/
+
+/* check create table */
+
+%mp_lockanytable(MAKETABLE, ctl_ds=work.controller)
+
+%mp_assertcols(work.controller,
+  cols=lock_status_cd lock_lib lock_ds lock_user_nm lock_ref lock_pid
+    lock_start_dttm lock_end_dttm,
+  test=ALL,
+  desc=check all control columns exist
+)
+
+/* check lock table */
+options dlcreatedir;
+libname tmp "%sysfunc(pathname(work))/tmp";
+data tmp.sometable;
+  x=1;
+run;
+
+%mp_lockanytable(LOCK,lib=tmp,ds=sometable,ref=This Ref, ctl_ds=work.controller)
+
+data work.checkds1;
+  checkval='SOMETABLE';
+run;
+%mp_assertcolvals(work.controller.lock_ds,
+  checkvals=work.checkds1.checkval,
+  desc=table is captured in lock,
+  test=ANYVAL
+)
+
+data work.checkds2;
+  checkval='LOCKED';
+run;
+%mp_assertcolvals(work.controller.lock_status_cd,
+  checkvals=work.checkds2.checkval,
+  desc=code is captured in lock,
+  test=ANYVAL
+)
+
+
+
+/* check for unsuccessful unlock */
+%mp_lockanytable(UNLOCK,lib=tmp,ds=sometable,ref=bye, ctl_ds=work.controller)
+
+data work.checkds3;
+  checkval='UNLOCKED';
+run;
+%mp_assertcolvals(work.controller.lock_status_cd,
+  checkvals=work.checkds3.checkval,
+  desc=Ref is captured in unlock,
+  test=ANYVAL
+)

tests/crossplatform/mp_lockfilecheck.test.sas

@@ -0,0 +1,39 @@
+/**
+  @file
+  @brief Testing mp_lockfilecheck macro
+
+  <h4> SAS Macros </h4>
+  @li mp_lockfilecheck.sas
+  @li mp_assert.sas
+  @li mp_reseterror.sas
+
+**/
+
+
+/* check for regular lock */
+data work.test; a=1;run;
+%mp_lockfilecheck(work.test)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking regular table can be locked,
+  outds=work.test_results
+)
+
+
+/* check for unsuccessful lock */
+%global success abortme;
+%let success=0;
+%macro mp_abort(iftrue=,mac=,msg=);
+  %if &abortme=1 %then %let success=1;
+%mend mp_abort;
+
+%mp_lockfilecheck(sashelp.class)
+
+%mp_reseterror()
+
+%mp_assert(
+  iftrue=(&success=1),
+  desc=Checking sashelp table cannot be locked,
+  outds=work.test_results
+)

tests/crossplatform/mp_reseterror.test.sas

@@ -0,0 +1,23 @@
+/**
+  @file
+  @brief Testing mp_reseterror macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mp_reseterror.sas
+
+**/
+
+
+/* cause an error */
+
+lock sashelp.class;
+
+/* recover ? */
+%mp_reseterror()
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking error condition was fixed,
+  outds=work.test_results
+)

tests/crossplatform/mp_sortinplace.test.sas

@@ -0,0 +1,42 @@
+/**
+  @file
+  @brief Testing mp_sortinplace.test.sas
+
+  <h4> SAS Macros </h4>
+  @li mp_sortinplace.sas
+  @li mp_assert.sas
+  @li mp_assertdsobs.sas
+  @li mp_getconstraints.sas
+
+**/
+
+
+/** Test 1 - regular usage  */
+proc sql;
+create table work.example as
+  select * from sashelp.classfit;
+alter table work.example
+  add constraint pk primary key(name);
+%mp_sortinplace(work.example)
+
+%mp_getconstraints(lib=work,ds=example,outds=work.testme)
+
+%mp_assertdsobs(work.testme,
+  desc=Test1 - check constraints recreated,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+%let test1=0;
+data _null_;
+  set work.example;
+  call symputx('test1',name);
+  stop;
+run;
+%mp_assert(
+  iftrue=(
+    %str(&test1)=%str(Alfred)
+  ),
+  desc=Check if sort was appplied,
+  outds=work.test_results
+)

tests/crossplatform/mp_zip.test.sas

@@ -0,0 +1,115 @@
+/**
+  @file
+  @brief Testing mp_zip macro
+
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mp_assert.sas
+  @li mp_zip.sas
+  @li mp_unzip.sas
+
+**/
+
+%let work=%sysfunc(pathname(work));
+%let root=&work/zipme;
+
+/* TEST 1 - zip a file */
+%mf_mkdir(&root)
+
+data _null_;
+  file "&root/test.txt";
+  put "houston, this is a test";
+run;
+
+%mp_zip(in=&root/test.txt
+  ,type=FILE
+  ,outpath=&work
+  ,outname=myFile
+)
+
+%mp_unzip(ziploc="&work/myFile.zip",outdir=&work)
+
+data _null_;
+  infile "&work/test.txt";
+  input;
+  call symputx('content1',_infile_);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(
+    %str(&content1)=%str(houston, this is a test)
+  ),
+  desc=Checking if file zip / unzip works,
+  outds=work.test_results
+)
+
+/* TEST 2 - zip a dataset of files */
+data _null_;
+  file "&root/test2.txt";
+  put "houston, this is test2";
+run;
+libname tmp "&root";
+data tmp.test;
+  filepath="&root/test2.txt";
+run;
+
+%mp_zip(in=tmp.test
+  ,incol=filepath
+  ,type=DATASET
+  ,outpath=&work
+  ,outname=myFile2
+)
+
+%mp_unzip(ziploc="&work/myFile2.zip",outdir=&work)
+
+data _null_;
+  infile "&work/test2.txt";
+  input;
+  call symputx('content2',_infile_);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(
+    %str(&content2)=%str(houston, this is test2)
+  ),
+  desc=Checking if file zip / unzip from a dataset works,
+  outds=work.test_results
+)
+
+/* TEST 3 - zip a dataset of files */
+%mf_mkdir(&work/out3)
+
+%mp_zip(in=&root
+  ,type=DIRECTORY
+  ,outpath=&work
+  ,outname=myFile3
+)
+
+%mp_unzip(ziploc="&work/myFile3.zip",outdir=&work/out3)
+
+data _null_;
+  infile "&work/out3/test.txt";
+  input;
+  call symputx('content3a',_infile_);
+  putlog _infile_;
+run;
+data _null_;
+  infile "&work/out3/test2.txt";
+  input;
+  call symputx('content3b',_infile_);
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(
+    %str(&content3a)=%str(houston, this is a test)
+    and
+    %str(&content3b)=%str(houston, this is test2)
+  ),
+  desc=Checking if file zip / unzip from a directory works,
+  outds=work.test_results
+)
+
+

tests/serveronly/ms_webout.test.sas

@@ -0,0 +1,35 @@
+/**
+  @file
+  @brief Testing ms_webout macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li ms_webout.sas
+  @li mp_assert.sas
+
+**/
+
+
+%let fref=%mf_getuniquefileref();
+%global _metaperson;
+data some datasets;
+  x=1;
+run;
+%ms_webout(OPEN,fref=&fref)
+%ms_webout(ARR,some,fref=&fref)
+%ms_webout(OBJ,datasets,fref=&fref)
+%ms_webout(CLOSE,fref=&fref)
+
+libname test JSON (&fref);
+data root;
+  set test.root;
+  call symputx('checkval',sysvlong);
+run;
+data alldata;
+  set test.alldata;
+run;
+
+%mp_assert(
+  iftrue=(%str(&checkval)=%str(&sysvlong)),
+  desc=Check if the sysvlong value was created
+)

tests/testinit.sas

@@ -2,7 +2,21 @@
   @file
   @brief init file for tests
 
+  <h4> SAS Macros </h4>
+  @li mp_init.sas
+
 **/
 
 /* location in metadata or SAS Drive for temporary files */
-%let mcTestAppLoc=/Public/temp/macrocore;
+%let mcTestAppLoc=/Public/temp/macrocore;
+
+/* set defaults */
+%mp_init()
+
+%macro loglevel();
+  %if &_debug=2477 %then %do;
+    options mprint;
+  %end;
+%mend loglevel;
+
+%loglevel()