Merge pull request #168 from sasjs/loadformat

feat: new mp_loadformat macro

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+  - package-ecosystem: npm
+    directory: '/'
+    schedule:
+      interval: monthly
+    open-pull-requests-limit: 3
+    allow:
+      - dependency-type: "production"

README.md

@@ -31,19 +31,28 @@ Documentation: https://core.sasjs.io
 
 ## Components
 
-### BASE library (SAS9/Viya)
+### BASE library (All Platforms)
 
 - OS independent
-- Not metadata aware
+- Works on all SAS Platforms
 - No X command
 - Prefixes: _mf_, _mp_
 
-#### FCMP library (SAS9/Viya)
+### DDL library (All Platforms)
+
+- OS independent
+- Works on all SAS Platforms
+- No X command
+- Prefixes: _mddl_(lib)_ -> where lib can be "SAS" (in relation to a SAS component) or "DC" (in relation to a Data Controller component)
+
+This library will not be used for storing data entries (such as formats or datalines).  Where this becomes necessary in the future, a new repo will be created, in order to keep the NPM bundle size down (for the benefit of those looking to embed purely macros in their applications).
+
+#### FCMP library (All Platforms)
+
 - Function and macro names are identical, except for special cases
 - Prefixes: _mcf_
 
-The fcmp macros are used to generate fcmp functions, and can be used with or
-without the `proc fcmp` wrapper.
+The fcmp macros are used to generate fcmp functions, and can be used with or without the `proc fcmp` wrapper.
 
 ### META library (SAS9 only)
 
@@ -81,7 +90,7 @@ Macros used for interfacing with SAS Viya.
 
 Wait - this is a macro library - what is LUA doing here?  Well, it is a little known fact that you CAN run LUA within a SAS Macro.  It has to be written to a text file with a `.lua` extension, from where you can `%include` it.  So, without using the `proc lua` wrapper.
 
-To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert your LUA into a data step with put statements, and create the macro wrapper with a `ml_` prefix.  You can then use your module in any program by running:
+To contribute, simply write your freeform LUA in the LUA folder.  Then run the `build.py`, which will convert all files with a ".lua" extension into a macro wrapper with an `ml_` prefix (embedding the necessary data step put statements).  You can then use your module in any program by running:
 
 ```sas
 /* compile the lua module */
@@ -95,8 +104,7 @@ endsubmit;
 run;
 ```
 
-- X command enabled
-- Prefixes: _mmw_,_mmu_,_mmx_
+- Prefixes: _ml_
 
 ## Installation
 
@@ -126,14 +134,15 @@ filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
 - one macro per file
 - prefixes:
   - _mcf_ for macro compiled functions (proc fcmp)
+  - _mddl_ for macros containing DDL (Data Definition Language)
   - _mf_ for macro functions (can be used in open code).
   - _ml_ for macros that are used to compile LUA modules
   - _mm_ for metadata macros (interface with the metadata server).
-  - _mmx_ for macros that use metadata and are XCMD enabled
+  - _mmx_ for macros that use metadata and are XCMD enabled (working on both windows and unix)
   - _mp_ for macro procedures (which generate sas code)
   - _ms_ for macro procedures that will only work with [@sasjs/server](https://github.com/sasjs/server)
   - _mv_ for macro procedures that will only work in Viya
-  - _mx_ for macros that are XCMD enabled
+  - _mx_ for macros that are XCMD enabled (working on both windows and unix)
 - follow verb-noun convention
 - unix style line endings (lf)
 - individual lines should be no more than 80 characters long
@@ -193,9 +202,13 @@ When contributing to this library, it is therefore important to ensure that all
 
 ## Breaking Changes
 
-We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major (breaking) release becomes necessary:
+We are currently on major release v4.  Breaking changes should be marked with the [deprecated](https://www.doxygen.nl/manual/commands.html#cmddeprecated) doxygen tag.  The following changes are planned when the next major/breaking release (v5) becomes necessary:
 
 * mp_testservice.sas to be renamed as mp_execute.sas (as it doesn't actually test anything)
+* `insert_cmplib` option of mcf_xxx macros will be deprecated (the option is now checked automatically with value inserted only if needed)
+* mcf_xxx macros to have `wrap=` option defaulted to YES for convenience.  Set this option explicitly to avoid issues.
+* mp_getddl.sas to be renamed to mp_ds2ddl.sas (consistent with other ds2xxx macros).  A wrapper macro is already in place, and you are able to use this immediately.  The default for SHOWLOG will also be YES instead of NO.
+* mp_coretable.sas will be replaced by the standalone macros in the `ddl` folder (which are already available)
 
 ## Star Gazing
 

base/mf_getapploc.sas

@@ -15,6 +15,7 @@
   @li /data
   @li /jobs
   @li /services
+  @li /tests
   @li /tests/jobs
   @li /tests/services
   @li /tests/macros
@@ -46,9 +47,13 @@
 
 /**
   * First check we are not in the tests/macros folder (which has no subfolders)
+  * or specifically in the testsetup or testteardown services
   */
-%if %index(&pgm,/tests/macros/) %then %do;
-  %let root=%substr(&pgm,1,%index(&pgm,/tests/macros)-1);
+%if %index(&pgm,/tests/macros/)
+or %index(&pgm,/tests/testsetup)
+or %index(&pgm,/tests/testteardown)
+%then %do;
+  %let root=%substr(&pgm,1,%index(&pgm,/tests)-1);
   &root
   %return;
 %end;

base/mf_getplatform.sas

@@ -22,6 +22,12 @@
 )/*/STORE SOURCE*/;
 %local a b c;
 %if &switch.NONE=NONE %then %do;
+  %if %symexist(sasjsprocessmode) %then %do;
+    %if &sasjsprocessmode=Stored Program %then %do;
+      SASJS
+      %return;
+    %end;
+  %end;
   %if %symexist(sysprocessmode) %then %do;
     %if "&sysprocessmode"="SAS Object Server"
     or "&sysprocessmode"= "SAS Compute Server" %then %do;

base/mf_verifymacvars.sas

@@ -1,6 +1,6 @@
 /**
   @file
-  @brief Checks if a set of macro variables exist / contain values.
+  @brief Checks if a set of macro variables exist AND contain values.
   @details Writes ERROR to log if abortType is SOFT, else will call %mf_abort.
   Usage:
 
@@ -14,10 +14,11 @@
   <h4> SAS Macros </h4>
   @li mf_abort.sas
 
-  @param verifyvars space separated list of macro variable names
-  @param makeupcase= set to YES to convert all variable VALUES to
+  @param [in] verifyvars Space separated list of macro variable names
+  @param [in] makeupcase= (NO) Set to YES to convert all variable VALUES to
     uppercase.
-  @param mAbort= Abort Type.  Default is SOFT (writes err to log).
+  @param [in] mAbort= (SOFT) Abort Type.  When SOFT, simply writes an err
+    message to the log.
     Set to any other value to call mf_abort (which can be configured to abort in
     various fashions according to context).
 
@@ -58,8 +59,14 @@
 
   %goto exit_success;
   %exit_err:
-    %if &mAbort=SOFT %then %put %str(ERR)OR: &abortmsg;
-    %else %mf_abort(mac=mf_verifymacvars,type=&mabort,msg=&abortmsg);
+    %put %str(ERR)OR: &abortmsg;
+    %mf_abort(iftrue=(&mabort ne SOFT),
+      mac=mf_verifymacvars,
+      msg=%str(&abortmsg)
+    )
+    0
+    %return;
   %exit_success:
+  1
 
 %mend mf_verifymacvars;

base/mp_abort.sas

@@ -66,7 +66,10 @@
   %if %length(&mac)>0 %then %put NOTE- called by &mac;
   %put NOTE - &msg;
 
-  %if %symexist(_SYSINCLUDEFILEDEVICE) %then %do;
+  %if %symexist(_SYSINCLUDEFILEDEVICE)
+  /* abort cancel FILE does not restart outside the INCLUDE on Viya 3.5 */
+  and "&SYSPROCESSNAME " ne "Compute Server "
+  %then %do;
     %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do;
       data &errds;
         iftrue='1=1';

base/mp_assertscope.sas

@@ -98,7 +98,7 @@
   create table _data_ as
     select name,offset,value
     from dictionary.macros
-    where scope="&scope" and name not in (%mf_getquotedstr(&ilist))
+    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
     order by name,offset;
 
   %let ds=&syslast;

base/mp_coretable.sas

@@ -5,24 +5,32 @@
     make use of permanent tables.  To avoid duplication in definitions, this
     macro provides a central location for managing the corresponding DDL.
 
+  Note - this macro is likely to be deprecated in future in favour of a
+  dedicated "datamodel" folder (prefix mddl)
+
+  Any corresponding data would go in a seperate repo, to avoid this one
+  ballooning in size!
+
   Example usage:
 
       %mp_coretable(LOCKTABLE,libds=work.locktable)
 
   @param [in] table_ref The type of table to create.  Example values:
-    @li DIFFTABLE - Used to store changes to tables.  Used by mp_storediffs.sas
-      and mp_stackdiffs.sas
-    @li FILTER_DETAIL - For storing detailed filter values.  Used by
-      mp_filterstore.sas.
-    @li FILTER_SUMMARY - For storing summary filter values.  Used by
-      mp_filterstore.sas.
-    @li LOCKANYTABLE - For "locking" tables prior to multipass loads. Used by
-      mp_lockanytable.sas
-    @li MAXKEYTABLE - For storing the maximum retained key information.  Used
-      by mp_retainedkey.sas
+    @li DIFFTABLE
+    @li FILTER_DETAIL
+    @li FILTER_SUMMARY
+    @li LOCKANYTABLE
+    @li MAXKEYTABLE
   @param [in] libds= (0) The library.dataset reference used to create the table.
     If not provided, then the DDL is simply printed to the log.
 
+  <h4> SAS Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mddl_dc_filterdetail.sas
+  @li mddl_dc_filtersummary.sas
+  @li mddl_dc_locktable.sas
+  @li mddl_dc_maxkeytable.sas
+
   <h4> Related Macros </h4>
   @li mp_filterstore.sas
   @li mp_lockanytable.sas
@@ -41,76 +49,21 @@
 %let outds=%sysfunc(ifc(&libds=0,_data_,&libds));
 proc sql;
 %if &table_ref=DIFFTABLE %then %do;
-  create table &outds(
-      load_ref char(36) label='unique load reference',
-      processed_dttm num format=E8601DT26.6 label='Processed at timestamp',
-      libref char(8) label='Library Reference (8 chars)',
-      dsn char(32) label='Dataset Name (32 chars)',
-      key_hash char(32) label=
-        'MD5 Hash of primary key values (pipe seperated)',
-      move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified',
-      is_pk num label='Is Primary Key Field? (1/0)',
-      is_diff num label=
-        'Did value change? (1/0/-1).  Always -1 for appends and deletes.',
-      tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
-      tgtvar_nm char(32) label='Target variable name (32 chars)',
-      oldval_num num format=best32. label='Old (numeric) value',
-      newval_num num format=best32. label='New (numeric) value',
-      oldval_char char(32765) label='Old (character) value',
-      newval_char char(32765) label='New (character) value',
-    constraint pk_mpe_audit
-      primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
-  );
+  %mddl_dc_difftable(libds=&outds)
 %end;
 %else %if &table_ref=LOCKTABLE %then %do;
-  create table &outds(
-      lock_lib char(8),
-      lock_ds char(32),
-      lock_status_cd char(10) not null,
-      lock_user_nm char(100) not null ,
-      lock_ref char(200),
-      lock_pid char(10),
-      lock_start_dttm num format=E8601DT26.6,
-      lock_end_dttm num format=E8601DT26.6,
-    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds));
+  %mddl_dc_locktable(libds=&outds)
 %end;
 %else %if &table_ref=FILTER_SUMMARY %then %do;
-  create table &outds(
-      filter_rk num not null,
-      filter_hash char(32) not null,
-      filter_table char(41) not null,
-      processed_dttm num not null format=E8601DT26.6,
-    constraint pk_mpe_filteranytable
-      primary key(filter_rk));
+  %mddl_dc_filtersummary(libds=&outds)
 %end;
 %else %if &table_ref=FILTER_DETAIL %then %do;
-  create table &outds(
-      filter_hash char(32) not null,
-      filter_line num not null,
-      group_logic char(3) not null,
-      subgroup_logic char(3) not null,
-      subgroup_id num not null,
-      variable_nm varchar(32) not null,
-      operator_nm varchar(12) not null,
-      raw_value varchar(4000) not null,
-      processed_dttm num not null format=E8601DT26.6,
-    constraint pk_mpe_filteranytable
-      primary key(filter_hash,filter_line));
+  %mddl_dc_filterdetail(libds=&outds)
 %end;
 %else %if &table_ref=MAXKEYTABLE %then %do;
-  create table &outds(
-      keytable varchar(41) label='Base table in libref.dataset format',
-      keycolumn char(32) format=$32.
-        label='The Retained key field containing the key values.',
-      max_key num label=
-        'Integer representing current max RK or SK value in the KEYTABLE',
-      processed_dttm num format=E8601DT26.6
-        label='Datetime this value was last updated',
-    constraint pk_mpe_maxkeyvalues
-        primary key(keytable));
+  %mddl_dc_maxkeytable(libds=&outds)
 %end;
 
-
 %if &libds=0 %then %do;
   describe table &syslast;
   drop table &syslast;

base/mp_ds2csv.sas

@@ -3,10 +3,28 @@
   @brief Export a dataset to a CSV file WITH leading blanks
   @details Export a dataset to a file or fileref, retaining leading blanks.
 
+  When using SASJS headerformat, the input statement is provided in the first
+  row of the CSV.
+
   Usage:
 
       %mp_ds2csv(sashelp.class,outref="%sysfunc(pathname(work))/file.csv")
 
+      filename example temp;
+      %mp_ds2csv(sashelp.air,outref=example,headerformat=SASJS)
+      data; infile example; input;put _infile_; if _n_>5 then stop;run;
+
+      data _null_;
+        infile example;
+        input;
+        call symputx('stmnt',_infile_);
+        stop;
+      run;
+      data work.want;
+        infile example dsd firstobs=2;
+        input &stmnt;
+      run;
+
   Why use mp_ds2csv over, say, proc export?
 
   1. Ability to retain leading blanks (this is a major one)
@@ -23,17 +41,24 @@
     @li LABEL - Use the variable label (or name, if blank)
     @li NAME - Use the variable name
     @li SASJS - Used to create sasjs-formatted input CSVs, eg for use in
-      mp_testservice.sas
+      mp_testservice.sas.  This format will supply an input statement in the
+      first row, making ingestion by datastep a breeze.  Special misisng values
+      will be prefixed with a period (eg `.A`) to enable ingestion on both SAS 9
+      and Viya.  Dates / Datetimes etc are identified by the format type (lookup
+      with mcf_getfmttype.sas) and converted to human readable formats (not
+      numbers).
   @param [out] outfile= The output filename - should be quoted.
   @param [out] outref= (0) The output fileref (takes precedence if provided)
-  @param [in] outencoding= (0) The output encoding to use (unquoted)
+  @param [in] outencoding= (0) The (quoted) output encoding to use, eg `"UTF-8"`
   @param [in] termstr= (CRLF) The line seperator to use.  For SASJS, will
     always be CRLF.  Valid values:
     @li CRLF
     @li LF
 
   <h4> SAS Macros </h4>
+  @li mcf_getfmttype.sas
   @li mf_getuniquename.sas
+  @li mf_getvarformat.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
 
@@ -50,7 +75,7 @@
   ,termstr=CRLF
 )/*/STORE SOURCE*/;
 
-%local outloc delim i varlist var vcnt vat dsv vcom vmiss;
+%local outloc delim i varlist var vcnt vat dsv vcom vmiss fmttype vfmt;
 
 %if not %sysfunc(exist(&ds)) %then %do;
   %put %str(WARN)ING:  &ds does not exist;
@@ -60,7 +85,7 @@
 %if %index(&ds,.)=0 %then %let ds=WORK.&ds;
 
 %if &outencoding=0 %then %let outencoding=;
-%else %let outencoding=encoding="&outencoding";
+%else %let outencoding=encoding=&outencoding;
 
 %if &outref=0 %then %let outloc=&outfile;
 %else %let outloc=&outref;
@@ -68,6 +93,7 @@
 %if &headerformat=SASJS %then %do;
   %let delim=",";
   %let termstr=CRLF;
+  %mcf_getfmttype(wrap=YES)
 %end;
 %else %if &dlm=COMMA %then %let delim=",";
 %else %let delim=";";
@@ -77,7 +103,8 @@
 /* first get headers */
 data _null_;
   file &outloc &outencoding lrecl=32767 termstr=&termstr;
-  length header $ 2000 varnm $32 dlm $1;
+  length header $ 2000 varnm vfmt $32 dlm $1 fmttype $8;
+  call missing(of _all_);
   dsid=open("&ds.","i");
   num=attrn(dsid,"nvars");
   dlm=&delim;
@@ -92,7 +119,14 @@ data _null_;
   %end;
   %else %if &headerformat=SASJS %then %do;
     if vartype(dsid,i)='C' then header=cats(varnm,':$char',varlen(dsid,i),'.');
-    else header=cats(varnm,':best.');
+    else do;
+      vfmt=coalescec(varfmt(dsid,i),'0');
+      fmttype=mcf_getfmttype(vfmt);
+      if fmttype='DATE' then header=cats(varnm,':date9.');
+      else if fmttype='DATETIME' then header=cats(varnm,':E8601DT26.6');
+      else if fmttype='TIME' then header=cats(varnm,':TIME12.');
+      else header=cats(varnm,':best.');
+    end;
   %end;
   %else %do;
     %put &sysmacroname: Invalid headerformat value (&headerformat);
@@ -147,14 +181,29 @@ data _null_;
       %let vcom=;
     %end;
     %if %mf_getvartype(&ds,&var)=N %then %do;
+      %if &headerformat = SASJS %then %do;
+        %let vcom=&delim;
+        %let fmttype=%sysfunc(mcf_getfmttype(%mf_getvarformat(&ds,&var)0));
+        %if &fmttype=DATE %then %let vfmt=DATE9.;
+        %else %if &fmttype=DATETIME %then %let vfmt=E8601DT26.6;
+        %else %if &fmttype=TIME %then %let vfmt=TIME12.;
+        %else %do;
+          %let vfmt=;
+          %let vcom=;
+        %end;
+      %end;
+      %else %let vcom=;
+
       /* must use period - in order to work in both 9.4 and Viya 3.5 */
       if missing(&var) and &var ne %sysfunc(getoption(MISSING)) then do;
         &vmiss=cats('.',&var);
         put &vmiss &vat;
       end;
-      else put &var &vat;
+      else put &var &vfmt &vcom &vat;
+
     %end;
     %else %do;
+      %if &i ne &vcnt %then %let vcom=&delim;
       put &var &&vlen&i &vcom &vat;
     %end;
   %end;

base/mp_ds2ddl.sas

@@ -0,0 +1,30 @@
+/**
+  @file
+  @brief A wrapper for mp_getddl.sas
+  @details In the next release, this will be the main version.
+
+
+  <h4> SAS Macros </h4>
+  @li mp_getddl.sas
+
+**/
+
+%macro mp_ds2ddl(libds,fref=getddl,flavour=SAS,showlog=YES,schema=
+  ,applydttm=NO
+)/*/STORE SOURCE*/;
+
+%local libref;
+%let libds=%upcase(&libds);
+%let libref=%scan(&libds,1,.);
+%if &libref=&libds %then %let libds=WORK.&libds;
+
+%mp_getddl(%scan(&libds,1,.)
+  ,%scan(&libds,2,.)
+  ,fref=&fref
+  ,flavour=SAS
+  ,showlog=&showlog
+  ,schema=&schema
+  ,applydttm=&applydttm
+)
+
+%mend mp_ds2ddl;

base/mp_filtercheck.sas

@@ -6,7 +6,8 @@
   SYSCC to 1008 if bad records are found, and call mp_abort.sas for a
   graceful service exit (configurable).
 
-  Used for dynamic filtering in [Data Controller for SAS®](https://datacontroller.io).
+  Used for dynamic filtering in [Data Controller for SAS®](
+  https://datacontroller.io).
 
   Usage:
 
@@ -125,7 +126,7 @@ data &outds;
     output;
   end;
   if OPERATOR_NM not in
-  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
+  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE')
   then do;
     REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;

base/mp_filterstore.sas

@@ -8,8 +8,13 @@
   https://sasapps.io)).  This macro is also used in [Data Controller for SAS](
   https://datacontroller.io).
 
+  A more recent feature of this macro is the ability to support filter queries
+  on Format Catalogs.  This is achieved by adding a `-FC` suffix to the `libds`
+  parameter - where the "ds" in this case is the catalog name.
 
-  @param [in] libds= The target dataset to be filtered (lib should be assigned)
+
+  @param [in] libds= The target dataset to be filtered (lib should be assigned).
+    If filtering a format catalog, add the following suffix:  `-FC`.
   @param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to
     be validated.  Has the following format:
 |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767|
@@ -45,6 +50,7 @@
 
 
   <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
   @li mf_getuniquename.sas
   @li mf_getvalue.sas
   @li mf_islibds.sas
@@ -78,7 +84,10 @@
 %put &sysmacroname entry vars:;
 %put _local_;
 
-%local ds1 ds2 ds3 ds4 filter_hash;
+%local ds0 ds1 ds2 ds3 ds4 filter_hash orig_libds;
+%let libds=%upcase(&libds);
+%let orig_libds=&libds;
+
 %mp_abort(iftrue= (&syscc ne 0)
   ,mac=mp_filterstore
   ,msg=%str(syscc=&syscc on macro entry)
@@ -96,12 +105,27 @@
   ,msg=%str(Invalid lock_table value: &lock_table)
 )
 
-/* validate query */
+/**
+  * validate query
+  * use format catalog export, if a format
+  */
+%if "%substr(&libds,%length(&libds)-2,3)"="-FC" %then %do;
+  %let libds=%scan(&libds,1,-); /* chop off -FC extension */
+  %let ds0=%mf_getuniquename(prefix=fmtds_);
+  %let libds=&ds0;
+  /*
+    There is no need to export the entire format catalog here - the validations
+    are done against the data model, not the data values.  So we can simply
+    hardcode the structure based on the cntlout dataset.
+  */
+  %mddl_sas_cntlout(libds=&ds0)
+
+%end;
 %mp_filtercheck(&queryds,targetds=&libds,abort=YES)
 
 /* hash the result */
 %let ds1=%mf_getuniquename(prefix=hashds);
-%mp_hashdataset(&queryds,outds=&ds1,salt=&libds)
+%mp_hashdataset(&queryds,outds=&ds1,salt=&orig_libds)
 %let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey));
 %if &mdebug=1 %then %do;
   data _null_;
@@ -132,7 +156,7 @@ run;
   %let ds3=%mf_getuniquename(prefix=filtersum);
   data work.&ds3;
     if 0 then set &filter_summary;
-    filter_table=symget('libds');
+    filter_table="&orig_libds";
     filter_hash="&filter_hash";
     PROCESSED_DTTM=%sysfunc(datetime());
     output;

base/mp_getformats.sas

@@ -40,6 +40,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
   |`WHICHPATH `|`**OTHER** `|`**OTHER** `|`big fat problem if not path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|`O `|` `|` `|` `|` `|
 
   <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
   @li mf_dedup.sas
   @li mf_getfmtlist.sas
   @li mf_getfmtname.sas
@@ -94,30 +95,7 @@ create table &outsummary as
 
 %if "&outdetail" ne "0" %then %do;
   /* ensure base table always exists */
-  proc sql;
-  create table &outdetail(
-      FMTNAME char(32)     label='Format name'
-      ,START char(16)     label='Starting value for format'
-      ,END char(16)     label='Ending value for format'
-      ,LABEL char(256)     label='Format value label'
-      ,MIN num length=3     label='Minimum length'
-      ,MAX num length=3     label='Maximum length'
-      ,DEFAULT num length=3     label='Default length'
-      ,LENGTH num length=3     label='Format length'
-      ,FUZZ num     label='Fuzz value'
-      ,PREFIX char(2)     label='Prefix characters'
-      ,MULT num     label='Multiplier'
-      ,FILL char(1)     label='Fill character'
-      ,NOEDIT num length=3     label='Is picture string noedit?'
-      ,TYPE char(1)     label='Type of format'
-      ,SEXCL char(1)     label='Start exclusion'
-      ,EEXCL char(1)     label='End exclusion'
-      ,HLO char(13)     label='Additional information'
-      ,DECSEP char(1)     label='Decimal separator'
-      ,DIG3SEP char(1)     label='Three-digit separator'
-      ,DATATYPE char(8)     label='Date/time/datetime?'
-      ,LANGUAGE char(8)     label='Language for date strings'
-  );
+  %mddl_sas_cntlout(libds=&outdetail)
   /* grab the location of each format */
   %let fmtcnt=0;
   data _null_;
@@ -134,6 +112,10 @@ create table &outsummary as
     proc format library=&&fmtloc&i CNTLOUT=&tempds;
       select &&fmtname&i;
     run;
+    data &tempds;
+      length label $256;
+      set &tempds;
+    run;
     proc append base=&outdetail data=&tempds;
     run;
   %end;

base/mp_include.sas

@@ -51,9 +51,10 @@ https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/mcrolref/n1j5tcc0n2xczyn1
     this dataset.
     It will then run an abort cancel FILE to stop the include running, and pass
     the dataset back.
-    NOTE - it is NOT possible to read this dataset as part of _this_ macro -
-    when running abort cancel FILE, ALL macros are closed, so instead it is
-    necessary to invoke "%mp_abort(mode=INCLUDE)" OUTSIDE of any macro wrappers.
+
+    IMPORTANT NOTE - it is NOT possible to read this dataset as part of _this_
+    macro! When running abort cancel FILE, ALL macros are closed, so instead it
+    is necessary to invoke "%mp_abort(mode=INCLUDE)" OUTSIDE of macro wrappers.
 
 
   @version 9.4

base/mp_loadformat.sas

@@ -0,0 +1,303 @@
+/**
+  @file
+  @brief Loads a format catalog from a staging dataset
+  @details When loading staged data, it is common to receive only the records
+  that have actually changed.  However, when loading a format catalog, if
+  records are missing they are presumed to be no longer required.
+
+  This macro will augment a staging dataset with other records from the same
+  format, to prevent loss of data - UNLESS the input dataset contains a marker
+  column, specifying that a particular row needs to be deleted (`delete_col=`).
+
+  This macro can also be used to identify which records would be (or were)
+  considered new, modified or deleted (`loadtarget=`) by creating the following
+  tables:
+
+  @li work.outds_add
+  @li work.outds_del
+  @li work.outds_mod
+
+  For example usage, see mp_loadformat.test.sas
+
+  @param [in] libcat The format catalog to be loaded
+  @param [in] libds The staging table to load
+  @param [in] loadtarget= (NO) Set to YES to actually load the target catalog
+  @param [in] delete_col= (_____DELETE__THIS__RECORD_____) The column used to
+    mark a record for deletion.  Values should be "Yes" or "No".
+  @param [out] auditlibds= (0) For change tracking, set to the libds of an audit
+    table as defined in mddl_dc_difftable.sas
+  @param [in] locklibds= (0) For multi-user (parallel) situations, set to the
+    libds of the DC lock table as defined in the mddl_dc_locktable.sas macro.
+  @param [out] outds_add= (0) Set a libds here to see the new records added
+  @param [out] outds_del= (0) Set a libds here to see the records deleted
+  @param [out] outds_mod= (0) Set a libds here to see the modified records
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_lockanytable.sas
+
+  <h4> Related Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mddl_dc_locktable.sas
+  @li mp_loadformat.test.sas
+  @li mp_lockanytable.sas
+  @li mp_storediffs.sas
+  @li mp_stackdiffs.sas
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_loadformat(libcat,libds
+  ,loadtarget=NO
+  ,auditlibds=0
+  ,locklibds=0
+  ,delete_col=_____DELETE__THIS__RECORD_____
+  ,outds_add=0
+  ,outds_del=0
+  ,outds_mod=0
+  ,mdebug=0
+);
+/* set up local macro variables and temporary tables (with a prefix) */
+%local err msg prefix dslist i var fmtlist ibufsize;
+%let dslist=base base_fmts template inlibds ds1 stagedata storediffs;
+%if &outds_add=0 %then %let dslist=&dslist outds_add;
+%if &outds_del=0 %then %let dslist=&dslist outds_del;
+%if &outds_mod=0 %then %let dslist=&dslist outds_mod;
+%let prefix=%substr(%mf_getuniquename(),1,22);
+%do i=1 %to %sysfunc(countw(&dslist));
+  %let var=%scan(&dslist,&i);
+  %local &var;
+  %let &var=%upcase(&prefix._&var);
+%end;
+
+
+/* perform input validations */
+%let err=0;
+%let msg=0;
+data _null_;
+  if _n_=1 then putlog "&sysmacroname entry vars:";
+  set sashelp.vmacro;
+  where scope="&sysmacroname";
+  value=upcase(value);
+  if &mdebug=0 then put name '=' value;
+  if name=:'LOAD' and value not in ('YES','NO') then do;
+    call symputx('msg',"invalid value for "!!name!!":"!!value);
+    call symputx('err',1);
+    stop;
+  end;
+  else if name='LIBCAT' then do;
+    if exist(value,'CATALOG') le 0 then do;
+      call symputx('msg',"Unable to open catalog: "!!value);
+      call symputx('err',1);
+      stop;
+    end;
+  end;
+  else if name='LIBDS' then do;
+    if exist(value) le 0 then do;
+      call symputx('msg',"Unable to open staging table: "!!value);
+      call symputx('err',1);
+      stop;
+    end;
+  end;
+  else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
+  and missing(value) then do;
+    call symputx('msg',"missing value in var: "!!name);
+    call symputx('err',1);
+    stop;
+  end;
+run;
+
+%mp_abort(
+  iftrue=(&err ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(&msg)
+)
+
+/**
+  * First, extract only relevant formats from the catalog
+  */
+proc sql noprint;
+select distinct fmtname into: fmtlist separated by ' ' from &libds;
+proc format lib=&libcat cntlout=&base;
+  select
+  /* send formats individually to avoid line truncation in the input stack */
+  %do i=1 %to %sysfunc(countw(&fmtlist));
+    %scan(&fmtlist,&i,%str( ))
+  %end;
+  ;
+run;
+proc sort data=&base;
+  by fmtname start;
+run;
+
+/**
+  * Ensure input table and base_formats have consistent lengths and types
+  */
+%mddl_sas_cntlout(libds=&template)
+data &inlibds;
+  if 0 then set &template;
+  set &libds;
+  if missing(type) then do;
+    if substr(fmtname,1,1)='$' then type='C';
+    else type='N';
+  end;
+  if type='N' then start=put(input(start,best.),best16.);
+run;
+data &base_fmts;
+  if 0 then set &template;
+  set &base;
+run;
+
+/*
+format values can be up to 32767 wide.  SQL joins on such a wide column can
+cause buffer issues.  Update ibufsize and reset at the end.
+*/
+%let ibufsize=%sysfunc(getoption(ibufsize));
+options ibufsize=32767 ;
+
+/**
+  * Identify new records
+  */
+proc sql;
+create table &outds_add(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  left join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where b.fmtname is null
+    and upcase(a.&delete_col) ne "YES"
+  order by fmtname, start;;
+
+/**
+  * Identify deleted records
+  */
+create table &outds_del(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where upcase(a.&delete_col)="YES"
+  order by fmtname, start;
+
+/**
+  * Identify modified records
+  */
+create table &outds_mod (drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where upcase(a.&delete_col) ne "YES"
+  order by fmtname, start;
+
+options ibufsize=&ibufsize;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc prior to load prep)
+)
+
+%if &loadtarget=YES %then %do;
+  data &ds1;
+    merge &base_fmts(in=base)
+      &outds_mod(in=mod)
+      &outds_add(in=add)
+      &outds_del(in=del);
+    if not del and not mod;
+    by fmtname start;
+  run;
+  data &stagedata;
+    set &ds1 &outds_mod;
+  run;
+  proc sort;
+    by fmtname start;
+  run;
+%end;
+/* mp abort needs to run outside of conditional blocks */
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc prior to actual load)
+)
+%if &loadtarget=YES %then %do;
+  %if %mf_nobs(&stagedata)=0 %then %do;
+    %put There are no changes to load in &libcat!;
+    %return;
+  %end;
+  %if &locklibds ne 0 %then %do;
+    /* prevent parallel updates */
+    %mp_lockanytable(LOCK,
+      lib=%scan(&libcat,1,.)
+      ,ds=%scan(&libcat,2,.)
+      ,ref=MP_LOADFORMAT commencing format load
+      ,ctl_ds=&locklibds
+    )
+  %end;
+  /* do the actual load */
+  proc format lib=&libcat cntlin=&stagedata;
+  run;
+  %if &locklibds ne 0 %then %do;
+    /* unlock the table */
+    %mp_lockanytable(UNLOCK
+      lib=%scan(&libcat,1,.)
+      ,ds=%scan(&libcat,2,.)
+      ,ref=MP_LOADFORMAT completed format load
+      ,ctl_ds=&locklibds
+    )
+  %end;
+  /* track the changes */
+  %if &auditlibds ne 0 %then %do;
+    %if &locklibds ne 0 %then %do;
+      %mp_lockanytable(LOCK,
+        lib=%scan(&auditlibds,1,.)
+        ,ds=%scan(&auditlibds,2,.)
+        ,ref=MP_LOADFORMAT commencing audit table load
+        ,ctl_ds=&locklibds
+      )
+    %end;
+
+    %mp_storediffs(&libcat
+      ,&stageds
+      ,FMTNAME START
+      ,delds=&outds_del
+      ,modds=&outds_mod
+      ,appds=&outds_add
+      ,outds=&storediffs
+      ,mdebug=&mdebug
+    )
+
+    %if &locklibds ne 0 %then %do;
+      %mp_lockanytable(UNLOCK
+        lib=%scan(&auditlibds,1,.)
+        ,ds=%scan(&auditlibds,2,.)
+        ,ref=MP_LOADFORMAT commencing audit table load
+        ,ctl_ds=&locklibds
+      )
+    %end;
+  %end;
+%end;
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc after load)
+)
+
+%if &mdebug=0 %then %do;
+  proc datasets lib=work;
+    delete &prefix:;
+  run;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%mend mp_loadformat;

base/mp_md5.sas

@@ -0,0 +1,58 @@
+/**
+  @file
+  @brief Generates an md5 expression for hashing a set of variables
+  @details This is the same algorithm used to hash records in
+  [Data Controller for SAS](https://datacontroller.io) (free for up
+  to 5 users).
+
+  It is not designed to be efficient - it is designed to be effective,
+  given the range of edge cases (large floating points, special missing
+  numerics, thousands of columns, very wide columns).
+
+  It can be used only in data step, eg as follows:
+
+      data _null_;
+        set sashelp.class;
+        hashvar=%mp_md5(cvars=name sex, nvars=age height weight);
+        put hashvar=;
+      run;
+
+  Unfortunately it will not run in SQL - it fails with the following message:
+
+  > The width value for HEX is out of bounds. It should be between 1 and 16
+
+  The macro will also cause errors if the data contains (non-special) missings
+  and the (undocumented) `options dsoptions=nonote2err;` is in effect.
+
+  This can be avoided in two ways:
+
+  @li Global option:  `options dsoptions=nonote2err;`
+  @li Data step option: `data YOURLIB.YOURDATASET /nonote2err;`
+
+  @param cvars= Space seperated list of character variables
+  @param nvars= Space seperated list of numeric variables
+
+  <h4> Related Programs </h4>
+  @li mp_init.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_md5(cvars=,nvars=);
+%local i var sep;
+put(md5(
+  %do i=1 %to %sysfunc(countw(&cvars));
+    %let var=%scan(&cvars,&i,%str( ));
+    &sep put(md5(trim(&var)),$hex32.)
+    %let sep=!!;
+  %end;
+  %do i=1 %to %sysfunc(countw(&nvars));
+    %let var=%scan(&nvars,&i,%str( ));
+    /* multiply by 1 to strip precision errors (eg 0 != 0) */
+    /* but ONLY if not missing, else will lose any special missing values */
+    &sep put(md5(trim(put(ifn(missing(&var),&var,&var*1),binary64.))),$hex32.)
+    %let sep=!!;
+  %end;
+),hex32.)
+%mend mp_md5;

base/mp_streamfile.sas

@@ -12,17 +12,19 @@
 
       %mp_streamfile(contenttype=csv,inloc=/some/where.txt,outname=myfile.txt)
 
+  @param [in] contenttype= (TEXTS) Either TEXT, ZIP, CSV, EXCEL
+  @param [in] inloc= /path/to/file.ext to be sent
+  @param [in] inref= fileref of file to be sent (if provided, overrides `inloc`)
+  @param [in] iftrue= (1=1) Provide a condition under which to execute.
+  @param [out] outname= the name of the file, as downloaded by the browser
+  @param [out] outref= (_webout) The destination where the file should be
+    streamed.
+
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
   @li mp_binarycopy.sas
 
-  @param contenttype= Either TEXT, ZIP, CSV, EXCEL (default TEXT)
-  @param inloc= /path/to/file.ext to be sent
-  @param inref= fileref of file to be sent (if provided, overrides `inloc`)
-  @param outname= the name of the file, as downloaded by the browser
-
   @author Allan Bowe
-  @source https://github.com/sasjs/core
 
 **/
 
@@ -30,12 +32,16 @@
   contenttype=TEXT
   ,inloc=
   ,inref=0
+  ,iftrue=%str(1=1)
   ,outname=
+  ,outref=_webout
 )/*/STORE SOURCE*/;
 
-%let contentype=%upcase(&contenttype);
-%local platform; %let platform=%mf_getplatform();
+%if not(%eval(%unquote(&iftrue))) %then %return;
 
+%let contentype=%upcase(&contenttype);
+%let outref=%upcase(&outref);
+%local platform; %let platform=%mf_getplatform();
 
 /**
   * check engine type to avoid the below err message:
@@ -44,39 +50,80 @@
 %local streamweb;
 %let streamweb=0;
 data _null_;
-  set sashelp.vextfl(where=(upcase(fileref)="_WEBOUT"));
+  set sashelp.vextfl(where=(upcase(fileref)="&outref"));
   if xengine='STREAM' then call symputx('streamweb',1,'l');
 run;
 
-%if &contentype=ZIP %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+%if &contentype=CSV %then %do;
+  %if (&platform=SASMETA and &streamweb=1) or &platform=SASJS %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/zip');
+      rc=stpsrv_header('Content-type','application/csv');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.zip'
-      contenttype='application/zip'
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
+      contenttype='application/csv'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
 %else %if &contentype=EXCEL %then %do;
   /* suitable for XLS format */
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) or &platform=SASJS %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/vnd.ms-excel');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
       contenttype='application/vnd.ms-excel'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
+%else %if &contentype=GIF or &contentype=JPEG or &contentype=PNG %then %do;
+  %if (&platform=SASMETA and &streamweb=1) or &platform=SASJS %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type',"image/%lowcase(&contenttype)");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI"
+      contenttype="image/%lowcase(&contenttype)";
+  %end;
+%end;
+%else %if &contentype=HTML %then %do;
+  %if &platform=SASVIYA %then %do;
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
+      contenttype="text/html";
+  %end;
+%end;
+%else %if &contentype=TEXT %then %do;
+  %if (&platform=SASMETA and &streamweb=1) or &platform=SASJS %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type','application/text');
+      rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
+      contenttype='application/text'
+      contentdisp="attachment; filename=&outname";
+  %end;
+%end;
+%else %if &contentype=WOFF or &contentype=WOFF2 or &contentype=TTF %then %do;
+  %if (&platform=SASMETA and &streamweb=1) or &platform=SASJS %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type',"font/%lowcase(&contenttype)");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI"
+      contenttype="font/%lowcase(&contenttype)";
+  %end;
+%end;
 %else %if &contentype=XLSX %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) or &platform=SASJS %then %do;
     data _null_;
       rc=stpsrv_header('Content-type',
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
@@ -84,54 +131,34 @@ run;
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
       contenttype=
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
-%else %if &contentype=TEXT %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+%else %if &contentype=ZIP %then %do;
+  %if (&platform=SASMETA and &streamweb=1) or &platform=SASJS %then %do;
     data _null_;
-      rc=stpsrv_header('Content-type','application/text');
+      rc=stpsrv_header('Content-type','application/zip');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
     run;
   %end;
   %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
-      contenttype='application/text'
+    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.zip'
+      contenttype='application/zip'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
-%else %if &contentype=CSV %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
-    data _null_;
-      rc=stpsrv_header('Content-type','application/csv');
-      rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
-    run;
-  %end;
-  %else %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
-      contenttype='application/csv'
-      contentdisp="attachment; filename=&outname";
-  %end;
-%end;
-%else %if &contentype=HTML %then %do;
-  %if &platform=SASVIYA %then %do;
-    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
-      contenttype="text/html";
-  %end;
-%end;
 %else %do;
   %put %str(ERR)OR: Content Type &contenttype NOT SUPPORTED by &sysmacroname!;
-  %return;
 %end;
 
 %if &inref ne 0 %then %do;
-  %mp_binarycopy(inref=&inref,outref=_webout)
+  %mp_binarycopy(inref=&inref,outref=&outref)
 %end;
 %else %do;
-  %mp_binarycopy(inloc="&inloc",outref=_webout)
+  %mp_binarycopy(inloc="&inloc",outref=&outref)
 %end;
 
 %mend mp_streamfile;

base/mp_wait4file.sas

@@ -4,9 +4,7 @@
   @details Loops with a `sleep()` command until a file arrives or the max wait
   period expires.
 
-  @example
-
-  Wait 3 minutes OR for /tmp/flag.txt to appear
+  Example: Wait 3 minutes OR for /tmp/flag.txt to appear
 
     %mp_wait4file(/tmp/flag.txt , maxwait=60*3)
 

build.py

@@ -84,7 +84,7 @@ options noquotelenmax;
 """
 f = open('all.sas', "w")             # r / r+ / rb / rb+ / w / wb
 f.write(header)
-folders=['base','meta','metax','server','viya','lua','fcmp']
+folders=['base','ddl','meta','metax','server','viya','lua','fcmp']
 for folder in folders:
     filenames = [fn for fn in Path('./' + folder).iterdir() if fn.match("*.sas")]
     filenames.sort()

ddl/mddl_dc_difftable.sas

@@ -0,0 +1,34 @@
+/**
+  @file
+  @brief Difftable DDL
+  @details Used to store changes to tables.  Used by mp_storediffs.sas
+      and mp_stackdiffs.sas
+
+**/
+
+
+%macro mddl_dc_difftable(libds=WORK.DIFFTABLE);
+
+  proc sql;
+  create table &libds(
+      load_ref char(36) label='unique load reference',
+      processed_dttm num format=E8601DT26.6 label='Processed at timestamp',
+      libref char(8) label='Library Reference (8 chars)',
+      dsn char(32) label='Dataset Name (32 chars)',
+      key_hash char(32) label=
+        'MD5 Hash of primary key values (pipe seperated)',
+      move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified',
+      is_pk num label='Is Primary Key Field? (1/0)',
+      is_diff num label=
+        'Did value change? (1/0/-1).  Always -1 for appends and deletes.',
+      tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
+      tgtvar_nm char(32) label='Target variable name (32 chars)',
+      oldval_num num format=best32. label='Old (numeric) value',
+      newval_num num format=best32. label='New (numeric) value',
+      oldval_char char(32765) label='Old (character) value',
+      newval_char char(32765) label='New (character) value',
+    constraint pk_mpe_audit
+      primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
+  );
+
+%mend mddl_dc_difftable;

ddl/mddl_dc_filterdetail.sas

@@ -0,0 +1,27 @@
+/**
+  @file
+  @brief Filtertable DDL
+  @details For storing detailed filter values.  Used by
+      mp_filterstore.sas.
+
+**/
+
+
+%macro mddl_dc_filterdetail(libds=WORK.FILTER_DETAIL);
+
+  proc sql;
+  create table &libds(
+      filter_hash char(32) not null,
+      filter_line num not null,
+      group_logic char(3) not null,
+      subgroup_logic char(3) not null,
+      subgroup_id num not null,
+      variable_nm varchar(32) not null,
+      operator_nm varchar(12) not null,
+      raw_value varchar(4000) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_hash,filter_line)
+  );
+
+%mend mddl_dc_filterdetail;

ddl/mddl_dc_filtersummary.sas

@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Filtersummary DDL
+  @details For storing summary filter values.  Used by
+      mp_filterstore.sas.
+
+**/
+
+
+%macro mddl_dc_filtersummary(libds=WORK.FILTER_SUMMARY);
+
+  proc sql;
+  create table &libds(
+      filter_rk num not null,
+      filter_hash char(32) not null,
+      filter_table char(41) not null,
+      processed_dttm num not null format=E8601DT26.6,
+    constraint pk_mpe_filteranytable
+      primary key(filter_rk)
+  );
+
+%mend mddl_dc_filtersummary;

ddl/mddl_dc_locktable.sas

@@ -0,0 +1,25 @@
+/**
+  @file
+  @brief Locktable DDL
+  @details For "locking" tables prior to multipass loads. Used by
+      mp_lockanytable.sas
+
+**/
+
+
+%macro mddl_dc_locktable(libds=WORK.LOCKTABLE);
+
+  proc sql;
+  create table &libds(
+      lock_lib char(8),
+      lock_ds char(32),
+      lock_status_cd char(10) not null,
+      lock_user_nm char(100) not null ,
+      lock_ref char(200),
+      lock_pid char(10),
+      lock_start_dttm num format=E8601DT26.6,
+      lock_end_dttm num format=E8601DT26.6,
+    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds)
+  );
+
+%mend mddl_dc_locktable;

ddl/mddl_dc_maxkeytable.sas

@@ -0,0 +1,24 @@
+/**
+  @file
+  @brief Maxkeytable DDL
+  @details For storing the maximum retained key information.  Used
+      by mp_retainedkey.sas
+
+**/
+
+
+%macro mddl_dc_maxkeytable(libds=WORK.MAXKEYTABLE);
+
+  proc sql;
+  create table &libds(
+      keytable varchar(41) label='Base table in libref.dataset format',
+      keycolumn char(32) format=$32.
+        label='The Retained key field containing the key values.',
+      max_key num label=
+        'Integer representing current max RK or SK value in the KEYTABLE',
+      processed_dttm num format=E8601DT26.6
+        label='Datetime this value was last updated',
+    constraint pk_mpe_maxkeyvalues
+        primary key(keytable));
+
+%mend mddl_dc_maxkeytable;

ddl/mddl_sas_cntlout.sas

@@ -0,0 +1,43 @@
+/**
+  @file
+  @brief The CNTLOUT table generated by proc format
+  @details This table will actually change format depending on the data values,
+  therefore the max possible lengths are described here to enable consistency
+  when dealing with format data.
+
+**/
+
+
+%macro mddl_sas_cntlout(libds=WORK.CNTLOUT);
+
+proc sql;
+create table &libds(
+    FMTNAME char(32)      label='Format name'
+    /*
+      to accomodate larger START values, mp_loadformat.sas will need the
+      SQL dependency removed (proc sql needs to accomodate 3 index values in
+      a 32767 ibufsize limit)
+    */
+    ,START char(10000)    label='Starting value for format'
+    ,END char(32767)      label='Ending value for format'
+    ,LABEL char(32767)    label='Format value label'
+    ,MIN num length=3     label='Minimum length'
+    ,MAX num length=3     label='Maximum length'
+    ,DEFAULT num length=3 label='Default length'
+    ,LENGTH num length=3  label='Format length'
+    ,FUZZ num             label='Fuzz value'
+    ,PREFIX char(2)       label='Prefix characters'
+    ,MULT num             label='Multiplier'
+    ,FILL char(1)         label='Fill character'
+    ,NOEDIT num length=3  label='Is picture string noedit?'
+    ,TYPE char(1)         label='Type of format'
+    ,SEXCL char(1)        label='Start exclusion'
+    ,EEXCL char(1)        label='End exclusion'
+    ,HLO char(13)         label='Additional information'
+    ,DECSEP char(1)       label='Decimal separator'
+    ,DIG3SEP char(1)      label='Three-digit separator'
+    ,DATATYPE char(8)     label='Date/time/datetime?'
+    ,LANGUAGE char(8)     label='Language for date strings'
+);
+
+%mend mddl_sas_cntlout;

fcmp/mcf_getfmttype.sas

@@ -31,12 +31,12 @@
   > fmt3=TIME
 
   @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
-  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
-    CMPLIB reference.
   @param [out] lib= (work) The output library in which to create the catalog.
   @param [out] cat= (sasjs) The output catalog in which to create the package.
   @param [out] pkg= (utils) The output package in which to create the function.
     Uses a 3 part format:  libref.catalog.package
+  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
+    values inserted only if needed.
 
   <h4> SAS Macros </h4>
   @li mcf_init.sas
@@ -51,11 +51,12 @@
 **/
 
 %macro mcf_getfmttype(wrap=NO
-  ,insert_cmplib=NO
+  ,insert_cmplib=DEPRECATED
   ,lib=WORK
   ,cat=SASJS
   ,pkg=UTILS
 )/*/STORE SOURCE*/;
+%local i var cmpval found;
 
 %if %mcf_init(mcf_getfmttype)=1 %then %return;
 
@@ -103,7 +104,14 @@ endsub;
   quit;
 %end;
 
-%if &insert_cmplib=YES %then %do;
+/* insert the CMPLIB if not already there */
+%let cmpval=%sysfunc(getoption(cmplib));
+%let found=0;
+%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
+  %let var=%scan(&cmpval,&i,%str( %(%)));
+  %if &var=&lib..&cat %then %let found=1;
+%end;
+%if &found=0 %then %do;
   options insert=(CMPLIB=(&lib..&cat));
 %end;
 

fcmp/mcf_length.sas

@@ -32,12 +32,12 @@
   > outa=3 outb=4 outc=5 outd=0
 
   @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
-  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
-    CMPLIB reference.
   @param [out] lib= (work) The output library in which to create the catalog.
   @param [out] cat= (sasjs) The output catalog in which to create the package.
   @param [out] pkg= (utils) The output package in which to create the function.
     Uses a 3 part format:  libref.catalog.package
+  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
+    values inserted only if needed.
 
   <h4> SAS Macros </h4>
   @li mcf_init.sas
@@ -49,12 +49,12 @@
 **/
 
 %macro mcf_length(wrap=NO
-  ,insert_cmplib=NO
+  ,insert_cmplib=DEPRECATED
   ,lib=WORK
   ,cat=SASJS
   ,pkg=UTILS
 )/*/STORE SOURCE*/;
-
+%local i var cmpval found;
 %if %mcf_init(mcf_length)=1 %then %return;
 
 %if &wrap=YES  %then %do;
@@ -76,7 +76,14 @@ endsub;
   quit;
 %end;
 
-%if &insert_cmplib=YES %then %do;
+/* insert the CMPLIB if not already there */
+%let cmpval=%sysfunc(getoption(cmplib));
+%let found=0;
+%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
+  %let var=%scan(&cmpval,&i,%str( %(%)));
+  %if &var=&lib..&cat %then %let found=1;
+%end;
+%if &found=0 %then %do;
   options insert=(CMPLIB=(&lib..&cat));
 %end;
 

fcmp/mcf_stpsrv_header.sas

@@ -47,12 +47,12 @@
 
 
   @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
-  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
-    CMPLIB reference.
   @param [out] lib= (work) The output library in which to create the catalog.
   @param [out] cat= (sasjs) The output catalog in which to create the package.
   @param [out] pkg= (utils) The output package in which to create the function.
     Uses a 3 part format:  libref.catalog.package
+  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
+    values inserted only if needed.
 
   <h4> SAS Macros </h4>
   @li mcf_init.sas
@@ -64,12 +64,12 @@
 **/
 
 %macro mcf_stpsrv_header(wrap=NO
-  ,insert_cmplib=NO
+  ,insert_cmplib=DEPRECATED
   ,lib=WORK
   ,cat=SASJS
   ,pkg=UTILS
 )/*/STORE SOURCE*/;
-
+%local i var cmpval found;
 %if %mcf_init(stpsrv_header)=1 %then %return;
 
 %if &wrap=YES  %then %do;
@@ -96,7 +96,14 @@ endsub;
   quit;
 %end;
 
-%if &insert_cmplib=YES %then %do;
+/* insert the CMPLIB if not already there */
+%let cmpval=%sysfunc(getoption(cmplib));
+%let found=0;
+%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
+  %let var=%scan(&cmpval,&i,%str( %(%)));
+  %if &var=&lib..&cat %then %let found=1;
+%end;
+%if &found=0 %then %do;
   options insert=(CMPLIB=(&lib..&cat));
 %end;
 

fcmp/mcf_string2file.sas

@@ -32,12 +32,12 @@
       run;
 
   @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
-  @param [out] insert_cmplib= (NO) Choose YES to insert the package into the
-    CMPLIB reference.
   @param [out] lib= (work) The output library in which to create the catalog.
   @param [out] cat= (sasjs) The output catalog in which to create the package.
   @param [out] pkg= (utils) The output package in which to create the function.
     Uses a 3 part format:  libref.catalog.package
+  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
+    values inserted only if needed.
 
   <h4> SAS Macros </h4>
   @li mcf_init.sas
@@ -49,12 +49,12 @@
 **/
 
 %macro mcf_string2file(wrap=NO
-  ,insert_cmplib=NO
+  ,insert_cmplib=DEPRECATED
   ,lib=WORK
   ,cat=SASJS
   ,pkg=UTILS
 )/*/STORE SOURCE*/;
-
+%local i var cmpval found;
 %if %mcf_init(mcf_string2file)=1 %then %return;
 
 %if &wrap=YES  %then %do;
@@ -81,7 +81,14 @@ endsub;
   quit;
 %end;
 
-%if &insert_cmplib=YES %then %do;
+/* insert the CMPLIB if not already there */
+%let cmpval=%sysfunc(getoption(cmplib));
+%let found=0;
+%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
+  %let var=%scan(&cmpval,&i,%str( %(%)));
+  %if &var=&lib..&cat %then %let found=1;
+%end;
+%if &found=0 %then %do;
   options insert=(CMPLIB=(&lib..&cat));
 %end;
 

meta/mm_createapplication.sas

@@ -59,7 +59,10 @@
 %&mD.put Executing &sysmacroname..sas;
 %&mD.put _local_;
 
-%mf_verifymacvars(tree name)
+%mp_abort(iftrue= (%mf_verifymacvars(tree name)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Empty inputs: tree name)
+)
 
 /**
   * check tree exists

meta/mm_createdocument.sas

@@ -47,7 +47,10 @@
 %&mD.put Executing &sysmacroname..sas;
 %&mD.put _local_;
 
-%mf_verifymacvars(tree name)
+%mp_abort(iftrue= (%mf_verifymacvars(tree name)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Empty inputs: tree name)
+)
 
 /**
   * check tree exists

meta/mm_createlibrary.sas

@@ -133,12 +133,14 @@ run;
 filename &frefin temp;
 filename &frefout temp;
 
+%mp_abort(iftrue= (
+    &engine=BASE & %mf_verifymacvars(libname libref engine servercontext tree)=0
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Empty inputs: libname libref engine servercontext tree)
+)
+
 %if &engine=BASE %then %do;
-
-  %mf_verifymacvars(libname libref engine servercontext tree)
-
-
-
   /**
     * Check that the ServerContext exists
     */

meta/mm_createstp.sas

@@ -108,7 +108,12 @@
 %&mD.put Executing mm_CreateSTP.sas;
 %&mD.put _local_;
 
-%mf_verifymacvars(stpname filename directory tree)
+%mp_abort(
+  iftrue=(%mf_verifymacvars(stpname filename directory tree)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Empty inputs: stpname filename directory tree)
+)
+
 %mp_dropmembers(%scan(&outds,2,.))
 
 /**

package.json

@@ -33,6 +33,7 @@
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "3.6.0"
+    "@sasjs/cli": "3.6.0",
+    "@sasjs/core": "4.4.7"
   }
 }

sasjs/sasjsconfig.json

@@ -2,13 +2,15 @@
   "$schema": "https://raw.githubusercontent.com/sasjs/utils/main/src/types/sasjsconfig-schema.json",
   "macroFolders": [
     "base",
+    "ddl",
     "fcmp",
     "meta",
     "metax",
     "server",
     "viya",
     "lua",
-    "tests/crossplatform"
+    "tests/crossplatform",
+    "tests/ddl"
   ],
   "docConfig": {
     "displayMacroCore": false,
@@ -59,6 +61,7 @@
         "tests/sas9only"
       ],
       "programFolders": [],
+      "binaryFolders": [],
       "deployConfig": {
         "deployServicePack": true,
         "deployScripts": []

tests/crossplatform/mf_getapploc.test.sas

@@ -46,4 +46,20 @@
   ),
   desc=Checking tests/macros appLoc matches (which has no subfolder),
   outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/tests/testsetup)"="/some/area"
+  ),
+  desc=Checking tests/testsetup operation,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_getapploc(/some/area/tests/testteardown)"="/some/area"
+  ),
+  desc=Checking tests/teardown operation,
+  outds=work.test_results
 )

tests/crossplatform/mf_verifymacvars.test.sas

@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Testing mf_verifymacvars macro
+
+  <h4> SAS Macros </h4>
+  @li mf_verifymacvars.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%let var1=x;
+%let var2=y;
+
+%mp_assertscope(SNAPSHOT)
+%mp_assert(
+  iftrue=(%mf_verifymacvars(var1 var2)=1),
+  desc=Checking macvars exist,
+  outds=work.test_results
+)
+%mp_assertscope(COMPARE)
+

tests/crossplatform/mp_ds2csv.test.sas

@@ -93,4 +93,74 @@ run;
   iftrue=("&test3b"="PASS"),
   desc=Checking data row Test 3,
   outds=work.test_results
-)
+)
+
+/* test 4 - sasjs with compare */
+filename example temp;
+%mp_ds2csv(sashelp.air,outref=example,headerformat=SASJS)
+data _null_; infile example; input;put _infile_; if _n_>5 then stop;run;
+
+data _null_;
+  infile example;
+  input;
+  call symputx('stmnt',_infile_);
+  stop;
+run;
+data work.want;
+  infile example dsd firstobs=2;
+  input &stmnt;
+run;
+
+%mp_assert(
+  iftrue=(&syscc =0),
+  desc=Checking syscc prior to compare of sashelp.air,
+  outds=work.test_results
+)
+
+proc compare base=want compare=sashelp.air;
+run;
+%mp_assert(
+  iftrue=(&sysinfo le 41),
+  desc=Checking compare of sashelp.air,
+  outds=work.test_results
+)
+
+/* test 5 - sasjs with time/datetime/date */
+filename f2 temp;
+data work.test5;
+  do x=1 to 5;
+    y=x;
+    z=x;
+  end;
+  format x date9. y datetime19. z time.;
+run;
+%mp_ds2csv(work.test5,outref=f2,headerformat=SASJS)
+data _null_; infile example; input;put _infile_; if _n_>5 then stop;run;
+
+data _null_;
+  infile f2;
+  input;
+  putlog _infile_;
+  call symputx('stmnt2',_infile_);
+  stop;
+run;
+data work.want5;
+  infile f2 dsd firstobs=2;
+  input &stmnt2;
+  putlog _infile_;
+run;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking syscc prior to compare of test5,
+  outds=work.test_results
+)
+
+proc compare base=want5 compare=work.test5;
+run;
+%mp_assert(
+  iftrue=(&sysinfo le 41),
+  desc=Checking compare of work.test5,
+  outds=work.test_results
+)
+

tests/crossplatform/mp_filterstore.test.2.sas

@@ -0,0 +1,112 @@
+/**
+  @file
+  @brief Testing mp_filterstore macro with a format catalog
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
+  @li mp_coretable.sas
+  @li mp_filterstore.sas
+
+**/
+
+libname permlib (work);
+
+%mp_coretable(LOCKTABLE,libds=permlib.locktable)
+%mp_coretable(FILTER_SUMMARY,libds=permlib.filtsum)
+%mp_coretable(FILTER_DETAIL,libds=permlib.filtdet)
+%mp_coretable(MAXKEYTABLE,libds=permlib.maxkey)
+
+/* valid filter */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$12. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,Start,>,"'2'"
+AND,AND,1,Fmtname,=,"'MORDOR'"
+OR,OR,2,Label,IN,"('Dragon1','Dragon2')"
+OR,OR,2,End,=,"'6'"
+OR,OR,2,Start,GE,"'10'"
+;;;;
+run;
+
+/* make some formats */
+PROC FORMAT library=permlib.testfmts;
+  picture MyMSdt other='%0Y-%0m-%0dT%0H:%0M:%0S' (datatype=datetime);
+RUN;
+data work.fmts;
+  length fmtname $32;
+  do fmtname='SMAUG','MORDOR','GOLLUM';
+    do start=1 to 10;
+      label= cats('Dragon ',start);
+      output;
+    end;
+  end;
+run;
+proc sort data=work.fmts nodupkey;
+  by fmtname start;
+run;
+proc format cntlin=work.fmts library=permlib.testfmts;
+run;
+proc format library=permlib.testfmts;
+  invalue indays (default=13) other=42;
+run;
+
+
+%mp_assertscope(SNAPSHOT)
+%mp_filterstore(libds=permlib.testfmts-fc,
+  queryds=work.inds,
+  filter_summary=permlib.filtsum,
+  filter_detail=permlib.filtdet,
+  lock_table=permlib.locktable,
+  maxkeytable=permlib.maxkey,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+)
+%mp_assertscope(COMPARE)
+
+%mp_assert(iftrue=(&syscc=0),
+  desc=Ensure macro runs without errors,
+  outds=work.test_results
+)
+/* ensure only one record created */
+%mp_assertdsobs(permlib.filtsum,
+  desc=Initial query,
+  test=ATMOST 1,
+  outds=work.test_results
+)
+/* check RK is correct */
+proc sql noprint;
+select max(filter_rk) into: test1 from work.result;
+%mp_assert(iftrue=(&test1=1),
+  desc=Ensure filter rk is correct,
+  outds=work.test_results
+)
+
+/* Test 2 - load same table again and ensure we get the same RK */
+%mp_filterstore(libds=permlib.testfmts-fc,
+  queryds=work.inds,
+  filter_summary=permlib.filtsum,
+  filter_detail=permlib.filtdet,
+  lock_table=permlib.locktable,
+  maxkeytable=permlib.maxkey,
+  outresult=work.result,
+  outquery=work.query,
+  mdebug=1
+)
+/* ensure only one record created */
+%mp_assertdsobs(permlib.filtsum,
+  desc=Initial query - same obs,
+  test=ATMOST 1,
+  outds=work.test_results
+)
+/* check RK is correct */
+proc sql noprint;
+select max(filter_rk) into: test2 from work.result;
+%mp_assert(iftrue=(&test2=1),
+  desc=Ensure filter rk is correct for second run,
+  outds=work.test_results
+)

tests/crossplatform/mp_loadformat.test.sas

@@ -0,0 +1,76 @@
+/**
+  @file
+  @brief Testing mp_loadformat.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_loadformat.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+/* prep format catalog */
+libname perm (work);
+
+data work.loadfmts;
+  length fmtname $32;
+  eexcl='Y';
+  type='N';
+  do i=1 to 100;
+    fmtname=cats('SASJS_',i,'X');
+    do j=1 to 100;
+      start=cats(j);
+      end=cats(j+1);
+      label= cats('Dummy ',start);
+      output;
+    end;
+  end;
+run;
+proc format cntlin=work.loadfmts library=perm.testcat;
+run;
+
+/* make some test data */
+data work.stagedata;
+  set work.loadfmts;
+  type='N';
+  eexcl='Y';
+  if _n_<150 then deleteme='Yes';
+  else if _n_<250 then label='mod'!!cats(_n_);
+  else if _n_<350 then do;
+    start=cats(_n_);
+    end=cats(_n_+1);
+    label='newval'!!cats(_N_);
+  end;
+  else stop;
+run;
+
+/* load the above */
+%mp_assertscope(SNAPSHOT)
+%mp_loadformat(perm.testcat
+  ,work.stagedata
+  ,loadtarget=YES
+  ,auditlibds=0
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_test1
+  ,outds_del=del_test1
+  ,outds_mod=mod_test1
+  ,mdebug=1
+)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(%mf_nobs(del_test1)=149),
+  desc=Test 1 - delete obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(add_test1)=100),
+  desc=Test 1 - add obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(mod_test1)=100),
+  desc=Test 1 - mod obs,
+  outds=work.test_results
+)

tests/crossplatform/mp_md5.test.sas

@@ -0,0 +1,41 @@
+/**
+  @file
+  @brief Testing mp_md5.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_md5.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+%global hash1 hash2 hash3;
+
+%mp_assertscope(SNAPSHOT)
+data work.test1 /nonote2err;
+  c1='';
+  c2=repeat('x',32767);
+  c3='  f';
+  n1=.a;
+  n2=.;
+  n3=1.0000000001;
+  hash=%mp_md5(cvars=c1 c2 c3,nvars=n1 n2 n3);
+  call symputx('hash1',hash);
+  n1=.b;
+  hash=%mp_md5(cvars=c1 c2 c3,nvars=n1 n2 n3);
+  call symputx('hash2',hash);
+  c3='f';
+  hash=%mp_md5(cvars=c1 c2 c3,nvars=n1 n2 n3);
+  call symputx('hash3',hash);
+run;
+%mp_assertscope(COMPARE,ignorelist=HASH1 HASH2 HASH3)
+
+%mp_assert(
+  iftrue=("&hash1" ne "&hash2"),
+  desc=Checking first hash diff,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&hash2" ne "&hash3"),
+  desc=Checking first hash diff,
+  outds=work.test_results
+)

tests/crossplatform/mp_streamfile.test.sas

@@ -0,0 +1,28 @@
+/**
+  @file
+  @brief Testing mp_streamfile.sas macro
+  @details This is tricky to test as it streams to webout.  For now just
+  check the compilation, and for macro leakage.
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mp_streamfile.sas
+
+**/
+
+%mp_assertscope(SNAPSHOT)
+
+%mp_streamfile(iftrue=(1=0)
+  ,contenttype=csv,inloc=/some/where.txt
+  ,outname=myfile.txt
+)
+
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking error condition,
+  outds=work.test_results
+)
+

tests/ddl/mddl_dc_difftable.test.sas

@@ -0,0 +1,19 @@
+/**
+  @file
+  @brief Difftable DDL test
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mf_existds.sas
+  @li mp_assert.sas
+
+**/
+
+
+%mddl_dc_difftable(libds=WORK.DIFFTABLE)
+
+%mp_assert(
+  iftrue=(%mf_existds(WORK.DIFFTABLE)=1),
+  desc=Checking table was created,
+  outds=work.test_results
+)

tests/ddl/mddl_dc_filterdetail.test.sas

@@ -0,0 +1,18 @@
+/**
+  @file
+  @brief Filtertable DDL test
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_filterdetail.sas
+  @li mf_existds.sas
+  @li mp_assert.sas
+
+**/
+
+%mddl_dc_filterdetail(libds=WORK.TEST)
+
+%mp_assert(
+  iftrue=(%mf_existds(WORK.TEST)=1),
+  desc=Checking table was created,
+  outds=work.test_results
+)

tests/ddl/mddl_dc_filtersummary.test.sas

@@ -0,0 +1,18 @@
+/**
+  @file
+  @brief Filtersummary DDL test
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_filtersummary.sas
+  @li mf_existds.sas
+  @li mp_assert.sas
+
+**/
+
+%mddl_dc_filtersummary(libds=WORK.TEST)
+
+%mp_assert(
+  iftrue=(%mf_existds(WORK.TEST)=1),
+  desc=Checking table was created,
+  outds=work.test_results
+)

tests/ddl/mddl_dc_locktable.test.sas

@@ -0,0 +1,18 @@
+/**
+  @file
+  @brief Locktable DDL test
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_locktable.sas
+  @li mf_existds.sas
+  @li mp_assert.sas
+
+**/
+
+%mddl_dc_locktable(libds=WORK.TEST)
+
+%mp_assert(
+  iftrue=(%mf_existds(WORK.TEST)=1),
+  desc=Checking table was created,
+  outds=work.test_results
+)

tests/ddl/mddl_dc_maxkeytable.test.sas

@@ -0,0 +1,18 @@
+/**
+  @file
+  @brief Maxkeytable DDL test
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_maxkeytable.sas
+  @li mf_existds.sas
+  @li mp_assert.sas
+
+**/
+
+%mddl_dc_maxkeytable(libds=WORK.TEST)
+
+%mp_assert(
+  iftrue=(%mf_existds(WORK.TEST)=1),
+  desc=Checking table was created,
+  outds=work.test_results
+)

tests/ddl/mddl_sas_cntlout.test.sas

@@ -0,0 +1,18 @@
+/**
+  @file
+  @brief mddl_sas_cntlout ddl test
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_existds.sas
+  @li mp_assert.sas
+
+**/
+
+%mddl_sas_cntlout(libds=WORK.TEST)
+
+%mp_assert(
+  iftrue=(%mf_existds(WORK.TEST)=1),
+  desc=Checking table was created,
+  outds=work.test_results
+)

tests/testinit.sas

@@ -19,7 +19,7 @@
 %macro loglevel();
   %if "&_debug"="2477" or "&_debug"="fields,log,trace" %then %do;
     %put debug mode activated;
-    options mprint;
+    options mprint mprintnest;
   %end;
 %mend loglevel;
 

tests/testterm.sas

@@ -2,8 +2,18 @@
   @file
   @brief term file for tests
 
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+
 **/
 
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking final error condition,
+  outds=work.test_results
+)
+
+
 %webout(OPEN)
 %webout(OBJ, TEST_RESULTS)
 %webout(CLOSE)

tests/viyaonly/mv_jobflow.test.1.sas

@@ -76,3 +76,6 @@ run;
   iftrue=(&syscc ne 0),
   desc=Check that non zero return code is returned if called job fails
 )
+
+/* set syscc to zero for final check in testterm */
+%let syscc=0;